Browse Source

API documentation, fixes, support not highlighting (closes #56).

tags/v1.0^2
Ben Kurtovic 9 years ago
parent
commit
e64c4924f4
10 changed files with 249 additions and 96 deletions
  1. +8
    -6
      app.py
  2. +12
    -5
      bitshift/codelet.py
  3. +2
    -15
      static/js/index.js
  4. +0
    -7
      static/sass/developers.sass
  5. +3
    -3
      static/sass/docs.sass
  6. +2
    -2
      static/sitemap.xml
  7. +0
    -18
      templates/developers.html
  8. +203
    -39
      templates/docs.html
  9. +0
    -1
      templates/layout.html
  10. +19
    -0
      test/find_function_def.py

+ 8
- 6
app.py View File

@@ -31,29 +31,31 @@ def search():
resp.mimetype = "application/json"
return resp

query, page = request.args.get("q"), request.args.get("p", 1)
query = request.args.get("q")
if not query:
return reply({"error": "No query given"})
try:
tree = parse_query(query)
except QueryParseException as exc:
return reply({"error": exc.args[0]})

page = request.args.get("p", 1)
try:
page = int(page)
except ValueError:
return reply({"error": u"Invalid page number: %s" % page})

highlight = request.args.get("hl", "0")
highlight = highlight.lower() not in ["0", "false", "no"]

count, codelets = database.search(tree, page)
results = [clt.serialize() for clt in codelets]
results = [clt.serialize(highlight) for clt in codelets]
return reply({"count": count, "results": results})

@app.route("/about")
def about():
return render_template("about.html")

@app.route("/developers")
def developers():
return render_template("developers.html")

@app.route("/docs")
def docs():
return render_template("docs.html")


+ 12
- 5
bitshift/codelet.py View File

@@ -74,18 +74,25 @@ class Codelet(object):
self.symbols = symbols or {}
self.origin = origin or (None, None)

def serialize(self):
def serialize(self, highlight=False):
"""
Convert the codelet into a dictionary that can be sent as JSON.

:param highlight: Whether to return code as pygments-highlighted HTML
or as plain source.
:type highlight: bool

:return: The codelet as a dictionary.
:rtype: str
"""
lang = LANGS[self.language]
lines = reduce(concat, [[loc[0] for loc in sym[1] + sym[2]] for sym in
reduce(concat, self.symbols.values(), [])], [])
formatter = HtmlFormatter(linenos=True, hl_lines=lines)
code = highlight(self.code, get_lexer_by_name(lang.lower()), formatter)
code = self.code
if highlight:
symbols = reduce(concat, self.symbols.values(), [])
lines = reduce(concat, [[loc[0] for loc in sym[1] + sym[2]]
for sym in symbols], [])
formatter = HtmlFormatter(linenos=True, hl_lines=lines)
code = highlight(code, get_lexer_by_name(lang.lower()), formatter)

return {
"name": self.name, "code": code, "lang": lang,


+ 2
- 15
static/js/index.js
File diff suppressed because it is too large
View File


+ 0
- 7
static/sass/developers.sass View File

@@ -1,7 +0,0 @@
@import variables

a
text-decoration: none

h1
color: $baseColor1

+ 3
- 3
static/sass/docs.sass View File

@@ -21,17 +21,17 @@ ul
margin-bottom: 2%

span
&#code
&.code
background-color: $baseColor3 * 1.2
font-family: monospace
padding: 5px

&#string
&.string
color: $baseColor1
font-family: monospace
font-size: 1.1em

&#title
&.title
color: $baseColor1
font-weight: bold



+ 2
- 2
static/sitemap.xml View File

@@ -4,13 +4,13 @@
<url>
<loc>http://bitshift.it/</loc>
<changefreq>monthly</changefreq>
</url>
</url>
<url>
<loc>http://bitshift.it/about</loc>
<changefreq>monthly</changefreq>
</url>
<url>
<loc>http://bitshift.it/developers</loc>
<loc>http://bitshift.it/docs</loc>
<changefreq>monthly</changefreq>
</url>
</urlset>

+ 0
- 18
templates/developers.html View File

@@ -1,18 +0,0 @@
= extends "layout.html"

= block title
About
= endblock

= block head
{{ assets.tag("developers.css") }}
= endblock

= block body
<div>
<h1>Page under construction</h1>
<p>
Check back later. Have you read our <a href="/about">about page</a> and the <a href="/docs">docs</a>?
</p>
</div>
= endblock

+ 203
- 39
templates/docs.html View File

@@ -5,6 +5,8 @@
= endblock

= block head
{{ assets.tag("lib/highlight.css") }}

{{ assets.tag("docs.css") }}
= endblock

@@ -21,84 +23,112 @@
<li>
<h1><span>&raquo;</span> Usage</h1>
<p>
<span id="title">bitshift</span> is a search-engine optimized for source code: beyond supporting searches with the full range of ASCII
symbols, the engine <em>understands</em> code, allowing users to query for metadata, like time of creation/last
modification, programming language, and even symbols like function names and variables. Basic use boils down to
general and advanced searches.
<span class="title">bitshift</span> is a search-engine optimized for
source code: beyond supporting searches with the full range of ASCII
symbols, the engine <em>understands</em> code, allowing users to query
for metadata, like time of creation/last modification, programming
language, and even symbols like function names and variables. Basic use
boils down to general and advanced searches.

<ul>
<li>
<h2>general search</h2>
<p>
To perform a "general search," simply place your cursor in the search bar on our home page and begin
entering text; when you stop typing for a short period of time, we'll automatically execute the query for you.
As you scroll down the results page, new codelets will be seamlessly downloaded from our server and
appended to the end.
To perform a "general search," simply place your cursor in the
search bar on our home page and begin entering text; when you
stop typing for a short period of time, we'll automatically
execute the query for you. As you scroll down the page, new
codelets, or results, will be seamlessly downloaded from our
server and appended to the end.
</p>
</li>

<li>
<h2>advanced search</h2>
<p>
General searches, though, are limited. To allow users to make the best of our engine, we created an advanced
search form that allows the creation of complex queries with the following specifiers:
General searches, though, are limited. To allow users to make the
best of our engine, we created an advanced search form that
allows the creation of complex queries with the following
specifiers:

<ul>
<li>
<h3>search fields</h3>
<ul id="search-fields">
<li>
<span id="code">languages</span> : The programming languages to search for.
<span class="code">languages</span> : The programming
languages to search for.
</li>
<li>
<span id="code">authors</span> : Search for code written/modified by a specific person.
<span class="code">authors</span> : Search for code
written/modified by a specific person.
</li>
<li>
<span id="code">date last modified</span> : Search for code last modified on a specific date (<span
id="code">mm/dd/yy</span> format).
<span class="code">date last modified</span> : Search for
code last modified on a specific date
(<span id="code">mm/dd/yy</span> format).
</li>
<li>
<span id="code">date created</span> : Search for code created on a specific date (<span
id="code">mm/dd/yy</span> format).
<span class="code">date created</span> : Search for code
created on a specific date
(<span id="code">mm/dd/yy</span> format).
</li>
<li>
<span id="code">symbols</span> : Search for specific symbols.
<span class="code">symbols</span> : Search for specific
symbols.
</li>
<li>
<span id="code">functions</span> : Search for functions with specific names.
<span class="code">functions</span> : Search for
functions with specific names.
</li>
<li>
<span id="code">classes</span> : Search for classes with specific names.
<span class="code">classes</span> : Search for classes
with specific names.
</li>
<li>
<span id="code">variables</span> : Search for variables with specific names.
<span class="code">variables</span> : Search for
variables with specific names.
</li>
</ul>

<p>
Each of the search fields allows for numerous values; just separate them with spaces. If you'd like
to search for a multi-word, space-delimited string, on the other hand, enclose it in double quotes.
Each of the search fields allows for numerous values; just
separate them with spaces. If you'd like to search for a
multi-word, space-delimited string, on the other hand,
enclose it in double quotes.

A query for <span id="code">foo bar</span> will search for occurrences of both <span
id="string">"foo"</span> and <span id="string">"bar"</span>, while <span id="code">"foo bar"</span>
will search for occurrences of <span id="string">"foo bar"</span>.
A query for <span class="code">foo bar</span> will search
for occurrences of both <span id="string">"foo"</span> and
<span class="string">"bar"</span>, while
<span class="code">"foo bar"</span> will search for
occurrences of <span class="string">"foo bar"</span>.
</p>
</li>

<li>
<h3>search groups</h3>
<p>
Search groups facilitate even more robust queries: they're like a bunch of individual searches
grouped into one. A user searching for occurrenes of symbol <span id="string">"curses"</span> in the
language <span id="string">"Python"</span>, and <span id="string">"ncurses"</span> in <span
id="string">"C"</span>, won't get away with: <span id="code">symbols: "curses ncurses"</span> and
<span id="code">languages: "Python C"</span>. The engine might return results <span
id="string">"curses"</span> in <span id="string">"C"</span> and <span id="string">"ncurses"</span> in
<span id="string">"Python"</span>!

To work around that, you can use two search groups: one for <span id="string">"curses"</span> in
<span id="string">"Python"</span>, and another for <span id="string">"curses"</span> in <span
id="string">"C"</span>. <span id="title">bitshift</span> will return the union of both sets of search results.
Search groups facilitate even more robust queries: they're
like a bunch of individual searches grouped into one. A
user searching for occurrenes of symbol
<span class="string">"curses"</span> in the language
<span class="string">"Python"</span>, and
<span class="string">"ncurses"</span> in
<span id="string">"C"</span>, won't get away with:
<span class="code">"symbols:curses ncurses"</span> and
<span class="code">"languages:Python C"</span>. The engine
might return results <span id="string">"curses"</span> in
<span class="string">"C"</span> and
<span class="string">"ncurses"</span> in
<span class="string">"Python"</span>!

To work around that, you can use two search groups: one for
<span class="string">"curses"</span> in
<span class="string">"Python"</span>, and another for
<span class="string">"curses"</span> in
<span id="string">"C"</span>.
<span class="title">bitshift</span> will return the union
of both sets of search results.
</p>
</li>
</ul>
@@ -115,11 +145,145 @@
</p>
</li>

<li>
<h1><span>&raquo;</span> API</h1>
<p>
<span class="title">bitshift</span> provides an API through GET
requests to
<a href="http://bitshift.it/search.json"><span class="code">/search.json</span></a>.
</p>
<h2>parameters</h2>
<ul>
<li>
<span class="code">q</span> : The search query, as entered into the
search bar.
</li>
<li>
<span class="code">p</span> : The result page to return. Defaults to
<span class="code">1</span>. Each page contains ten results, so this
effectively offsets the search by
<span class="code">10 * (p - 1)</span> codelets.
</li>
<li>
<span class="code">hl</span> : Whether to return code as
<a href="http://pygments.org/">pygments</a>-highlighted HTML or as
plain source. Defaults to <span class="code">false</span>.
</li>
</ul>
<h2>output</h2>
<p>
<span class="code">/search.json</span> returns a JSON-encoded
dictionary. If there was an error, it will contain a single key,
<span class="code">"error"</span>, whose value will contain a
human-readable description of the error. Otherwise, there will be two
keys: <span class="code">"count"</span>, storing the number of results,
and <span class="code">"results"</span>, storing a list of codelets.
Each codelet is a dictionary with the following key–value pairs:
</p>
<ul>
<li>
<span class="code">name</span> : The name of the codelet.
</li>
<li>
<span class="code">code</span> : The actual source code if
<span class="code">hl</span> was not given or was
<span class="code">false</span>; HTML code otherwise.
</li>
<li>
<span class="code">lang</span> : The language of the code.
</li>
<li>
<span class="code">authors</span> : A list of authors. Each author is
a list of two items: their name, and URL (or
<span class="code">null</span> if none is known).
</li>
<li>
<span class="code">url</span> : The URL of the page where the code
was crawled from.
</li>
<li>
<span class="code">created</span> : The date the code was created, as
a
<a href="https://en.wikipedia.org/wiki/ISO_8601">ISO 8601</a>-formatted
string (e.g. <span class="code">"2014-06-01T12:41:28"</span>).
</li>
<li>
<span class="code">modified</span> : The date the code was last
modified, as a
<a href="https://en.wikipedia.org/wiki/ISO_8601">ISO 8601</a>-formatted
string (e.g. <span class="code">"2014-06-01T12:41:28"</span>).
</li>
<li>
<span class="code">origin</span> : A list of two items: the
originating site's name (e.g. <span class="code">"GitHub"</span>) and
URL (e.g. <span class="code">"https://github.com"</span>).
</li>
</ul>
<h2>example</h2>
<p>
The following example Python 2 code searches for a given Python
function definition and prints the URL of the first result:
</p>
<table class="highlighttable">
<tr>
<td class="linenos">
<div class="linenodiv">
<pre> 1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19</pre>
</div>
</td>
<td class="code">
<div class="highlight">
<pre><span class="c">#!/usr/bin/env python</span>

<span class="kn">from</span> <span class="nn">json</span> <span class="kn">import</span> <span class="n">loads</span>
<span class="kn">from</span> <span class="nn">sys</span> <span class="kn">import</span> <span class="n">argv</span>
<span class="kn">from</span> <span class="nn">urllib</span> <span class="kn">import</span> <span class="n">urlencode</span>
<span class="kn">from</span> <span class="nn">urllib2</span> <span class="kn">import</span> <span class="n">urlopen</span>

<span class="k">def</span> <span class="nf">get_function</span><span class="p">(</span><span class="n">name</span><span class="p">):</span>
<span class="n">params</span> <span class="o">=</span> <span class="p">{</span><span class="s">&quot;q&quot;</span><span class="p">:</span> <span class="s">&quot;lang:python and func:def:</span><span class="si">%s</span><span class="s">&quot;</span> <span class="o">%</span> <span class="n">name</span><span class="p">}</span>
<span class="n">request</span> <span class="o">=</span> <span class="n">urlopen</span><span class="p">(</span><span class="s">&quot;http://bitshift.it/search.json?&quot;</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">(</span><span class="n">params</span><span class="p">))</span>
<span class="n">res</span> <span class="o">=</span> <span class="n">loads</span><span class="p">(</span><span class="n">request</span><span class="o">.</span><span class="n">read</span><span class="p">())[</span><span class="s">&quot;results&quot;</span><span class="p">]</span>
<span class="k">if</span> <span class="n">res</span><span class="p">:</span>
<span class="k">print</span> <span class="s">&quot;</span><span class="si">%s</span><span class="s">: </span><span class="si">%s</span><span class="s">&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">res</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="s">&quot;url&quot;</span><span class="p">])</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">print</span> <span class="s">&quot;</span><span class="si">%s</span><span class="s"> not found.&quot;</span> <span class="o">%</span> <span class="n">name</span>

<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s">&quot;__main__&quot;</span><span class="p">:</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">argv</span><span class="p">)</span> <span class="o">==</span> <span class="mi">2</span><span class="p">:</span>
<span class="n">get_function</span><span class="p">(</span><span class="n">argv</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span>
</pre>
</div>
</td>
</tr>
</table>
</li>

<li id="sec3">
<h1><span>&raquo;</span> Get involved</h1>
<h1><span>&raquo;</span> Get Involved</h1>
<p>
<span id="title">bitshift</span> is <span id="gasp">(gasp)</span> open-source! The project is hosted on
<a href="https://github.com/earwig/bitshift">GitHub</a>; feel free to file an issue or submit a pull request!
<span class="title">bitshift</span> is <span id="gasp">(gasp)</span>
open-source! The project is hosted on
<a href="https://github.com/earwig/bitshift">GitHub</a>; feel free to
file an issue or submit a pull request.
</p>
</li>
</ul>


+ 0
- 1
templates/layout.html View File

@@ -41,7 +41,6 @@
<a href="/">home</a>
<a href="/about">about</a>
<a href="/docs">docs</a>
<a href="/developers">developers</a>
</div>
</div>
</body>


+ 19
- 0
test/find_function_def.py View File

@@ -0,0 +1,19 @@
#!/usr/bin/env python

from json import loads
from sys import argv
from urllib import urlencode
from urllib2 import urlopen

def get_function(name):
params = {"q": "lang:python and func:def:%s" % name}
request = urlopen("http://bitshift.it/search.json?" + urlencode(params))
res = loads(request.read())["results"]
if res:
print "%s: %s" % (name, res[0]["url"])
else:
print "%s not found." % name

if __name__ == "__main__":
if len(argv) == 2:
get_function(argv[1])

Loading…
Cancel
Save