diff --git a/.gitignore b/.gitignore index 7663424..3c8288c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,10 @@ +static/css/* +!lib + +*.swp .sass-cache +.DS_Store +.my.cnf # github premade rules *.py[cod] @@ -18,7 +24,6 @@ var sdist develop-eggs .installed.cfg -lib lib64 __pycache__ @@ -37,3 +42,15 @@ nosetests.xml .mr.developer.cfg .project .pydevproject + +# Maven +target + +# Ruby +!parsers/ruby/lib + +# Ctags +*/tags +logs +Gemfile.lock +parsing.jar diff --git a/LICENSE b/LICENSE index edcb6e5..feaf8b8 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2014 Ben Kurtovic +Copyright (c) 2014 Benjamin Attal, Ben Kurtovic, Severyn Kozak Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file +SOFTWARE. diff --git a/README.md b/README.md index e6de751..6e8bacd 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,44 @@ bitshift ======== -bitshift is an online code snippet exchange. +bitshift is a semantic search engine for source code developed by Benjamin +Attal, Ben Kurtovic, and Severyn Kozak. This README is intended for developers +only. For a user overview of the project: + + * read our [about page](http://bitshift.it/) + * watch our [demo video](https://vimeo.com/98697078) + +Branches +-------- + +- `master`: working, tested, version-numbered code - no direct commits; should + only accept merges from `develop` when ready to release +- `develop`: integration branch with unreleased but mostly functional code - + direct commits allowed but should be minor +- `feature/*`: individual components of the project with untested, likely + horribly broken code - branch off from and merge into `develop` when done + +Style +----- +bitshift uses [SASS][SASS] for styling; compile the stylesheets to CSS with +`sass --watch static/sass/:static/css`. + +Documentation +------------- + +To build documentation, run `make html` from the `docs` subdirectory. You can +then browse from `docs/build/html/index.html`. + +To automatically update the API documentation structure (necessary when adding +new modules or packages, but *not* when adding functions or changing +docstrings), run `sphinx-apidoc -fo docs/source/api bitshift` from the project +root. Note that this will revert any custom changes made to the files in +`docs/source/api`, so you might want to update them by hand instead. + +[SASS]: http://sass-lang.com/guide + +Releasing +--------- + +- Update `__version__` in `bitshift/__init__.py`, `version` in `setup.py`, and + `version` and `release` in `docs/conf.py`. diff --git a/app.py b/app.py index a7508c4..f05ba67 100644 --- a/app.py +++ b/app.py @@ -2,21 +2,67 @@ Module to contain all the project's Flask server plumbing. """ -from flask import Flask -from flask import render_template, session +from json import dumps -from bitshift import * +from flask import Flask, make_response, render_template, request -app = Flask(__name__) +from bitshift import assets +from bitshift.database import Database +from bitshift.languages import LANGS +from bitshift.query import parse_query, QueryParseException + +app = Flask(__name__, static_folder="static", static_url_path="") app.config.from_object("bitshift.config") app_env = app.jinja_env app_env.line_statement_prefix = "=" -app_env.globals.update(assets = assets) +app_env.globals.update(assets=assets) + +database = Database() @app.route("/") def index(): - return render_template("index.html") + return render_template("index.html", autocomplete_languages=LANGS) + +@app.route("/search.json") +def search(): + def reply(json): + resp = make_response(dumps(json)) + resp.mimetype = "application/json" + return resp + + query = request.args.get("q") + if not query: + return reply({"error": "No query given"}) + try: + tree = parse_query(query) + except QueryParseException as exc: + return reply({"error": exc.args[0]}) + + page = request.args.get("p", 1) + try: + page = int(page) + except ValueError: + return reply({"error": u"Invalid page number: %s" % page}) + + highlight = request.args.get("hl", "0") + highlight = highlight.lower() not in ["0", "false", "no"] + + count, codelets = database.search(tree, page) + results = [clt.serialize(highlight) for clt in codelets] + return reply({"count": count, "results": results}) + +@app.route("/about") +def about(): + return render_template("about.html") + +@app.route("/docs") +def docs(): + return render_template("docs.html") + +@app.errorhandler(404) +def error404(error): + return render_template("error404.html"), 404 if __name__ == "__main__": - app.run() + app.run(debug=True) diff --git a/bitshift/__init__.py b/bitshift/__init__.py index d51957e..0bd031c 100644 --- a/bitshift/__init__.py +++ b/bitshift/__init__.py @@ -1 +1,8 @@ -__all__ = ["config", "assets"] +# -*- coding: utf-8 -*- + +__author__ = "Benjamin Attal, Ben Kurtovic, Severyn Kozak" +__copyright__ = "Copyright (c) 2014 Benjamin Attal, Ben Kurtovic, Severyn Kozak" +__license__ = "MIT License" +__version__ = "0.1.dev" + +from . import assets, codelet, config, crawler, database, parser, query diff --git a/bitshift/assets.py b/bitshift/assets.py index 4754036..c7b7c11 100644 --- a/bitshift/assets.py +++ b/bitshift/assets.py @@ -1,22 +1,46 @@ """ -Module contains helper functions to be used inside the project's Jinja -templates. +:synopsis: Helper functions for use inside the project's Jinja templates. """ +import re + from flask import Markup ASSET_HTML_TEMPLATES = { - 'css': "", - 'js': "" + 'css': "", + 'js': "" } def tag(filename): """ - Return HTML tag for asset named filename. + Generate an HTML tag for a CSS/JS asset, based on its file extension. + + :param filename: The filename of the asset to create a tag for. + + :type filename: str - Return either a = endblock = block body -

Hello, world.

+
+ + + +
+ +
+ + {{ assets.tag("index.js") }} + {{ assets.tag("index.advanced-search-form.js") }} += endblock + += block after_body + = endblock diff --git a/templates/layout.html b/templates/layout.html index 9ab3eac..b8cd7f5 100644 --- a/templates/layout.html +++ b/templates/layout.html @@ -4,24 +4,44 @@ - = block title - = endblock + bitshift « + = filter lower + = block title + = endblock + = endfilter - - + + {{ assets.tag("main.css") }} + {{ assets.tag("main.js") }} = block head = endblock - = block body - = endblock +
+ + +
+
+ = block body + = endblock +
+
+ + = block after_body + = endblock + + +
diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/find_function_def.py b/test/find_function_def.py new file mode 100755 index 0000000..187370b --- /dev/null +++ b/test/find_function_def.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python + +from json import loads +from sys import argv +from urllib import urlencode +from urllib2 import urlopen + +def get_function(name): + params = {"q": "lang:python and func:def:%s" % name} + request = urlopen("http://bitshift.it/search.json?" + urlencode(params)) + res = loads(request.read())["results"] + if res: + print "%s: %s" % (name, res[0]["url"]) + else: + print "%s not found." % name + +if __name__ == "__main__": + if len(argv) == 2: + get_function(argv[1]) diff --git a/test/parser_test.py b/test/parser_test.py new file mode 100644 index 0000000..ffee75c --- /dev/null +++ b/test/parser_test.py @@ -0,0 +1,56 @@ +import socket, sys, struct + +file_name = 'resources/.c' +server_socket_number = 5001 +recv_size = 8192 + +if __name__ == '__main__': + if len(sys.argv) == 1: + print "Please input a parser to test." + + elif len(sys.argv) > 2: + print "Too many arguments." + + else: + if sys.argv[1] == 'c': + pass + + elif sys.argv[1] == 'java': + file_name = "resources/Matrix.java" + server_socket_number = 5002 + + elif sys.argv[1] == 'ruby': + file_name = "resources/parser.rb" + server_socket_number = 5065 + + server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server_socket.connect(("localhost", server_socket_number)) + + with open(file_name, "r") as source_file: + source = source_file.read() + server_socket.send("%d\n%s" % (len(source), source)); + + total_data = []; size_data = cur_data = '' + total_size = 0; size = sys.maxint + + while total_size < size: + cur_data = server_socket.recv(recv_size) + + if not total_data: + if len(size_data) > 4: + size_data += cur_data + size = struct.unpack('>i', size_data[:4])[0] + recv_size = size + if recv_size > sys.maxint: recv_size = sys.maxint + total_data.append(size_data[4:]) + else: + size_data += cur_data + + else: + total_data.append(cur_data) + + total_size = sum([len(s) for s in total_data]) + + + server_socket.close() + print ''.join(total_data); diff --git a/test/resources/Matrix.java b/test/resources/Matrix.java new file mode 100644 index 0000000..5d641e0 --- /dev/null +++ b/test/resources/Matrix.java @@ -0,0 +1,218 @@ +package battlechap; + +import java.io.PrintStream; + +public class Matrix { + private Object[][] _datmatrix; + + public Matrix(int paramInt){ + this._datmatrix = new Object[paramInt][paramInt]; + } + + public int size() { + return this._datmatrix.length; + } + + public Object get(int paramInt1, int paramInt2) { + return this._datmatrix[paramInt1][paramInt2]; + } + + public boolean isEmpty(int paramInt1, int paramInt2) { + return this._datmatrix[paramInt1][paramInt2] == null; + } + + public boolean equals(Object paramObject) { + boolean bool = true; + if ((paramObject instanceof Matrix)) { + Matrix localMatrix = (Matrix)paramObject; + if (localMatrix.size() == size()) { + for (int i = 0; i < size(); i++) { + for (int j = 0; j < size(); j++) { + if (!localMatrix.get(i, j).equals(get(i, j))) { + bool = false; + break; + } + } + if (!bool) + break; + } + } + else + bool = false; + } + else + { + bool = false; + } + return bool; + } + + public Object set(int paramInt1, int paramInt2, Object paramObject) { + Object localObject = this._datmatrix[paramInt1][paramInt2]; + this._datmatrix[paramInt1][paramInt2] = paramObject; + return localObject; + } + + public void transpose() { + int i = 0; + for (int j = 0; j < size(); j++) { + for (int k = i; k < size(); k++) { + set(j, k, set(k, j, get(j, k))); + } + i++; + } + } + + public static void swapRows(int paramInt1, int paramInt2, Object[][] paramArrayOfObject) { + for (int i = 0; i < paramArrayOfObject[paramInt1].length; i++) { + Object localObject = paramArrayOfObject[paramInt1][i]; + paramArrayOfObject[paramInt1][i] = paramArrayOfObject[paramInt2][i]; + paramArrayOfObject[paramInt2][i] = localObject; + } + } + + public static void swapCols(int paramInt1, int paramInt2, Object[][] paramArrayOfObject) { + for (int i = 0; i < paramArrayOfObject.length; i++) { + Object localObject = paramArrayOfObject[i][paramInt1]; + paramArrayOfObject[i][paramInt1] = paramArrayOfObject[i][paramInt2]; + paramArrayOfObject[i][paramInt2] = localObject; + } + } + + public Object[] getRow(int paramInt) { + Object[] arrayOfObject = new Object[this._datmatrix[paramInt].length]; + for (int i = 0; i < arrayOfObject.length; i++) { + arrayOfObject[i] = this._datmatrix[paramInt][i]; + } + return arrayOfObject; + } + + public Object[] getCol(int paramInt) { + Object[] arrayOfObject = new Object[this._datmatrix[paramInt].length]; + for (int i = 0; i < arrayOfObject.length; i++) { + arrayOfObject[i] = this._datmatrix[i][paramInt]; + } + return arrayOfObject; + } + + public Object[] setRow(int paramInt, Object[] paramArrayOfObject) { + Object[] arrayOfObject = getRow(paramInt); + + for (int i = 0; i < size(); i++) { + set(paramInt, i, paramArrayOfObject[i]); + } + + return arrayOfObject; + } + + public Object[] setCol(int paramInt, Object[] paramArrayOfObject) { + Object[] arrayOfObject = getCol(paramInt); + + for (int i = 0; i < size(); i++) { + set(i, paramInt, paramArrayOfObject[i]); + } + + return arrayOfObject; + } + + public String toString() + { + String str1 = ""; + for (int i = 0; i < this._datmatrix.length; i++) { + if (i < 9) + str1 = str1 + (i + 1) + ": "; + else + str1 = str1 + (i + 1) + ":"; + for (int j = 0; j < this._datmatrix[i].length; j++) { + int k = (this._datmatrix[i][j] + "").length(); + String str2 = " ".substring(k); + str1 = str1 + this._datmatrix[i][j] + str2; + } + str1 = str1 + "\n"; + } + return str1; + } + + public static void print(Object[][] paramArrayOfObject) { + for (int i = 0; i < paramArrayOfObject.length; i++) { + for (int j = 0; j < paramArrayOfObject[i].length; j++) { + int k = (paramArrayOfObject[i][j] + "").length(); + String str = " ".substring(k); + System.out.print(paramArrayOfObject[i][j] + str); + } + System.out.print("\n"); + } + } + + public static void printArray(Object[] paramArrayOfObject) { + for (int i = 0; i < paramArrayOfObject.length; i++) { + int j = (paramArrayOfObject[i] + "").length(); + String str = " ".substring(j); + System.out.print(paramArrayOfObject[i] + str); + } + System.out.print("\n"); + } + + public static void main(String[] paramArrayOfString) { + Matrix localMatrix1 = new Matrix(5); + Matrix localMatrix2 = new Matrix(5); + for (int i = 0; i < localMatrix1.size(); i++) { + for (int j = 0; j < localMatrix1.size(); j++) { + Integer localInteger1 = new Integer((int)(Math.random() * 20.0D)); + localMatrix1.set(i, j, localInteger1); + localMatrix2.set(i, j, localInteger1); + } + } + + System.out.println("\nDemonstrating equals method (should be true)\t" + localMatrix2.equals(localMatrix1) + "\n"); + + System.out.println("Demonstrating get method\n" + localMatrix1.get(0, 0) + "\n"); + System.out.println("Demonstrating is empty method\n" + localMatrix1.isEmpty(1, 0) + "\n"); + System.out.println("Demonstrating size method \n" + localMatrix1.size() + "\n"); + System.out.println("Demonstrating toString method\n" + localMatrix1 + "\n"); + localMatrix1.transpose(); + System.out.println("Blop has been transposed\n" + localMatrix1 + "\n"); + + Object[][] arrayOfObject = new Object[4][4]; + for (int j = 0; j < arrayOfObject.length; j++) { + for (int k = 0; k < arrayOfObject[j].length; k++) { + Integer localInteger2 = new Integer((int)(Math.random() * 20.0D)); + arrayOfObject[j][k] = localInteger2; + } + } + System.out.println("\n\n**Swapping Rows Demo**"); + print(arrayOfObject); + System.out.println("\nRows 1 and 2 have been Swapped \n"); + swapRows(1, 2, arrayOfObject); + print(arrayOfObject); + + System.out.println("\n**Swapping Columns Demo**"); + print(arrayOfObject); + System.out.println("\n\nColumns 1 and 2 have been Swapped \n"); + swapCols(1, 2, arrayOfObject); + print(arrayOfObject); + + System.out.println("\n**Getting rows demo (from blop)**"); + System.out.println(localMatrix1); + System.out.println("\nGetting row 1\n"); + printArray(localMatrix1.getRow(1)); + + System.out.println("\n**Getting cols demo (from blop)**"); + System.out.println(localMatrix1); + System.out.println("\nGetting col 1\n"); + printArray(localMatrix1.getCol(1)); + + System.out.println("\n**Demonstrating set row method**"); + System.out.println(localMatrix1); + System.out.println("\nSwitching row 1 of blop to 1st column of blop\n"); + localMatrix1.setRow(1, localMatrix1.getCol(1)); + System.out.println(localMatrix1 + "\n"); + + System.out.println("\n**Demonstrating set col method**"); + System.out.println(localMatrix1); + System.out.println("\nSwitching col 1 of blop to 2nd row of blop\n"); + localMatrix1.setCol(1, localMatrix1.getRow(2)); + System.out.println(localMatrix1 + "\n"); + } +} + diff --git a/test/resources/app.py b/test/resources/app.py new file mode 100644 index 0000000..3fae72f --- /dev/null +++ b/test/resources/app.py @@ -0,0 +1,40 @@ +""" +Module to contain all the project's Flask server plumbing. +""" + +from flask import Flask +from flask import render_template, session + +from bitshift import assets +# from bitshift.database import Database +# from bitshift.query import parse_query + +app = Flask(__name__) +app.config.from_object("bitshift.config") + +app_env = app.jinja_env +app_env.line_statement_prefix = "=" +app_env.globals.update(assets=assets) + +# database = Database() + +@app.route("/") +def index(): + return render_template("index.html") + +@app.route("/search/") +def search(query): + # tree = parse_query(query) + # database.search(tree) + pass + +@app.route("/about") +def about(): + return render_template("about.html") + +@app.route("/developers") +def developers(): + return render_template("developers.html") + +if __name__ == "__main__": + app.run(debug=True) diff --git a/test/resources/parser.rb b/test/resources/parser.rb new file mode 100644 index 0000000..01d934b --- /dev/null +++ b/test/resources/parser.rb @@ -0,0 +1,126 @@ +require 'socket' +require 'ruby_parser' +require 'sexp_processor' + +module Bitshift + class Parser + def initialize(source) + @source = source + end + + def parse + parser = RubyParser.new + tree = parser.parse(@source) + puts tree.inspect + offset = tree.line - 1 + processor = NodeVisitor.new offset + processor.process tree + return processor.symbols + end + end + + class NodeVisitor < SexpProcessor + attr_accessor :symbols + attr_accessor :offset + + def initialize(offset) + super() + @require_empty = false + @offset = offset + + module_hash = Hash.new {|hash, key| hash[key] = Hash.new} + class_hash = module_hash.clone + function_hash = Hash.new {|hash, key| hash[key] = { calls: [] } } + var_hash = Hash.new {|hash, key| hash[key] = [] } + + @symbols = { + modules: module_hash, + classes: class_hash, + functions: function_hash, + vars: var_hash + } + end + + def block_position(exp) + pos = Hash.new + end_ln = (start_ln = exp.line - offset) + cur_exp = exp + + while cur_exp.is_a? Sexp + end_ln = cur_exp.line - offset + cur_exp = cur_exp.last + break if cur_exp == nil + end + + pos[:coord] = { + start_ln: start_ln, + end_ln: end_ln } + return pos + end + + def statement_position(exp) + pos = Hash.new + end_ln = start_ln = exp.line - offset + + pos[:coord] = { + start_ln: start_ln, + end_ln: end_ln } + return pos + end + + def process_module(exp) + pos = block_position exp + exp.shift + name = exp.shift + symbols[:modules][name] = pos + exp.each_sexp {|s| process(s)} + return exp.clear + end + + def process_class(exp) + pos = block_position exp + exp.shift + name = exp.shift + symbols[:classes][name] = pos + exp.each_sexp {|s| process(s)} + return exp.clear + end + + def process_defn(exp) + pos = block_position exp + exp.shift + name = exp.shift + symbols[:functions][name][:declaration] = pos + exp.each_sexp {|s| process(s)} + return exp.clear + end + + def process_call(exp) + pos = statement_position exp + exp.shift + exp.shift + name = exp.shift + symbols[:functions][name][:calls] << pos + exp.each_sexp {|s| process(s)} + return exp.clear + end + + def process_iasgn(exp) + pos = statement_position exp + exp.shift + name = exp.shift + symbols[:vars][name] << pos + exp.each_sexp {|s| process(s)} + return exp.clear + end + + def process_lasgn(exp) + pos = statement_position exp + exp.shift + name = exp.shift + symbols[:vars][name] << pos + exp.each_sexp {|s| process(s)} + return exp.clear + end + end +end diff --git a/test/test_query_parser.py b/test/test_query_parser.py new file mode 100644 index 0000000..7156a6f --- /dev/null +++ b/test/test_query_parser.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- + +from __future__ import unicode_literals +import unittest + +from bitshift.query import parse_query + +TESTS = [ + # Text + ("test", "Tree(Text(String(u'test')))"), + ("re:test", "Tree(Text(Regex(u'test')))"), + + # Language + ("language:python", "Tree(Language(Python))"), + ("language:py", "Tree(Language(Python))"), + ("l:r:r..y", "Tree(Language(Ruby))"), + (r'"lang:re:python|^c$"', + "Tree(BinaryOp(Language(C), OR, Language(Python)))"), + + # Author + ('"author:Ben Kurtovic"', "Tree(Author(String(u'Ben Kurtovic')))"), + (r"'a:re:b.*?\sk.*?'", r"Tree(Author(Regex(u'b.*?\\sk.*?')))"), + + # Date + ("'create:before:Jan 1, 2014'", + "Tree(Date(CREATE, BEFORE, 2014-01-01 00:00:00))"), + ("'modify:after:2010-05-09 10:11:12'", + "Tree(Date(MODIFY, AFTER, 2010-05-09 10:11:12))"), + + # Symbol + ("sym:foobar", "Tree(Symbol(ALL, ALL, String(u'foobar')))"), + ("func:foo_bar", "Tree(Symbol(ALL, FUNCTION, String(u'foo_bar')))"), + ("func:foo_bar()", "Tree(Symbol(ALL, FUNCTION, String(u'foo_bar')))"), + ("class:FooBar", "Tree(Symbol(ALL, CLASS, String(u'FooBar')))"), + ("var:foobar", "Tree(Symbol(ALL, VARIABLE, String(u'foobar')))"), + ("var:r:foobar", "Tree(Symbol(ALL, VARIABLE, Regex(u'foobar')))"), + + # Composition + ("(a and b) or (c and d)", ", ".join([ + "Tree(BinaryOp(BinaryOp(Text(String(u'a'))", "AND", + "Text(String(u'b')))", "OR", "BinaryOp(Text(String(u'c'))", "AND", + "Text(String(u'd')))))"])), + ("a and b or c and d", ", ".join([ + "Tree(BinaryOp(BinaryOp(Text(String(u'a'))", "AND", + "Text(String(u'b')))", "OR", "BinaryOp(Text(String(u'c'))", "AND", + "Text(String(u'd')))))"])), + ("a and b or c or d", ", ".join([ + "Tree(BinaryOp(BinaryOp(Text(String(u'a'))", "AND", + "Text(String(u'b')))", "OR", "BinaryOp(Text(String(u'c'))", "OR", + "Text(String(u'd')))))"])), + ("a and (b or c or d)", ", ".join([ + "Tree(BinaryOp(Text(String(u'a'))", "AND", + "BinaryOp(Text(String(u'b'))", "OR", "BinaryOp(Text(String(u'c'))", "OR", + "Text(String(u'd'))))))"])), + ("a not b", ", ".join([ + "Tree(BinaryOp(Text(String(u'a'))", "AND", "UnaryOp(NOT", + "Text(String(u'b')))))"])), + + # Unicode, Escaping + (r'lang:py "author:fo\\o \"bar\" baz\\"', ", ".join([ + "Tree(BinaryOp(Language(Python)", "AND", + "Author(String(u'fo\\\\o \"bar\" baz\\\\'))))"])), + ('"author:Ben Kurtović"', "Tree(Author(String(u'Ben Kurtovi\\u0107')))") +] + +class TestQueryParser(unittest.TestCase): + """Unit tests for the query parser in :py:mod:`bitshift.query`.""" + + def test_parse(self): + """test full query parsing""" + for test, expected in TESTS: + self.assertEqual(expected, parse_query(test).serialize()) + + +if __name__ == "__main__": + unittest.main(verbosity=2)