Stop ruby parser from failing. Add other parser fixes. Should be good

to go now.

bitshift/crawler/indexer.py

@@ -17,6 +17,7 @@ import bs4
 
 from ..database import Database
 from ..parser import parse, UnsupportedFileError
+from ..languages import LANGS
 from ..codelet import Codelet
 
 GIT_CLONE_DIR = "/tmp/bitshift"

bitshift/parser/__init__.py

@@ -15,10 +15,10 @@ __all__ = ["parse", "UnsupportedFileError", "start_parse_servers"]
 PARSER_COMMANDS = [
         ('Java', ['mvn', '-f',
             path.join(path.dirname(__file__), "../../parsers/java/pom.xml"),
-            'exec:java', '-Dexec.args="%d"']),
+            'exec:java', '-Dexec.args=%d']),
         ('Ruby', ['rake', '-f',
             path.join(path.dirname(__file__), "../../parsers/ruby/Rakefile"),
-            "'start_server[%d]'"])
+            'start_server[%d]'])
 ]
 
 class UnsupportedFileError(Exception):
@@ -94,9 +94,8 @@ def start_parse_servers():
     procs = []
 
     for (lang, cmd) in PARSER_COMMANDS:
-        procs.append(
-                subprocess.Popen(' '.join(cmd) % (5001 + LANGS.index(lang)),
-                    shell=True))
+        cmd[-1] = cmd[-1] % (5001 + LANGS.index(lang))
+        procs.append(subprocess.Popen(cmd))
 
     return procs
 
@@ -130,12 +129,20 @@ def parse(codelet):
     lang_string = LANGS[lang]
     codelet.language = lang
 
+    def loc_helper(l):
+        for i in l:
+            if i == -1:
+                yield None
+            else:
+                yield i
+
+
     if lang_string in PARSERS:
         symbols = PARSERS[lang_string](codelet)
         symbols = {
             key: [(name,
-                   [tuple(loc) for loc in syms[name]["assignments"]],
-                   [tuple(loc) for loc in syms[name]["uses"]])
+                   [tuple(loc_helper(loc)) for loc in syms[name]["assignments"]],
+                   [tuple(loc_helper(loc)) for loc in syms[name]["uses"]])
                   for name in syms]
             for key, syms in symbols.iteritems()}
         codelet.symbols = symbols

bitshift/parser/python.py

@@ -58,7 +58,7 @@ class _CachedWalker(ast.NodeVisitor):
         """
 
         line, col = node.lineno, node.col_offset
-        pos = (line, col, line, col)
+        pos = (line, col, -1, -1)
 
         self.cache.append({'nodes': []})
         self.generic_visit(node)
@@ -107,7 +107,7 @@ class _CachedWalker(ast.NodeVisitor):
         """
 
         line, col = node.lineno, node.col_offset
-        pos = (line, col, line, col)
+        pos = (line, col, -1, -1)
 
         if isinstance(node.func, ast.Name):
             name = node.func.id
@@ -154,7 +154,7 @@ def parse_py(codelet):
 
     :param codelet: The codelet object to parsed.
 
-    :type code: list
+    :type code: Codelet
     """
 
     def strip_encoding(lines):

parsers/java/src/main/java/com/bitshift/parsing/parsers/JavaParser.java

@@ -80,8 +80,8 @@ public class JavaParser extends Parser {
 
             int sl = this.root.getLineNumber(node.getStartPosition());
             int sc = this.root.getColumnNumber(node.getStartPosition());
-            int el = sl;
-            int ec = sc;
+            Integer el = -1;
+            Integer ec = -1;
 
             if (statements.size() > 0) {
                 Statement last = statements.get(statements.size() - 1);
@@ -89,7 +89,7 @@ public class JavaParser extends Parser {
                 ec = this.root.getColumnNumber(last.getStartPosition());
             }
 
-            data.put("coord", Symbols.createCoord(sl, sc, sl, sc));
+            data.put("coord", Symbols.createCoord(sl, sc, el, ec));
             data.put("name", name);
             this._cache.push(data);
             return true;
@@ -110,7 +110,7 @@ public class JavaParser extends Parser {
             int sl = this.root.getLineNumber(node.getStartPosition());
             int sc = this.root.getColumnNumber(node.getStartPosition());
 
-            data.put("coord", Symbols.createCoord(sl, sc, sl, sc));
+            data.put("coord", Symbols.createCoord(sl, sc, -1, -1));
             data.put("name", name);
             this._cache.push(data);
             return true;
@@ -140,7 +140,7 @@ public class JavaParser extends Parser {
             int sl = this.root.getLineNumber(node.getStartPosition());
             int sc = this.root.getColumnNumber(node.getStartPosition());
 
-            data.put("coord", Symbols.createCoord(sl, sc, sl, sc));
+            data.put("coord", Symbols.createCoord(sl, sc, -1, -1));
             this._cache.push(data);
             return true;
         }
@@ -161,7 +161,7 @@ public class JavaParser extends Parser {
             int sl = this.root.getLineNumber(node.getStartPosition());
             int sc = this.root.getColumnNumber(node.getStartPosition());
 
-            data.put("coord", Symbols.createCoord(sl, sc, sl, sc));
+            data.put("coord", Symbols.createCoord(sl, sc, -1, -1));
             this._cache.push(data);
             return true;
         }

parsers/ruby/lib/parser.rb

@@ -11,6 +11,8 @@ module Bitshift
         def parse
             parser = RubyParser.new
             tree = parser.parse(@source)
+            return '{}' if tree.nil?
+
             offset = tree.line - 1
             processor = CachedWalker.new offset, tree
             processor.process(tree)
@@ -51,7 +53,7 @@ module Bitshift
                 break if cur_exp == nil
             end
 
-            pos = [start_ln, 1, end_ln, 1]
+            pos = [start_ln, -1, end_ln, -1]
             return pos
         end
 
@@ -59,7 +61,7 @@ module Bitshift
             pos = Hash.new
             end_ln = start_ln = exp.line - offset
 
-            pos = [start_ln, 1, end_ln, 1]
+            pos = [start_ln, -1, end_ln, -1]
             return pos
         end