From 2d643b106998ce9bdc449dfa1d32670538af9fdb Mon Sep 17 00:00:00 2001 From: Benjamin Attal Date: Thu, 5 Jun 2014 23:38:11 -0400 Subject: [PATCH] Stop ruby parser from failing. Add other parser fixes. Should be good to go now. --- bitshift/crawler/indexer.py | 1 + bitshift/parser/__init__.py | 21 ++++++++++++++------- bitshift/parser/python.py | 6 +++--- .../com/bitshift/parsing/parsers/JavaParser.java | 12 ++++++------ parsers/ruby/lib/parser.rb | 6 ++++-- 5 files changed, 28 insertions(+), 18 deletions(-) diff --git a/bitshift/crawler/indexer.py b/bitshift/crawler/indexer.py index ba70b7f..209078d 100644 --- a/bitshift/crawler/indexer.py +++ b/bitshift/crawler/indexer.py @@ -17,6 +17,7 @@ import bs4 from ..database import Database from ..parser import parse, UnsupportedFileError +from ..languages import LANGS from ..codelet import Codelet GIT_CLONE_DIR = "/tmp/bitshift" diff --git a/bitshift/parser/__init__.py b/bitshift/parser/__init__.py index 44e4c40..6d4229b 100644 --- a/bitshift/parser/__init__.py +++ b/bitshift/parser/__init__.py @@ -15,10 +15,10 @@ __all__ = ["parse", "UnsupportedFileError", "start_parse_servers"] PARSER_COMMANDS = [ ('Java', ['mvn', '-f', path.join(path.dirname(__file__), "../../parsers/java/pom.xml"), - 'exec:java', '-Dexec.args="%d"']), + 'exec:java', '-Dexec.args=%d']), ('Ruby', ['rake', '-f', path.join(path.dirname(__file__), "../../parsers/ruby/Rakefile"), - "'start_server[%d]'"]) + 'start_server[%d]']) ] class UnsupportedFileError(Exception): @@ -94,9 +94,8 @@ def start_parse_servers(): procs = [] for (lang, cmd) in PARSER_COMMANDS: - procs.append( - subprocess.Popen(' '.join(cmd) % (5001 + LANGS.index(lang)), - shell=True)) + cmd[-1] = cmd[-1] % (5001 + LANGS.index(lang)) + procs.append(subprocess.Popen(cmd)) return procs @@ -130,12 +129,20 @@ def parse(codelet): lang_string = LANGS[lang] codelet.language = lang + def loc_helper(l): + for i in l: + if i == -1: + yield None + else: + yield i + + if lang_string in PARSERS: symbols = PARSERS[lang_string](codelet) symbols = { key: [(name, - [tuple(loc) for loc in syms[name]["assignments"]], - [tuple(loc) for loc in syms[name]["uses"]]) + [tuple(loc_helper(loc)) for loc in syms[name]["assignments"]], + [tuple(loc_helper(loc)) for loc in syms[name]["uses"]]) for name in syms] for key, syms in symbols.iteritems()} codelet.symbols = symbols diff --git a/bitshift/parser/python.py b/bitshift/parser/python.py index 67d2631..d7ac108 100644 --- a/bitshift/parser/python.py +++ b/bitshift/parser/python.py @@ -58,7 +58,7 @@ class _CachedWalker(ast.NodeVisitor): """ line, col = node.lineno, node.col_offset - pos = (line, col, line, col) + pos = (line, col, -1, -1) self.cache.append({'nodes': []}) self.generic_visit(node) @@ -107,7 +107,7 @@ class _CachedWalker(ast.NodeVisitor): """ line, col = node.lineno, node.col_offset - pos = (line, col, line, col) + pos = (line, col, -1, -1) if isinstance(node.func, ast.Name): name = node.func.id @@ -154,7 +154,7 @@ def parse_py(codelet): :param codelet: The codelet object to parsed. - :type code: list + :type code: Codelet """ def strip_encoding(lines): diff --git a/parsers/java/src/main/java/com/bitshift/parsing/parsers/JavaParser.java b/parsers/java/src/main/java/com/bitshift/parsing/parsers/JavaParser.java index bea65d4..989c0dd 100644 --- a/parsers/java/src/main/java/com/bitshift/parsing/parsers/JavaParser.java +++ b/parsers/java/src/main/java/com/bitshift/parsing/parsers/JavaParser.java @@ -80,8 +80,8 @@ public class JavaParser extends Parser { int sl = this.root.getLineNumber(node.getStartPosition()); int sc = this.root.getColumnNumber(node.getStartPosition()); - int el = sl; - int ec = sc; + Integer el = -1; + Integer ec = -1; if (statements.size() > 0) { Statement last = statements.get(statements.size() - 1); @@ -89,7 +89,7 @@ public class JavaParser extends Parser { ec = this.root.getColumnNumber(last.getStartPosition()); } - data.put("coord", Symbols.createCoord(sl, sc, sl, sc)); + data.put("coord", Symbols.createCoord(sl, sc, el, ec)); data.put("name", name); this._cache.push(data); return true; @@ -110,7 +110,7 @@ public class JavaParser extends Parser { int sl = this.root.getLineNumber(node.getStartPosition()); int sc = this.root.getColumnNumber(node.getStartPosition()); - data.put("coord", Symbols.createCoord(sl, sc, sl, sc)); + data.put("coord", Symbols.createCoord(sl, sc, -1, -1)); data.put("name", name); this._cache.push(data); return true; @@ -140,7 +140,7 @@ public class JavaParser extends Parser { int sl = this.root.getLineNumber(node.getStartPosition()); int sc = this.root.getColumnNumber(node.getStartPosition()); - data.put("coord", Symbols.createCoord(sl, sc, sl, sc)); + data.put("coord", Symbols.createCoord(sl, sc, -1, -1)); this._cache.push(data); return true; } @@ -161,7 +161,7 @@ public class JavaParser extends Parser { int sl = this.root.getLineNumber(node.getStartPosition()); int sc = this.root.getColumnNumber(node.getStartPosition()); - data.put("coord", Symbols.createCoord(sl, sc, sl, sc)); + data.put("coord", Symbols.createCoord(sl, sc, -1, -1)); this._cache.push(data); return true; } diff --git a/parsers/ruby/lib/parser.rb b/parsers/ruby/lib/parser.rb index e4ff4eb..2a94317 100644 --- a/parsers/ruby/lib/parser.rb +++ b/parsers/ruby/lib/parser.rb @@ -11,6 +11,8 @@ module Bitshift def parse parser = RubyParser.new tree = parser.parse(@source) + return '{}' if tree.nil? + offset = tree.line - 1 processor = CachedWalker.new offset, tree processor.process(tree) @@ -51,7 +53,7 @@ module Bitshift break if cur_exp == nil end - pos = [start_ln, 1, end_ln, 1] + pos = [start_ln, -1, end_ln, -1] return pos end @@ -59,7 +61,7 @@ module Bitshift pos = Hash.new end_ln = start_ln = exp.line - offset - pos = [start_ln, 1, end_ln, 1] + pos = [start_ln, -1, end_ln, -1] return pos end