Browse Source

Stop ruby parser from failing. Add other parser fixes. Should be good

to go now.
tags/v1.0^2
Benjamin Attal 10 years ago
parent
commit
2d643b1069
5 changed files with 28 additions and 18 deletions
  1. +1
    -0
      bitshift/crawler/indexer.py
  2. +14
    -7
      bitshift/parser/__init__.py
  3. +3
    -3
      bitshift/parser/python.py
  4. +6
    -6
      parsers/java/src/main/java/com/bitshift/parsing/parsers/JavaParser.java
  5. +4
    -2
      parsers/ruby/lib/parser.rb

+ 1
- 0
bitshift/crawler/indexer.py View File

@@ -17,6 +17,7 @@ import bs4

from ..database import Database
from ..parser import parse, UnsupportedFileError
from ..languages import LANGS
from ..codelet import Codelet

GIT_CLONE_DIR = "/tmp/bitshift"


+ 14
- 7
bitshift/parser/__init__.py View File

@@ -15,10 +15,10 @@ __all__ = ["parse", "UnsupportedFileError", "start_parse_servers"]
PARSER_COMMANDS = [
('Java', ['mvn', '-f',
path.join(path.dirname(__file__), "../../parsers/java/pom.xml"),
'exec:java', '-Dexec.args="%d"']),
'exec:java', '-Dexec.args=%d']),
('Ruby', ['rake', '-f',
path.join(path.dirname(__file__), "../../parsers/ruby/Rakefile"),
"'start_server[%d]'"])
'start_server[%d]'])
]

class UnsupportedFileError(Exception):
@@ -94,9 +94,8 @@ def start_parse_servers():
procs = []

for (lang, cmd) in PARSER_COMMANDS:
procs.append(
subprocess.Popen(' '.join(cmd) % (5001 + LANGS.index(lang)),
shell=True))
cmd[-1] = cmd[-1] % (5001 + LANGS.index(lang))
procs.append(subprocess.Popen(cmd))

return procs

@@ -130,12 +129,20 @@ def parse(codelet):
lang_string = LANGS[lang]
codelet.language = lang

def loc_helper(l):
for i in l:
if i == -1:
yield None
else:
yield i


if lang_string in PARSERS:
symbols = PARSERS[lang_string](codelet)
symbols = {
key: [(name,
[tuple(loc) for loc in syms[name]["assignments"]],
[tuple(loc) for loc in syms[name]["uses"]])
[tuple(loc_helper(loc)) for loc in syms[name]["assignments"]],
[tuple(loc_helper(loc)) for loc in syms[name]["uses"]])
for name in syms]
for key, syms in symbols.iteritems()}
codelet.symbols = symbols

+ 3
- 3
bitshift/parser/python.py View File

@@ -58,7 +58,7 @@ class _CachedWalker(ast.NodeVisitor):
"""

line, col = node.lineno, node.col_offset
pos = (line, col, line, col)
pos = (line, col, -1, -1)

self.cache.append({'nodes': []})
self.generic_visit(node)
@@ -107,7 +107,7 @@ class _CachedWalker(ast.NodeVisitor):
"""

line, col = node.lineno, node.col_offset
pos = (line, col, line, col)
pos = (line, col, -1, -1)

if isinstance(node.func, ast.Name):
name = node.func.id
@@ -154,7 +154,7 @@ def parse_py(codelet):

:param codelet: The codelet object to parsed.

:type code: list
:type code: Codelet
"""

def strip_encoding(lines):


+ 6
- 6
parsers/java/src/main/java/com/bitshift/parsing/parsers/JavaParser.java View File

@@ -80,8 +80,8 @@ public class JavaParser extends Parser {

int sl = this.root.getLineNumber(node.getStartPosition());
int sc = this.root.getColumnNumber(node.getStartPosition());
int el = sl;
int ec = sc;
Integer el = -1;
Integer ec = -1;

if (statements.size() > 0) {
Statement last = statements.get(statements.size() - 1);
@@ -89,7 +89,7 @@ public class JavaParser extends Parser {
ec = this.root.getColumnNumber(last.getStartPosition());
}

data.put("coord", Symbols.createCoord(sl, sc, sl, sc));
data.put("coord", Symbols.createCoord(sl, sc, el, ec));
data.put("name", name);
this._cache.push(data);
return true;
@@ -110,7 +110,7 @@ public class JavaParser extends Parser {
int sl = this.root.getLineNumber(node.getStartPosition());
int sc = this.root.getColumnNumber(node.getStartPosition());

data.put("coord", Symbols.createCoord(sl, sc, sl, sc));
data.put("coord", Symbols.createCoord(sl, sc, -1, -1));
data.put("name", name);
this._cache.push(data);
return true;
@@ -140,7 +140,7 @@ public class JavaParser extends Parser {
int sl = this.root.getLineNumber(node.getStartPosition());
int sc = this.root.getColumnNumber(node.getStartPosition());

data.put("coord", Symbols.createCoord(sl, sc, sl, sc));
data.put("coord", Symbols.createCoord(sl, sc, -1, -1));
this._cache.push(data);
return true;
}
@@ -161,7 +161,7 @@ public class JavaParser extends Parser {
int sl = this.root.getLineNumber(node.getStartPosition());
int sc = this.root.getColumnNumber(node.getStartPosition());

data.put("coord", Symbols.createCoord(sl, sc, sl, sc));
data.put("coord", Symbols.createCoord(sl, sc, -1, -1));
this._cache.push(data);
return true;
}


+ 4
- 2
parsers/ruby/lib/parser.rb View File

@@ -11,6 +11,8 @@ module Bitshift
def parse
parser = RubyParser.new
tree = parser.parse(@source)
return '{}' if tree.nil?

offset = tree.line - 1
processor = CachedWalker.new offset, tree
processor.process(tree)
@@ -51,7 +53,7 @@ module Bitshift
break if cur_exp == nil
end

pos = [start_ln, 1, end_ln, 1]
pos = [start_ln, -1, end_ln, -1]
return pos
end

@@ -59,7 +61,7 @@ module Bitshift
pos = Hash.new
end_ln = start_ln = exp.line - offset

pos = [start_ln, 1, end_ln, 1]
pos = [start_ln, -1, end_ln, -1]
return pos
end



Loading…
Cancel
Save