diff --git a/bitshift/parser/__init__.py b/bitshift/parser/__init__.py
index 5522ae6..55c76e1 100644
--- a/bitshift/parser/__init__.py
+++ b/bitshift/parser/__init__.py
@@ -1,4 +1,4 @@
-import ast, pygments.lexers as pgl, sys, socket, struct
+import json, pygments.lexers as pgl, sys, socket, struct
from ..languages import LANGS
from .python import parse_py
@@ -19,13 +19,13 @@ def _lang(codelet):
Modify function to incorporate tags from stackoverflow.
"""
- try:
- if codelet.filename is not None:
+ if codelet.filename is not None:
+ try:
return pgl.guess_lexer_for_filename(codelet.filename, '').name
+ except:
+ raise UnsupportedFileError('Could not find a lexer for the codelet\'s filename')
- return LANGS.index(pgl.guess_lexer(codelet.code))
- except:
- raise UnsupportedFileError('Could not find a lexer for the codelet\'s filename')
+ return LANGS.index(pgl.guess_lexer(codelet.code))
def _recv_data(server_socket):
"""
@@ -88,6 +88,6 @@ def parse(codelet):
server_socket.connect(("localhost", server_socket_number))
server_socket.send("%d\n%s" % (len(source), source));
- symbols = ast.literal_eval(_recv_data(server_socket))
+ symbols = json.loads(_recv_data(server_socket))
codelet.symbols = symbols
diff --git a/bitshift/parser/python.py b/bitshift/parser/python.py
index ac71e29..7e9b109 100644
--- a/bitshift/parser/python.py
+++ b/bitshift/parser/python.py
@@ -1,6 +1,6 @@
import ast
-class _TreeCutter(ast.NodeVisitor):
+class _CachedWalker(ast.NodeVisitor):
"""
Local node visitor for python abstract syntax trees.
@@ -22,9 +22,9 @@ class _TreeCutter(ast.NodeVisitor):
"""
self.accum = {'vars': {}, 'functions': {}, 'classes': {}}
- self.cache = None
+ self.cache = []
- def start_n_end(self, node):
+ def block_position(self, node):
"""
Helper function to get the start and end lines of an AST node.
@@ -54,32 +54,19 @@ class _TreeCutter(ast.NodeVisitor):
Add value and type metadata to accum.
"""
- for t in node.targets:
- if isinstance(t, ast.Tuple):
- for n in t.elts:
- line, col = n.lineno, n.col_offset
+ line, col = node.lineno, node.col_offset
+ pos = (line, col, line, col)
- if not self.accum['vars'].has_key(node.name):
- self.accum['vars'][node.name] = {'declaration': {}, 'uses': []}
-
- pos = {'coord': {}}
- pos['coord']['start_line'] = line
- pos['coord']['start_col'] = col
- pos['coord']['end_line'] = line
- pos['coord']['end_col'] = col
- self.accum['vars'][n.id]['declaration'] = pos
+ self.cache.append({'nodes': []})
+ self.generic_visit(node)
+ last = self.cache.pop()
- else:
- line, col = t.lineno, t.col_offset
+ for name in last['nodes']:
+ if not self.accum['vars'].has_key(name):
+ self.accum['vars'][name] = {'assignments': [], 'uses': []}
- pos = {'coord': {}}
- pos['coord']['start_line'] = line
- pos['coord']['start_col'] = col
- pos['coord']['end_line'] = line
- pos['coord']['end_col'] = col
- self.accum['vars'][t.id]['declaration'] = pos
+ self.accum['vars'][name]['assignments'].append(pos)
- self.generic_visit(node)
def visit_FunctionDef(self, node):
"""
@@ -93,17 +80,13 @@ class _TreeCutter(ast.NodeVisitor):
Add arguments and decorators metadata to accum.
"""
- start_line, start_col, end_line, end_col = self.start_n_end(node)
+ start_line, start_col, end_line, end_col = self.block_position(node)
if not self.accum['functions'].has_key(node.name):
- self.accum['functions'][node.name] = {'declaration': {}, 'calls': []}
+ self.accum['functions'][node.name] = {'assignments': [], 'uses': []}
- pos = {'coord': {}}
- pos['coord']['start_ln']= start_line
- pos['coord']['start_col'] = start_col
- pos['coord']['end_ln'] = end_line
- pos['coord']['end_col'] = end_col
- self.accum['functions'][node.name]['declaration'] = pos
+ pos = (start_line, start_col, end_line, end_col)
+ self.accum['functions'][node.name]['assignments'].append(pos)
self.generic_visit(node)
@@ -120,20 +103,18 @@ class _TreeCutter(ast.NodeVisitor):
Add arguments and decorators metadata to accum.
"""
- line, col = node.line_no, node.col_offset
+ line, col = node.lineno, node.col_offset
+ pos = (line, col, line, col)
- if not self.accum['functions'].has_key(node.name):
- self.accum['functions'][node.name] = {'declaration': {}, 'calls': []}
+ if isinstance(node.func, ast.Name):
+ name = node.func.id
+ else:
+ name = node.func.attr
- pos = {'coord': {}}
- pos['coord']['start_line'] = line
- pos['coord']['start_col'] = col
- pos['coord']['end_line'] = line
- pos['coord']['end_col'] = col
- self.accum['functions'][node.name]['calls'].append(pos)
-
- self.generic_visit(node)
+ if not self.accum['functions'].has_key(name):
+ self.accum['functions'][name] = {'assignments': [], 'uses': []}
+ self.accum['functions'][name]['uses'].append(pos)
def visit_ClassDef(self, node):
"""
@@ -147,19 +128,22 @@ class _TreeCutter(ast.NodeVisitor):
Add arguments, inherits, and decorators metadata to accum.
"""
- start_line, start_col, end_line, end_col = self.start_n_end(node)
+ start_line, start_col, end_line, end_col = self.block_position(node)
- pos = {'coord': {}}
- pos['coord']['start_ln']= start_line
- pos['coord']['start_col'] = start_col
- pos['coord']['end_ln'] = end_line
- pos['coord']['end_col'] = end_col
+ pos = (start_line, start_col, end_line, end_col)
self.accum['classes'][node.name] = pos
self.generic_visit(node)
def visit_Name(self, node):
- pass
+ if self.cache:
+ last = self.cache[-1]
+ last['nodes'].append(node.id)
+
+ def visit_Attribute(self, node):
+ if self.cache:
+ last = self.cache[-1]
+ last['nodes'].append(node.attr)
def parse_py(codelet):
"""
@@ -171,6 +155,6 @@ def parse_py(codelet):
"""
tree = ast.parse(codelet.code)
- cutter = _TreeCutter()
+ cutter = _CachedWalker()
cutter.visit(tree)
codelet.symbols = cutter.accum
diff --git a/parsers/java/pom.xml b/parsers/java/pom.xml
index 340feb0..cfecc30 100644
--- a/parsers/java/pom.xml
+++ b/parsers/java/pom.xml
@@ -16,8 +16,7 @@
3.8.1
test
-
-
+
org.eclipse.jdt
core
3.3.0-v_771
diff --git a/parsers/java/src/main/java/com/bitshift/parsing/symbols/JavaSymbols.java b/parsers/java/src/main/java/com/bitshift/parsing/symbols/JavaSymbols.java
index 9265feb..dd15468 100644
--- a/parsers/java/src/main/java/com/bitshift/parsing/symbols/JavaSymbols.java
+++ b/parsers/java/src/main/java/com/bitshift/parsing/symbols/JavaSymbols.java
@@ -42,9 +42,17 @@ public class JavaSymbols extends Symbols {
HashMap method = this._methods.get(name);
if (method == null) {
method = new HashMap();
- method.put("declaration", data);
+ ArrayList