@@ -1,4 +1,4 @@ | |||
import ast, pygments.lexers as pgl, sys, socket, struct | |||
import json, pygments.lexers as pgl, sys, socket, struct | |||
from ..languages import LANGS | |||
from .python import parse_py | |||
@@ -19,13 +19,13 @@ def _lang(codelet): | |||
Modify function to incorporate tags from stackoverflow. | |||
""" | |||
try: | |||
if codelet.filename is not None: | |||
if codelet.filename is not None: | |||
try: | |||
return pgl.guess_lexer_for_filename(codelet.filename, '').name | |||
except: | |||
raise UnsupportedFileError('Could not find a lexer for the codelet\'s filename') | |||
return LANGS.index(pgl.guess_lexer(codelet.code)) | |||
except: | |||
raise UnsupportedFileError('Could not find a lexer for the codelet\'s filename') | |||
return LANGS.index(pgl.guess_lexer(codelet.code)) | |||
def _recv_data(server_socket): | |||
""" | |||
@@ -88,6 +88,6 @@ def parse(codelet): | |||
server_socket.connect(("localhost", server_socket_number)) | |||
server_socket.send("%d\n%s" % (len(source), source)); | |||
symbols = ast.literal_eval(_recv_data(server_socket)) | |||
symbols = json.loads(_recv_data(server_socket)) | |||
codelet.symbols = symbols | |||
@@ -1,6 +1,6 @@ | |||
import ast | |||
class _TreeCutter(ast.NodeVisitor): | |||
class _CachedWalker(ast.NodeVisitor): | |||
""" | |||
Local node visitor for python abstract syntax trees. | |||
@@ -22,9 +22,9 @@ class _TreeCutter(ast.NodeVisitor): | |||
""" | |||
self.accum = {'vars': {}, 'functions': {}, 'classes': {}} | |||
self.cache = None | |||
self.cache = [] | |||
def start_n_end(self, node): | |||
def block_position(self, node): | |||
""" | |||
Helper function to get the start and end lines of an AST node. | |||
@@ -54,32 +54,19 @@ class _TreeCutter(ast.NodeVisitor): | |||
Add value and type metadata to accum. | |||
""" | |||
for t in node.targets: | |||
if isinstance(t, ast.Tuple): | |||
for n in t.elts: | |||
line, col = n.lineno, n.col_offset | |||
line, col = node.lineno, node.col_offset | |||
pos = (line, col, line, col) | |||
if not self.accum['vars'].has_key(node.name): | |||
self.accum['vars'][node.name] = {'declaration': {}, 'uses': []} | |||
pos = {'coord': {}} | |||
pos['coord']['start_line'] = line | |||
pos['coord']['start_col'] = col | |||
pos['coord']['end_line'] = line | |||
pos['coord']['end_col'] = col | |||
self.accum['vars'][n.id]['declaration'] = pos | |||
self.cache.append({'nodes': []}) | |||
self.generic_visit(node) | |||
last = self.cache.pop() | |||
else: | |||
line, col = t.lineno, t.col_offset | |||
for name in last['nodes']: | |||
if not self.accum['vars'].has_key(name): | |||
self.accum['vars'][name] = {'assignments': [], 'uses': []} | |||
pos = {'coord': {}} | |||
pos['coord']['start_line'] = line | |||
pos['coord']['start_col'] = col | |||
pos['coord']['end_line'] = line | |||
pos['coord']['end_col'] = col | |||
self.accum['vars'][t.id]['declaration'] = pos | |||
self.accum['vars'][name]['assignments'].append(pos) | |||
self.generic_visit(node) | |||
def visit_FunctionDef(self, node): | |||
""" | |||
@@ -93,17 +80,13 @@ class _TreeCutter(ast.NodeVisitor): | |||
Add arguments and decorators metadata to accum. | |||
""" | |||
start_line, start_col, end_line, end_col = self.start_n_end(node) | |||
start_line, start_col, end_line, end_col = self.block_position(node) | |||
if not self.accum['functions'].has_key(node.name): | |||
self.accum['functions'][node.name] = {'declaration': {}, 'calls': []} | |||
self.accum['functions'][node.name] = {'assignments': [], 'uses': []} | |||
pos = {'coord': {}} | |||
pos['coord']['start_ln']= start_line | |||
pos['coord']['start_col'] = start_col | |||
pos['coord']['end_ln'] = end_line | |||
pos['coord']['end_col'] = end_col | |||
self.accum['functions'][node.name]['declaration'] = pos | |||
pos = (start_line, start_col, end_line, end_col) | |||
self.accum['functions'][node.name]['assignments'].append(pos) | |||
self.generic_visit(node) | |||
@@ -120,20 +103,18 @@ class _TreeCutter(ast.NodeVisitor): | |||
Add arguments and decorators metadata to accum. | |||
""" | |||
line, col = node.line_no, node.col_offset | |||
line, col = node.lineno, node.col_offset | |||
pos = (line, col, line, col) | |||
if not self.accum['functions'].has_key(node.name): | |||
self.accum['functions'][node.name] = {'declaration': {}, 'calls': []} | |||
if isinstance(node.func, ast.Name): | |||
name = node.func.id | |||
else: | |||
name = node.func.attr | |||
pos = {'coord': {}} | |||
pos['coord']['start_line'] = line | |||
pos['coord']['start_col'] = col | |||
pos['coord']['end_line'] = line | |||
pos['coord']['end_col'] = col | |||
self.accum['functions'][node.name]['calls'].append(pos) | |||
self.generic_visit(node) | |||
if not self.accum['functions'].has_key(name): | |||
self.accum['functions'][name] = {'assignments': [], 'uses': []} | |||
self.accum['functions'][name]['uses'].append(pos) | |||
def visit_ClassDef(self, node): | |||
""" | |||
@@ -147,19 +128,22 @@ class _TreeCutter(ast.NodeVisitor): | |||
Add arguments, inherits, and decorators metadata to accum. | |||
""" | |||
start_line, start_col, end_line, end_col = self.start_n_end(node) | |||
start_line, start_col, end_line, end_col = self.block_position(node) | |||
pos = {'coord': {}} | |||
pos['coord']['start_ln']= start_line | |||
pos['coord']['start_col'] = start_col | |||
pos['coord']['end_ln'] = end_line | |||
pos['coord']['end_col'] = end_col | |||
pos = (start_line, start_col, end_line, end_col) | |||
self.accum['classes'][node.name] = pos | |||
self.generic_visit(node) | |||
def visit_Name(self, node): | |||
pass | |||
if self.cache: | |||
last = self.cache[-1] | |||
last['nodes'].append(node.id) | |||
def visit_Attribute(self, node): | |||
if self.cache: | |||
last = self.cache[-1] | |||
last['nodes'].append(node.attr) | |||
def parse_py(codelet): | |||
""" | |||
@@ -171,6 +155,6 @@ def parse_py(codelet): | |||
""" | |||
tree = ast.parse(codelet.code) | |||
cutter = _TreeCutter() | |||
cutter = _CachedWalker() | |||
cutter.visit(tree) | |||
codelet.symbols = cutter.accum |
@@ -16,8 +16,7 @@ | |||
<version>3.8.1</version> | |||
<scope>test</scope> | |||
</dependency> | |||
<dependency> | |||
<dependency> | |||
<groupId>org.eclipse.jdt</groupId> | |||
<artifactId>core</artifactId> | |||
<version>3.3.0-v_771</version> | |||
@@ -42,9 +42,17 @@ public class JavaSymbols extends Symbols { | |||
HashMap<String, Object> method = this._methods.get(name); | |||
if (method == null) { | |||
method = new HashMap<String, Object>(); | |||
method.put("declaration", data); | |||
ArrayList<Object> assignments = new ArrayList<Object>(10); | |||
ArrayList<Object> uses = new ArrayList<Object>(10); | |||
assignments.add(data.get("coord")); | |||
method.put("assignments", assignments); | |||
method.put("uses", uses); | |||
} else { | |||
method.put("declaration", data); | |||
ArrayList<Object> assignments = (ArrayList<Object>)method.get("assignments"); | |||
assignments.add(data.get("coord")); | |||
method.put("assignments", assignments); | |||
} | |||
this._methods.put(name, method); | |||
@@ -54,14 +62,17 @@ public class JavaSymbols extends Symbols { | |||
HashMap<String, Object> method = this._methods.get(name); | |||
if (method == null) { | |||
method = new HashMap<String, Object>(); | |||
ArrayList<Object> calls = new ArrayList<Object>(10); | |||
calls.add(data); | |||
method.put("calls", calls); | |||
ArrayList<Object> assignments = new ArrayList<Object>(10); | |||
ArrayList<Object> uses = new ArrayList<Object>(10); | |||
uses.add(data.get("coord")); | |||
method.put("assignments", assignments); | |||
method.put("uses", uses); | |||
} else { | |||
ArrayList<Object> calls = (ArrayList<Object>)method.get("calls"); | |||
calls = (calls == null) ? new ArrayList<Object>(10) : calls; | |||
calls.add(data); | |||
method.put("calls", calls); | |||
ArrayList<Object> uses = (ArrayList<Object>)method.get("uses"); | |||
uses.add(data.get("coord")); | |||
method.put("uses", uses); | |||
} | |||
this._methods.put(name, method); | |||
@@ -77,9 +88,17 @@ public class JavaSymbols extends Symbols { | |||
HashMap<String, Object> var = this._vars.get(name); | |||
if (var == null) { | |||
var = new HashMap<String, Object>(); | |||
var.put("declaration", data); | |||
ArrayList<Object> assignments = new ArrayList<Object>(10); | |||
ArrayList<Object> uses = new ArrayList<Object>(10); | |||
assignments.add(data.get("coord")); | |||
var.put("assignments", assignments); | |||
var.put("uses", uses); | |||
} else { | |||
var.put("declaration", data); | |||
ArrayList<Object> assignments = (ArrayList<Object>)var.get("assignments"); | |||
assignments.add(data.get("coord")); | |||
var.put("assignments", assignments); | |||
} | |||
this._vars.put(name, var); | |||
@@ -89,13 +108,16 @@ public class JavaSymbols extends Symbols { | |||
HashMap<String, Object> var = this._vars.get(name); | |||
if (var == null) { | |||
var = new HashMap<String, Object>(); | |||
ArrayList<Object> assignments = new ArrayList<Object>(10); | |||
ArrayList<Object> uses = new ArrayList<Object>(10); | |||
uses.add(data); | |||
uses.add(data.get("coord")); | |||
var.put("assignments", assignments); | |||
var.put("uses", uses); | |||
} else { | |||
ArrayList<Object> uses = (ArrayList<Object>)var.get("uses"); | |||
uses = (uses == null) ? new ArrayList<Object>(10) : uses; | |||
uses.add(data); | |||
uses.add(data.get("coord")); | |||
var.put("uses", uses); | |||
} | |||
@@ -12,23 +12,23 @@ module Bitshift | |||
parser = RubyParser.new | |||
tree = parser.parse(@source) | |||
offset = tree.line - 1 | |||
processor = NodeVisitor.new offset, tree | |||
processor = CachedWalker.new offset, tree | |||
processor.process(tree) | |||
return processor.to_s | |||
end | |||
end | |||
class NodeVisitor < SexpProcessor | |||
class CachedWalker < SexpProcessor | |||
attr_accessor :symbols | |||
attr_accessor :offset | |||
def initialize(offset, tree) | |||
super() | |||
module_hash = Hash.new {|hash, key| hash[key] = Hash.new} | |||
module_hash = Hash.new {|hash, key| hash[key] = { assignments: [], uses: [] }} | |||
class_hash = module_hash.clone | |||
function_hash = Hash.new {|hash, key| hash[key] = { calls: [] } } | |||
var_hash = Hash.new {|hash, key| hash[key] = [] } | |||
function_hash = module_hash.clone | |||
var_hash = module_hash.clone | |||
@require_empty = false | |||
@offset = offset | |||
@@ -41,7 +41,6 @@ module Bitshift | |||
end | |||
def block_position(exp) | |||
pos = Hash.new | |||
end_ln = (start_ln = exp.line - offset) | |||
cur_exp = exp | |||
@@ -51,9 +50,7 @@ module Bitshift | |||
break if cur_exp == nil | |||
end | |||
pos[:coord] = { | |||
start_ln: start_ln, | |||
end_ln: end_ln } | |||
pos = [start_ln, -1, end_ln, -1] | |||
return pos | |||
end | |||
@@ -61,9 +58,7 @@ module Bitshift | |||
pos = Hash.new | |||
end_ln = start_ln = exp.line - offset | |||
pos[:coord] = { | |||
start_ln: start_ln, | |||
end_ln: end_ln } | |||
pos = [start_ln, -1, end_ln, -1] | |||
return pos | |||
end | |||
@@ -71,7 +66,7 @@ module Bitshift | |||
pos = block_position(exp) | |||
exp.shift | |||
name = exp.shift | |||
symbols[:modules][name] = pos | |||
symbols[:modules][name][:assignments] << pos | |||
exp.each_sexp {|s| process(s)} | |||
return exp.clear | |||
end | |||
@@ -80,7 +75,7 @@ module Bitshift | |||
pos = block_position(exp) | |||
exp.shift | |||
name = exp.shift | |||
symbols[:classes][name] = pos | |||
symbols[:classes][name][:assignments] << pos | |||
exp.each_sexp {|s| process(s)} | |||
return exp.clear | |||
end | |||
@@ -89,7 +84,7 @@ module Bitshift | |||
pos = block_position(exp) | |||
exp.shift | |||
name = exp.shift | |||
symbols[:functions][name][:declaration] = pos | |||
symbols[:functions][name][:assignments] << pos | |||
exp.each_sexp {|s| process(s)} | |||
return exp.clear | |||
end | |||
@@ -99,7 +94,7 @@ module Bitshift | |||
exp.shift | |||
exp.shift | |||
name = exp.shift | |||
symbols[:functions][name][:calls] << pos | |||
symbols[:functions][name][:uses] << pos | |||
exp.each_sexp {|s| process(s)} | |||
return exp.clear | |||
end | |||
@@ -108,7 +103,7 @@ module Bitshift | |||
pos = statement_position(exp) | |||
exp.shift | |||
name = exp.shift | |||
symbols[:vars][name] << pos | |||
symbols[:vars][name][:assignments] << pos | |||
exp.each_sexp {|s| process(s)} | |||
return exp.clear | |||
end | |||
@@ -117,7 +112,7 @@ module Bitshift | |||
pos = statement_position(exp) | |||
exp.shift | |||
name = exp.shift | |||
symbols[:vars][name] << pos | |||
symbols[:vars][name][:assignments] << pos | |||
exp.each_sexp {|s| process(s)} | |||
return exp.clear | |||
end | |||