Browse Source

Change the format of the symbols to fit with earwig's issue.

tags/v1.0^2
Benjamin Attal 10 years ago
parent
commit
044a448602
5 changed files with 93 additions and 93 deletions
  1. +7
    -7
      bitshift/parser/__init__.py
  2. +36
    -52
      bitshift/parser/python.py
  3. +1
    -2
      parsers/java/pom.xml
  4. +36
    -14
      parsers/java/src/main/java/com/bitshift/parsing/symbols/JavaSymbols.java
  5. +13
    -18
      parsers/ruby/lib/parser.rb

+ 7
- 7
bitshift/parser/__init__.py View File

@@ -1,4 +1,4 @@
import ast, pygments.lexers as pgl, sys, socket, struct
import json, pygments.lexers as pgl, sys, socket, struct
from ..languages import LANGS
from .python import parse_py

@@ -19,13 +19,13 @@ def _lang(codelet):
Modify function to incorporate tags from stackoverflow.
"""

try:
if codelet.filename is not None:
if codelet.filename is not None:
try:
return pgl.guess_lexer_for_filename(codelet.filename, '').name
except:
raise UnsupportedFileError('Could not find a lexer for the codelet\'s filename')

return LANGS.index(pgl.guess_lexer(codelet.code))
except:
raise UnsupportedFileError('Could not find a lexer for the codelet\'s filename')
return LANGS.index(pgl.guess_lexer(codelet.code))

def _recv_data(server_socket):
"""
@@ -88,6 +88,6 @@ def parse(codelet):
server_socket.connect(("localhost", server_socket_number))
server_socket.send("%d\n%s" % (len(source), source));

symbols = ast.literal_eval(_recv_data(server_socket))
symbols = json.loads(_recv_data(server_socket))
codelet.symbols = symbols


+ 36
- 52
bitshift/parser/python.py View File

@@ -1,6 +1,6 @@
import ast

class _TreeCutter(ast.NodeVisitor):
class _CachedWalker(ast.NodeVisitor):
"""
Local node visitor for python abstract syntax trees.

@@ -22,9 +22,9 @@ class _TreeCutter(ast.NodeVisitor):
"""

self.accum = {'vars': {}, 'functions': {}, 'classes': {}}
self.cache = None
self.cache = []

def start_n_end(self, node):
def block_position(self, node):
"""
Helper function to get the start and end lines of an AST node.

@@ -54,32 +54,19 @@ class _TreeCutter(ast.NodeVisitor):
Add value and type metadata to accum.
"""

for t in node.targets:
if isinstance(t, ast.Tuple):
for n in t.elts:
line, col = n.lineno, n.col_offset
line, col = node.lineno, node.col_offset
pos = (line, col, line, col)

if not self.accum['vars'].has_key(node.name):
self.accum['vars'][node.name] = {'declaration': {}, 'uses': []}

pos = {'coord': {}}
pos['coord']['start_line'] = line
pos['coord']['start_col'] = col
pos['coord']['end_line'] = line
pos['coord']['end_col'] = col
self.accum['vars'][n.id]['declaration'] = pos
self.cache.append({'nodes': []})
self.generic_visit(node)
last = self.cache.pop()

else:
line, col = t.lineno, t.col_offset
for name in last['nodes']:
if not self.accum['vars'].has_key(name):
self.accum['vars'][name] = {'assignments': [], 'uses': []}

pos = {'coord': {}}
pos['coord']['start_line'] = line
pos['coord']['start_col'] = col
pos['coord']['end_line'] = line
pos['coord']['end_col'] = col
self.accum['vars'][t.id]['declaration'] = pos
self.accum['vars'][name]['assignments'].append(pos)

self.generic_visit(node)

def visit_FunctionDef(self, node):
"""
@@ -93,17 +80,13 @@ class _TreeCutter(ast.NodeVisitor):
Add arguments and decorators metadata to accum.
"""

start_line, start_col, end_line, end_col = self.start_n_end(node)
start_line, start_col, end_line, end_col = self.block_position(node)

if not self.accum['functions'].has_key(node.name):
self.accum['functions'][node.name] = {'declaration': {}, 'calls': []}
self.accum['functions'][node.name] = {'assignments': [], 'uses': []}

pos = {'coord': {}}
pos['coord']['start_ln']= start_line
pos['coord']['start_col'] = start_col
pos['coord']['end_ln'] = end_line
pos['coord']['end_col'] = end_col
self.accum['functions'][node.name]['declaration'] = pos
pos = (start_line, start_col, end_line, end_col)
self.accum['functions'][node.name]['assignments'].append(pos)

self.generic_visit(node)

@@ -120,20 +103,18 @@ class _TreeCutter(ast.NodeVisitor):
Add arguments and decorators metadata to accum.
"""

line, col = node.line_no, node.col_offset
line, col = node.lineno, node.col_offset
pos = (line, col, line, col)

if not self.accum['functions'].has_key(node.name):
self.accum['functions'][node.name] = {'declaration': {}, 'calls': []}
if isinstance(node.func, ast.Name):
name = node.func.id
else:
name = node.func.attr

pos = {'coord': {}}
pos['coord']['start_line'] = line
pos['coord']['start_col'] = col
pos['coord']['end_line'] = line
pos['coord']['end_col'] = col
self.accum['functions'][node.name]['calls'].append(pos)

self.generic_visit(node)
if not self.accum['functions'].has_key(name):
self.accum['functions'][name] = {'assignments': [], 'uses': []}

self.accum['functions'][name]['uses'].append(pos)

def visit_ClassDef(self, node):
"""
@@ -147,19 +128,22 @@ class _TreeCutter(ast.NodeVisitor):
Add arguments, inherits, and decorators metadata to accum.
"""

start_line, start_col, end_line, end_col = self.start_n_end(node)
start_line, start_col, end_line, end_col = self.block_position(node)

pos = {'coord': {}}
pos['coord']['start_ln']= start_line
pos['coord']['start_col'] = start_col
pos['coord']['end_ln'] = end_line
pos['coord']['end_col'] = end_col
pos = (start_line, start_col, end_line, end_col)
self.accum['classes'][node.name] = pos

self.generic_visit(node)

def visit_Name(self, node):
pass
if self.cache:
last = self.cache[-1]
last['nodes'].append(node.id)

def visit_Attribute(self, node):
if self.cache:
last = self.cache[-1]
last['nodes'].append(node.attr)

def parse_py(codelet):
"""
@@ -171,6 +155,6 @@ def parse_py(codelet):
"""

tree = ast.parse(codelet.code)
cutter = _TreeCutter()
cutter = _CachedWalker()
cutter.visit(tree)
codelet.symbols = cutter.accum

+ 1
- 2
parsers/java/pom.xml View File

@@ -16,8 +16,7 @@
<version>3.8.1</version>
<scope>test</scope>
</dependency>

<dependency>
<dependency>
<groupId>org.eclipse.jdt</groupId>
<artifactId>core</artifactId>
<version>3.3.0-v_771</version>


+ 36
- 14
parsers/java/src/main/java/com/bitshift/parsing/symbols/JavaSymbols.java View File

@@ -42,9 +42,17 @@ public class JavaSymbols extends Symbols {
HashMap<String, Object> method = this._methods.get(name);
if (method == null) {
method = new HashMap<String, Object>();
method.put("declaration", data);
ArrayList<Object> assignments = new ArrayList<Object>(10);
ArrayList<Object> uses = new ArrayList<Object>(10);

assignments.add(data.get("coord"));
method.put("assignments", assignments);
method.put("uses", uses);
} else {
method.put("declaration", data);
ArrayList<Object> assignments = (ArrayList<Object>)method.get("assignments");

assignments.add(data.get("coord"));
method.put("assignments", assignments);
}

this._methods.put(name, method);
@@ -54,14 +62,17 @@ public class JavaSymbols extends Symbols {
HashMap<String, Object> method = this._methods.get(name);
if (method == null) {
method = new HashMap<String, Object>();
ArrayList<Object> calls = new ArrayList<Object>(10);
calls.add(data);
method.put("calls", calls);
ArrayList<Object> assignments = new ArrayList<Object>(10);
ArrayList<Object> uses = new ArrayList<Object>(10);

uses.add(data.get("coord"));
method.put("assignments", assignments);
method.put("uses", uses);
} else {
ArrayList<Object> calls = (ArrayList<Object>)method.get("calls");
calls = (calls == null) ? new ArrayList<Object>(10) : calls;
calls.add(data);
method.put("calls", calls);
ArrayList<Object> uses = (ArrayList<Object>)method.get("uses");
uses.add(data.get("coord"));
method.put("uses", uses);
}

this._methods.put(name, method);
@@ -77,9 +88,17 @@ public class JavaSymbols extends Symbols {
HashMap<String, Object> var = this._vars.get(name);
if (var == null) {
var = new HashMap<String, Object>();
var.put("declaration", data);
ArrayList<Object> assignments = new ArrayList<Object>(10);
ArrayList<Object> uses = new ArrayList<Object>(10);

assignments.add(data.get("coord"));
var.put("assignments", assignments);
var.put("uses", uses);
} else {
var.put("declaration", data);
ArrayList<Object> assignments = (ArrayList<Object>)var.get("assignments");

assignments.add(data.get("coord"));
var.put("assignments", assignments);
}

this._vars.put(name, var);
@@ -89,13 +108,16 @@ public class JavaSymbols extends Symbols {
HashMap<String, Object> var = this._vars.get(name);
if (var == null) {
var = new HashMap<String, Object>();
ArrayList<Object> assignments = new ArrayList<Object>(10);
ArrayList<Object> uses = new ArrayList<Object>(10);
uses.add(data);

uses.add(data.get("coord"));
var.put("assignments", assignments);
var.put("uses", uses);
} else {
ArrayList<Object> uses = (ArrayList<Object>)var.get("uses");
uses = (uses == null) ? new ArrayList<Object>(10) : uses;
uses.add(data);
uses.add(data.get("coord"));
var.put("uses", uses);
}



+ 13
- 18
parsers/ruby/lib/parser.rb View File

@@ -12,23 +12,23 @@ module Bitshift
parser = RubyParser.new
tree = parser.parse(@source)
offset = tree.line - 1
processor = NodeVisitor.new offset, tree
processor = CachedWalker.new offset, tree
processor.process(tree)
return processor.to_s
end
end

class NodeVisitor < SexpProcessor
class CachedWalker < SexpProcessor
attr_accessor :symbols
attr_accessor :offset

def initialize(offset, tree)
super()

module_hash = Hash.new {|hash, key| hash[key] = Hash.new}
module_hash = Hash.new {|hash, key| hash[key] = { assignments: [], uses: [] }}
class_hash = module_hash.clone
function_hash = Hash.new {|hash, key| hash[key] = { calls: [] } }
var_hash = Hash.new {|hash, key| hash[key] = [] }
function_hash = module_hash.clone
var_hash = module_hash.clone

@require_empty = false
@offset = offset
@@ -41,7 +41,6 @@ module Bitshift
end

def block_position(exp)
pos = Hash.new
end_ln = (start_ln = exp.line - offset)
cur_exp = exp

@@ -51,9 +50,7 @@ module Bitshift
break if cur_exp == nil
end

pos[:coord] = {
start_ln: start_ln,
end_ln: end_ln }
pos = [start_ln, -1, end_ln, -1]
return pos
end

@@ -61,9 +58,7 @@ module Bitshift
pos = Hash.new
end_ln = start_ln = exp.line - offset

pos[:coord] = {
start_ln: start_ln,
end_ln: end_ln }
pos = [start_ln, -1, end_ln, -1]
return pos
end

@@ -71,7 +66,7 @@ module Bitshift
pos = block_position(exp)
exp.shift
name = exp.shift
symbols[:modules][name] = pos
symbols[:modules][name][:assignments] << pos
exp.each_sexp {|s| process(s)}
return exp.clear
end
@@ -80,7 +75,7 @@ module Bitshift
pos = block_position(exp)
exp.shift
name = exp.shift
symbols[:classes][name] = pos
symbols[:classes][name][:assignments] << pos
exp.each_sexp {|s| process(s)}
return exp.clear
end
@@ -89,7 +84,7 @@ module Bitshift
pos = block_position(exp)
exp.shift
name = exp.shift
symbols[:functions][name][:declaration] = pos
symbols[:functions][name][:assignments] << pos
exp.each_sexp {|s| process(s)}
return exp.clear
end
@@ -99,7 +94,7 @@ module Bitshift
exp.shift
exp.shift
name = exp.shift
symbols[:functions][name][:calls] << pos
symbols[:functions][name][:uses] << pos
exp.each_sexp {|s| process(s)}
return exp.clear
end
@@ -108,7 +103,7 @@ module Bitshift
pos = statement_position(exp)
exp.shift
name = exp.shift
symbols[:vars][name] << pos
symbols[:vars][name][:assignments] << pos
exp.each_sexp {|s| process(s)}
return exp.clear
end
@@ -117,7 +112,7 @@ module Bitshift
pos = statement_position(exp)
exp.shift
name = exp.shift
symbols[:vars][name] << pos
symbols[:vars][name][:assignments] << pos
exp.each_sexp {|s| process(s)}
return exp.clear
end


Loading…
Cancel
Save