From 0a7cccc74c351b05aeb64c00274695dc9cc33590 Mon Sep 17 00:00:00 2001 From: Benjamin Attal Date: Fri, 30 May 2014 23:54:51 -0400 Subject: [PATCH 01/12] Change format of string returned from ruby server to correct symbol table format. --- parsers/ruby/lib/parser.rb | 42 ++++++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/parsers/ruby/lib/parser.rb b/parsers/ruby/lib/parser.rb index c757fa0..6dfa175 100644 --- a/parsers/ruby/lib/parser.rb +++ b/parsers/ruby/lib/parser.rb @@ -3,6 +3,20 @@ require 'ruby_parser' require 'sexp_processor' module Bitshift + class Tuple + attr_accessor :objects + + def initialize(arr) + @objects = arr + end + + def inspect + s = "(" + @objects.each {|o| s += "#{o},"} + s = s[0..-2] + ')' + end + end + class Parser def initialize(source) @source = source @@ -25,7 +39,7 @@ module Bitshift def initialize(offset, tree) super() - module_hash = Hash.new {|hash, key| hash[key] = { assignments: [], uses: [] }} + module_hash = Hash.new {|hash, key| hash[key] = { decls: [], uses: [] }} class_hash = module_hash.clone function_hash = module_hash.clone var_hash = module_hash.clone @@ -50,7 +64,7 @@ module Bitshift break if cur_exp == nil end - pos = [start_ln, -1, end_ln, -1] + pos = Tuple.new([start_ln, -1, end_ln, -1]) return pos end @@ -58,7 +72,7 @@ module Bitshift pos = Hash.new end_ln = start_ln = exp.line - offset - pos = [start_ln, -1, end_ln, -1] + pos = Tuple.new([start_ln, -1, end_ln, -1]) return pos end @@ -66,7 +80,7 @@ module Bitshift pos = block_position(exp) exp.shift name = exp.shift - symbols[:modules][name][:assignments] << pos + symbols[:modules][name][:decls] << pos exp.each_sexp {|s| process(s)} return exp.clear end @@ -75,7 +89,7 @@ module Bitshift pos = block_position(exp) exp.shift name = exp.shift - symbols[:classes][name][:assignments] << pos + symbols[:classes][name][:decls] << pos exp.each_sexp {|s| process(s)} return exp.clear end @@ -84,7 +98,7 @@ module Bitshift pos = block_position(exp) exp.shift name = exp.shift - symbols[:functions][name][:assignments] << pos + symbols[:functions][name][:decls] << pos exp.each_sexp {|s| process(s)} return exp.clear end @@ -103,7 +117,7 @@ module Bitshift pos = statement_position(exp) exp.shift name = exp.shift - symbols[:vars][name][:assignments] << pos + symbols[:vars][name][:decls] << pos exp.each_sexp {|s| process(s)} return exp.clear end @@ -112,14 +126,22 @@ module Bitshift pos = statement_position(exp) exp.shift name = exp.shift - symbols[:vars][name][:assignments] << pos + symbols[:vars][name][:decls] << pos exp.each_sexp {|s| process(s)} return exp.clear end def to_s - str = symbols.to_s - str = str.gsub(/:(\w*)=>/, '"\1":') + new_symbols = Hash.new {|hash, key| hash[key] = []} + + symbols.each do |type, sym_list| + sym_list.each do |name, sym| + new_symbols[type.to_s] << Tuple.new(["'#{name}'", sym[:decls], sym[:uses]]) + end + end + + str = new_symbols.to_s + str = str.gsub(/=>/, ":") return str end end From 0fb2cacbd4f13e881387d2fef152c3cb8fb92b5f Mon Sep 17 00:00:00 2001 From: Benjamin Attal Date: Sat, 31 May 2014 23:42:10 -0400 Subject: [PATCH 02/12] Make sure parse servers return json loadable data. Make sure Java server closes client connection. --- .../src/main/java/com/bitshift/parsing/Parse.java | 13 +--- .../com/bitshift/parsing/parsers/JavaParser.java | 27 ++------ .../java/com/bitshift/parsing/parsers/Parser.java | 15 +++-- .../com/bitshift/parsing/symbols/JavaSymbols.java | 71 ++++++++++++---------- .../com/bitshift/parsing/utils/PackableMemory.java | 2 +- .../java/com/bitshift/parsing/utils/Tuple.java | 23 +++++++ parsers/ruby/lib/parse_server.rb | 2 +- parsers/ruby/lib/parser.rb | 37 ++++------- 8 files changed, 93 insertions(+), 97 deletions(-) create mode 100644 parsers/java/src/main/java/com/bitshift/parsing/utils/Tuple.java diff --git a/parsers/java/src/main/java/com/bitshift/parsing/Parse.java b/parsers/java/src/main/java/com/bitshift/parsing/Parse.java index fc1d36f..302c083 100644 --- a/parsers/java/src/main/java/com/bitshift/parsing/Parse.java +++ b/parsers/java/src/main/java/com/bitshift/parsing/Parse.java @@ -1,8 +1,5 @@ package com.bitshift.parsing; -import java.io.BufferedReader; -import java.io.InputStreamReader; -import java.io.PrintWriter; import java.io.IOException; import java.net.ServerSocket; @@ -13,18 +10,12 @@ import com.bitshift.parsing.parsers.JavaParser; public class Parse { public static void main(String[] args) { - String fromClient; - String toClient; - try { - ServerSocket server = new ServerSocket(5002); + ServerSocket server = new ServerSocket(5033); while(true) { Socket clientSocket = server.accept(); - - JavaParser parser = new JavaParser(clientSocket); - Thread parserTask = new Thread(parser); - parserTask.start(); + new Thread(new JavaParser(clientSocket)).start(); } } catch (IOException ex) { } diff --git a/parsers/java/src/main/java/com/bitshift/parsing/parsers/JavaParser.java b/parsers/java/src/main/java/com/bitshift/parsing/parsers/JavaParser.java index 4ba3623..989c0dd 100644 --- a/parsers/java/src/main/java/com/bitshift/parsing/parsers/JavaParser.java +++ b/parsers/java/src/main/java/com/bitshift/parsing/parsers/JavaParser.java @@ -13,7 +13,6 @@ import org.eclipse.jdt.core.dom.ASTParser; import org.eclipse.jdt.core.dom.ASTVisitor; import org.eclipse.jdt.core.dom.CompilationUnit; import org.eclipse.jdt.core.dom.ClassInstanceCreation; -import org.eclipse.jdt.core.dom.FieldDeclaration; import org.eclipse.jdt.core.dom.MethodDeclaration; import org.eclipse.jdt.core.dom.MethodInvocation; import org.eclipse.jdt.core.dom.Name; @@ -71,22 +70,6 @@ public class JavaParser extends Parser { this._cache = new Stack>(); } - public boolean visit(FieldDeclaration node) { - HashMap data = new HashMap(); - int sl = this.root.getLineNumber(node.getStartPosition()); - int sc = this.root.getColumnNumber(node.getStartPosition()); - - data.put("coord", Symbols.createCoord(sl, sc, -1, -1)); - this._cache.push(data); - return true; - } - - public void endVisit(FieldDeclaration node) { - HashMap data = this._cache.pop(); - String name = (String)data.remove("name"); - this.symbols.insertFieldDeclaration(name, data); - } - public boolean visit(MethodDeclaration node) { HashMap data = new HashMap(); Name nameObj = node.getName(); @@ -115,7 +98,7 @@ public class JavaParser extends Parser { public void endVisit(MethodDeclaration node) { HashMap data = this._cache.pop(); String name = (String)data.remove("name"); - this.symbols.insertMethodDeclaration(name, data); + this.symbols.insertMethodDeclaration("\"" + name + "\"", data); } public boolean visit(MethodInvocation node) { @@ -136,7 +119,7 @@ public class JavaParser extends Parser { public void endVisit(MethodInvocation node) { HashMap data = this._cache.pop(); String name = (String)data.remove("name"); - this.symbols.insertMethodInvocation(name, data); + this.symbols.insertMethodInvocation("\"" + name + "\"", data); } public boolean visit(PackageDeclaration node) { @@ -167,9 +150,9 @@ public class JavaParser extends Parser { String name = (String)data.remove("name"); if (node.isInterface()) { - this.symbols.insertInterfaceDeclaration(name, data); + this.symbols.insertInterfaceDeclaration("\"" + name + "\"", data); } else { - this.symbols.insertClassDeclaration(name, data); + this.symbols.insertClassDeclaration("\"" + name + "\"", data); } } @@ -186,7 +169,7 @@ public class JavaParser extends Parser { public void endVisit(VariableDeclarationFragment node) { HashMap data = this._cache.pop(); String name = (String)data.remove("name"); - this.symbols.insertVariableDeclaration(name, data); + this.symbols.insertVariableDeclaration("\"" + name + "\"", data); } public boolean visit(QualifiedName node) { diff --git a/parsers/java/src/main/java/com/bitshift/parsing/parsers/Parser.java b/parsers/java/src/main/java/com/bitshift/parsing/parsers/Parser.java index 9d00954..83100f5 100644 --- a/parsers/java/src/main/java/com/bitshift/parsing/parsers/Parser.java +++ b/parsers/java/src/main/java/com/bitshift/parsing/parsers/Parser.java @@ -1,8 +1,9 @@ package com.bitshift.parsing.parsers; import java.io.BufferedReader; +import java.io.BufferedWriter; import java.io.InputStreamReader; -import java.io.PrintWriter; +import java.io.OutputStreamWriter; import java.io.IOException; import java.net.Socket; @@ -46,12 +47,16 @@ public abstract class Parser implements Runnable { protected void writeToClient(String toClient) { try { - PrintWriter clientWriter = new PrintWriter( - this.clientSocket.getOutputStream(), true); + BufferedWriter clientWriter = new BufferedWriter( + new OutputStreamWriter(this.clientSocket.getOutputStream())); - PackableMemory mem = new PackableMemory(toClient.length()); + PackableMemory mem = new PackableMemory(4); + mem.pack(toClient.length(), 0); String dataSize = new String(mem.mem); - clientWriter.println(dataSize + toClient); + + clientWriter.write(dataSize + toClient); + clientWriter.flush(); + this.clientSocket.close(); } catch (IOException ex) { } } diff --git a/parsers/java/src/main/java/com/bitshift/parsing/symbols/JavaSymbols.java b/parsers/java/src/main/java/com/bitshift/parsing/symbols/JavaSymbols.java index 5419d5a..6f0caf1 100644 --- a/parsers/java/src/main/java/com/bitshift/parsing/symbols/JavaSymbols.java +++ b/parsers/java/src/main/java/com/bitshift/parsing/symbols/JavaSymbols.java @@ -11,15 +11,16 @@ public class JavaSymbols extends Symbols { private HashMap> _classes; private HashMap> _interfaces; private HashMap> _methods; - private HashMap> _fields; private HashMap> _vars; + private final String assignKey = "\"assignments\""; + private final String useKey = "\"uses\""; + public JavaSymbols() { _packageName = null; _classes = new HashMap>(); _interfaces = new HashMap>(); _methods = new HashMap>(); - _fields = new HashMap>(); _vars = new HashMap>(); } @@ -34,15 +35,23 @@ public class JavaSymbols extends Symbols { HashMap klass = new HashMap(); assignments.add(data.get("coord")); - klass.put("assignments", assignments); - klass.put("uses", uses); + klass.put(assignKey, assignments); + klass.put(useKey, uses); this._classes.put(name, klass); return true; } public boolean insertInterfaceDeclaration(String name, HashMap data) { - this._interfaces.put(name, data); + ArrayList assignments = new ArrayList(10); + ArrayList uses = new ArrayList(10); + HashMap klass = new HashMap(); + + assignments.add(data.get("coord")); + klass.put(assignKey, assignments); + klass.put(useKey, uses); + + this._interfaces.put(name, klass); return true; } @@ -54,13 +63,13 @@ public class JavaSymbols extends Symbols { ArrayList uses = new ArrayList(10); assignments.add(data.get("coord")); - method.put("assignments", assignments); - method.put("uses", uses); + method.put(assignKey, assignments); + method.put(useKey, uses); } else { - ArrayList assignments = (ArrayList)method.get("assignments"); + ArrayList assignments = (ArrayList)method.get(assignKey); assignments.add(data.get("coord")); - method.put("assignments", assignments); + method.put(assignKey, assignments); } this._methods.put(name, method); @@ -74,24 +83,19 @@ public class JavaSymbols extends Symbols { ArrayList uses = new ArrayList(10); uses.add(data.get("coord")); - method.put("assignments", assignments); - method.put("uses", uses); + method.put(assignKey, assignments); + method.put(useKey, uses); } else { - ArrayList uses = (ArrayList)method.get("uses"); + ArrayList uses = (ArrayList)method.get(useKey); uses.add(data.get("coord")); - method.put("uses", uses); + method.put(useKey, uses); } this._methods.put(name, method); return true; } - public boolean insertFieldDeclaration(String name, HashMap data) { - this._fields.put(name, data); - return true; - } - public boolean insertVariableDeclaration(String name, HashMap data) { HashMap var = this._vars.get(name); if (var == null) { @@ -100,13 +104,13 @@ public class JavaSymbols extends Symbols { ArrayList uses = new ArrayList(10); assignments.add(data.get("coord")); - var.put("assignments", assignments); - var.put("uses", uses); + var.put(assignKey, assignments); + var.put(useKey, uses); } else { - ArrayList assignments = (ArrayList)var.get("assignments"); + ArrayList assignments = (ArrayList)var.get(assignKey); assignments.add(data.get("coord")); - var.put("assignments", assignments); + var.put(assignKey, assignments); } this._vars.put(name, var); @@ -120,13 +124,13 @@ public class JavaSymbols extends Symbols { ArrayList uses = new ArrayList(10); uses.add(data.get("coord")); - var.put("assignments", assignments); - var.put("uses", uses); + var.put(assignKey, assignments); + var.put(useKey, uses); } else { - ArrayList uses = (ArrayList)var.get("uses"); + ArrayList uses = (ArrayList)var.get(useKey); uses.add(data.get("coord")); - var.put("uses", uses); + var.put(useKey, uses); } this._vars.put(name, var); @@ -135,13 +139,14 @@ public class JavaSymbols extends Symbols { public String toString() { StringBuilder builder = new StringBuilder(); - builder.append("classes:" + this._classes + ","); - builder.append("interfaces:" + this._interfaces + ","); - builder.append("methods:" + this._methods + ","); - builder.append("fields:" + this._fields + ","); - builder.append("vars:" + this._vars + ","); - - return "{" + builder.toString() + "}"; + builder.append("\"classes\":" + this._classes + ","); + builder.append("\"interfaces\":" + this._interfaces + ","); + builder.append("\"methods\":" + this._methods + ","); + builder.append("\"vars\":" + this._vars + ","); + + String s = builder.toString().replaceAll("=", ":"); + s = s.substring(0, s.length() - 1); + return "{" + s + "}"; } } diff --git a/parsers/java/src/main/java/com/bitshift/parsing/utils/PackableMemory.java b/parsers/java/src/main/java/com/bitshift/parsing/utils/PackableMemory.java index 24d883c..1f54d99 100644 --- a/parsers/java/src/main/java/com/bitshift/parsing/utils/PackableMemory.java +++ b/parsers/java/src/main/java/com/bitshift/parsing/utils/PackableMemory.java @@ -22,7 +22,7 @@ public class PackableMemory { // The most significant porion of the integer is stored in mem[loc]. // Bytes are masked out of the integer and stored in the array, working // from right(least significant) to left (most significant). - void pack(int val, int loc) + public void pack(int val, int loc) { final int MASK = 0xff; for (int i = 3; i >= 0; i--) diff --git a/parsers/java/src/main/java/com/bitshift/parsing/utils/Tuple.java b/parsers/java/src/main/java/com/bitshift/parsing/utils/Tuple.java new file mode 100644 index 0000000..115a3c6 --- /dev/null +++ b/parsers/java/src/main/java/com/bitshift/parsing/utils/Tuple.java @@ -0,0 +1,23 @@ +package com.bitshift.parsing.utils; + +import java.util.List; +import java.util.Arrays; + +public class Tuple { + private List _objects; + + public Tuple(T... args) { + _objects = Arrays.asList(args); + } + + public String toString() { + StringBuilder builder = new StringBuilder(); + + for(T o: this._objects) { + builder.append(o + ","); + } + + String s = builder.toString(); + return "(" + s.substring(0, s.length() - 1) + ")"; + } +} diff --git a/parsers/ruby/lib/parse_server.rb b/parsers/ruby/lib/parse_server.rb index 916f434..9a929aa 100644 --- a/parsers/ruby/lib/parse_server.rb +++ b/parsers/ruby/lib/parse_server.rb @@ -14,7 +14,7 @@ end def start_server - server = TCPServer.new 5003 + server = TCPServer.new 5065 loop do # Start a new thread for each client accepted diff --git a/parsers/ruby/lib/parser.rb b/parsers/ruby/lib/parser.rb index 6dfa175..eec293b 100644 --- a/parsers/ruby/lib/parser.rb +++ b/parsers/ruby/lib/parser.rb @@ -3,20 +3,6 @@ require 'ruby_parser' require 'sexp_processor' module Bitshift - class Tuple - attr_accessor :objects - - def initialize(arr) - @objects = arr - end - - def inspect - s = "(" - @objects.each {|o| s += "#{o},"} - s = s[0..-2] + ')' - end - end - class Parser def initialize(source) @source = source @@ -39,7 +25,8 @@ module Bitshift def initialize(offset, tree) super() - module_hash = Hash.new {|hash, key| hash[key] = { decls: [], uses: [] }} + module_hash = Hash.new {|hash, key| + hash[key] = { assignments: [], uses: [] }} class_hash = module_hash.clone function_hash = module_hash.clone var_hash = module_hash.clone @@ -64,7 +51,7 @@ module Bitshift break if cur_exp == nil end - pos = Tuple.new([start_ln, -1, end_ln, -1]) + pos = [start_ln, -1, end_ln, -1] return pos end @@ -72,7 +59,7 @@ module Bitshift pos = Hash.new end_ln = start_ln = exp.line - offset - pos = Tuple.new([start_ln, -1, end_ln, -1]) + pos = [start_ln, -1, end_ln, -1] return pos end @@ -80,7 +67,7 @@ module Bitshift pos = block_position(exp) exp.shift name = exp.shift - symbols[:modules][name][:decls] << pos + symbols[:modules][name][:assignments] << pos exp.each_sexp {|s| process(s)} return exp.clear end @@ -89,7 +76,7 @@ module Bitshift pos = block_position(exp) exp.shift name = exp.shift - symbols[:classes][name][:decls] << pos + symbols[:classes][name][:assignments] << pos exp.each_sexp {|s| process(s)} return exp.clear end @@ -98,7 +85,7 @@ module Bitshift pos = block_position(exp) exp.shift name = exp.shift - symbols[:functions][name][:decls] << pos + symbols[:functions][name][:assignments] << pos exp.each_sexp {|s| process(s)} return exp.clear end @@ -117,7 +104,7 @@ module Bitshift pos = statement_position(exp) exp.shift name = exp.shift - symbols[:vars][name][:decls] << pos + symbols[:vars][name][:assignments] << pos exp.each_sexp {|s| process(s)} return exp.clear end @@ -126,17 +113,19 @@ module Bitshift pos = statement_position(exp) exp.shift name = exp.shift - symbols[:vars][name][:decls] << pos + symbols[:vars][name][:assignments] << pos exp.each_sexp {|s| process(s)} return exp.clear end def to_s - new_symbols = Hash.new {|hash, key| hash[key] = []} + new_symbols = Hash.new {|hash, key| hash[key] = Hash.new} symbols.each do |type, sym_list| sym_list.each do |name, sym| - new_symbols[type.to_s] << Tuple.new(["'#{name}'", sym[:decls], sym[:uses]]) + new_symbols[type.to_s][name.to_s] = { + "assignments" => sym[:assignments], + "uses" => sym[:uses]} end end From 5e6e3fcbf59532a68c0cb3ec8c788050ae6ac814 Mon Sep 17 00:00:00 2001 From: Benjamin Attal Date: Sat, 31 May 2014 23:42:44 -0400 Subject: [PATCH 03/12] Adjust test for new language dict. --- bitshift/parser/__init__.py | 3 ++- test/parser_test.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bitshift/parser/__init__.py b/bitshift/parser/__init__.py index bc22514..c3af22e 100644 --- a/bitshift/parser/__init__.py +++ b/bitshift/parser/__init__.py @@ -21,7 +21,8 @@ def _lang(codelet): if codelet.filename is not None: try: - return pgl.guess_lexer_for_filename(codelet.filename, codelet.code).name + return LANGS.index( + pgl.get_lexer_for_filename(codelet.filename).name) except: raise UnsupportedFileError('Could not find a lexer for the codelet\'s filename') diff --git a/test/parser_test.py b/test/parser_test.py index a1cfad3..ffee75c 100644 --- a/test/parser_test.py +++ b/test/parser_test.py @@ -21,7 +21,7 @@ if __name__ == '__main__': elif sys.argv[1] == 'ruby': file_name = "resources/parser.rb" - server_socket_number = 5003 + server_socket_number = 5065 server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server_socket.connect(("localhost", server_socket_number)) From 5d3baf49d142edb91150e61a13113fb04ffdf536 Mon Sep 17 00:00:00 2001 From: Benjamin Attal Date: Fri, 30 May 2014 23:54:51 -0400 Subject: [PATCH 04/12] Change format of string returned from ruby server to correct symbol table format. --- parsers/ruby/lib/parser.rb | 42 ++++++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/parsers/ruby/lib/parser.rb b/parsers/ruby/lib/parser.rb index c757fa0..6dfa175 100644 --- a/parsers/ruby/lib/parser.rb +++ b/parsers/ruby/lib/parser.rb @@ -3,6 +3,20 @@ require 'ruby_parser' require 'sexp_processor' module Bitshift + class Tuple + attr_accessor :objects + + def initialize(arr) + @objects = arr + end + + def inspect + s = "(" + @objects.each {|o| s += "#{o},"} + s = s[0..-2] + ')' + end + end + class Parser def initialize(source) @source = source @@ -25,7 +39,7 @@ module Bitshift def initialize(offset, tree) super() - module_hash = Hash.new {|hash, key| hash[key] = { assignments: [], uses: [] }} + module_hash = Hash.new {|hash, key| hash[key] = { decls: [], uses: [] }} class_hash = module_hash.clone function_hash = module_hash.clone var_hash = module_hash.clone @@ -50,7 +64,7 @@ module Bitshift break if cur_exp == nil end - pos = [start_ln, -1, end_ln, -1] + pos = Tuple.new([start_ln, -1, end_ln, -1]) return pos end @@ -58,7 +72,7 @@ module Bitshift pos = Hash.new end_ln = start_ln = exp.line - offset - pos = [start_ln, -1, end_ln, -1] + pos = Tuple.new([start_ln, -1, end_ln, -1]) return pos end @@ -66,7 +80,7 @@ module Bitshift pos = block_position(exp) exp.shift name = exp.shift - symbols[:modules][name][:assignments] << pos + symbols[:modules][name][:decls] << pos exp.each_sexp {|s| process(s)} return exp.clear end @@ -75,7 +89,7 @@ module Bitshift pos = block_position(exp) exp.shift name = exp.shift - symbols[:classes][name][:assignments] << pos + symbols[:classes][name][:decls] << pos exp.each_sexp {|s| process(s)} return exp.clear end @@ -84,7 +98,7 @@ module Bitshift pos = block_position(exp) exp.shift name = exp.shift - symbols[:functions][name][:assignments] << pos + symbols[:functions][name][:decls] << pos exp.each_sexp {|s| process(s)} return exp.clear end @@ -103,7 +117,7 @@ module Bitshift pos = statement_position(exp) exp.shift name = exp.shift - symbols[:vars][name][:assignments] << pos + symbols[:vars][name][:decls] << pos exp.each_sexp {|s| process(s)} return exp.clear end @@ -112,14 +126,22 @@ module Bitshift pos = statement_position(exp) exp.shift name = exp.shift - symbols[:vars][name][:assignments] << pos + symbols[:vars][name][:decls] << pos exp.each_sexp {|s| process(s)} return exp.clear end def to_s - str = symbols.to_s - str = str.gsub(/:(\w*)=>/, '"\1":') + new_symbols = Hash.new {|hash, key| hash[key] = []} + + symbols.each do |type, sym_list| + sym_list.each do |name, sym| + new_symbols[type.to_s] << Tuple.new(["'#{name}'", sym[:decls], sym[:uses]]) + end + end + + str = new_symbols.to_s + str = str.gsub(/=>/, ":") return str end end From 1f56300c73ef236936d340256333fb635b6c14dd Mon Sep 17 00:00:00 2001 From: Benjamin Attal Date: Sat, 31 May 2014 23:42:10 -0400 Subject: [PATCH 05/12] Make sure parse servers return json loadable data. Make sure Java server closes client connection. --- .../src/main/java/com/bitshift/parsing/Parse.java | 13 +--- .../com/bitshift/parsing/parsers/JavaParser.java | 27 ++------ .../java/com/bitshift/parsing/parsers/Parser.java | 15 +++-- .../com/bitshift/parsing/symbols/JavaSymbols.java | 71 ++++++++++++---------- .../com/bitshift/parsing/utils/PackableMemory.java | 2 +- .../java/com/bitshift/parsing/utils/Tuple.java | 23 +++++++ parsers/ruby/lib/parse_server.rb | 2 +- parsers/ruby/lib/parser.rb | 37 ++++------- 8 files changed, 93 insertions(+), 97 deletions(-) create mode 100644 parsers/java/src/main/java/com/bitshift/parsing/utils/Tuple.java diff --git a/parsers/java/src/main/java/com/bitshift/parsing/Parse.java b/parsers/java/src/main/java/com/bitshift/parsing/Parse.java index fc1d36f..302c083 100644 --- a/parsers/java/src/main/java/com/bitshift/parsing/Parse.java +++ b/parsers/java/src/main/java/com/bitshift/parsing/Parse.java @@ -1,8 +1,5 @@ package com.bitshift.parsing; -import java.io.BufferedReader; -import java.io.InputStreamReader; -import java.io.PrintWriter; import java.io.IOException; import java.net.ServerSocket; @@ -13,18 +10,12 @@ import com.bitshift.parsing.parsers.JavaParser; public class Parse { public static void main(String[] args) { - String fromClient; - String toClient; - try { - ServerSocket server = new ServerSocket(5002); + ServerSocket server = new ServerSocket(5033); while(true) { Socket clientSocket = server.accept(); - - JavaParser parser = new JavaParser(clientSocket); - Thread parserTask = new Thread(parser); - parserTask.start(); + new Thread(new JavaParser(clientSocket)).start(); } } catch (IOException ex) { } diff --git a/parsers/java/src/main/java/com/bitshift/parsing/parsers/JavaParser.java b/parsers/java/src/main/java/com/bitshift/parsing/parsers/JavaParser.java index 4ba3623..989c0dd 100644 --- a/parsers/java/src/main/java/com/bitshift/parsing/parsers/JavaParser.java +++ b/parsers/java/src/main/java/com/bitshift/parsing/parsers/JavaParser.java @@ -13,7 +13,6 @@ import org.eclipse.jdt.core.dom.ASTParser; import org.eclipse.jdt.core.dom.ASTVisitor; import org.eclipse.jdt.core.dom.CompilationUnit; import org.eclipse.jdt.core.dom.ClassInstanceCreation; -import org.eclipse.jdt.core.dom.FieldDeclaration; import org.eclipse.jdt.core.dom.MethodDeclaration; import org.eclipse.jdt.core.dom.MethodInvocation; import org.eclipse.jdt.core.dom.Name; @@ -71,22 +70,6 @@ public class JavaParser extends Parser { this._cache = new Stack>(); } - public boolean visit(FieldDeclaration node) { - HashMap data = new HashMap(); - int sl = this.root.getLineNumber(node.getStartPosition()); - int sc = this.root.getColumnNumber(node.getStartPosition()); - - data.put("coord", Symbols.createCoord(sl, sc, -1, -1)); - this._cache.push(data); - return true; - } - - public void endVisit(FieldDeclaration node) { - HashMap data = this._cache.pop(); - String name = (String)data.remove("name"); - this.symbols.insertFieldDeclaration(name, data); - } - public boolean visit(MethodDeclaration node) { HashMap data = new HashMap(); Name nameObj = node.getName(); @@ -115,7 +98,7 @@ public class JavaParser extends Parser { public void endVisit(MethodDeclaration node) { HashMap data = this._cache.pop(); String name = (String)data.remove("name"); - this.symbols.insertMethodDeclaration(name, data); + this.symbols.insertMethodDeclaration("\"" + name + "\"", data); } public boolean visit(MethodInvocation node) { @@ -136,7 +119,7 @@ public class JavaParser extends Parser { public void endVisit(MethodInvocation node) { HashMap data = this._cache.pop(); String name = (String)data.remove("name"); - this.symbols.insertMethodInvocation(name, data); + this.symbols.insertMethodInvocation("\"" + name + "\"", data); } public boolean visit(PackageDeclaration node) { @@ -167,9 +150,9 @@ public class JavaParser extends Parser { String name = (String)data.remove("name"); if (node.isInterface()) { - this.symbols.insertInterfaceDeclaration(name, data); + this.symbols.insertInterfaceDeclaration("\"" + name + "\"", data); } else { - this.symbols.insertClassDeclaration(name, data); + this.symbols.insertClassDeclaration("\"" + name + "\"", data); } } @@ -186,7 +169,7 @@ public class JavaParser extends Parser { public void endVisit(VariableDeclarationFragment node) { HashMap data = this._cache.pop(); String name = (String)data.remove("name"); - this.symbols.insertVariableDeclaration(name, data); + this.symbols.insertVariableDeclaration("\"" + name + "\"", data); } public boolean visit(QualifiedName node) { diff --git a/parsers/java/src/main/java/com/bitshift/parsing/parsers/Parser.java b/parsers/java/src/main/java/com/bitshift/parsing/parsers/Parser.java index 9d00954..83100f5 100644 --- a/parsers/java/src/main/java/com/bitshift/parsing/parsers/Parser.java +++ b/parsers/java/src/main/java/com/bitshift/parsing/parsers/Parser.java @@ -1,8 +1,9 @@ package com.bitshift.parsing.parsers; import java.io.BufferedReader; +import java.io.BufferedWriter; import java.io.InputStreamReader; -import java.io.PrintWriter; +import java.io.OutputStreamWriter; import java.io.IOException; import java.net.Socket; @@ -46,12 +47,16 @@ public abstract class Parser implements Runnable { protected void writeToClient(String toClient) { try { - PrintWriter clientWriter = new PrintWriter( - this.clientSocket.getOutputStream(), true); + BufferedWriter clientWriter = new BufferedWriter( + new OutputStreamWriter(this.clientSocket.getOutputStream())); - PackableMemory mem = new PackableMemory(toClient.length()); + PackableMemory mem = new PackableMemory(4); + mem.pack(toClient.length(), 0); String dataSize = new String(mem.mem); - clientWriter.println(dataSize + toClient); + + clientWriter.write(dataSize + toClient); + clientWriter.flush(); + this.clientSocket.close(); } catch (IOException ex) { } } diff --git a/parsers/java/src/main/java/com/bitshift/parsing/symbols/JavaSymbols.java b/parsers/java/src/main/java/com/bitshift/parsing/symbols/JavaSymbols.java index 5419d5a..6f0caf1 100644 --- a/parsers/java/src/main/java/com/bitshift/parsing/symbols/JavaSymbols.java +++ b/parsers/java/src/main/java/com/bitshift/parsing/symbols/JavaSymbols.java @@ -11,15 +11,16 @@ public class JavaSymbols extends Symbols { private HashMap> _classes; private HashMap> _interfaces; private HashMap> _methods; - private HashMap> _fields; private HashMap> _vars; + private final String assignKey = "\"assignments\""; + private final String useKey = "\"uses\""; + public JavaSymbols() { _packageName = null; _classes = new HashMap>(); _interfaces = new HashMap>(); _methods = new HashMap>(); - _fields = new HashMap>(); _vars = new HashMap>(); } @@ -34,15 +35,23 @@ public class JavaSymbols extends Symbols { HashMap klass = new HashMap(); assignments.add(data.get("coord")); - klass.put("assignments", assignments); - klass.put("uses", uses); + klass.put(assignKey, assignments); + klass.put(useKey, uses); this._classes.put(name, klass); return true; } public boolean insertInterfaceDeclaration(String name, HashMap data) { - this._interfaces.put(name, data); + ArrayList assignments = new ArrayList(10); + ArrayList uses = new ArrayList(10); + HashMap klass = new HashMap(); + + assignments.add(data.get("coord")); + klass.put(assignKey, assignments); + klass.put(useKey, uses); + + this._interfaces.put(name, klass); return true; } @@ -54,13 +63,13 @@ public class JavaSymbols extends Symbols { ArrayList uses = new ArrayList(10); assignments.add(data.get("coord")); - method.put("assignments", assignments); - method.put("uses", uses); + method.put(assignKey, assignments); + method.put(useKey, uses); } else { - ArrayList assignments = (ArrayList)method.get("assignments"); + ArrayList assignments = (ArrayList)method.get(assignKey); assignments.add(data.get("coord")); - method.put("assignments", assignments); + method.put(assignKey, assignments); } this._methods.put(name, method); @@ -74,24 +83,19 @@ public class JavaSymbols extends Symbols { ArrayList uses = new ArrayList(10); uses.add(data.get("coord")); - method.put("assignments", assignments); - method.put("uses", uses); + method.put(assignKey, assignments); + method.put(useKey, uses); } else { - ArrayList uses = (ArrayList)method.get("uses"); + ArrayList uses = (ArrayList)method.get(useKey); uses.add(data.get("coord")); - method.put("uses", uses); + method.put(useKey, uses); } this._methods.put(name, method); return true; } - public boolean insertFieldDeclaration(String name, HashMap data) { - this._fields.put(name, data); - return true; - } - public boolean insertVariableDeclaration(String name, HashMap data) { HashMap var = this._vars.get(name); if (var == null) { @@ -100,13 +104,13 @@ public class JavaSymbols extends Symbols { ArrayList uses = new ArrayList(10); assignments.add(data.get("coord")); - var.put("assignments", assignments); - var.put("uses", uses); + var.put(assignKey, assignments); + var.put(useKey, uses); } else { - ArrayList assignments = (ArrayList)var.get("assignments"); + ArrayList assignments = (ArrayList)var.get(assignKey); assignments.add(data.get("coord")); - var.put("assignments", assignments); + var.put(assignKey, assignments); } this._vars.put(name, var); @@ -120,13 +124,13 @@ public class JavaSymbols extends Symbols { ArrayList uses = new ArrayList(10); uses.add(data.get("coord")); - var.put("assignments", assignments); - var.put("uses", uses); + var.put(assignKey, assignments); + var.put(useKey, uses); } else { - ArrayList uses = (ArrayList)var.get("uses"); + ArrayList uses = (ArrayList)var.get(useKey); uses.add(data.get("coord")); - var.put("uses", uses); + var.put(useKey, uses); } this._vars.put(name, var); @@ -135,13 +139,14 @@ public class JavaSymbols extends Symbols { public String toString() { StringBuilder builder = new StringBuilder(); - builder.append("classes:" + this._classes + ","); - builder.append("interfaces:" + this._interfaces + ","); - builder.append("methods:" + this._methods + ","); - builder.append("fields:" + this._fields + ","); - builder.append("vars:" + this._vars + ","); - - return "{" + builder.toString() + "}"; + builder.append("\"classes\":" + this._classes + ","); + builder.append("\"interfaces\":" + this._interfaces + ","); + builder.append("\"methods\":" + this._methods + ","); + builder.append("\"vars\":" + this._vars + ","); + + String s = builder.toString().replaceAll("=", ":"); + s = s.substring(0, s.length() - 1); + return "{" + s + "}"; } } diff --git a/parsers/java/src/main/java/com/bitshift/parsing/utils/PackableMemory.java b/parsers/java/src/main/java/com/bitshift/parsing/utils/PackableMemory.java index 24d883c..1f54d99 100644 --- a/parsers/java/src/main/java/com/bitshift/parsing/utils/PackableMemory.java +++ b/parsers/java/src/main/java/com/bitshift/parsing/utils/PackableMemory.java @@ -22,7 +22,7 @@ public class PackableMemory { // The most significant porion of the integer is stored in mem[loc]. // Bytes are masked out of the integer and stored in the array, working // from right(least significant) to left (most significant). - void pack(int val, int loc) + public void pack(int val, int loc) { final int MASK = 0xff; for (int i = 3; i >= 0; i--) diff --git a/parsers/java/src/main/java/com/bitshift/parsing/utils/Tuple.java b/parsers/java/src/main/java/com/bitshift/parsing/utils/Tuple.java new file mode 100644 index 0000000..115a3c6 --- /dev/null +++ b/parsers/java/src/main/java/com/bitshift/parsing/utils/Tuple.java @@ -0,0 +1,23 @@ +package com.bitshift.parsing.utils; + +import java.util.List; +import java.util.Arrays; + +public class Tuple { + private List _objects; + + public Tuple(T... args) { + _objects = Arrays.asList(args); + } + + public String toString() { + StringBuilder builder = new StringBuilder(); + + for(T o: this._objects) { + builder.append(o + ","); + } + + String s = builder.toString(); + return "(" + s.substring(0, s.length() - 1) + ")"; + } +} diff --git a/parsers/ruby/lib/parse_server.rb b/parsers/ruby/lib/parse_server.rb index 916f434..9a929aa 100644 --- a/parsers/ruby/lib/parse_server.rb +++ b/parsers/ruby/lib/parse_server.rb @@ -14,7 +14,7 @@ end def start_server - server = TCPServer.new 5003 + server = TCPServer.new 5065 loop do # Start a new thread for each client accepted diff --git a/parsers/ruby/lib/parser.rb b/parsers/ruby/lib/parser.rb index 6dfa175..eec293b 100644 --- a/parsers/ruby/lib/parser.rb +++ b/parsers/ruby/lib/parser.rb @@ -3,20 +3,6 @@ require 'ruby_parser' require 'sexp_processor' module Bitshift - class Tuple - attr_accessor :objects - - def initialize(arr) - @objects = arr - end - - def inspect - s = "(" - @objects.each {|o| s += "#{o},"} - s = s[0..-2] + ')' - end - end - class Parser def initialize(source) @source = source @@ -39,7 +25,8 @@ module Bitshift def initialize(offset, tree) super() - module_hash = Hash.new {|hash, key| hash[key] = { decls: [], uses: [] }} + module_hash = Hash.new {|hash, key| + hash[key] = { assignments: [], uses: [] }} class_hash = module_hash.clone function_hash = module_hash.clone var_hash = module_hash.clone @@ -64,7 +51,7 @@ module Bitshift break if cur_exp == nil end - pos = Tuple.new([start_ln, -1, end_ln, -1]) + pos = [start_ln, -1, end_ln, -1] return pos end @@ -72,7 +59,7 @@ module Bitshift pos = Hash.new end_ln = start_ln = exp.line - offset - pos = Tuple.new([start_ln, -1, end_ln, -1]) + pos = [start_ln, -1, end_ln, -1] return pos end @@ -80,7 +67,7 @@ module Bitshift pos = block_position(exp) exp.shift name = exp.shift - symbols[:modules][name][:decls] << pos + symbols[:modules][name][:assignments] << pos exp.each_sexp {|s| process(s)} return exp.clear end @@ -89,7 +76,7 @@ module Bitshift pos = block_position(exp) exp.shift name = exp.shift - symbols[:classes][name][:decls] << pos + symbols[:classes][name][:assignments] << pos exp.each_sexp {|s| process(s)} return exp.clear end @@ -98,7 +85,7 @@ module Bitshift pos = block_position(exp) exp.shift name = exp.shift - symbols[:functions][name][:decls] << pos + symbols[:functions][name][:assignments] << pos exp.each_sexp {|s| process(s)} return exp.clear end @@ -117,7 +104,7 @@ module Bitshift pos = statement_position(exp) exp.shift name = exp.shift - symbols[:vars][name][:decls] << pos + symbols[:vars][name][:assignments] << pos exp.each_sexp {|s| process(s)} return exp.clear end @@ -126,17 +113,19 @@ module Bitshift pos = statement_position(exp) exp.shift name = exp.shift - symbols[:vars][name][:decls] << pos + symbols[:vars][name][:assignments] << pos exp.each_sexp {|s| process(s)} return exp.clear end def to_s - new_symbols = Hash.new {|hash, key| hash[key] = []} + new_symbols = Hash.new {|hash, key| hash[key] = Hash.new} symbols.each do |type, sym_list| sym_list.each do |name, sym| - new_symbols[type.to_s] << Tuple.new(["'#{name}'", sym[:decls], sym[:uses]]) + new_symbols[type.to_s][name.to_s] = { + "assignments" => sym[:assignments], + "uses" => sym[:uses]} end end From 65a2688be9175de2beae9bca833f501e8a683b59 Mon Sep 17 00:00:00 2001 From: Benjamin Attal Date: Sat, 31 May 2014 23:42:44 -0400 Subject: [PATCH 06/12] Adjust test for new language dict. --- bitshift/parser/__init__.py | 3 ++- test/parser_test.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bitshift/parser/__init__.py b/bitshift/parser/__init__.py index 79fca78..927b397 100644 --- a/bitshift/parser/__init__.py +++ b/bitshift/parser/__init__.py @@ -27,11 +27,12 @@ def _lang(codelet): try: if codelet.filename: - lex = pgl.guess_lexer_for_filename(codelet.filename, codelet.code) + lex = pgl.get_lexer_for_filename(codelet.filename) else: lex = pgl.guess_lexer(codelet.code) except util.ClassNotFound: raise UnsupportedFileError(codelet.filename) + return LANGS.index(lex.name) def _recv_data(server_socket): diff --git a/test/parser_test.py b/test/parser_test.py index a1cfad3..ffee75c 100644 --- a/test/parser_test.py +++ b/test/parser_test.py @@ -21,7 +21,7 @@ if __name__ == '__main__': elif sys.argv[1] == 'ruby': file_name = "resources/parser.rb" - server_socket_number = 5003 + server_socket_number = 5065 server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server_socket.connect(("localhost", server_socket_number)) From 6edb142b4a7e9f854f9c0c0f32dcf7a2572384c5 Mon Sep 17 00:00:00 2001 From: Benjamin Attal Date: Sun, 1 Jun 2014 00:18:02 -0400 Subject: [PATCH 07/12] Can specify port number to run java server on. --- .../src/main/java/com/bitshift/parsing/Parse.java | 18 +----- .../com/bitshift/parsing/utils/ParseServer.java | 65 ++++++++++++++++++++++ .../java/com/bitshift/parsing/utils/Tuple.java | 23 -------- 3 files changed, 68 insertions(+), 38 deletions(-) create mode 100644 parsers/java/src/main/java/com/bitshift/parsing/utils/ParseServer.java delete mode 100644 parsers/java/src/main/java/com/bitshift/parsing/utils/Tuple.java diff --git a/parsers/java/src/main/java/com/bitshift/parsing/Parse.java b/parsers/java/src/main/java/com/bitshift/parsing/Parse.java index 302c083..ae54a9e 100644 --- a/parsers/java/src/main/java/com/bitshift/parsing/Parse.java +++ b/parsers/java/src/main/java/com/bitshift/parsing/Parse.java @@ -1,24 +1,12 @@ package com.bitshift.parsing; -import java.io.IOException; - -import java.net.ServerSocket; -import java.net.Socket; - -import com.bitshift.parsing.parsers.JavaParser; +import com.bitshift.parsing.utils.ParseServer; public class Parse { public static void main(String[] args) { - try { - ServerSocket server = new ServerSocket(5033); - - while(true) { - Socket clientSocket = server.accept(); - new Thread(new JavaParser(clientSocket)).start(); - } - } catch (IOException ex) { - } + ParseServer server = new ParseServer(Integer.parseInt(args[0])); + new Thread(server).start(); } } diff --git a/parsers/java/src/main/java/com/bitshift/parsing/utils/ParseServer.java b/parsers/java/src/main/java/com/bitshift/parsing/utils/ParseServer.java new file mode 100644 index 0000000..291be34 --- /dev/null +++ b/parsers/java/src/main/java/com/bitshift/parsing/utils/ParseServer.java @@ -0,0 +1,65 @@ +/* Code for multithreaded server taken from Jakob Jenkov */ +package com.bitshift.parsing.utils; + +import java.net.ServerSocket; +import java.net.Socket; +import java.io.IOException; + +import com.bitshift.parsing.parsers.JavaParser; + +public class ParseServer implements Runnable{ + + protected int serverPort = 8080; + protected ServerSocket serverSocket = null; + protected boolean isStopped = false; + protected Thread runningThread= null; + + public ParseServer(int port){ + this.serverPort = port; + } + + public void run(){ + synchronized(this){ + this.runningThread = Thread.currentThread(); + } + openServerSocket(); + while(! isStopped()){ + Socket clientSocket = null; + try { + clientSocket = this.serverSocket.accept(); + } catch (IOException e) { + if(isStopped()) { + System.out.println("Server Stopped.") ; + return; + } + throw new RuntimeException( + "Error accepting client connection", e); + } + new Thread(new JavaParser(clientSocket)).start(); + } + System.out.println("Server Stopped.") ; + } + + + private synchronized boolean isStopped() { + return this.isStopped; + } + + public synchronized void stop(){ + this.isStopped = true; + try { + this.serverSocket.close(); + } catch (IOException e) { + throw new RuntimeException("Error closing server", e); + } + } + + private void openServerSocket() { + try { + this.serverSocket = new ServerSocket(this.serverPort); + } catch (IOException e) { + throw new RuntimeException("Cannot open port 8080", e); + } + } + +} diff --git a/parsers/java/src/main/java/com/bitshift/parsing/utils/Tuple.java b/parsers/java/src/main/java/com/bitshift/parsing/utils/Tuple.java deleted file mode 100644 index 115a3c6..0000000 --- a/parsers/java/src/main/java/com/bitshift/parsing/utils/Tuple.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.bitshift.parsing.utils; - -import java.util.List; -import java.util.Arrays; - -public class Tuple { - private List _objects; - - public Tuple(T... args) { - _objects = Arrays.asList(args); - } - - public String toString() { - StringBuilder builder = new StringBuilder(); - - for(T o: this._objects) { - builder.append(o + ","); - } - - String s = builder.toString(); - return "(" + s.substring(0, s.length() - 1) + ")"; - } -} From 73dee778c58e30bdd5796257ceb05960fa287187 Mon Sep 17 00:00:00 2001 From: Benjamin Attal Date: Sun, 1 Jun 2014 01:29:01 -0400 Subject: [PATCH 08/12] Can specify which port to run ruby server on. --- parsers/java/src/main/java/com/bitshift/parsing/Parse.java | 1 + parsers/ruby/Rakefile | 4 ++-- parsers/ruby/lib/parse_server.rb | 5 +++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/parsers/java/src/main/java/com/bitshift/parsing/Parse.java b/parsers/java/src/main/java/com/bitshift/parsing/Parse.java index ae54a9e..1964b59 100644 --- a/parsers/java/src/main/java/com/bitshift/parsing/Parse.java +++ b/parsers/java/src/main/java/com/bitshift/parsing/Parse.java @@ -6,6 +6,7 @@ public class Parse { public static void main(String[] args) { ParseServer server = new ParseServer(Integer.parseInt(args[0])); + System.out.println("Java Server listening on port " + args[0]); new Thread(server).start(); } diff --git a/parsers/ruby/Rakefile b/parsers/ruby/Rakefile index e66f695..f8cdf64 100644 --- a/parsers/ruby/Rakefile +++ b/parsers/ruby/Rakefile @@ -1,5 +1,5 @@ require File.expand_path('../lib/parse_server.rb', __FILE__) -task :start_server do |t| - start_server +task :start_server, [:port_number] do |t, args| + start_server Integer(args[:port_number]) end diff --git a/parsers/ruby/lib/parse_server.rb b/parsers/ruby/lib/parse_server.rb index 9a929aa..2c87e49 100644 --- a/parsers/ruby/lib/parse_server.rb +++ b/parsers/ruby/lib/parse_server.rb @@ -13,8 +13,9 @@ def pack_int(i) end -def start_server - server = TCPServer.new 5065 +def start_server(port_number) + server = TCPServer.new port_number + puts "Ruby Server listening on port #{port_number}\n" loop do # Start a new thread for each client accepted From c9520fd4e3a562e30a1cdb50fa4ab9c72854076f Mon Sep 17 00:00:00 2001 From: Benjamin Attal Date: Sun, 1 Jun 2014 01:29:28 -0400 Subject: [PATCH 09/12] Add function to start parse servers in parser/__init__.py --- bitshift/parser/__init__.py | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/bitshift/parser/__init__.py b/bitshift/parser/__init__.py index 927b397..7979f09 100644 --- a/bitshift/parser/__init__.py +++ b/bitshift/parser/__init__.py @@ -2,13 +2,24 @@ import json import sys import socket import struct +import subprocess +from os import path from pygments import lexers as pgl, util from ..languages import LANGS from .python import parse_py -_all__ = ["parse"] +_all__ = ["parse", "start_parse_servers"] + +PARSER_COMMANDS = [ + ('Java', ['mvn', '-f', + path.join(path.dirname(__file__), "../../parsers/java/pom.xml"), + 'exec:java', '-Dexec.args="%d"']), + ('Ruby', ['rake', '-f', + path.join(path.dirname(__file__), "../../parsers/ruby/Rakefile"), + "'start_server[%d]'"]) +] class UnsupportedFileError(Exception): pass @@ -72,6 +83,22 @@ def _recv_data(server_socket): server_socket.close() return ''.join(total_data) +def start_parse_servers(): + """ + Starts all the parse servers for languages besides python. + + :rtype: list + """ + + procs = [] + + for (lang, cmd) in PARSER_COMMANDS: + procs.append( + subprocess.Popen(' '.join(cmd) % (5001 + LANGS.index(lang)), + shell=True)) + + return procs + def parse(codelet): """ Dispatches the codelet to the correct parser based on its language. @@ -87,7 +114,7 @@ def parse(codelet): lang = _lang(codelet) source = codelet.code codelet.language = lang - server_socket_number = 5000 + lang + server_socket_number = 5001 + lang if lang == LANGS.index('Python'): parse_py(codelet) From 21cf52ea65dbe68f2529e70c799d8afa8796088e Mon Sep 17 00:00:00 2001 From: Benjamin Attal Date: Sun, 1 Jun 2014 01:29:41 -0400 Subject: [PATCH 10/12] Call start_parse_servers from crawl.py --- bitshift/crawler/crawl.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bitshift/crawler/crawl.py b/bitshift/crawler/crawl.py index c121866..b91fc95 100644 --- a/bitshift/crawler/crawl.py +++ b/bitshift/crawler/crawl.py @@ -7,6 +7,7 @@ Contains functions for initializing all subsidiary, threaded crawlers. import logging, logging.handlers, os, Queue from bitshift.crawler import crawler, indexer +from bitshift.parser import parse, start_parse_servers __all__ = ["crawl"] @@ -32,6 +33,8 @@ def crawl(): for thread in threads: thread.start() + parse_servers = start_parse_servers() + def _configure_logging(): # This isn't ideal, since it means the bitshift python package must be kept # inside the app, but it works for now: From ca4bb8ff424b5acde1472f71a56a3c87007e154e Mon Sep 17 00:00:00 2001 From: Benjamin Attal Date: Sun, 1 Jun 2014 01:52:35 -0400 Subject: [PATCH 11/12] Change format of symbols to fit databse model. --- bitshift/parser/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bitshift/parser/__init__.py b/bitshift/parser/__init__.py index 7979f09..295d45a 100644 --- a/bitshift/parser/__init__.py +++ b/bitshift/parser/__init__.py @@ -125,4 +125,11 @@ def parse(codelet): server_socket.send("%d\n%s" % (len(source), source)) symbols = json.loads(_recv_data(server_socket)) + symbols = {key: [(name, [tuple(loc) + for loc in syms[name]['assignments']], + [tuple(loc) for loc in syms[name]['uses']]) + for name in syms.keys()] + for key, syms in symbols.iteritems()} + + print symbols codelet.symbols = symbols From 7337638b72e2a78c8b1cc33302415ef3da370324 Mon Sep 17 00:00:00 2001 From: Benjamin Attal Date: Sun, 1 Jun 2014 01:55:00 -0400 Subject: [PATCH 12/12] Change form get_lexer to guess_lexer -- guess_lexer is more accurate. --- bitshift/parser/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bitshift/parser/__init__.py b/bitshift/parser/__init__.py index 295d45a..66484a4 100644 --- a/bitshift/parser/__init__.py +++ b/bitshift/parser/__init__.py @@ -38,7 +38,7 @@ def _lang(codelet): try: if codelet.filename: - lex = pgl.get_lexer_for_filename(codelet.filename) + lex = pgl.guess_lexer_for_filename(codelet.filename, codelet.code) else: lex = pgl.guess_lexer(codelet.code) except util.ClassNotFound: