@@ -7,6 +7,7 @@ Contains functions for initializing all subsidiary, threaded crawlers. | |||
import logging, logging.handlers, os, Queue | |||
from bitshift.crawler import crawler, indexer | |||
from bitshift.parser import parse, start_parse_servers | |||
__all__ = ["crawl"] | |||
@@ -32,6 +33,8 @@ def crawl(): | |||
for thread in threads: | |||
thread.start() | |||
parse_servers = start_parse_servers() | |||
def _configure_logging(): | |||
# This isn't ideal, since it means the bitshift python package must be kept | |||
# inside the app, but it works for now: | |||
@@ -2,13 +2,24 @@ import json | |||
import sys | |||
import socket | |||
import struct | |||
import subprocess | |||
from os import path | |||
from pygments import lexers as pgl, util | |||
from ..languages import LANGS | |||
from .python import parse_py | |||
_all__ = ["parse", "UnsupportedFileError"] | |||
_all__ = ["parse", "UnsupportedFileError", "start_parse_servers"] | |||
PARSER_COMMANDS = [ | |||
('Java', ['mvn', '-f', | |||
path.join(path.dirname(__file__), "../../parsers/java/pom.xml"), | |||
'exec:java', '-Dexec.args="%d"']), | |||
('Ruby', ['rake', '-f', | |||
path.join(path.dirname(__file__), "../../parsers/ruby/Rakefile"), | |||
"'start_server[%d]'"]) | |||
] | |||
class UnsupportedFileError(Exception): | |||
pass | |||
@@ -32,6 +43,7 @@ def _lang(codelet): | |||
lex = pgl.guess_lexer(codelet.code) | |||
except util.ClassNotFound: | |||
raise UnsupportedFileError(codelet.filename) | |||
return LANGS.index(lex.name) | |||
def _recv_data(server_socket): | |||
@@ -71,6 +83,22 @@ def _recv_data(server_socket): | |||
server_socket.close() | |||
return ''.join(total_data) | |||
def start_parse_servers(): | |||
""" | |||
Starts all the parse servers for languages besides python. | |||
:rtype: list | |||
""" | |||
procs = [] | |||
for (lang, cmd) in PARSER_COMMANDS: | |||
procs.append( | |||
subprocess.Popen(' '.join(cmd) % (5001 + LANGS.index(lang)), | |||
shell=True)) | |||
return procs | |||
def parse(codelet): | |||
""" | |||
Dispatches the codelet to the correct parser based on its language. | |||
@@ -86,7 +114,7 @@ def parse(codelet): | |||
lang = _lang(codelet) | |||
source = codelet.code | |||
codelet.language = lang | |||
server_socket_number = 5000 + lang | |||
server_socket_number = 5001 + lang | |||
if lang == LANGS.index('Python'): | |||
parse_py(codelet) | |||
@@ -97,4 +125,10 @@ def parse(codelet): | |||
server_socket.send("%d\n%s" % (len(source), source)) | |||
symbols = json.loads(_recv_data(server_socket)) | |||
symbols = {key: [(name, [tuple(loc) | |||
for loc in syms[name]['assignments']], | |||
[tuple(loc) for loc in syms[name]['uses']]) | |||
for name in syms.keys()] | |||
for key, syms in symbols.iteritems()} | |||
codelet.symbols = symbols |
@@ -1,33 +1,13 @@ | |||
package com.bitshift.parsing; | |||
import java.io.BufferedReader; | |||
import java.io.InputStreamReader; | |||
import java.io.PrintWriter; | |||
import java.io.IOException; | |||
import java.net.ServerSocket; | |||
import java.net.Socket; | |||
import com.bitshift.parsing.parsers.JavaParser; | |||
import com.bitshift.parsing.utils.ParseServer; | |||
public class Parse { | |||
public static void main(String[] args) { | |||
String fromClient; | |||
String toClient; | |||
try { | |||
ServerSocket server = new ServerSocket(5002); | |||
while(true) { | |||
Socket clientSocket = server.accept(); | |||
JavaParser parser = new JavaParser(clientSocket); | |||
Thread parserTask = new Thread(parser); | |||
parserTask.start(); | |||
} | |||
} catch (IOException ex) { | |||
} | |||
ParseServer server = new ParseServer(Integer.parseInt(args[0])); | |||
System.out.println("Java Server listening on port " + args[0]); | |||
new Thread(server).start(); | |||
} | |||
} |
@@ -13,7 +13,6 @@ import org.eclipse.jdt.core.dom.ASTParser; | |||
import org.eclipse.jdt.core.dom.ASTVisitor; | |||
import org.eclipse.jdt.core.dom.CompilationUnit; | |||
import org.eclipse.jdt.core.dom.ClassInstanceCreation; | |||
import org.eclipse.jdt.core.dom.FieldDeclaration; | |||
import org.eclipse.jdt.core.dom.MethodDeclaration; | |||
import org.eclipse.jdt.core.dom.MethodInvocation; | |||
import org.eclipse.jdt.core.dom.Name; | |||
@@ -71,22 +70,6 @@ public class JavaParser extends Parser { | |||
this._cache = new Stack<HashMap<String, Object>>(); | |||
} | |||
public boolean visit(FieldDeclaration node) { | |||
HashMap<String, Object> data = new HashMap<String, Object>(); | |||
int sl = this.root.getLineNumber(node.getStartPosition()); | |||
int sc = this.root.getColumnNumber(node.getStartPosition()); | |||
data.put("coord", Symbols.createCoord(sl, sc, -1, -1)); | |||
this._cache.push(data); | |||
return true; | |||
} | |||
public void endVisit(FieldDeclaration node) { | |||
HashMap<String, Object> data = this._cache.pop(); | |||
String name = (String)data.remove("name"); | |||
this.symbols.insertFieldDeclaration(name, data); | |||
} | |||
public boolean visit(MethodDeclaration node) { | |||
HashMap<String, Object> data = new HashMap<String, Object>(); | |||
Name nameObj = node.getName(); | |||
@@ -115,7 +98,7 @@ public class JavaParser extends Parser { | |||
public void endVisit(MethodDeclaration node) { | |||
HashMap<String, Object> data = this._cache.pop(); | |||
String name = (String)data.remove("name"); | |||
this.symbols.insertMethodDeclaration(name, data); | |||
this.symbols.insertMethodDeclaration("\"" + name + "\"", data); | |||
} | |||
public boolean visit(MethodInvocation node) { | |||
@@ -136,7 +119,7 @@ public class JavaParser extends Parser { | |||
public void endVisit(MethodInvocation node) { | |||
HashMap<String, Object> data = this._cache.pop(); | |||
String name = (String)data.remove("name"); | |||
this.symbols.insertMethodInvocation(name, data); | |||
this.symbols.insertMethodInvocation("\"" + name + "\"", data); | |||
} | |||
public boolean visit(PackageDeclaration node) { | |||
@@ -167,9 +150,9 @@ public class JavaParser extends Parser { | |||
String name = (String)data.remove("name"); | |||
if (node.isInterface()) { | |||
this.symbols.insertInterfaceDeclaration(name, data); | |||
this.symbols.insertInterfaceDeclaration("\"" + name + "\"", data); | |||
} else { | |||
this.symbols.insertClassDeclaration(name, data); | |||
this.symbols.insertClassDeclaration("\"" + name + "\"", data); | |||
} | |||
} | |||
@@ -186,7 +169,7 @@ public class JavaParser extends Parser { | |||
public void endVisit(VariableDeclarationFragment node) { | |||
HashMap<String, Object> data = this._cache.pop(); | |||
String name = (String)data.remove("name"); | |||
this.symbols.insertVariableDeclaration(name, data); | |||
this.symbols.insertVariableDeclaration("\"" + name + "\"", data); | |||
} | |||
public boolean visit(QualifiedName node) { | |||
@@ -1,8 +1,9 @@ | |||
package com.bitshift.parsing.parsers; | |||
import java.io.BufferedReader; | |||
import java.io.BufferedWriter; | |||
import java.io.InputStreamReader; | |||
import java.io.PrintWriter; | |||
import java.io.OutputStreamWriter; | |||
import java.io.IOException; | |||
import java.net.Socket; | |||
@@ -46,12 +47,16 @@ public abstract class Parser implements Runnable { | |||
protected void writeToClient(String toClient) { | |||
try { | |||
PrintWriter clientWriter = new PrintWriter( | |||
this.clientSocket.getOutputStream(), true); | |||
BufferedWriter clientWriter = new BufferedWriter( | |||
new OutputStreamWriter(this.clientSocket.getOutputStream())); | |||
PackableMemory mem = new PackableMemory(toClient.length()); | |||
PackableMemory mem = new PackableMemory(4); | |||
mem.pack(toClient.length(), 0); | |||
String dataSize = new String(mem.mem); | |||
clientWriter.println(dataSize + toClient); | |||
clientWriter.write(dataSize + toClient); | |||
clientWriter.flush(); | |||
this.clientSocket.close(); | |||
} catch (IOException ex) { | |||
} | |||
} | |||
@@ -11,15 +11,16 @@ public class JavaSymbols extends Symbols { | |||
private HashMap<String, HashMap<String, Object>> _classes; | |||
private HashMap<String, HashMap<String, Object>> _interfaces; | |||
private HashMap<String, HashMap<String, Object>> _methods; | |||
private HashMap<String, HashMap<String, Object>> _fields; | |||
private HashMap<String, HashMap<String, Object>> _vars; | |||
private final String assignKey = "\"assignments\""; | |||
private final String useKey = "\"uses\""; | |||
public JavaSymbols() { | |||
_packageName = null; | |||
_classes = new HashMap<String, HashMap<String, Object>>(); | |||
_interfaces = new HashMap<String, HashMap<String, Object>>(); | |||
_methods = new HashMap<String, HashMap<String, Object>>(); | |||
_fields = new HashMap<String, HashMap<String, Object>>(); | |||
_vars = new HashMap<String, HashMap<String, Object>>(); | |||
} | |||
@@ -34,15 +35,23 @@ public class JavaSymbols extends Symbols { | |||
HashMap<String, Object> klass = new HashMap<String, Object>(); | |||
assignments.add(data.get("coord")); | |||
klass.put("assignments", assignments); | |||
klass.put("uses", uses); | |||
klass.put(assignKey, assignments); | |||
klass.put(useKey, uses); | |||
this._classes.put(name, klass); | |||
return true; | |||
} | |||
public boolean insertInterfaceDeclaration(String name, HashMap<String, Object> data) { | |||
this._interfaces.put(name, data); | |||
ArrayList<Object> assignments = new ArrayList<Object>(10); | |||
ArrayList<Object> uses = new ArrayList<Object>(10); | |||
HashMap<String, Object> klass = new HashMap<String, Object>(); | |||
assignments.add(data.get("coord")); | |||
klass.put(assignKey, assignments); | |||
klass.put(useKey, uses); | |||
this._interfaces.put(name, klass); | |||
return true; | |||
} | |||
@@ -54,13 +63,13 @@ public class JavaSymbols extends Symbols { | |||
ArrayList<Object> uses = new ArrayList<Object>(10); | |||
assignments.add(data.get("coord")); | |||
method.put("assignments", assignments); | |||
method.put("uses", uses); | |||
method.put(assignKey, assignments); | |||
method.put(useKey, uses); | |||
} else { | |||
ArrayList<Object> assignments = (ArrayList<Object>)method.get("assignments"); | |||
ArrayList<Object> assignments = (ArrayList<Object>)method.get(assignKey); | |||
assignments.add(data.get("coord")); | |||
method.put("assignments", assignments); | |||
method.put(assignKey, assignments); | |||
} | |||
this._methods.put(name, method); | |||
@@ -74,24 +83,19 @@ public class JavaSymbols extends Symbols { | |||
ArrayList<Object> uses = new ArrayList<Object>(10); | |||
uses.add(data.get("coord")); | |||
method.put("assignments", assignments); | |||
method.put("uses", uses); | |||
method.put(assignKey, assignments); | |||
method.put(useKey, uses); | |||
} else { | |||
ArrayList<Object> uses = (ArrayList<Object>)method.get("uses"); | |||
ArrayList<Object> uses = (ArrayList<Object>)method.get(useKey); | |||
uses.add(data.get("coord")); | |||
method.put("uses", uses); | |||
method.put(useKey, uses); | |||
} | |||
this._methods.put(name, method); | |||
return true; | |||
} | |||
public boolean insertFieldDeclaration(String name, HashMap<String, Object> data) { | |||
this._fields.put(name, data); | |||
return true; | |||
} | |||
public boolean insertVariableDeclaration(String name, HashMap<String, Object> data) { | |||
HashMap<String, Object> var = this._vars.get(name); | |||
if (var == null) { | |||
@@ -100,13 +104,13 @@ public class JavaSymbols extends Symbols { | |||
ArrayList<Object> uses = new ArrayList<Object>(10); | |||
assignments.add(data.get("coord")); | |||
var.put("assignments", assignments); | |||
var.put("uses", uses); | |||
var.put(assignKey, assignments); | |||
var.put(useKey, uses); | |||
} else { | |||
ArrayList<Object> assignments = (ArrayList<Object>)var.get("assignments"); | |||
ArrayList<Object> assignments = (ArrayList<Object>)var.get(assignKey); | |||
assignments.add(data.get("coord")); | |||
var.put("assignments", assignments); | |||
var.put(assignKey, assignments); | |||
} | |||
this._vars.put(name, var); | |||
@@ -120,13 +124,13 @@ public class JavaSymbols extends Symbols { | |||
ArrayList<Object> uses = new ArrayList<Object>(10); | |||
uses.add(data.get("coord")); | |||
var.put("assignments", assignments); | |||
var.put("uses", uses); | |||
var.put(assignKey, assignments); | |||
var.put(useKey, uses); | |||
} else { | |||
ArrayList<Object> uses = (ArrayList<Object>)var.get("uses"); | |||
ArrayList<Object> uses = (ArrayList<Object>)var.get(useKey); | |||
uses.add(data.get("coord")); | |||
var.put("uses", uses); | |||
var.put(useKey, uses); | |||
} | |||
this._vars.put(name, var); | |||
@@ -135,13 +139,14 @@ public class JavaSymbols extends Symbols { | |||
public String toString() { | |||
StringBuilder builder = new StringBuilder(); | |||
builder.append("classes:" + this._classes + ","); | |||
builder.append("interfaces:" + this._interfaces + ","); | |||
builder.append("methods:" + this._methods + ","); | |||
builder.append("fields:" + this._fields + ","); | |||
builder.append("vars:" + this._vars + ","); | |||
return "{" + builder.toString() + "}"; | |||
builder.append("\"classes\":" + this._classes + ","); | |||
builder.append("\"interfaces\":" + this._interfaces + ","); | |||
builder.append("\"methods\":" + this._methods + ","); | |||
builder.append("\"vars\":" + this._vars + ","); | |||
String s = builder.toString().replaceAll("=", ":"); | |||
s = s.substring(0, s.length() - 1); | |||
return "{" + s + "}"; | |||
} | |||
} | |||
@@ -22,7 +22,7 @@ public class PackableMemory { | |||
// The most significant porion of the integer is stored in mem[loc]. | |||
// Bytes are masked out of the integer and stored in the array, working | |||
// from right(least significant) to left (most significant). | |||
void pack(int val, int loc) | |||
public void pack(int val, int loc) | |||
{ | |||
final int MASK = 0xff; | |||
for (int i = 3; i >= 0; i--) | |||
@@ -0,0 +1,65 @@ | |||
/* Code for multithreaded server taken from Jakob Jenkov */ | |||
package com.bitshift.parsing.utils; | |||
import java.net.ServerSocket; | |||
import java.net.Socket; | |||
import java.io.IOException; | |||
import com.bitshift.parsing.parsers.JavaParser; | |||
public class ParseServer implements Runnable{ | |||
protected int serverPort = 8080; | |||
protected ServerSocket serverSocket = null; | |||
protected boolean isStopped = false; | |||
protected Thread runningThread= null; | |||
public ParseServer(int port){ | |||
this.serverPort = port; | |||
} | |||
public void run(){ | |||
synchronized(this){ | |||
this.runningThread = Thread.currentThread(); | |||
} | |||
openServerSocket(); | |||
while(! isStopped()){ | |||
Socket clientSocket = null; | |||
try { | |||
clientSocket = this.serverSocket.accept(); | |||
} catch (IOException e) { | |||
if(isStopped()) { | |||
System.out.println("Server Stopped.") ; | |||
return; | |||
} | |||
throw new RuntimeException( | |||
"Error accepting client connection", e); | |||
} | |||
new Thread(new JavaParser(clientSocket)).start(); | |||
} | |||
System.out.println("Server Stopped.") ; | |||
} | |||
private synchronized boolean isStopped() { | |||
return this.isStopped; | |||
} | |||
public synchronized void stop(){ | |||
this.isStopped = true; | |||
try { | |||
this.serverSocket.close(); | |||
} catch (IOException e) { | |||
throw new RuntimeException("Error closing server", e); | |||
} | |||
} | |||
private void openServerSocket() { | |||
try { | |||
this.serverSocket = new ServerSocket(this.serverPort); | |||
} catch (IOException e) { | |||
throw new RuntimeException("Cannot open port 8080", e); | |||
} | |||
} | |||
} |
@@ -0,0 +1,23 @@ | |||
package com.bitshift.parsing.utils; | |||
import java.util.List; | |||
import java.util.Arrays; | |||
public class Tuple<T> { | |||
private List<T> _objects; | |||
public Tuple(T... args) { | |||
_objects = Arrays.asList(args); | |||
} | |||
public String toString() { | |||
StringBuilder builder = new StringBuilder(); | |||
for(T o: this._objects) { | |||
builder.append(o + ","); | |||
} | |||
String s = builder.toString(); | |||
return "(" + s.substring(0, s.length() - 1) + ")"; | |||
} | |||
} |
@@ -1,5 +1,5 @@ | |||
require File.expand_path('../lib/parse_server.rb', __FILE__) | |||
task :start_server do |t| | |||
start_server | |||
task :start_server, [:port_number] do |t, args| | |||
start_server Integer(args[:port_number]) | |||
end |
@@ -13,8 +13,9 @@ def pack_int(i) | |||
end | |||
def start_server | |||
server = TCPServer.new 5003 | |||
def start_server(port_number) | |||
server = TCPServer.new port_number | |||
puts "Ruby Server listening on port #{port_number}\n" | |||
loop do | |||
# Start a new thread for each client accepted | |||
@@ -25,7 +25,8 @@ module Bitshift | |||
def initialize(offset, tree) | |||
super() | |||
module_hash = Hash.new {|hash, key| hash[key] = { assignments: [], uses: [] }} | |||
module_hash = Hash.new {|hash, key| | |||
hash[key] = { assignments: [], uses: [] }} | |||
class_hash = module_hash.clone | |||
function_hash = module_hash.clone | |||
var_hash = module_hash.clone | |||
@@ -118,8 +119,18 @@ module Bitshift | |||
end | |||
def to_s | |||
str = symbols.to_s | |||
str = str.gsub(/:(\w*)=>/, '"\1":') | |||
new_symbols = Hash.new {|hash, key| hash[key] = Hash.new} | |||
symbols.each do |type, sym_list| | |||
sym_list.each do |name, sym| | |||
new_symbols[type.to_s][name.to_s] = { | |||
"assignments" => sym[:assignments], | |||
"uses" => sym[:uses]} | |||
end | |||
end | |||
str = new_symbols.to_s | |||
str = str.gsub(/=>/, ":") | |||
return str | |||
end | |||
end | |||
@@ -21,7 +21,7 @@ if __name__ == '__main__': | |||
elif sys.argv[1] == 'ruby': | |||
file_name = "resources/parser.rb" | |||
server_socket_number = 5003 | |||
server_socket_number = 5065 | |||
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) | |||
server_socket.connect(("localhost", server_socket_number)) | |||