Fix merge conflicts

bitshift/crawler/crawl.py

@@ -7,6 +7,7 @@ Contains functions for initializing all subsidiary, threaded crawlers.
 import logging, logging.handlers, os, Queue
 
 from bitshift.crawler import crawler, indexer
+from bitshift.parser import parse, start_parse_servers
 
 __all__ = ["crawl"]
 
@@ -32,6 +33,8 @@ def crawl():
     for thread in threads:
         thread.start()
 
+    parse_servers = start_parse_servers()
+
 def _configure_logging():
     # This isn't ideal, since it means the bitshift python package must be kept
     # inside the app, but it works for now:

bitshift/parser/__init__.py

@@ -2,13 +2,24 @@ import json
 import sys
 import socket
 import struct
+import subprocess
 
+from os import path
 from pygments import lexers as pgl, util
 
 from ..languages import LANGS
 from .python import parse_py
 
-_all__ = ["parse", "UnsupportedFileError"]
+_all__ = ["parse", "UnsupportedFileError", "start_parse_servers"]
+
+PARSER_COMMANDS = [
+        ('Java', ['mvn', '-f',
+            path.join(path.dirname(__file__), "../../parsers/java/pom.xml"),
+            'exec:java', '-Dexec.args="%d"']),
+        ('Ruby', ['rake', '-f',
+            path.join(path.dirname(__file__), "../../parsers/ruby/Rakefile"),
+            "'start_server[%d]'"])
+]
 
 class UnsupportedFileError(Exception):
     pass
@@ -32,6 +43,7 @@ def _lang(codelet):
             lex = pgl.guess_lexer(codelet.code)
     except util.ClassNotFound:
         raise UnsupportedFileError(codelet.filename)
+
     return LANGS.index(lex.name)
 
 def _recv_data(server_socket):
@@ -71,6 +83,22 @@ def _recv_data(server_socket):
     server_socket.close()
     return ''.join(total_data)
 
+def start_parse_servers():
+    """
+    Starts all the parse servers for languages besides python.
+
+    :rtype: list
+    """
+
+    procs = []
+
+    for (lang, cmd) in PARSER_COMMANDS:
+        procs.append(
+                subprocess.Popen(' '.join(cmd) % (5001 + LANGS.index(lang)),
+                    shell=True))
+
+    return procs
+
 def parse(codelet):
     """
     Dispatches the codelet to the correct parser based on its language.
@@ -86,7 +114,7 @@ def parse(codelet):
     lang = _lang(codelet)
     source = codelet.code
     codelet.language = lang
-    server_socket_number = 5000 + lang
+    server_socket_number = 5001 + lang
 
     if lang == LANGS.index('Python'):
         parse_py(codelet)
@@ -97,4 +125,10 @@ def parse(codelet):
         server_socket.send("%d\n%s" % (len(source), source))
 
         symbols = json.loads(_recv_data(server_socket))
+        symbols = {key: [(name, [tuple(loc)
+            for loc in syms[name]['assignments']],
+            [tuple(loc) for loc in syms[name]['uses']])
+            for name in syms.keys()]
+            for key, syms in symbols.iteritems()}
+
         codelet.symbols = symbols

parsers/java/src/main/java/com/bitshift/parsing/Parse.java

@@ -1,33 +1,13 @@
 package com.bitshift.parsing;
 
-import java.io.BufferedReader;
-import java.io.InputStreamReader;
-import java.io.PrintWriter;
-import java.io.IOException;
-
-import java.net.ServerSocket;
-import java.net.Socket;
-
-import com.bitshift.parsing.parsers.JavaParser;
+import com.bitshift.parsing.utils.ParseServer;
 
 public class Parse {
 
     public static void main(String[] args) {
-        String fromClient;
-        String toClient;
-
-        try {
-            ServerSocket server = new ServerSocket(5002);
-
-            while(true) {
-                Socket clientSocket = server.accept();
-
-                JavaParser parser = new JavaParser(clientSocket);
-                Thread parserTask = new Thread(parser);
-                parserTask.start();
-            }
-        } catch (IOException ex) {
-        }
+        ParseServer server = new ParseServer(Integer.parseInt(args[0]));
+        System.out.println("Java Server listening on port " + args[0]);
+        new Thread(server).start();
     }
 
 }

parsers/java/src/main/java/com/bitshift/parsing/parsers/JavaParser.java

@@ -13,7 +13,6 @@ import org.eclipse.jdt.core.dom.ASTParser;
 import org.eclipse.jdt.core.dom.ASTVisitor;
 import org.eclipse.jdt.core.dom.CompilationUnit;
 import org.eclipse.jdt.core.dom.ClassInstanceCreation;
-import org.eclipse.jdt.core.dom.FieldDeclaration;
 import org.eclipse.jdt.core.dom.MethodDeclaration;
 import org.eclipse.jdt.core.dom.MethodInvocation;
 import org.eclipse.jdt.core.dom.Name;
@@ -71,22 +70,6 @@ public class JavaParser extends Parser {
             this._cache = new Stack<HashMap<String, Object>>();
         }
 
-        public boolean visit(FieldDeclaration node) {
-            HashMap<String, Object> data = new HashMap<String, Object>();
-            int sl = this.root.getLineNumber(node.getStartPosition());
-            int sc = this.root.getColumnNumber(node.getStartPosition());
-
-            data.put("coord", Symbols.createCoord(sl, sc, -1, -1));
-            this._cache.push(data);
-            return true;
-        }
-
-        public void endVisit(FieldDeclaration node) {
-            HashMap<String, Object> data = this._cache.pop();
-            String name = (String)data.remove("name");
-            this.symbols.insertFieldDeclaration(name, data);
-        }
-
         public boolean visit(MethodDeclaration node) {
             HashMap<String, Object> data = new HashMap<String, Object>();
             Name nameObj = node.getName();
@@ -115,7 +98,7 @@ public class JavaParser extends Parser {
         public void endVisit(MethodDeclaration node) {
             HashMap<String, Object> data = this._cache.pop();
             String name = (String)data.remove("name");
-            this.symbols.insertMethodDeclaration(name, data);
+            this.symbols.insertMethodDeclaration("\"" + name + "\"", data);
         }
 
         public boolean visit(MethodInvocation node) {
@@ -136,7 +119,7 @@ public class JavaParser extends Parser {
         public void endVisit(MethodInvocation node) {
             HashMap<String, Object> data = this._cache.pop();
             String name = (String)data.remove("name");
-            this.symbols.insertMethodInvocation(name, data);
+            this.symbols.insertMethodInvocation("\"" + name + "\"", data);
         }
 
         public boolean visit(PackageDeclaration node) {
@@ -167,9 +150,9 @@ public class JavaParser extends Parser {
             String name = (String)data.remove("name");
 
             if (node.isInterface()) {
-                this.symbols.insertInterfaceDeclaration(name, data);
+                this.symbols.insertInterfaceDeclaration("\"" + name + "\"", data);
             } else {
-                this.symbols.insertClassDeclaration(name, data);
+                this.symbols.insertClassDeclaration("\"" + name + "\"", data);
             }
         }
 
@@ -186,7 +169,7 @@ public class JavaParser extends Parser {
         public void endVisit(VariableDeclarationFragment node) {
             HashMap<String, Object> data = this._cache.pop();
             String name = (String)data.remove("name");
-            this.symbols.insertVariableDeclaration(name, data);
+            this.symbols.insertVariableDeclaration("\"" + name + "\"", data);
         }
 
         public boolean visit(QualifiedName node) {

parsers/java/src/main/java/com/bitshift/parsing/parsers/Parser.java

@@ -1,8 +1,9 @@
 package com.bitshift.parsing.parsers;
 
 import java.io.BufferedReader;
+import java.io.BufferedWriter;
 import java.io.InputStreamReader;
-import java.io.PrintWriter;
+import java.io.OutputStreamWriter;
 import java.io.IOException;
 
 import java.net.Socket;
@@ -46,12 +47,16 @@ public abstract class Parser implements Runnable {
 
     protected void writeToClient(String toClient) {
         try {
-            PrintWriter clientWriter = new PrintWriter(
-                    this.clientSocket.getOutputStream(), true);
+            BufferedWriter clientWriter = new BufferedWriter(
+                    new OutputStreamWriter(this.clientSocket.getOutputStream()));
 
-            PackableMemory mem = new PackableMemory(toClient.length());
+            PackableMemory mem = new PackableMemory(4);
+            mem.pack(toClient.length(), 0);
             String dataSize = new String(mem.mem);
-            clientWriter.println(dataSize + toClient);
+
+            clientWriter.write(dataSize + toClient);
+            clientWriter.flush();
+            this.clientSocket.close();
         } catch (IOException ex) {
         }
     }

parsers/java/src/main/java/com/bitshift/parsing/symbols/JavaSymbols.java

@@ -11,15 +11,16 @@ public class JavaSymbols extends Symbols {
     private HashMap<String, HashMap<String, Object>> _classes;
     private HashMap<String, HashMap<String, Object>> _interfaces;
     private HashMap<String, HashMap<String, Object>> _methods;
-    private HashMap<String, HashMap<String, Object>> _fields;
     private HashMap<String, HashMap<String, Object>> _vars;
 
+    private final String assignKey = "\"assignments\"";
+    private final String useKey = "\"uses\"";
+
     public JavaSymbols() {
         _packageName = null;
         _classes = new HashMap<String, HashMap<String, Object>>();
         _interfaces = new HashMap<String, HashMap<String, Object>>();
         _methods = new HashMap<String, HashMap<String, Object>>();
-        _fields = new HashMap<String, HashMap<String, Object>>();
         _vars = new HashMap<String, HashMap<String, Object>>();
     }
 
@@ -34,15 +35,23 @@ public class JavaSymbols extends Symbols {
         HashMap<String, Object> klass = new HashMap<String, Object>();
 
         assignments.add(data.get("coord"));
-        klass.put("assignments", assignments);
-        klass.put("uses", uses);
+        klass.put(assignKey, assignments);
+        klass.put(useKey, uses);
 
         this._classes.put(name, klass);
         return true;
     }
 
     public boolean insertInterfaceDeclaration(String name, HashMap<String, Object> data) {
-        this._interfaces.put(name, data);
+        ArrayList<Object> assignments = new ArrayList<Object>(10);
+        ArrayList<Object> uses = new ArrayList<Object>(10);
+        HashMap<String, Object> klass = new HashMap<String, Object>();
+
+        assignments.add(data.get("coord"));
+        klass.put(assignKey, assignments);
+        klass.put(useKey, uses);
+
+        this._interfaces.put(name, klass);
         return true;
     }
 
@@ -54,13 +63,13 @@ public class JavaSymbols extends Symbols {
             ArrayList<Object> uses = new ArrayList<Object>(10);
 
             assignments.add(data.get("coord"));
-            method.put("assignments", assignments);
-            method.put("uses", uses);
+            method.put(assignKey, assignments);
+            method.put(useKey, uses);
         } else {
-            ArrayList<Object> assignments = (ArrayList<Object>)method.get("assignments");
+            ArrayList<Object> assignments = (ArrayList<Object>)method.get(assignKey);
 
             assignments.add(data.get("coord"));
-            method.put("assignments", assignments);
+            method.put(assignKey, assignments);
         }
 
         this._methods.put(name, method);
@@ -74,24 +83,19 @@ public class JavaSymbols extends Symbols {
             ArrayList<Object> uses = new ArrayList<Object>(10);
 
             uses.add(data.get("coord"));
-            method.put("assignments", assignments);
-            method.put("uses", uses);
+            method.put(assignKey, assignments);
+            method.put(useKey, uses);
         } else {
-            ArrayList<Object> uses = (ArrayList<Object>)method.get("uses");
+            ArrayList<Object> uses = (ArrayList<Object>)method.get(useKey);
 
             uses.add(data.get("coord"));
-            method.put("uses", uses);
+            method.put(useKey, uses);
         }
 
         this._methods.put(name, method);
         return true;
     }
 
-    public boolean insertFieldDeclaration(String name, HashMap<String, Object> data) {
-        this._fields.put(name, data);
-        return true;
-    }
-
     public boolean insertVariableDeclaration(String name, HashMap<String, Object> data) {
         HashMap<String, Object> var = this._vars.get(name);
         if (var == null) {
@@ -100,13 +104,13 @@ public class JavaSymbols extends Symbols {
             ArrayList<Object> uses = new ArrayList<Object>(10);
 
             assignments.add(data.get("coord"));
-            var.put("assignments", assignments);
-            var.put("uses", uses);
+            var.put(assignKey, assignments);
+            var.put(useKey, uses);
         } else {
-            ArrayList<Object> assignments = (ArrayList<Object>)var.get("assignments");
+            ArrayList<Object> assignments = (ArrayList<Object>)var.get(assignKey);
 
             assignments.add(data.get("coord"));
-            var.put("assignments", assignments);
+            var.put(assignKey, assignments);
         }
 
         this._vars.put(name, var);
@@ -120,13 +124,13 @@ public class JavaSymbols extends Symbols {
             ArrayList<Object> uses = new ArrayList<Object>(10);
 
             uses.add(data.get("coord"));
-            var.put("assignments", assignments);
-            var.put("uses", uses);
+            var.put(assignKey, assignments);
+            var.put(useKey, uses);
         } else {
-            ArrayList<Object> uses = (ArrayList<Object>)var.get("uses");
+            ArrayList<Object> uses = (ArrayList<Object>)var.get(useKey);
 
             uses.add(data.get("coord"));
-            var.put("uses", uses);
+            var.put(useKey, uses);
         }
 
         this._vars.put(name, var);
@@ -135,13 +139,14 @@ public class JavaSymbols extends Symbols {
 
     public String toString() {
         StringBuilder builder = new StringBuilder();
-        builder.append("classes:" + this._classes + ",");
-        builder.append("interfaces:" + this._interfaces + ",");
-        builder.append("methods:" + this._methods + ",");
-        builder.append("fields:" + this._fields + ",");
-        builder.append("vars:" + this._vars + ",");
-
-        return "{" + builder.toString() + "}";
+        builder.append("\"classes\":" + this._classes + ",");
+        builder.append("\"interfaces\":" + this._interfaces + ",");
+        builder.append("\"methods\":" + this._methods + ",");
+        builder.append("\"vars\":" + this._vars + ",");
+
+        String s = builder.toString().replaceAll("=", ":");
+        s = s.substring(0, s.length() - 1);
+        return "{" + s + "}";
     }
 }
 

parsers/java/src/main/java/com/bitshift/parsing/utils/PackableMemory.java

@@ -22,7 +22,7 @@ public class PackableMemory {
    // The most significant porion of the integer is stored in mem[loc].
    // Bytes are masked out of the integer and stored in the array, working
    // from right(least significant) to left (most significant).
-   void pack(int val, int loc)
+   public void pack(int val, int loc)
    {
       final int MASK = 0xff;
       for (int i = 3; i >= 0; i--)

parsers/java/src/main/java/com/bitshift/parsing/utils/ParseServer.java

@@ -0,0 +1,65 @@
+/* Code for multithreaded server taken from Jakob Jenkov */
+package com.bitshift.parsing.utils;
+
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.io.IOException;
+
+import com.bitshift.parsing.parsers.JavaParser;
+
+public class ParseServer implements Runnable{
+
+    protected int          serverPort   = 8080;
+    protected ServerSocket serverSocket = null;
+    protected boolean      isStopped    = false;
+    protected Thread       runningThread= null;
+
+    public ParseServer(int port){
+        this.serverPort = port;
+    }
+
+    public void run(){
+        synchronized(this){
+            this.runningThread = Thread.currentThread();
+        }
+        openServerSocket();
+        while(! isStopped()){
+            Socket clientSocket = null;
+            try {
+                clientSocket = this.serverSocket.accept();
+            } catch (IOException e) {
+                if(isStopped()) {
+                    System.out.println("Server Stopped.") ;
+                    return;
+                }
+                throw new RuntimeException(
+                        "Error accepting client connection", e);
+            }
+            new Thread(new JavaParser(clientSocket)).start();
+        }
+        System.out.println("Server Stopped.") ;
+    }
+
+
+    private synchronized boolean isStopped() {
+        return this.isStopped;
+    }
+
+    public synchronized void stop(){
+        this.isStopped = true;
+        try {
+            this.serverSocket.close();
+        } catch (IOException e) {
+            throw new RuntimeException("Error closing server", e);
+        }
+    }
+
+    private void openServerSocket() {
+        try {
+            this.serverSocket = new ServerSocket(this.serverPort);
+        } catch (IOException e) {
+            throw new RuntimeException("Cannot open port 8080", e);
+        }
+    }
+
+}

parsers/java/src/main/java/com/bitshift/parsing/utils/Tuple.java

@@ -0,0 +1,23 @@
+package com.bitshift.parsing.utils;
+
+import java.util.List;
+import java.util.Arrays;
+
+public class Tuple<T> {
+    private List<T> _objects;
+
+    public Tuple(T... args) {
+        _objects = Arrays.asList(args);
+    }
+
+    public String toString() {
+        StringBuilder builder = new StringBuilder();
+
+        for(T o: this._objects) {
+            builder.append(o + ",");
+        }
+
+        String s = builder.toString();
+        return "(" + s.substring(0, s.length() - 1) + ")";
+    }
+}

parsers/ruby/Rakefile

@@ -1,5 +1,5 @@
 require File.expand_path('../lib/parse_server.rb', __FILE__)
 
-task :start_server do |t|
-    start_server
+task :start_server, [:port_number] do |t, args|
+    start_server Integer(args[:port_number])
 end

parsers/ruby/lib/parse_server.rb

@@ -13,8 +13,9 @@ def pack_int(i)
 end
 
 
-def start_server
-    server = TCPServer.new 5003
+def start_server(port_number)
+    server = TCPServer.new port_number
+    puts "Ruby Server listening on port #{port_number}\n"
 
     loop do
         # Start a new thread for each client accepted

parsers/ruby/lib/parser.rb

@@ -25,7 +25,8 @@ module Bitshift
         def initialize(offset, tree)
             super()
 
-            module_hash = Hash.new {|hash, key| hash[key] = { assignments: [], uses: [] }}
+            module_hash = Hash.new {|hash, key|
+                hash[key] = { assignments: [], uses: [] }}
             class_hash = module_hash.clone
             function_hash = module_hash.clone
             var_hash = module_hash.clone
@@ -118,8 +119,18 @@ module Bitshift
         end
 
         def to_s
-            str = symbols.to_s
-            str = str.gsub(/:(\w*)=>/, '"\1":')
+            new_symbols = Hash.new {|hash, key| hash[key] = Hash.new}
+
+            symbols.each do |type, sym_list|
+                sym_list.each do |name, sym|
+                    new_symbols[type.to_s][name.to_s] = {
+                        "assignments" => sym[:assignments],
+                        "uses" => sym[:uses]}
+                end
+            end
+
+            str = new_symbols.to_s
+            str = str.gsub(/=>/, ":")
             return str
         end
     end

test/parser_test.py

@@ -21,7 +21,7 @@ if __name__ == '__main__':
 
         elif sys.argv[1] == 'ruby':
             file_name = "resources/parser.rb"
-            server_socket_number = 5003
+            server_socket_number = 5065
 
         server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         server_socket.connect(("localhost", server_socket_number))