Browse Source

Make sure parse servers return json loadable data. Make sure Java server closes client connection.

tags/v1.0^2
Benjamin Attal 10 years ago
parent
commit
1f56300c73
8 changed files with 93 additions and 97 deletions
  1. +2
    -11
      parsers/java/src/main/java/com/bitshift/parsing/Parse.java
  2. +5
    -22
      parsers/java/src/main/java/com/bitshift/parsing/parsers/JavaParser.java
  3. +10
    -5
      parsers/java/src/main/java/com/bitshift/parsing/parsers/Parser.java
  4. +38
    -33
      parsers/java/src/main/java/com/bitshift/parsing/symbols/JavaSymbols.java
  5. +1
    -1
      parsers/java/src/main/java/com/bitshift/parsing/utils/PackableMemory.java
  6. +23
    -0
      parsers/java/src/main/java/com/bitshift/parsing/utils/Tuple.java
  7. +1
    -1
      parsers/ruby/lib/parse_server.rb
  8. +13
    -24
      parsers/ruby/lib/parser.rb

+ 2
- 11
parsers/java/src/main/java/com/bitshift/parsing/Parse.java View File

@@ -1,8 +1,5 @@
package com.bitshift.parsing; package com.bitshift.parsing;


import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.io.IOException; import java.io.IOException;


import java.net.ServerSocket; import java.net.ServerSocket;
@@ -13,18 +10,12 @@ import com.bitshift.parsing.parsers.JavaParser;
public class Parse { public class Parse {


public static void main(String[] args) { public static void main(String[] args) {
String fromClient;
String toClient;

try { try {
ServerSocket server = new ServerSocket(5002);
ServerSocket server = new ServerSocket(5033);


while(true) { while(true) {
Socket clientSocket = server.accept(); Socket clientSocket = server.accept();

JavaParser parser = new JavaParser(clientSocket);
Thread parserTask = new Thread(parser);
parserTask.start();
new Thread(new JavaParser(clientSocket)).start();
} }
} catch (IOException ex) { } catch (IOException ex) {
} }


+ 5
- 22
parsers/java/src/main/java/com/bitshift/parsing/parsers/JavaParser.java View File

@@ -13,7 +13,6 @@ import org.eclipse.jdt.core.dom.ASTParser;
import org.eclipse.jdt.core.dom.ASTVisitor; import org.eclipse.jdt.core.dom.ASTVisitor;
import org.eclipse.jdt.core.dom.CompilationUnit; import org.eclipse.jdt.core.dom.CompilationUnit;
import org.eclipse.jdt.core.dom.ClassInstanceCreation; import org.eclipse.jdt.core.dom.ClassInstanceCreation;
import org.eclipse.jdt.core.dom.FieldDeclaration;
import org.eclipse.jdt.core.dom.MethodDeclaration; import org.eclipse.jdt.core.dom.MethodDeclaration;
import org.eclipse.jdt.core.dom.MethodInvocation; import org.eclipse.jdt.core.dom.MethodInvocation;
import org.eclipse.jdt.core.dom.Name; import org.eclipse.jdt.core.dom.Name;
@@ -71,22 +70,6 @@ public class JavaParser extends Parser {
this._cache = new Stack<HashMap<String, Object>>(); this._cache = new Stack<HashMap<String, Object>>();
} }


public boolean visit(FieldDeclaration node) {
HashMap<String, Object> data = new HashMap<String, Object>();
int sl = this.root.getLineNumber(node.getStartPosition());
int sc = this.root.getColumnNumber(node.getStartPosition());

data.put("coord", Symbols.createCoord(sl, sc, -1, -1));
this._cache.push(data);
return true;
}

public void endVisit(FieldDeclaration node) {
HashMap<String, Object> data = this._cache.pop();
String name = (String)data.remove("name");
this.symbols.insertFieldDeclaration(name, data);
}

public boolean visit(MethodDeclaration node) { public boolean visit(MethodDeclaration node) {
HashMap<String, Object> data = new HashMap<String, Object>(); HashMap<String, Object> data = new HashMap<String, Object>();
Name nameObj = node.getName(); Name nameObj = node.getName();
@@ -115,7 +98,7 @@ public class JavaParser extends Parser {
public void endVisit(MethodDeclaration node) { public void endVisit(MethodDeclaration node) {
HashMap<String, Object> data = this._cache.pop(); HashMap<String, Object> data = this._cache.pop();
String name = (String)data.remove("name"); String name = (String)data.remove("name");
this.symbols.insertMethodDeclaration(name, data);
this.symbols.insertMethodDeclaration("\"" + name + "\"", data);
} }


public boolean visit(MethodInvocation node) { public boolean visit(MethodInvocation node) {
@@ -136,7 +119,7 @@ public class JavaParser extends Parser {
public void endVisit(MethodInvocation node) { public void endVisit(MethodInvocation node) {
HashMap<String, Object> data = this._cache.pop(); HashMap<String, Object> data = this._cache.pop();
String name = (String)data.remove("name"); String name = (String)data.remove("name");
this.symbols.insertMethodInvocation(name, data);
this.symbols.insertMethodInvocation("\"" + name + "\"", data);
} }


public boolean visit(PackageDeclaration node) { public boolean visit(PackageDeclaration node) {
@@ -167,9 +150,9 @@ public class JavaParser extends Parser {
String name = (String)data.remove("name"); String name = (String)data.remove("name");


if (node.isInterface()) { if (node.isInterface()) {
this.symbols.insertInterfaceDeclaration(name, data);
this.symbols.insertInterfaceDeclaration("\"" + name + "\"", data);
} else { } else {
this.symbols.insertClassDeclaration(name, data);
this.symbols.insertClassDeclaration("\"" + name + "\"", data);
} }
} }


@@ -186,7 +169,7 @@ public class JavaParser extends Parser {
public void endVisit(VariableDeclarationFragment node) { public void endVisit(VariableDeclarationFragment node) {
HashMap<String, Object> data = this._cache.pop(); HashMap<String, Object> data = this._cache.pop();
String name = (String)data.remove("name"); String name = (String)data.remove("name");
this.symbols.insertVariableDeclaration(name, data);
this.symbols.insertVariableDeclaration("\"" + name + "\"", data);
} }


public boolean visit(QualifiedName node) { public boolean visit(QualifiedName node) {


+ 10
- 5
parsers/java/src/main/java/com/bitshift/parsing/parsers/Parser.java View File

@@ -1,8 +1,9 @@
package com.bitshift.parsing.parsers; package com.bitshift.parsing.parsers;


import java.io.BufferedReader; import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.InputStreamReader; import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.io.OutputStreamWriter;
import java.io.IOException; import java.io.IOException;


import java.net.Socket; import java.net.Socket;
@@ -46,12 +47,16 @@ public abstract class Parser implements Runnable {


protected void writeToClient(String toClient) { protected void writeToClient(String toClient) {
try { try {
PrintWriter clientWriter = new PrintWriter(
this.clientSocket.getOutputStream(), true);
BufferedWriter clientWriter = new BufferedWriter(
new OutputStreamWriter(this.clientSocket.getOutputStream()));


PackableMemory mem = new PackableMemory(toClient.length());
PackableMemory mem = new PackableMemory(4);
mem.pack(toClient.length(), 0);
String dataSize = new String(mem.mem); String dataSize = new String(mem.mem);
clientWriter.println(dataSize + toClient);

clientWriter.write(dataSize + toClient);
clientWriter.flush();
this.clientSocket.close();
} catch (IOException ex) { } catch (IOException ex) {
} }
} }


+ 38
- 33
parsers/java/src/main/java/com/bitshift/parsing/symbols/JavaSymbols.java View File

@@ -11,15 +11,16 @@ public class JavaSymbols extends Symbols {
private HashMap<String, HashMap<String, Object>> _classes; private HashMap<String, HashMap<String, Object>> _classes;
private HashMap<String, HashMap<String, Object>> _interfaces; private HashMap<String, HashMap<String, Object>> _interfaces;
private HashMap<String, HashMap<String, Object>> _methods; private HashMap<String, HashMap<String, Object>> _methods;
private HashMap<String, HashMap<String, Object>> _fields;
private HashMap<String, HashMap<String, Object>> _vars; private HashMap<String, HashMap<String, Object>> _vars;


private final String assignKey = "\"assignments\"";
private final String useKey = "\"uses\"";

public JavaSymbols() { public JavaSymbols() {
_packageName = null; _packageName = null;
_classes = new HashMap<String, HashMap<String, Object>>(); _classes = new HashMap<String, HashMap<String, Object>>();
_interfaces = new HashMap<String, HashMap<String, Object>>(); _interfaces = new HashMap<String, HashMap<String, Object>>();
_methods = new HashMap<String, HashMap<String, Object>>(); _methods = new HashMap<String, HashMap<String, Object>>();
_fields = new HashMap<String, HashMap<String, Object>>();
_vars = new HashMap<String, HashMap<String, Object>>(); _vars = new HashMap<String, HashMap<String, Object>>();
} }


@@ -34,15 +35,23 @@ public class JavaSymbols extends Symbols {
HashMap<String, Object> klass = new HashMap<String, Object>(); HashMap<String, Object> klass = new HashMap<String, Object>();


assignments.add(data.get("coord")); assignments.add(data.get("coord"));
klass.put("assignments", assignments);
klass.put("uses", uses);
klass.put(assignKey, assignments);
klass.put(useKey, uses);


this._classes.put(name, klass); this._classes.put(name, klass);
return true; return true;
} }


public boolean insertInterfaceDeclaration(String name, HashMap<String, Object> data) { public boolean insertInterfaceDeclaration(String name, HashMap<String, Object> data) {
this._interfaces.put(name, data);
ArrayList<Object> assignments = new ArrayList<Object>(10);
ArrayList<Object> uses = new ArrayList<Object>(10);
HashMap<String, Object> klass = new HashMap<String, Object>();

assignments.add(data.get("coord"));
klass.put(assignKey, assignments);
klass.put(useKey, uses);

this._interfaces.put(name, klass);
return true; return true;
} }


@@ -54,13 +63,13 @@ public class JavaSymbols extends Symbols {
ArrayList<Object> uses = new ArrayList<Object>(10); ArrayList<Object> uses = new ArrayList<Object>(10);


assignments.add(data.get("coord")); assignments.add(data.get("coord"));
method.put("assignments", assignments);
method.put("uses", uses);
method.put(assignKey, assignments);
method.put(useKey, uses);
} else { } else {
ArrayList<Object> assignments = (ArrayList<Object>)method.get("assignments");
ArrayList<Object> assignments = (ArrayList<Object>)method.get(assignKey);


assignments.add(data.get("coord")); assignments.add(data.get("coord"));
method.put("assignments", assignments);
method.put(assignKey, assignments);
} }


this._methods.put(name, method); this._methods.put(name, method);
@@ -74,24 +83,19 @@ public class JavaSymbols extends Symbols {
ArrayList<Object> uses = new ArrayList<Object>(10); ArrayList<Object> uses = new ArrayList<Object>(10);


uses.add(data.get("coord")); uses.add(data.get("coord"));
method.put("assignments", assignments);
method.put("uses", uses);
method.put(assignKey, assignments);
method.put(useKey, uses);
} else { } else {
ArrayList<Object> uses = (ArrayList<Object>)method.get("uses");
ArrayList<Object> uses = (ArrayList<Object>)method.get(useKey);


uses.add(data.get("coord")); uses.add(data.get("coord"));
method.put("uses", uses);
method.put(useKey, uses);
} }


this._methods.put(name, method); this._methods.put(name, method);
return true; return true;
} }


public boolean insertFieldDeclaration(String name, HashMap<String, Object> data) {
this._fields.put(name, data);
return true;
}

public boolean insertVariableDeclaration(String name, HashMap<String, Object> data) { public boolean insertVariableDeclaration(String name, HashMap<String, Object> data) {
HashMap<String, Object> var = this._vars.get(name); HashMap<String, Object> var = this._vars.get(name);
if (var == null) { if (var == null) {
@@ -100,13 +104,13 @@ public class JavaSymbols extends Symbols {
ArrayList<Object> uses = new ArrayList<Object>(10); ArrayList<Object> uses = new ArrayList<Object>(10);


assignments.add(data.get("coord")); assignments.add(data.get("coord"));
var.put("assignments", assignments);
var.put("uses", uses);
var.put(assignKey, assignments);
var.put(useKey, uses);
} else { } else {
ArrayList<Object> assignments = (ArrayList<Object>)var.get("assignments");
ArrayList<Object> assignments = (ArrayList<Object>)var.get(assignKey);


assignments.add(data.get("coord")); assignments.add(data.get("coord"));
var.put("assignments", assignments);
var.put(assignKey, assignments);
} }


this._vars.put(name, var); this._vars.put(name, var);
@@ -120,13 +124,13 @@ public class JavaSymbols extends Symbols {
ArrayList<Object> uses = new ArrayList<Object>(10); ArrayList<Object> uses = new ArrayList<Object>(10);


uses.add(data.get("coord")); uses.add(data.get("coord"));
var.put("assignments", assignments);
var.put("uses", uses);
var.put(assignKey, assignments);
var.put(useKey, uses);
} else { } else {
ArrayList<Object> uses = (ArrayList<Object>)var.get("uses");
ArrayList<Object> uses = (ArrayList<Object>)var.get(useKey);


uses.add(data.get("coord")); uses.add(data.get("coord"));
var.put("uses", uses);
var.put(useKey, uses);
} }


this._vars.put(name, var); this._vars.put(name, var);
@@ -135,13 +139,14 @@ public class JavaSymbols extends Symbols {


public String toString() { public String toString() {
StringBuilder builder = new StringBuilder(); StringBuilder builder = new StringBuilder();
builder.append("classes:" + this._classes + ",");
builder.append("interfaces:" + this._interfaces + ",");
builder.append("methods:" + this._methods + ",");
builder.append("fields:" + this._fields + ",");
builder.append("vars:" + this._vars + ",");

return "{" + builder.toString() + "}";
builder.append("\"classes\":" + this._classes + ",");
builder.append("\"interfaces\":" + this._interfaces + ",");
builder.append("\"methods\":" + this._methods + ",");
builder.append("\"vars\":" + this._vars + ",");

String s = builder.toString().replaceAll("=", ":");
s = s.substring(0, s.length() - 1);
return "{" + s + "}";
} }
} }



+ 1
- 1
parsers/java/src/main/java/com/bitshift/parsing/utils/PackableMemory.java View File

@@ -22,7 +22,7 @@ public class PackableMemory {
// The most significant porion of the integer is stored in mem[loc]. // The most significant porion of the integer is stored in mem[loc].
// Bytes are masked out of the integer and stored in the array, working // Bytes are masked out of the integer and stored in the array, working
// from right(least significant) to left (most significant). // from right(least significant) to left (most significant).
void pack(int val, int loc)
public void pack(int val, int loc)
{ {
final int MASK = 0xff; final int MASK = 0xff;
for (int i = 3; i >= 0; i--) for (int i = 3; i >= 0; i--)


+ 23
- 0
parsers/java/src/main/java/com/bitshift/parsing/utils/Tuple.java View File

@@ -0,0 +1,23 @@
package com.bitshift.parsing.utils;

import java.util.List;
import java.util.Arrays;

public class Tuple<T> {
private List<T> _objects;

public Tuple(T... args) {
_objects = Arrays.asList(args);
}

public String toString() {
StringBuilder builder = new StringBuilder();

for(T o: this._objects) {
builder.append(o + ",");
}

String s = builder.toString();
return "(" + s.substring(0, s.length() - 1) + ")";
}
}

+ 1
- 1
parsers/ruby/lib/parse_server.rb View File

@@ -14,7 +14,7 @@ end




def start_server def start_server
server = TCPServer.new 5003
server = TCPServer.new 5065


loop do loop do
# Start a new thread for each client accepted # Start a new thread for each client accepted


+ 13
- 24
parsers/ruby/lib/parser.rb View File

@@ -3,20 +3,6 @@ require 'ruby_parser'
require 'sexp_processor' require 'sexp_processor'


module Bitshift module Bitshift
class Tuple
attr_accessor :objects

def initialize(arr)
@objects = arr
end

def inspect
s = "("
@objects.each {|o| s += "#{o},"}
s = s[0..-2] + ')'
end
end

class Parser class Parser
def initialize(source) def initialize(source)
@source = source @source = source
@@ -39,7 +25,8 @@ module Bitshift
def initialize(offset, tree) def initialize(offset, tree)
super() super()


module_hash = Hash.new {|hash, key| hash[key] = { decls: [], uses: [] }}
module_hash = Hash.new {|hash, key|
hash[key] = { assignments: [], uses: [] }}
class_hash = module_hash.clone class_hash = module_hash.clone
function_hash = module_hash.clone function_hash = module_hash.clone
var_hash = module_hash.clone var_hash = module_hash.clone
@@ -64,7 +51,7 @@ module Bitshift
break if cur_exp == nil break if cur_exp == nil
end end


pos = Tuple.new([start_ln, -1, end_ln, -1])
pos = [start_ln, -1, end_ln, -1]
return pos return pos
end end


@@ -72,7 +59,7 @@ module Bitshift
pos = Hash.new pos = Hash.new
end_ln = start_ln = exp.line - offset end_ln = start_ln = exp.line - offset


pos = Tuple.new([start_ln, -1, end_ln, -1])
pos = [start_ln, -1, end_ln, -1]
return pos return pos
end end


@@ -80,7 +67,7 @@ module Bitshift
pos = block_position(exp) pos = block_position(exp)
exp.shift exp.shift
name = exp.shift name = exp.shift
symbols[:modules][name][:decls] << pos
symbols[:modules][name][:assignments] << pos
exp.each_sexp {|s| process(s)} exp.each_sexp {|s| process(s)}
return exp.clear return exp.clear
end end
@@ -89,7 +76,7 @@ module Bitshift
pos = block_position(exp) pos = block_position(exp)
exp.shift exp.shift
name = exp.shift name = exp.shift
symbols[:classes][name][:decls] << pos
symbols[:classes][name][:assignments] << pos
exp.each_sexp {|s| process(s)} exp.each_sexp {|s| process(s)}
return exp.clear return exp.clear
end end
@@ -98,7 +85,7 @@ module Bitshift
pos = block_position(exp) pos = block_position(exp)
exp.shift exp.shift
name = exp.shift name = exp.shift
symbols[:functions][name][:decls] << pos
symbols[:functions][name][:assignments] << pos
exp.each_sexp {|s| process(s)} exp.each_sexp {|s| process(s)}
return exp.clear return exp.clear
end end
@@ -117,7 +104,7 @@ module Bitshift
pos = statement_position(exp) pos = statement_position(exp)
exp.shift exp.shift
name = exp.shift name = exp.shift
symbols[:vars][name][:decls] << pos
symbols[:vars][name][:assignments] << pos
exp.each_sexp {|s| process(s)} exp.each_sexp {|s| process(s)}
return exp.clear return exp.clear
end end
@@ -126,17 +113,19 @@ module Bitshift
pos = statement_position(exp) pos = statement_position(exp)
exp.shift exp.shift
name = exp.shift name = exp.shift
symbols[:vars][name][:decls] << pos
symbols[:vars][name][:assignments] << pos
exp.each_sexp {|s| process(s)} exp.each_sexp {|s| process(s)}
return exp.clear return exp.clear
end end


def to_s def to_s
new_symbols = Hash.new {|hash, key| hash[key] = []}
new_symbols = Hash.new {|hash, key| hash[key] = Hash.new}


symbols.each do |type, sym_list| symbols.each do |type, sym_list|
sym_list.each do |name, sym| sym_list.each do |name, sym|
new_symbols[type.to_s] << Tuple.new(["'#{name}'", sym[:decls], sym[:uses]])
new_symbols[type.to_s][name.to_s] = {
"assignments" => sym[:assignments],
"uses" => sym[:uses]}
end end
end end




Loading…
Cancel
Save