ben
/
bitshift
огледало от https://github.com/earwig/bitshift
1
1
Разклонения 0
Код Версии 1 Activity
Преглед на файлове

Make Parser implement runnable so parsing tasks can be started in separate threads. Make Parser constructor accept a client socket, add reading and writing methods for the socket to JavaParser. Parse main method sets up a server for accepting parse jobs from the crawler, and starts threads for each parse task.

tags/v1.0^2
Benjamin Attal преди 12 години
родител
77e2b6f524
ревизия
306875dae7
променени са 8 файла, в които са добавени 460 реда и са изтрити 78 реда
Разделен изглед
Diff Options
Показване на статистика Download Patch File Download Diff File
  1. +21
    -78
      parsers/java/pom.xml
  2. +33
    -0
      parsers/java/src/main/java/com/bitshift/parsing/Parse.java
  3. +3
    -0
      parsers/java/src/main/java/com/bitshift/parsing/parsers/CParser.java
  4. +215
    -0
      parsers/java/src/main/java/com/bitshift/parsing/parsers/JavaParser.java
  5. +19
    -0
      parsers/java/src/main/java/com/bitshift/parsing/parsers/Parser.java
  6. +1
    -0
      parsers/java/src/main/java/com/bitshift/parsing/symbols/CSymbols.java
  7. +159
    -0
      parsers/java/src/main/java/com/bitshift/parsing/symbols/JavaSymbols.java
  8. +9
    -0
      parsers/java/src/main/java/com/bitshift/parsing/symbols/Symbols.java

+ 21
- 78
parsers/java/pom.xml Целия файл

@@ -1,22 +1,23 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<prerequisites>
<maven>3.0.0</maven>
</prerequisites>

<groupId>org.bitshift.parsing</groupId>
<groupId>com.bitshift.parsing</groupId>
<artifactId>parsing</artifactId>
<version>0.1</version>
<name>Java Parser for Bitshift</name>

<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
</properties>
<packaging>jar</packaging>
<version>1.0-SNAPSHOT</version>
<name>parsing</name>
<url>http://maven.apache.org</url>

<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>3.8.1</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.eclipse.jdt</groupId>
<artifactId>core</artifactId>
<version>3.3.0-v_771</version>
@@ -26,74 +27,16 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.1</version>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>1.2.1</version>
<configuration>
<source>1.7</source>
<target>1.7</target>
<encoding>UTF-8</encoding>
<mainClass>com.bitshift.parsing.Parse</mainClass>
<arguments>
</arguments>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<version>2.2.1</version>
<executions>
<execution>
<id>attach-sources</id>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>2.4</version>
<configuration>
<archive>
<manifest>
<addDefaultImplementationEntries>true</addDefaultImplementationEntries>
</manifest>
</archive>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>1.6</version>
<configuration>
<createDependencyReducedPom>true</createDependencyReducedPom>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>org.bishift.parsing.Parse</mainClass>
</transformer>
</transformers>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>

</project>

+ 33
- 0
parsers/java/src/main/java/com/bitshift/parsing/Parse.java Целия файл

@@ -0,0 +1,33 @@
package com.bitshift.parsing;

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.io.IOException;

import java.net.ServerSocket;
import java.net.Socket;

import com.bitshift.parsing.parsers.JavaParser;

public class Parse {

public static void main(String[] args) {
String fromClient;
String toClient;

try {
ServerSocket server = new ServerSocket(5002);

while(true) {
Socket clientSocket = server.accept();

JavaParser parser = new JavaParser(clientSocket);
Thread parserTask = new Thread(parser);
parserTask.start();
}
} catch (IOException ex) {
}
}

}

+ 3
- 0
parsers/java/src/main/java/com/bitshift/parsing/parsers/CParser.java Целия файл

@@ -0,0 +1,3 @@
package com.bitshift.parsing.parsers;

import com.bitshift.parsing.parsers.Parser;

+ 215
- 0
parsers/java/src/main/java/com/bitshift/parsing/parsers/JavaParser.java Целия файл

@@ -0,0 +1,215 @@
package com.bitshift.parsing.parsers;

import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Stack;

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.io.IOException;

import java.net.Socket;

import org.eclipse.jdt.core.JavaCore;
import org.eclipse.jdt.core.dom.AST;
import org.eclipse.jdt.core.dom.ASTParser;
import org.eclipse.jdt.core.dom.ASTVisitor;
import org.eclipse.jdt.core.dom.CompilationUnit;
import org.eclipse.jdt.core.dom.ClassInstanceCreation;
import org.eclipse.jdt.core.dom.FieldAccess;
import org.eclipse.jdt.core.dom.FieldDeclaration;
import org.eclipse.jdt.core.dom.MethodDeclaration;
import org.eclipse.jdt.core.dom.MethodInvocation;
import org.eclipse.jdt.core.dom.Name;
import org.eclipse.jdt.core.dom.PackageDeclaration;
import org.eclipse.jdt.core.dom.QualifiedName;
import org.eclipse.jdt.core.dom.QualifiedType;
import org.eclipse.jdt.core.dom.SimpleName;
import org.eclipse.jdt.core.dom.SimpleType;
import org.eclipse.jdt.core.dom.Statement;
import org.eclipse.jdt.core.dom.Type;
import org.eclipse.jdt.core.dom.TypeDeclaration;
import org.eclipse.jdt.core.dom.VariableDeclarationFragment;

import com.bitshift.parsing.parsers.Parser;
import com.bitshift.parsing.symbols.Symbols;
import com.bitshift.parsing.symbols.JavaSymbols;

/*TODO: Work on parsing partial java code.
* Change visits to endVisit and implement a cache for more concise code structure.
* Get rid of unecessary imports.
* Fix column and line numbers.*/
public class JavaParser extends Parser {

public JavaParser(Socket clientSocket) {
super(clientSocket);
}

private String readFromClient() {
String fromClient = "";

try {
BufferedReader clientReader = new BufferedReader(
new InputStreamReader(this.clientSocket.getInputStream()));

int bytes = Integer.parseInt(clientReader.readLine());
System.out.println(bytes);

StringBuilder builder = new StringBuilder();
int i = 0;

while(i < bytes) {
char aux = (char)clientReader.read();
builder.append(aux);
i++;
}

fromClient = builder.toString();

} catch (IOException ex) {
}

return fromClient;
}

private void writeToClient(String toClient) {
try {
PrintWriter clientWriter = new PrintWriter(
this.clientSocket.getOutputStream(), true);

clientWriter.println(toClient);
} catch (IOException ex) {
}
}

@Override
public Symbols genSymbols() {
char[] source = this.readFromClient().toCharArray();

ASTParser parser = ASTParser.newParser(AST.JLS3);
parser.setSource(source);

Map options = JavaCore.getOptions();
parser.setCompilerOptions(options);

CompilationUnit root = (CompilationUnit) parser.createAST(null);

NodeVisitor visitor = new NodeVisitor(root);
root.accept(visitor);

return visitor.symbols;
}

@Override
public void run() {
JavaSymbols symbols = (JavaSymbols) this.genSymbols();
System.out.println(symbols.toString());
writeToClient(symbols.toString());
}

class NodeVisitor extends ASTVisitor {

protected CompilationUnit root;
protected JavaSymbols symbols;
private Stack<HashMap<String, Object>> _cache;

public NodeVisitor(CompilationUnit root) {
this.root = root;
this.symbols = new JavaSymbols();
this._cache = new Stack<HashMap<String, Object>>();
}

public boolean visit(ClassInstanceCreation node) {
return true;
}

public boolean visit(FieldAccess node) {
Name nameObj = node.getName();
String name = nameObj.isQualifiedName() ?
((QualifiedName) nameObj).getFullyQualifiedName() :
((SimpleName) nameObj).getIdentifier();

int sl = this.root.getLineNumber(node.getStartPosition()) - 1;
int sc = this.root.getColumnNumber(node.getStartPosition()) - 1;

this.symbols.insertFieldAccess(name, sl, sc, null, null);
return true;
}

public boolean visit(FieldDeclaration node) {
return true;
}

public boolean visit(MethodDeclaration node) {
Name nameObj = node.getName();
String name = nameObj.isQualifiedName() ?
((QualifiedName) nameObj).getFullyQualifiedName() :
((SimpleName) nameObj).getIdentifier();
List<Statement> statements = node.getBody().statements();
Statement last = statements.get(statements.size() - 1);

int sl = this.root.getLineNumber(node.getStartPosition()) - 1;
int sc = this.root.getColumnNumber(node.getStartPosition()) - 1;
int el = this.root.getLineNumber(last.getStartPosition()) - 1;
int ec = this.root.getColumnNumber(last.getStartPosition()) - 1;

this.symbols.insertMethodDeclaration(name, sl, sc, el, ec);
return true;
}

public boolean visit(MethodInvocation node) {
Name nameObj = node.getName();
String name = nameObj.isQualifiedName() ?
((QualifiedName) nameObj).getFullyQualifiedName() :
((SimpleName) nameObj).getIdentifier();

int sl = this.root.getLineNumber(node.getStartPosition()) - 1;
int sc = this.root.getColumnNumber(node.getStartPosition()) - 1;

this.symbols.insertMethodInvocation(name, sl, sc, null, null);
return true;
}

public boolean visit(PackageDeclaration node) {
Name nameObj = node.getName();
String name = nameObj.isQualifiedName() ?
((QualifiedName) nameObj).getFullyQualifiedName() :
((SimpleName) nameObj).getIdentifier();

this.symbols.setPackage(name);
return true;
}

public boolean visit(TypeDeclaration node) {
Name nameObj = node.getName();
String name = nameObj.isQualifiedName() ?
((QualifiedName) nameObj).getFullyQualifiedName() :
((SimpleName) nameObj).getIdentifier();

int sl = this.root.getLineNumber(node.getStartPosition()) - 1;
int sc = this.root.getColumnNumber(node.getStartPosition()) - 1;

if (node.isInterface()) {
this.symbols.insertInterfaceDeclaration(name, sl, sc, null, null);
} else {
this.symbols.insertClassDeclaration(name, sl, sc, null, null);
}
return true;
}

public boolean visit(VariableDeclarationFragment node) {
Name nameObj = node.getName();
String name = nameObj.isQualifiedName() ?
((QualifiedName) nameObj).getFullyQualifiedName() :
((SimpleName) nameObj).getIdentifier();

int sl = this.root.getLineNumber(node.getStartPosition()) - 1;
int sc = this.root.getColumnNumber(node.getStartPosition()) - 1;
this.symbols.insertVariableDeclaration(name, sl, sc, null, null);
return true;
}

}
}

+ 19
- 0
parsers/java/src/main/java/com/bitshift/parsing/parsers/Parser.java Целия файл

@@ -0,0 +1,19 @@
package com.bitshift.parsing.parsers;

import java.net.Socket;
import com.bitshift.parsing.symbols.Symbols;

public abstract class Parser implements Runnable {

protected Socket clientSocket;

public Parser(Socket clientSocket) {
this.clientSocket = clientSocket;
}

abstract Symbols genSymbols();

public abstract void run();
}


+ 1
- 0
parsers/java/src/main/java/com/bitshift/parsing/symbols/CSymbols.java Целия файл

@@ -0,0 +1 @@
package com.bitshift.parsing.symbols;

+ 159
- 0
parsers/java/src/main/java/com/bitshift/parsing/symbols/JavaSymbols.java Целия файл

@@ -0,0 +1,159 @@
package com.bitshift.parsing.symbols;

import java.util.List;
import java.util.Map;
import java.util.HashMap;
import java.util.ArrayList;
import com.bitshift.parsing.symbols.Symbols;

/*TODO: Overwrite toString.*/
public class JavaSymbols extends Symbols {

private String _packageName;
private Map<String, Object> _classes;
private Map<String, Object> _interfaces;
private Map<String, Object> _methods;
private Map<String, Object> _fields;
private Map<String, Object> _vars;

public JavaSymbols() {
_packageName = null;
_classes = new HashMap<String, Object>();
_interfaces = new HashMap<String, Object>();
_methods = new HashMap<String, Object>();
_fields = new HashMap<String, Object>();
_vars = new HashMap<String, Object>();
}

public boolean setPackage(String name) {
_packageName = name;
return true;
}

public boolean insertClassDeclaration(String name, Integer startLine, Integer startCol, Integer endLine, Integer endCol) {
List<Integer> pos = new ArrayList<Integer>(4);
pos.add(startLine); pos.add(startCol); pos.add(endLine); pos.add(endCol);
List<List<Integer>> copy = (List<List<Integer>>)_classes.get(name);
copy = (copy == null) ? new ArrayList<List<Integer>>() : copy;

copy.add(0, pos);
this._classes.put(name, copy);
return true;
}
public boolean insertClassInstance(String name, Integer startLine, Integer startCol, Integer endLine, Integer endCol) {
List<Integer> pos = new ArrayList<Integer>(4);
pos.add(startLine); pos.add(startCol); pos.add(endLine); pos.add(endCol);
List<List<Integer>> copy = (List<List<Integer>>)_classes.get(name);
copy = (copy == null) ? new ArrayList<List<Integer>>() : copy;

copy.add(pos);
this._classes.put(name, copy);
return true;
}

public boolean insertInterfaceDeclaration(String name, Integer startLine, Integer startCol, Integer endLine, Integer endCol) {
List<Integer> pos = new ArrayList<Integer>(4);
pos.add(startLine); pos.add(startCol); pos.add(endLine); pos.add(endCol);
List<List<Integer>> copy = (List<List<Integer>>)_classes.get(name);
copy = (copy == null) ? new ArrayList<List<Integer>>() : copy;

copy.add(0, pos);
this._interfaces.put(name, copy);
return true;
}
public boolean insertInterfaceInstance(String name, Integer startLine, Integer startCol, Integer endLine, Integer endCol) {
List<Integer> pos = new ArrayList<Integer>(4);
pos.add(startLine); pos.add(startCol); pos.add(endLine); pos.add(endCol);
List<List<Integer>> copy = (List<List<Integer>>)_classes.get(name);
copy = (copy == null) ? new ArrayList<List<Integer>>() : copy;

copy.add(pos);
this._interfaces.put(name, copy);
return true;
}

public boolean insertMethodDeclaration(String name, Integer startLine, Integer startCol, Integer endLine, Integer endCol) {
List<Integer> pos = new ArrayList<Integer>(4);
pos.add(startLine); pos.add(startCol); pos.add(endLine); pos.add(endCol);
List<List<Integer>> copy = (List<List<Integer>>)_classes.get(name);
copy = (copy == null) ? new ArrayList<List<Integer>>() : copy;

copy.add(0, pos);
this._methods.put(name, copy);
return true;
}
public boolean insertMethodInvocation(String name, Integer startLine, Integer startCol, Integer endLine, Integer endCol) {
List<Integer> pos = new ArrayList<Integer>(4);
pos.add(startLine); pos.add(startCol); pos.add(endLine); pos.add(endCol);
List<List<Integer>> copy = (List<List<Integer>>)_classes.get(name);
copy = (copy == null) ? new ArrayList<List<Integer>>() : copy;

copy.add(pos);
this._methods.put(name, copy);
return true;
}

public boolean insertFieldDeclaration(String name, Integer startLine, Integer startCol, Integer endLine, Integer endCol) {
List<Integer> pos = new ArrayList<Integer>(4);
pos.add(startLine); pos.add(startCol); pos.add(endLine); pos.add(endCol);
List<List<Integer>> copy = (List<List<Integer>>)_classes.get(name);
copy = (copy == null) ? new ArrayList<List<Integer>>() : copy;

copy.add(0, pos);
this._fields.put(name, copy);
return true;
}
public boolean insertFieldAccess(String name, Integer startLine, Integer startCol, Integer endLine, Integer endCol) {
List<Integer> pos = new ArrayList<Integer>(4);
pos.add(startLine); pos.add(startCol); pos.add(endLine); pos.add(endCol);
List<List<Integer>> copy = (List<List<Integer>>)_classes.get(name);
copy = (copy == null) ? new ArrayList<List<Integer>>() : copy;

copy.add(pos);
this._fields.put(name, copy);
return true;
}

public boolean insertVariableDeclaration(String name, Integer startLine, Integer startCol, Integer endLine, Integer endCol) {
List<Integer> pos = new ArrayList<Integer>(4);
pos.add(startLine); pos.add(startCol); pos.add(endLine); pos.add(endCol);
List<List<Integer>> copy = (List<List<Integer>>)_classes.get(name);
copy = (copy == null) ? new ArrayList<List<Integer>>() : copy;

copy.add(0, pos);
this._vars.put(name, copy);
return true;
}
public boolean insertVariableAccess(String name, Integer startLine, Integer startCol, Integer endLine, Integer endCol) {
List<Integer> pos = new ArrayList<Integer>(4);
pos.add(startLine); pos.add(startCol); pos.add(endLine); pos.add(endCol);
List<List<Integer>> copy = (List<List<Integer>>)_classes.get(name);
copy = (copy == null) ? new ArrayList<List<Integer>>() : copy;

copy.add(pos);
this._vars.put(name, copy);
return true;
}

public String toString() {
StringBuilder builder = new StringBuilder();
builder.append("classes:" + this._classes + ",");
builder.append("interfaces:" + this._interfaces + ",");
builder.append("methods:" + this._methods + ",");
builder.append("fields:" + this._fields + ",");
builder.append("vars:" + this._vars + ",");

return "{" + builder.toString() + "}";
}
}


+ 9
- 0
parsers/java/src/main/java/com/bitshift/parsing/symbols/Symbols.java Целия файл

@@ -0,0 +1,9 @@
package com.bitshift.parsing.symbols;

public abstract class Symbols {

public Symbols() {
}

}

Пиши Преглед
Зареждане…
Отказ
Запис