Browse Source

Java server tells python client how much data to read.

tags/v1.0^2
Benjamin Attal 10 years ago
parent
commit
6e54eb5147
4 changed files with 248 additions and 14 deletions
  1. +5
    -2
      parsers/java/src/main/java/com/bitshift/parsing/parsers/Parser.java
  2. +89
    -0
      parsers/java/src/main/java/com/bitshift/parsing/utils/PackableMemory.java
  3. +28
    -12
      test/parser_test.py
  4. +126
    -0
      test/resources/parser.rb

+ 5
- 2
parsers/java/src/main/java/com/bitshift/parsing/parsers/Parser.java View File

@@ -8,6 +8,7 @@ import java.io.IOException;
import java.net.Socket;

import com.bitshift.parsing.symbols.Symbols;
import com.bitshift.parsing.utils.PackableMemory;

public abstract class Parser implements Runnable {

@@ -48,7 +49,9 @@ public abstract class Parser implements Runnable {
PrintWriter clientWriter = new PrintWriter(
this.clientSocket.getOutputStream(), true);

clientWriter.println(toClient);
PackableMemory mem = new PackableMemory(toClient.length());
String dataSize = new String(mem.mem);
clientWriter.println(dataSize + toClient);
} catch (IOException ex) {
}
}
@@ -56,6 +59,6 @@ public abstract class Parser implements Runnable {
protected abstract Symbols genSymbols();

public abstract void run();
}


+ 89
- 0
parsers/java/src/main/java/com/bitshift/parsing/utils/PackableMemory.java View File

@@ -0,0 +1,89 @@
package com.bitshift.parsing.utils;

//This class contains implementations of methods to
// -- pack an integer into 4 consecutive bytes of a byte array
// -- unpack an integer from 4 consecutive bytes of a byte array
// -- exhaustively test the pack and unpack methods.
//
// This file should be saved as PackableMemory.java. Once it has been
// compiled, the tester can be invoked by typing "java PackableMemory"

public class PackableMemory {
int size;
public byte mem[] = null;

public PackableMemory(int size)
{
this.size = size;
this.mem = new byte[size];
}

// Pack the 4-byte integer val into the four bytes mem[loc]...mem[loc+3].
// The most significant porion of the integer is stored in mem[loc].
// Bytes are masked out of the integer and stored in the array, working
// from right(least significant) to left (most significant).
void pack(int val, int loc)
{
final int MASK = 0xff;
for (int i = 3; i >= 0; i--)
{
mem[loc+i] = (byte)(val & MASK);
val = val >> 8;
}
}

// Unpack the four bytes mem[loc]...mem[loc+3] into a 4-byte integer,
// and return the resulting integer value.
// The most significant porion of the integer is stored in mem[loc].
// Bytes are 'OR'ed into the integer, working from left (most significant)
// to right (least significant)
int unpack(int loc)
{
final int MASK = 0xff;
int v = (int)mem[loc] & MASK;
for (int i = 1; i < 4; i++)
{
v = v << 8;
v = v | ((int)mem[loc+i] & MASK);
}
return v;
}



// Test the above pack and unpack methods by iterating the following
// over all possible 4-byte integers: pack the integer,
// then unpack it, and then verify that the unpacked integer equals the
// original integer. It tests all nonnegative numbers in ascending order
// and then all negative numbers in ascending order. The transition from
// positive to negative numbers happens implicitly due to integer overflow.
public void packTest()
{

int i = 0;
long k = 0;
do
{
this.pack(i,4);
int j = this.unpack(4);
if (j != i)
{
System.out.printf("pack/unpack test failed: i = %d, j = %d\n",i,j);
System.exit(0);
}
i++; k++;
}
while (i != 0);
System.out.printf("pack/unpack test successful, %d iterations\n",k);
}

// main routine to test the PackableMemory class by running the
// packTest() method.
public static void main(String[] args)
{
PackableMemory pm = new PackableMemory(100);
pm.packTest();
System.exit(0);
}
}


+ 28
- 12
test/parser_test.py View File

@@ -1,7 +1,8 @@
import socket, sys
import socket, sys, struct

file_name = 'resources/<name>.c'
server_socket_number = 5001
recv_size = 8192

if __name__ == '__main__':
if len(sys.argv) == 1:
@@ -19,22 +20,37 @@ if __name__ == '__main__':
server_socket_number = 5002

elif sys.argv[1] == 'ruby':
file_name = "resources/<name>.rb"
file_name = "resources/parser.rb"
server_socket_number = 5003

client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client_socket.connect(("localhost", server_socket_number))
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.connect(("localhost", server_socket_number))

with open(file_name, "r") as source_file:
source = source_file.read()
client_socket.send("%d\n%s" % (len(source), source));
server_socket.send("%d\n%s" % (len(source), source));

data = ''
while True:
data = client_socket.recv(10000)
total_data = []; size_data = cur_data = ''
total_size = 0; size = sys.maxint

if data != '':
client_socket.close()
break;
while total_size < size:
cur_data = server_socket.recv(recv_size)

print data;
if not total_data:
if len(size_data) > 4:
size_data += cur_data
size = struct.unpack('>i', size_data[:4])[0]
recv_size = size
if recv_size > sys.maxint: recv_size = sys.maxint
total_data.append(size_data[4:])
else:
size_data += cur_data

else:
total_data.append(cur_data)

total_size = sum([len(s) for s in total_data])


server_socket.close()
print ''.join(total_data);

+ 126
- 0
test/resources/parser.rb View File

@@ -0,0 +1,126 @@
require 'socket'
require 'ruby_parser'
require 'sexp_processor'

module Bitshift
class Parser
def initialize(source)
@source = source
end

def parse
parser = RubyParser.new
tree = parser.parse(@source)
puts tree.inspect
offset = tree.line - 1
processor = NodeVisitor.new offset
processor.process tree
return processor.symbols
end
end

class NodeVisitor < SexpProcessor
attr_accessor :symbols
attr_accessor :offset

def initialize(offset)
super()
@require_empty = false
@offset = offset

module_hash = Hash.new {|hash, key| hash[key] = Hash.new}
class_hash = module_hash.clone
function_hash = Hash.new {|hash, key| hash[key] = { calls: [] } }
var_hash = Hash.new {|hash, key| hash[key] = [] }

@symbols = {
modules: module_hash,
classes: class_hash,
functions: function_hash,
vars: var_hash
}
end

def block_position(exp)
pos = Hash.new
end_ln = (start_ln = exp.line - offset)
cur_exp = exp

while cur_exp.is_a? Sexp
end_ln = cur_exp.line - offset
cur_exp = cur_exp.last
break if cur_exp == nil
end

pos[:coord] = {
start_ln: start_ln,
end_ln: end_ln }
return pos
end

def statement_position(exp)
pos = Hash.new
end_ln = start_ln = exp.line - offset

pos[:coord] = {
start_ln: start_ln,
end_ln: end_ln }
return pos
end

def process_module(exp)
pos = block_position exp
exp.shift
name = exp.shift
symbols[:modules][name] = pos
exp.each_sexp {|s| process(s)}
return exp.clear
end

def process_class(exp)
pos = block_position exp
exp.shift
name = exp.shift
symbols[:classes][name] = pos
exp.each_sexp {|s| process(s)}
return exp.clear
end

def process_defn(exp)
pos = block_position exp
exp.shift
name = exp.shift
symbols[:functions][name][:declaration] = pos
exp.each_sexp {|s| process(s)}
return exp.clear
end

def process_call(exp)
pos = statement_position exp
exp.shift
exp.shift
name = exp.shift
symbols[:functions][name][:calls] << pos
exp.each_sexp {|s| process(s)}
return exp.clear
end

def process_iasgn(exp)
pos = statement_position exp
exp.shift
name = exp.shift
symbols[:vars][name] << pos
exp.each_sexp {|s| process(s)}
return exp.clear
end

def process_lasgn(exp)
pos = statement_position exp
exp.shift
name = exp.shift
symbols[:vars][name] << pos
exp.each_sexp {|s| process(s)}
return exp.clear
end
end
end

Loading…
Cancel
Save