From 6e54eb51473b1053c9d17ccc51e78d01e2bd64c0 Mon Sep 17 00:00:00 2001 From: Benjamin Attal Date: Sun, 27 Apr 2014 22:48:02 -0400 Subject: [PATCH] Java server tells python client how much data to read. --- .../java/com/bitshift/parsing/parsers/Parser.java | 7 +- .../com/bitshift/parsing/utils/PackableMemory.java | 89 +++++++++++++++ test/parser_test.py | 40 +++++-- test/resources/parser.rb | 126 +++++++++++++++++++++ 4 files changed, 248 insertions(+), 14 deletions(-) create mode 100644 parsers/java/src/main/java/com/bitshift/parsing/utils/PackableMemory.java create mode 100644 test/resources/parser.rb diff --git a/parsers/java/src/main/java/com/bitshift/parsing/parsers/Parser.java b/parsers/java/src/main/java/com/bitshift/parsing/parsers/Parser.java index 088c185..9d00954 100644 --- a/parsers/java/src/main/java/com/bitshift/parsing/parsers/Parser.java +++ b/parsers/java/src/main/java/com/bitshift/parsing/parsers/Parser.java @@ -8,6 +8,7 @@ import java.io.IOException; import java.net.Socket; import com.bitshift.parsing.symbols.Symbols; +import com.bitshift.parsing.utils.PackableMemory; public abstract class Parser implements Runnable { @@ -48,7 +49,9 @@ public abstract class Parser implements Runnable { PrintWriter clientWriter = new PrintWriter( this.clientSocket.getOutputStream(), true); - clientWriter.println(toClient); + PackableMemory mem = new PackableMemory(toClient.length()); + String dataSize = new String(mem.mem); + clientWriter.println(dataSize + toClient); } catch (IOException ex) { } } @@ -56,6 +59,6 @@ public abstract class Parser implements Runnable { protected abstract Symbols genSymbols(); public abstract void run(); - + } diff --git a/parsers/java/src/main/java/com/bitshift/parsing/utils/PackableMemory.java b/parsers/java/src/main/java/com/bitshift/parsing/utils/PackableMemory.java new file mode 100644 index 0000000..24d883c --- /dev/null +++ b/parsers/java/src/main/java/com/bitshift/parsing/utils/PackableMemory.java @@ -0,0 +1,89 @@ +package com.bitshift.parsing.utils; + +//This class contains implementations of methods to +// -- pack an integer into 4 consecutive bytes of a byte array +// -- unpack an integer from 4 consecutive bytes of a byte array +// -- exhaustively test the pack and unpack methods. +// +// This file should be saved as PackableMemory.java. Once it has been +// compiled, the tester can be invoked by typing "java PackableMemory" + +public class PackableMemory { + int size; + public byte mem[] = null; + + public PackableMemory(int size) + { + this.size = size; + this.mem = new byte[size]; + } + + // Pack the 4-byte integer val into the four bytes mem[loc]...mem[loc+3]. + // The most significant porion of the integer is stored in mem[loc]. + // Bytes are masked out of the integer and stored in the array, working + // from right(least significant) to left (most significant). + void pack(int val, int loc) + { + final int MASK = 0xff; + for (int i = 3; i >= 0; i--) + { + mem[loc+i] = (byte)(val & MASK); + val = val >> 8; + } + } + + // Unpack the four bytes mem[loc]...mem[loc+3] into a 4-byte integer, + // and return the resulting integer value. + // The most significant porion of the integer is stored in mem[loc]. + // Bytes are 'OR'ed into the integer, working from left (most significant) + // to right (least significant) + int unpack(int loc) + { + final int MASK = 0xff; + int v = (int)mem[loc] & MASK; + for (int i = 1; i < 4; i++) + { + v = v << 8; + v = v | ((int)mem[loc+i] & MASK); + } + return v; + } + + + + // Test the above pack and unpack methods by iterating the following + // over all possible 4-byte integers: pack the integer, + // then unpack it, and then verify that the unpacked integer equals the + // original integer. It tests all nonnegative numbers in ascending order + // and then all negative numbers in ascending order. The transition from + // positive to negative numbers happens implicitly due to integer overflow. + public void packTest() + { + + int i = 0; + long k = 0; + do + { + this.pack(i,4); + int j = this.unpack(4); + if (j != i) + { + System.out.printf("pack/unpack test failed: i = %d, j = %d\n",i,j); + System.exit(0); + } + i++; k++; + } + while (i != 0); + System.out.printf("pack/unpack test successful, %d iterations\n",k); + } + + // main routine to test the PackableMemory class by running the + // packTest() method. + public static void main(String[] args) + { + PackableMemory pm = new PackableMemory(100); + pm.packTest(); + System.exit(0); + } +} + diff --git a/test/parser_test.py b/test/parser_test.py index 253da10..a1cfad3 100644 --- a/test/parser_test.py +++ b/test/parser_test.py @@ -1,7 +1,8 @@ -import socket, sys +import socket, sys, struct file_name = 'resources/.c' server_socket_number = 5001 +recv_size = 8192 if __name__ == '__main__': if len(sys.argv) == 1: @@ -19,22 +20,37 @@ if __name__ == '__main__': server_socket_number = 5002 elif sys.argv[1] == 'ruby': - file_name = "resources/.rb" + file_name = "resources/parser.rb" server_socket_number = 5003 - client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - client_socket.connect(("localhost", server_socket_number)) + server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server_socket.connect(("localhost", server_socket_number)) with open(file_name, "r") as source_file: source = source_file.read() - client_socket.send("%d\n%s" % (len(source), source)); + server_socket.send("%d\n%s" % (len(source), source)); - data = '' - while True: - data = client_socket.recv(10000) + total_data = []; size_data = cur_data = '' + total_size = 0; size = sys.maxint - if data != '': - client_socket.close() - break; + while total_size < size: + cur_data = server_socket.recv(recv_size) - print data; + if not total_data: + if len(size_data) > 4: + size_data += cur_data + size = struct.unpack('>i', size_data[:4])[0] + recv_size = size + if recv_size > sys.maxint: recv_size = sys.maxint + total_data.append(size_data[4:]) + else: + size_data += cur_data + + else: + total_data.append(cur_data) + + total_size = sum([len(s) for s in total_data]) + + + server_socket.close() + print ''.join(total_data); diff --git a/test/resources/parser.rb b/test/resources/parser.rb new file mode 100644 index 0000000..01d934b --- /dev/null +++ b/test/resources/parser.rb @@ -0,0 +1,126 @@ +require 'socket' +require 'ruby_parser' +require 'sexp_processor' + +module Bitshift + class Parser + def initialize(source) + @source = source + end + + def parse + parser = RubyParser.new + tree = parser.parse(@source) + puts tree.inspect + offset = tree.line - 1 + processor = NodeVisitor.new offset + processor.process tree + return processor.symbols + end + end + + class NodeVisitor < SexpProcessor + attr_accessor :symbols + attr_accessor :offset + + def initialize(offset) + super() + @require_empty = false + @offset = offset + + module_hash = Hash.new {|hash, key| hash[key] = Hash.new} + class_hash = module_hash.clone + function_hash = Hash.new {|hash, key| hash[key] = { calls: [] } } + var_hash = Hash.new {|hash, key| hash[key] = [] } + + @symbols = { + modules: module_hash, + classes: class_hash, + functions: function_hash, + vars: var_hash + } + end + + def block_position(exp) + pos = Hash.new + end_ln = (start_ln = exp.line - offset) + cur_exp = exp + + while cur_exp.is_a? Sexp + end_ln = cur_exp.line - offset + cur_exp = cur_exp.last + break if cur_exp == nil + end + + pos[:coord] = { + start_ln: start_ln, + end_ln: end_ln } + return pos + end + + def statement_position(exp) + pos = Hash.new + end_ln = start_ln = exp.line - offset + + pos[:coord] = { + start_ln: start_ln, + end_ln: end_ln } + return pos + end + + def process_module(exp) + pos = block_position exp + exp.shift + name = exp.shift + symbols[:modules][name] = pos + exp.each_sexp {|s| process(s)} + return exp.clear + end + + def process_class(exp) + pos = block_position exp + exp.shift + name = exp.shift + symbols[:classes][name] = pos + exp.each_sexp {|s| process(s)} + return exp.clear + end + + def process_defn(exp) + pos = block_position exp + exp.shift + name = exp.shift + symbols[:functions][name][:declaration] = pos + exp.each_sexp {|s| process(s)} + return exp.clear + end + + def process_call(exp) + pos = statement_position exp + exp.shift + exp.shift + name = exp.shift + symbols[:functions][name][:calls] << pos + exp.each_sexp {|s| process(s)} + return exp.clear + end + + def process_iasgn(exp) + pos = statement_position exp + exp.shift + name = exp.shift + symbols[:vars][name] << pos + exp.each_sexp {|s| process(s)} + return exp.clear + end + + def process_lasgn(exp) + pos = statement_position exp + exp.shift + name = exp.shift + symbols[:vars][name] << pos + exp.each_sexp {|s| process(s)} + return exp.clear + end + end +end