Java server tells python client how much data to read.

parsers/java/src/main/java/com/bitshift/parsing/parsers/Parser.java

@@ -8,6 +8,7 @@ import java.io.IOException;
 import java.net.Socket;
 
 import com.bitshift.parsing.symbols.Symbols;
+import com.bitshift.parsing.utils.PackableMemory;
 
 public abstract class Parser implements Runnable {
 
@@ -48,7 +49,9 @@ public abstract class Parser implements Runnable {
             PrintWriter clientWriter = new PrintWriter(
                     this.clientSocket.getOutputStream(), true);
 
-            clientWriter.println(toClient);
+            PackableMemory mem = new PackableMemory(toClient.length());
+            String dataSize = new String(mem.mem);
+            clientWriter.println(dataSize + toClient);
         } catch (IOException ex) {
         }
     }
@@ -56,6 +59,6 @@ public abstract class Parser implements Runnable {
     protected abstract Symbols genSymbols();
 
     public abstract void run();
-    
+
 }
 

parsers/java/src/main/java/com/bitshift/parsing/utils/PackableMemory.java

@@ -0,0 +1,89 @@
+package com.bitshift.parsing.utils;
+
+//This class contains implementations of methods to
+//   -- pack an integer into 4 consecutive bytes of a byte array
+//   -- unpack an integer from 4 consecutive bytes of a byte array
+//   -- exhaustively test the pack and unpack methods.
+//
+// This file should be saved as PackableMemory.java.  Once it has been
+//  compiled, the tester can be invoked by typing "java PackableMemory"
+
+public class PackableMemory {
+   int size;
+   public byte mem[] = null;
+
+   public PackableMemory(int size)
+   {
+      this.size = size;
+      this.mem = new byte[size];
+   }
+
+   // Pack the 4-byte integer val into the four bytes mem[loc]...mem[loc+3].
+   // The most significant porion of the integer is stored in mem[loc].
+   // Bytes are masked out of the integer and stored in the array, working
+   // from right(least significant) to left (most significant).
+   void pack(int val, int loc)
+   {
+      final int MASK = 0xff;
+      for (int i = 3; i >= 0; i--)
+      {
+         mem[loc+i] = (byte)(val & MASK);
+         val = val >> 8;
+      }
+   }
+
+   // Unpack the four bytes mem[loc]...mem[loc+3] into a 4-byte integer,
+   //  and return the resulting integer value.
+   // The most significant porion of the integer is stored in mem[loc].
+   // Bytes are 'OR'ed into the integer, working from left (most significant)
+   //  to right (least significant)
+   int unpack(int loc)
+   {
+      final int MASK = 0xff;
+      int v = (int)mem[loc] & MASK;
+      for (int i = 1; i < 4; i++)
+      {
+         v = v << 8;
+         v = v | ((int)mem[loc+i] & MASK);
+      }
+      return v;
+   }
+
+
+
+   // Test the above pack and unpack methods by iterating the following
+   //  over all possible 4-byte integers: pack the integer,
+   //  then unpack it, and then verify that the unpacked integer equals the
+   //  original integer.  It tests all nonnegative numbers in ascending order
+   //  and then all negative numbers in ascending order.  The transition from
+   //  positive to negative numbers happens implicitly due to integer overflow.
+   public void packTest()
+   {
+
+      int i = 0;
+      long k = 0;
+      do
+      {
+         this.pack(i,4);
+         int j = this.unpack(4);
+         if (j != i)
+         {
+            System.out.printf("pack/unpack test failed: i = %d, j = %d\n",i,j);
+            System.exit(0);
+         }
+         i++; k++;
+      }
+      while (i != 0);
+      System.out.printf("pack/unpack test successful, %d iterations\n",k);
+   }
+
+   // main routine to test the PackableMemory class by running the
+   //  packTest() method.
+   public static void main(String[] args)
+   {
+      PackableMemory pm = new PackableMemory(100);
+      pm.packTest();
+      System.exit(0);
+   }
+}
+

test/parser_test.py

@@ -1,7 +1,8 @@
-import socket, sys
+import socket, sys, struct
 
 file_name = 'resources/<name>.c'
 server_socket_number = 5001
+recv_size = 8192
 
 if __name__ == '__main__':
     if len(sys.argv) == 1:
@@ -19,22 +20,37 @@ if __name__ == '__main__':
             server_socket_number = 5002
 
         elif sys.argv[1] == 'ruby':
-            file_name = "resources/<name>.rb"
+            file_name = "resources/parser.rb"
             server_socket_number = 5003
 
-        client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        client_socket.connect(("localhost", server_socket_number))
+        server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        server_socket.connect(("localhost", server_socket_number))
 
         with open(file_name, "r") as source_file:
             source = source_file.read()
-            client_socket.send("%d\n%s" % (len(source), source));
+            server_socket.send("%d\n%s" % (len(source), source));
 
-        data = ''
-        while True:
-            data = client_socket.recv(10000)
+        total_data = []; size_data = cur_data = ''
+        total_size = 0; size = sys.maxint
 
-            if data != '':
-                client_socket.close()
-                break;
+        while total_size < size:
+            cur_data = server_socket.recv(recv_size)
 
-        print data;
+            if not total_data:
+                if len(size_data) > 4:
+                    size_data += cur_data
+                    size = struct.unpack('>i', size_data[:4])[0]
+                    recv_size = size
+                    if recv_size > sys.maxint: recv_size = sys.maxint
+                    total_data.append(size_data[4:])
+                else:
+                    size_data += cur_data
+
+            else:
+                total_data.append(cur_data)
+
+            total_size = sum([len(s) for s in total_data])
+
+
+        server_socket.close()
+        print ''.join(total_data);

test/resources/parser.rb

@@ -0,0 +1,126 @@
+require 'socket'
+require 'ruby_parser'
+require 'sexp_processor'
+
+module Bitshift
+    class Parser
+        def initialize(source)
+            @source = source
+        end
+
+        def parse
+            parser = RubyParser.new
+            tree = parser.parse(@source)
+            puts tree.inspect
+            offset = tree.line - 1
+            processor = NodeVisitor.new offset
+            processor.process tree
+            return processor.symbols
+        end
+    end
+
+    class NodeVisitor < SexpProcessor
+        attr_accessor :symbols
+        attr_accessor :offset
+
+        def initialize(offset)
+            super()
+            @require_empty = false
+            @offset = offset
+
+            module_hash = Hash.new {|hash, key| hash[key] = Hash.new}
+            class_hash = module_hash.clone
+            function_hash = Hash.new {|hash, key| hash[key] = { calls: [] } }
+            var_hash = Hash.new {|hash, key| hash[key] = [] }
+
+            @symbols = {
+                modules: module_hash,
+                classes: class_hash,
+                functions: function_hash,
+                vars: var_hash
+            }
+        end
+
+        def block_position(exp)
+            pos = Hash.new
+            end_ln = (start_ln = exp.line - offset)
+            cur_exp = exp
+
+            while cur_exp.is_a? Sexp
+                end_ln = cur_exp.line - offset
+                cur_exp = cur_exp.last
+                break if cur_exp == nil
+            end
+
+            pos[:coord] = {
+                start_ln: start_ln,
+                end_ln: end_ln }
+            return pos
+        end
+
+        def statement_position(exp)
+            pos = Hash.new
+            end_ln = start_ln = exp.line - offset
+
+            pos[:coord] = {
+                start_ln: start_ln,
+                end_ln: end_ln }
+            return pos
+        end
+
+        def process_module(exp)
+            pos = block_position exp
+            exp.shift
+            name = exp.shift
+            symbols[:modules][name] = pos
+            exp.each_sexp {|s| process(s)}
+            return exp.clear
+        end
+
+        def process_class(exp)
+            pos = block_position exp
+            exp.shift
+            name = exp.shift
+            symbols[:classes][name] = pos
+            exp.each_sexp {|s| process(s)}
+            return exp.clear
+        end
+
+        def process_defn(exp)
+            pos = block_position exp
+            exp.shift
+            name = exp.shift
+            symbols[:functions][name][:declaration] = pos
+            exp.each_sexp {|s| process(s)}
+            return exp.clear
+        end
+
+        def process_call(exp)
+            pos = statement_position exp
+            exp.shift
+            exp.shift
+            name = exp.shift
+            symbols[:functions][name][:calls] << pos
+            exp.each_sexp {|s| process(s)}
+            return exp.clear
+        end
+
+        def process_iasgn(exp)
+            pos = statement_position exp
+            exp.shift
+            name = exp.shift
+            symbols[:vars][name] << pos
+            exp.each_sexp {|s| process(s)}
+            return exp.clear
+        end
+
+        def process_lasgn(exp)
+            pos = statement_position exp
+            exp.shift
+            name = exp.shift
+            symbols[:vars][name] << pos
+            exp.each_sexp {|s| process(s)}
+            return exp.clear
+        end
+    end
+end