ENH: rview and report system introduced

.gitignore

@@ -12,6 +12,7 @@ _build
 *.so.*
 *_pch.h.cpp
 *_resource.rc
+.*
 
 # qtcreator generated files
 *.pro.user*

src/client/client.py

-from loomcomm_pb2 import Register, Data, ClientMessage
+from loomcomm_pb2 import Register, Data, ClientMessage, ClientSubmit
+from loomreport_pb2 import Report
 
 import socket
 from connection import Connection
@@ -23,7 +24,7 @@ class TaskFailed(LoomException):
 
 class Client(object):
 
-    def __init__(self, address, port, info=False):
+    def __init__(self, address, port):
         self.server_address = address
         self.server_port = port
 
@@ -36,32 +37,27 @@ class Client(object):
         s.connect((address, port))
         self.connection = Connection(s)
 
-        if info:
-            self.info = []
-        else:
-            self.info = None
-
         msg = Register()
         msg.type = Register.REGISTER_CLIENT
         msg.protocol_version = LOOM_PROTOCOL_VERSION
-        msg.info = info
         self._send_message(msg)
 
         while self.symbols is None:
             self._read_symbols()
 
-    def submit(self, plan, results):
-
-        msg = plan.create_message(self.symbols)
+    def submit(self, plan, results, report=None):
+        msg = ClientSubmit()
+        msg.report = bool(report)
+        plan.set_message(msg.plan, self.symbols)
 
         if isinstance(results, Task):
             single_result = True
-            msg.result_ids.extend((results.id,))
+            msg.plan.result_ids.extend((results.id,))
             expected = 1
         else:
             single_result = False
             r = set(results)
-            msg.result_ids.extend(r.id for r in r)
+            msg.plan.result_ids.extend(r.id for r in r)
             expected = len(r)
 
         self._send_message(msg)
@@ -74,18 +70,35 @@ class Client(object):
             if cmsg.type == ClientMessage.DATA:
                 prologue = cmsg.data
                 data[prologue.id] = self._receive_data()
-            elif cmsg.type == ClientMessage.INFO:
-                self.add_info(cmsg.info)
+            elif cmsg.type == ClientMessage.EVENT:
+                self.process_event(cmsg.event)
             elif cmsg.type == ClientMessage.ERROR:
                 self.process_error(cmsg)
             else:
                 assert 0
 
+        if report:
+            self._write_report(report, plan)
+
         if single_result:
             return data[results.id]
         else:
             return [data[task.id] for task in results]
 
+    def _symbol_list(self):
+        symbols = [None] * len(self.symbols)
+        for name, index in self.symbols.items():
+            symbols[index] = name
+        return symbols
+
+    def _write_report(self, report_filename, plan):
+        report_msg = Report()
+        report_msg.symbols.extend(self._symbol_list())
+        plan.set_message(report_msg.plan, self.symbols)
+
+        with open(report_filename + ".report", "w") as f:
+            f.write(report_msg.SerializeToString())
+
     def _read_symbols(self):
         msg = self.connection.receive_message()
         cmsg = ClientMessage()
@@ -102,8 +115,8 @@ class Client(object):
         error = cmsg.error
         raise TaskFailed(error.id, error.worker, error.error_msg)
 
-    def add_info(self, info):
-        self.info.append((info.id, info.worker))
+    def process_event(self, event):
+        assert 0
 
     def _receive_data(self):
         msg_data = Data()

src/client/loomcomm_pb2.py

@@ -13,13 +13,15 @@ from google.protobuf import descriptor_pb2
 _sym_db = _symbol_database.Default()
 
 
+import loomplan_pb2
 
 
 DESCRIPTOR = _descriptor.FileDescriptor(
   name='loomcomm.proto',
   package='loomcomm',
-  serialized_pb=_b('\n\x0eloomcomm.proto\x12\x08loomcomm\"\xcf\x01\n\x08Register\x12\x18\n\x10protocol_version\x18\x01 \x02(\x05\x12%\n\x04type\x18\x02 \x02(\x0e\x32\x17.loomcomm.Register.Type\x12\x0c\n\x04port\x18\x03 \x01(\x05\x12\x12\n\ntask_types\x18\x04 \x03(\t\x12\x12\n\ndata_types\x18\x05 \x03(\t\x12\x0c\n\x04\x63pus\x18\x06 \x01(\x05\x12\x0c\n\x04info\x18\n \x01(\x08\"0\n\x04Type\x12\x13\n\x0fREGISTER_WORKER\x10\x01\x12\x13\n\x0fREGISTER_CLIENT\x10\x02\"&\n\rServerMessage\"\x15\n\x04Type\x12\r\n\tSTART_JOB\x10\x01\"\xf1\x01\n\rWorkerCommand\x12*\n\x04type\x18\x01 \x02(\x0e\x32\x1c.loomcomm.WorkerCommand.Type\x12\n\n\x02id\x18\x02 \x01(\x05\x12\x11\n\ttask_type\x18\x03 \x01(\x05\x12\x13\n\x0btask_config\x18\x04 \x01(\t\x12\x13\n\x0btask_inputs\x18\x05 \x03(\x05\x12\x0f\n\x07\x61\x64\x64ress\x18\n \x01(\t\x12\x11\n\twith_size\x18\x0b \x01(\x08\x12\x0f\n\x07symbols\x18\x64 \x03(\t\"6\n\x04Type\x12\x08\n\x04TASK\x10\x01\x12\x08\n\x04SEND\x10\x02\x12\n\n\x06REMOVE\x10\x03\x12\x0e\n\nDICTIONARY\x10\x04\"\x9a\x01\n\x0eWorkerResponse\x12+\n\x04type\x18\x01 \x02(\x0e\x32\x1d.loomcomm.WorkerResponse.Type\x12\n\n\x02id\x18\x02 \x02(\x05\x12\x0c\n\x04size\x18\x03 \x01(\x04\x12\x0e\n\x06length\x18\x04 \x01(\x04\x12\x11\n\terror_msg\x18\x64 \x01(\t\"\x1e\n\x04Type\x12\n\n\x06\x46INISH\x10\x01\x12\n\n\x06\x46\x41ILED\x10\x02\"\x18\n\x08\x41nnounce\x12\x0c\n\x04port\x18\x01 \x02(\x05\"-\n\x0c\x44\x61taPrologue\x12\n\n\x02id\x18\x01 \x02(\x05\x12\x11\n\tdata_size\x18\x03 \x01(\x04\"Y\n\x04\x44\x61ta\x12\x0f\n\x07type_id\x18\x01 \x02(\x05\x12\x0c\n\x04size\x18\x02 \x02(\x04\x12\x0e\n\x06length\x18\x03 \x01(\x04\x12\x10\n\x08\x61rg0_u64\x18\x08 \x01(\x04\x12\x10\n\x08\x61rg1_u64\x18\t \x01(\x04\"\"\n\x04Info\x12\n\n\x02id\x18\x01 \x02(\x05\x12\x0e\n\x06worker\x18\x02 \x02(\t\"6\n\x05\x45rror\x12\n\n\x02id\x18\x01 \x02(\x05\x12\x0e\n\x06worker\x18\x02 \x02(\t\x12\x11\n\terror_msg\x18\x03 \x02(\t\"\xe7\x01\n\rClientMessage\x12*\n\x04type\x18\x01 \x02(\x0e\x32\x1c.loomcomm.ClientMessage.Type\x12$\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x16.loomcomm.DataPrologue\x12\x1c\n\x04info\x18\x03 \x01(\x0b\x32\x0e.loomcomm.Info\x12\x1e\n\x05\x65rror\x18\x04 \x01(\x0b\x32\x0f.loomcomm.Error\x12\x0f\n\x07symbols\x18\x05 \x03(\t\"5\n\x04Type\x12\x08\n\x04\x44\x41TA\x10\x01\x12\x08\n\x04INFO\x10\x02\x12\t\n\x05\x45RROR\x10\x03\x12\x0e\n\nDICTIONARY\x10\x04\x42\x02H\x03')
-)
+  serialized_pb=_b('\n\x0eloomcomm.proto\x12\x08loomcomm\x1a\x0eloomplan.proto\"\xc1\x01\n\x08Register\x12\x18\n\x10protocol_version\x18\x01 \x02(\x05\x12%\n\x04type\x18\x02 \x02(\x0e\x32\x17.loomcomm.Register.Type\x12\x0c\n\x04port\x18\x03 \x01(\x05\x12\x12\n\ntask_types\x18\x04 \x03(\t\x12\x12\n\ndata_types\x18\x05 \x03(\t\x12\x0c\n\x04\x63pus\x18\x06 \x01(\x05\"0\n\x04Type\x12\x13\n\x0fREGISTER_WORKER\x10\x01\x12\x13\n\x0fREGISTER_CLIENT\x10\x02\"&\n\rServerMessage\"\x15\n\x04Type\x12\r\n\tSTART_JOB\x10\x01\"\xf1\x01\n\rWorkerCommand\x12*\n\x04type\x18\x01 \x02(\x0e\x32\x1c.loomcomm.WorkerCommand.Type\x12\n\n\x02id\x18\x02 \x01(\x05\x12\x11\n\ttask_type\x18\x03 \x01(\x05\x12\x13\n\x0btask_config\x18\x04 \x01(\t\x12\x13\n\x0btask_inputs\x18\x05 \x03(\x05\x12\x0f\n\x07\x61\x64\x64ress\x18\n \x01(\t\x12\x11\n\twith_size\x18\x0b \x01(\x08\x12\x0f\n\x07symbols\x18\x64 \x03(\t\"6\n\x04Type\x12\x08\n\x04TASK\x10\x01\x12\x08\n\x04SEND\x10\x02\x12\n\n\x06REMOVE\x10\x03\x12\x0e\n\nDICTIONARY\x10\x04\"\x9a\x01\n\x0eWorkerResponse\x12+\n\x04type\x18\x01 \x02(\x0e\x32\x1d.loomcomm.WorkerResponse.Type\x12\n\n\x02id\x18\x02 \x02(\x05\x12\x0c\n\x04size\x18\x03 \x01(\x04\x12\x0e\n\x06length\x18\x04 \x01(\x04\x12\x11\n\terror_msg\x18\x64 \x01(\t\"\x1e\n\x04Type\x12\n\n\x06\x46INISH\x10\x01\x12\n\n\x06\x46\x41ILED\x10\x02\"\x18\n\x08\x41nnounce\x12\x0c\n\x04port\x18\x01 \x02(\x05\"-\n\x0c\x44\x61taPrologue\x12\n\n\x02id\x18\x01 \x02(\x05\x12\x11\n\tdata_size\x18\x03 \x01(\x04\"Y\n\x04\x44\x61ta\x12\x0f\n\x07type_id\x18\x01 \x02(\x05\x12\x0c\n\x04size\x18\x02 \x02(\x04\x12\x0e\n\x06length\x18\x03 \x01(\x04\x12\x10\n\x08\x61rg0_u64\x18\x08 \x01(\x04\x12\x10\n\x08\x61rg1_u64\x18\t \x01(\x04\"\x9f\x01\n\x05\x45vent\x12\x0c\n\x04time\x18\x01 \x02(\x04\x12\"\n\x04type\x18\x02 \x02(\x0e\x32\x14.loomcomm.Event.Type\x12\n\n\x02id\x18\x03 \x02(\x05\x12\x14\n\x0cworker_index\x18\x04 \x01(\x05\"B\n\x04Type\x12\x0e\n\nTASK_START\x10\x01\x12\x0c\n\x08TASK_END\x10\x02\x12\x0e\n\nSEND_START\x10\x03\x12\x0c\n\x08SEND_END\x10\x04\"6\n\x05\x45rror\x12\n\n\x02id\x18\x01 \x02(\x05\x12\x0e\n\x06worker\x18\x02 \x02(\t\x12\x11\n\terror_msg\x18\x03 \x02(\t\"\xea\x01\n\rClientMessage\x12*\n\x04type\x18\x01 \x02(\x0e\x32\x1c.loomcomm.ClientMessage.Type\x12$\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x16.loomcomm.DataPrologue\x12\x1e\n\x05\x65vent\x18\x03 \x01(\x0b\x32\x0f.loomcomm.Event\x12\x1e\n\x05\x65rror\x18\x04 \x01(\x0b\x32\x0f.loomcomm.Error\x12\x0f\n\x07symbols\x18\x05 \x03(\t\"6\n\x04Type\x12\x08\n\x04\x44\x41TA\x10\x01\x12\t\n\x05\x45VENT\x10\x02\x12\t\n\x05\x45RROR\x10\x03\x12\x0e\n\nDICTIONARY\x10\x04\"<\n\x0c\x43lientSubmit\x12\x1c\n\x04plan\x18\x01 \x02(\x0b\x32\x0e.loomplan.Plan\x12\x0e\n\x06report\x18\x02 \x02(\x08\x42\x02H\x03')
+  ,
+  dependencies=[loomplan_pb2.DESCRIPTOR,])
 _sym_db.RegisterFileDescriptor(DESCRIPTOR)
 
 
@@ -41,8 +43,8 @@ _REGISTER_TYPE = _descriptor.EnumDescriptor(
   ],
   containing_type=None,
   options=None,
-  serialized_start=188,
-  serialized_end=236,
+  serialized_start=190,
+  serialized_end=238,
 )
 _sym_db.RegisterEnumDescriptor(_REGISTER_TYPE)
 
@@ -59,8 +61,8 @@ _SERVERMESSAGE_TYPE = _descriptor.EnumDescriptor(
   ],
   containing_type=None,
   options=None,
-  serialized_start=255,
-  serialized_end=276,
+  serialized_start=257,
+  serialized_end=278,
 )
 _sym_db.RegisterEnumDescriptor(_SERVERMESSAGE_TYPE)
 
@@ -89,8 +91,8 @@ _WORKERCOMMAND_TYPE = _descriptor.EnumDescriptor(
   ],
   containing_type=None,
   options=None,
-  serialized_start=466,
-  serialized_end=520,
+  serialized_start=468,
+  serialized_end=522,
 )
 _sym_db.RegisterEnumDescriptor(_WORKERCOMMAND_TYPE)
 
@@ -111,11 +113,41 @@ _WORKERRESPONSE_TYPE = _descriptor.EnumDescriptor(
   ],
   containing_type=None,
   options=None,
-  serialized_start=647,
-  serialized_end=677,
+  serialized_start=649,
+  serialized_end=679,
 )
 _sym_db.RegisterEnumDescriptor(_WORKERRESPONSE_TYPE)
 
+_EVENT_TYPE = _descriptor.EnumDescriptor(
+  name='Type',
+  full_name='loomcomm.Event.Type',
+  filename=None,
+  file=DESCRIPTOR,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='TASK_START', index=0, number=1,
+      options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='TASK_END', index=1, number=2,
+      options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='SEND_START', index=2, number=3,
+      options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='SEND_END', index=3, number=4,
+      options=None,
+      type=None),
+  ],
+  containing_type=None,
+  options=None,
+  serialized_start=939,
+  serialized_end=1005,
+)
+_sym_db.RegisterEnumDescriptor(_EVENT_TYPE)
+
 _CLIENTMESSAGE_TYPE = _descriptor.EnumDescriptor(
   name='Type',
   full_name='loomcomm.ClientMessage.Type',
@@ -127,7 +159,7 @@ _CLIENTMESSAGE_TYPE = _descriptor.EnumDescriptor(
       options=None,
       type=None),
     _descriptor.EnumValueDescriptor(
-      name='INFO', index=1, number=2,
+      name='EVENT', index=1, number=2,
       options=None,
       type=None),
     _descriptor.EnumValueDescriptor(
@@ -141,8 +173,8 @@ _CLIENTMESSAGE_TYPE = _descriptor.EnumDescriptor(
   ],
   containing_type=None,
   options=None,
-  serialized_start=1114,
-  serialized_end=1167,
+  serialized_start=1244,
+  serialized_end=1298,
 )
 _sym_db.RegisterEnumDescriptor(_CLIENTMESSAGE_TYPE)
 
@@ -196,13 +228,6 @@ _REGISTER = _descriptor.Descriptor(
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       options=None),
-    _descriptor.FieldDescriptor(
-      name='info', full_name='loomcomm.Register.info', index=6,
-      number=10, type=8, cpp_type=7, label=1,
-      has_default_value=False, default_value=False,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      options=None),
   ],
   extensions=[
   ],
@@ -215,8 +240,8 @@ _REGISTER = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=29,
-  serialized_end=236,
+  serialized_start=45,
+  serialized_end=238,
 )
 
 
@@ -239,8 +264,8 @@ _SERVERMESSAGE = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=238,
-  serialized_end=276,
+  serialized_start=240,
+  serialized_end=278,
 )
 
 
@@ -319,8 +344,8 @@ _WORKERCOMMAND = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=279,
-  serialized_end=520,
+  serialized_start=281,
+  serialized_end=522,
 )
 
 
@@ -378,8 +403,8 @@ _WORKERRESPONSE = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=523,
-  serialized_end=677,
+  serialized_start=525,
+  serialized_end=679,
 )
 
 
@@ -408,8 +433,8 @@ _ANNOUNCE = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=679,
-  serialized_end=703,
+  serialized_start=681,
+  serialized_end=705,
 )
 
 
@@ -445,8 +470,8 @@ _DATAPROLOGUE = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=705,
-  serialized_end=750,
+  serialized_start=707,
+  serialized_end=752,
 )
 
 
@@ -503,29 +528,43 @@ _DATA = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=752,
-  serialized_end=841,
+  serialized_start=754,
+  serialized_end=843,
 )
 
 
-_INFO = _descriptor.Descriptor(
-  name='Info',
-  full_name='loomcomm.Info',
+_EVENT = _descriptor.Descriptor(
+  name='Event',
+  full_name='loomcomm.Event',
   filename=None,
   file=DESCRIPTOR,
   containing_type=None,
   fields=[
     _descriptor.FieldDescriptor(
-      name='id', full_name='loomcomm.Info.id', index=0,
-      number=1, type=5, cpp_type=1, label=2,
+      name='time', full_name='loomcomm.Event.time', index=0,
+      number=1, type=4, cpp_type=4, label=2,
       has_default_value=False, default_value=0,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       options=None),
     _descriptor.FieldDescriptor(
-      name='worker', full_name='loomcomm.Info.worker', index=1,
-      number=2, type=9, cpp_type=9, label=2,
-      has_default_value=False, default_value=_b("").decode('utf-8'),
+      name='type', full_name='loomcomm.Event.type', index=1,
+      number=2, type=14, cpp_type=8, label=2,
+      has_default_value=False, default_value=1,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='id', full_name='loomcomm.Event.id', index=2,
+      number=3, type=5, cpp_type=1, label=2,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='worker_index', full_name='loomcomm.Event.worker_index', index=3,
+      number=4, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       options=None),
@@ -534,14 +573,15 @@ _INFO = _descriptor.Descriptor(
   ],
   nested_types=[],
   enum_types=[
+    _EVENT_TYPE,
   ],
   options=None,
   is_extendable=False,
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=843,
-  serialized_end=877,
+  serialized_start=846,
+  serialized_end=1005,
 )
 
 
@@ -584,8 +624,8 @@ _ERROR = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=879,
-  serialized_end=933,
+  serialized_start=1007,
+  serialized_end=1061,
 )
 
 
@@ -611,7 +651,7 @@ _CLIENTMESSAGE = _descriptor.Descriptor(
       is_extension=False, extension_scope=None,
       options=None),
     _descriptor.FieldDescriptor(
-      name='info', full_name='loomcomm.ClientMessage.info', index=2,
+      name='event', full_name='loomcomm.ClientMessage.event', index=2,
       number=3, type=11, cpp_type=10, label=1,
       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
@@ -643,8 +683,45 @@ _CLIENTMESSAGE = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=936,
-  serialized_end=1167,
+  serialized_start=1064,
+  serialized_end=1298,
+)
+
+
+_CLIENTSUBMIT = _descriptor.Descriptor(
+  name='ClientSubmit',
+  full_name='loomcomm.ClientSubmit',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='plan', full_name='loomcomm.ClientSubmit.plan', index=0,
+      number=1, type=11, cpp_type=10, label=2,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='report', full_name='loomcomm.ClientSubmit.report', index=1,
+      number=2, type=8, cpp_type=7, label=2,
+      has_default_value=False, default_value=False,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=1300,
+  serialized_end=1360,
 )
 
 _REGISTER.fields_by_name['type'].enum_type = _REGISTER_TYPE
@@ -654,11 +731,14 @@ _WORKERCOMMAND.fields_by_name['type'].enum_type = _WORKERCOMMAND_TYPE
 _WORKERCOMMAND_TYPE.containing_type = _WORKERCOMMAND
 _WORKERRESPONSE.fields_by_name['type'].enum_type = _WORKERRESPONSE_TYPE
 _WORKERRESPONSE_TYPE.containing_type = _WORKERRESPONSE
+_EVENT.fields_by_name['type'].enum_type = _EVENT_TYPE
+_EVENT_TYPE.containing_type = _EVENT
 _CLIENTMESSAGE.fields_by_name['type'].enum_type = _CLIENTMESSAGE_TYPE
 _CLIENTMESSAGE.fields_by_name['data'].message_type = _DATAPROLOGUE
-_CLIENTMESSAGE.fields_by_name['info'].message_type = _INFO
+_CLIENTMESSAGE.fields_by_name['event'].message_type = _EVENT
 _CLIENTMESSAGE.fields_by_name['error'].message_type = _ERROR
 _CLIENTMESSAGE_TYPE.containing_type = _CLIENTMESSAGE
+_CLIENTSUBMIT.fields_by_name['plan'].message_type = loomplan_pb2._PLAN
 DESCRIPTOR.message_types_by_name['Register'] = _REGISTER
 DESCRIPTOR.message_types_by_name['ServerMessage'] = _SERVERMESSAGE
 DESCRIPTOR.message_types_by_name['WorkerCommand'] = _WORKERCOMMAND
@@ -666,9 +746,10 @@ DESCRIPTOR.message_types_by_name['WorkerResponse'] = _WORKERRESPONSE
 DESCRIPTOR.message_types_by_name['Announce'] = _ANNOUNCE
 DESCRIPTOR.message_types_by_name['DataPrologue'] = _DATAPROLOGUE
 DESCRIPTOR.message_types_by_name['Data'] = _DATA
-DESCRIPTOR.message_types_by_name['Info'] = _INFO
+DESCRIPTOR.message_types_by_name['Event'] = _EVENT
 DESCRIPTOR.message_types_by_name['Error'] = _ERROR
 DESCRIPTOR.message_types_by_name['ClientMessage'] = _CLIENTMESSAGE
+DESCRIPTOR.message_types_by_name['ClientSubmit'] = _CLIENTSUBMIT
 
 Register = _reflection.GeneratedProtocolMessageType('Register', (_message.Message,), dict(
   DESCRIPTOR = _REGISTER,
@@ -719,12 +800,12 @@ Data = _reflection.GeneratedProtocolMessageType('Data', (_message.Message,), dic
   ))
 _sym_db.RegisterMessage(Data)
 
-Info = _reflection.GeneratedProtocolMessageType('Info', (_message.Message,), dict(
-  DESCRIPTOR = _INFO,
+Event = _reflection.GeneratedProtocolMessageType('Event', (_message.Message,), dict(
+  DESCRIPTOR = _EVENT,
   __module__ = 'loomcomm_pb2'
-  # @@protoc_insertion_point(class_scope:loomcomm.Info)
+  # @@protoc_insertion_point(class_scope:loomcomm.Event)
   ))
-_sym_db.RegisterMessage(Info)
+_sym_db.RegisterMessage(Event)
 
 Error = _reflection.GeneratedProtocolMessageType('Error', (_message.Message,), dict(
   DESCRIPTOR = _ERROR,
@@ -740,6 +821,13 @@ ClientMessage = _reflection.GeneratedProtocolMessageType('ClientMessage', (_mess
   ))
 _sym_db.RegisterMessage(ClientMessage)
 
+ClientSubmit = _reflection.GeneratedProtocolMessageType('ClientSubmit', (_message.Message,), dict(
+  DESCRIPTOR = _CLIENTSUBMIT,
+  __module__ = 'loomcomm_pb2'
+  # @@protoc_insertion_point(class_scope:loomcomm.ClientSubmit)
+  ))
+_sym_db.RegisterMessage(ClientSubmit)
+
 
 DESCRIPTOR.has_options = True
 DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('H\003'))

src/client/loomreport_pb2.py

+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: loomreport.proto
+
+import sys
+_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf import descriptor_pb2
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+import loomplan_pb2
+import loomcomm_pb2
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='loomreport.proto',
+  package='loomcomm',
+  serialized_pb=_b('\n\x10loomreport.proto\x12\x08loomcomm\x1a\x0eloomplan.proto\x1a\x0eloomcomm.proto\"X\n\x06Report\x12\x0f\n\x07symbols\x18\x01 \x03(\t\x12\x1c\n\x04plan\x18\x02 \x02(\x0b\x32\x0e.loomplan.Plan\x12\x1f\n\x06\x65vents\x18\x03 \x03(\x0b\x32\x0f.loomcomm.EventB\x02H\x03')
+  ,
+  dependencies=[loomplan_pb2.DESCRIPTOR,loomcomm_pb2.DESCRIPTOR,])
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+
+
+
+_REPORT = _descriptor.Descriptor(
+  name='Report',
+  full_name='loomcomm.Report',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='symbols', full_name='loomcomm.Report.symbols', index=0,
+      number=1, type=9, cpp_type=9, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='plan', full_name='loomcomm.Report.plan', index=1,
+      number=2, type=11, cpp_type=10, label=2,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='events', full_name='loomcomm.Report.events', index=2,
+      number=3, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=62,
+  serialized_end=150,
+)
+
+_REPORT.fields_by_name['plan'].message_type = loomplan_pb2._PLAN
+_REPORT.fields_by_name['events'].message_type = loomcomm_pb2._EVENT
+DESCRIPTOR.message_types_by_name['Report'] = _REPORT
+
+Report = _reflection.GeneratedProtocolMessageType('Report', (_message.Message,), dict(
+  DESCRIPTOR = _REPORT,
+  __module__ = 'loomreport_pb2'
+  # @@protoc_insertion_point(class_scope:loomcomm.Report)
+  ))
+_sym_db.RegisterMessage(Report)
+
+
+DESCRIPTOR.has_options = True
+DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('H\003'))
+# @@protoc_insertion_point(module_scope)

src/client/plan.py

 
 import loomplan_pb2
 import loomrun_pb2
-import gv
 
 import struct
 
@@ -204,9 +203,7 @@ class Plan(object):
         task.policy = POLICY_SIMPLE
         return self.add(task)
 
-    def create_message(self, symbols):
-        msg = loomplan_pb2.Plan()
-
+    def set_message(self, msg, symbols):
         requests = set()
         for task in self.tasks:
             if task.resource_request:
@@ -221,23 +218,3 @@ class Plan(object):
             t = msg.tasks.add()
             task.set_message(t, symbols, requests)
         return msg
-
-    def write_dot(self, filename, info=None):
-        colors = ["red", "green", "blue", "orange", "violet"]
-        if info:
-            w = sorted(set(worker for id, worker in info))
-            workers = {}
-            for id, worker in info:
-                workers[id] = w.index(worker)
-            del w
-        else:
-            workers = None
-        graph = gv.Graph()
-        for task in self.tasks:
-            node = graph.node(task.id)
-            if workers:
-                node.color = colors[workers[task.id] % len(colors)]
-            node.label = "{}\n{}".format(str(task.id), task.task_type)
-            for inp in task.inputs:
-                graph.node(inp.id).add_arc(node)
-        graph.write(filename)

src/proto/loomcomm.proto

 package loomcomm;
 
 option optimize_for = LITE_RUNTIME;
+import "loomplan.proto";
 
 message Register {
 	enum Type {
@@ -15,9 +16,6 @@ message Register {
 	repeated string task_types = 4;
 	repeated string data_types = 5;
 	optional int32 cpus = 6;
-
-	// Client
-	optional bool info = 10;
 }
 
 message ServerMessage {
@@ -85,9 +83,17 @@ message Data
 	optional uint64 arg1_u64 = 9;
 }
 
-message Info {
-	required int32 id = 1;
-	required string worker = 2;
+message Event {
+	enum Type {
+		TASK_START = 1;
+		TASK_END = 2;
+		SEND_START = 3;
+		SEND_END = 4;
+	}
+	required uint64 time = 1;
+	required Type type = 2;
+	required int32 id = 3;
+	optional int32 worker_index = 4;
 }
 
 message Error {
@@ -99,13 +105,18 @@ message Error {
 message ClientMessage {
 	enum Type {
 		DATA = 1;
-		INFO = 2;
+		EVENT = 2;
 		ERROR = 3;
 		DICTIONARY = 4;
 	}
 	required Type type = 1;
 	optional DataPrologue data = 2;
-	optional Info info = 3;
+	optional Event event = 3;
 	optional Error error = 4;
 	repeated string symbols = 5;
 }
+
+message ClientSubmit {
+	required loomplan.Plan plan = 1;
+	required bool report = 2;
+}

src/proto/loomreport.proto

+package loomcomm;
+
+option optimize_for = LITE_RUNTIME;
+
+import "loomplan.proto";
+import "loomcomm.proto";
+
+message Report {
+	repeated string symbols = 1;
+	required loomplan.Plan plan = 2;
+	repeated loomcomm.Event events = 3;
+}

src/proto/rebuild.sh

@@ -15,3 +15,4 @@ protoc loomrun.proto --cpp_out=${LIBLOOM_DIR}/tasks
 protoc loomcomm.proto --python_out=${CLIENT_DIR}
 protoc loomplan.proto --python_out=${CLIENT_DIR}
 protoc loomrun.proto --python_out=${CLIENT_DIR}
+protoc loomreport.proto --python_out=${CLIENT_DIR}

src/rview/report.py

+
+# HACK! We need to fix this
+import sys
+import os
+sys.path.insert(0,
+                os.path.join(
+                    os.path.dirname(__file__),
+                    "..",
+                    "client"))
+
+import loomreport_pb2  # noqa
+import gv  # noqas
+
+
+class Report:
+
+    def __init__(self, filename):
+        with open(filename) as f:
+            raw_data = f.read()
+
+        self.report_msg = loomreport_pb2.Report()
+        self.report_msg.ParseFromString(raw_data)
+
+        self.symbols = [s.replace("loom", "L")
+                        for s in self.report_msg.symbols]
+
+    def create_graph(self):
+        graph = gv.Graph()
+        symbols = self.symbols
+        for i, task in enumerate(self.report_msg.plan.tasks):
+            node = graph.node(i)
+            node.label = symbols[task.task_type]
+            for j in task.input_ids:
+                graph.node(j).add_arc(node)
+        return graph

src/rview/rview.py

+
+from report import Report
+import argparse
+import subprocess
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description="rview -- Loom report inscpector")
+
+    parser.add_argument("report",
+                        metavar="REPORT",
+                        type=str,
+                        help="Path to report")
+
+    parser.add_argument("--show-symbols",
+                        action="store_true")
+
+    parser.add_argument("--show-graph",
+                        action="store_true")
+
+    return parser.parse_args()
+
+
+def run_program(args, stdin=None):
+    p = subprocess.Popen(args, stdin=subprocess.PIPE)
+    p.communicate(input=stdin)
+
+
+def show_symbols(report):
+    for i, symbol in enumerate(report.symbols):
+        print "{}: {}".format(i, symbol)
+
+
+def show_graph(report):
+    dot = report.create_graph().make_dot("Plan")
+    run_program(("xdot", "-"), dot)
+
+
+def main():
+    args = parse_args()
+    report = Report(args.report)
+
+    if args.show_symbols:
+        show_symbols(report)
+
+    if args.show_graph:
+        show_graph(report)
+
+
+if __name__ == "__main__":
+    main()

src/server/clientconn.cpp

@@ -7,8 +7,9 @@
 
 using namespace loom;
 
-ClientConnection::ClientConnection(Server &server, std::unique_ptr<loom::Connection> connection, bool info_flag)
-    : server(server), connection(std::move(connection)), info_flag(info_flag)
+ClientConnection::ClientConnection(Server &server,
+                                   std::unique_ptr<loom::Connection> connection)
+    : server(server), connection(std::move(connection))
 {
     this->connection->set_callback(this);
     llog->info("Client {} connected", this->connection->get_peername());
@@ -36,13 +37,14 @@ ClientConnection::~ClientConnection()
 void ClientConnection::on_message(const char *buffer, size_t size)
 {
     llog->debug("Plan received");
-    loomplan::Plan plan;
-    plan.ParseFromArray(buffer, size);
+    loomcomm::ClientSubmit submit;
+    submit.ParseFromArray(buffer, size);
     auto& task_manager = server.get_task_manager();
 
+    const loomplan::Plan &plan = submit.plan();
     loom::Id id_base = server.new_id(plan.tasks_size());
     task_manager.add_plan(Plan(plan, id_base, server.get_dictionary()));
-    llog->info("Plan submitted tasks={}", plan.tasks_size());
+    llog->info("Plan submitted tasks={} report={}", plan.tasks_size(), submit.report());
 }
 
 void ClientConnection::on_close()

src/server/clientconn.h

@@ -13,7 +13,7 @@ class Server;
 class ClientConnection : public loom::ConnectionCallback {
 public:
     ClientConnection(Server &server,
-                     std::unique_ptr<loom::Connection> connection, bool info_flag);
+                     std::unique_ptr<loom::Connection> connection);
     ~ClientConnection();
     void on_message(const char *buffer, size_t size);
     void on_close();
@@ -22,14 +22,9 @@ public:
         connection->send_buffer(buffer);
     }
 
-    bool has_info_flag() const {
-        return info_flag;
-    }
-
 protected:
     Server &server;
     std::unique_ptr<loom::Connection> connection;
-    bool info_flag;
 };
 
 

src/server/compstate.cpp

@@ -16,6 +16,8 @@ ComputationState::ComputationState(Server &server) : server(server)
    get_task_id = dictionary.find_or_create("loom/base/get");
    dslice_task_id = dictionary.find_or_create("loom/scheduler/dslice");
    dget_task_id = dictionary.find_or_create("loom/scheduler/dget");
+
+   base_time = uv_now(server.get_loop());
 }
 
 void ComputationState::set_plan(Plan &&plan)

src/server/compstate.h

@@ -62,12 +62,15 @@ private:
     Plan plan;
     Server &server;
 
+    uint64_t base_time;
+
     loom::Id dslice_task_id;
     loom::Id dget_task_id;
 
     loom::Id slice_task_id;
     loom::Id get_task_id;
 
+
     WorkerConnection *get_best_holder_of_deps(PlanNode *task);
     WorkerConnection *find_best_worker_for_node(PlanNode *task);
 

src/server/freshconn.cpp

@@ -61,10 +61,8 @@ void FreshConnection::on_message(const char *buffer, size_t size)
         return;
     }
     if (msg.type() == loomcomm::Register_Type_REGISTER_CLIENT) {
-        bool info_flag = msg.has_info() && msg.info();
         auto cconn = std::make_unique<ClientConnection>(server,
-                                                        std::move(connection),
-                                                        info_flag);
+                                                        std::move(connection));
         server.add_client_connection(std::move(cconn));
         assert(connection.get() == nullptr);
         server.remove_freshconnection(*this);

src/server/server.cpp

@@ -81,9 +81,10 @@ void Server::remove_freshconnection(FreshConnection &conn)
 void Server::on_task_finished(loom::Id id, size_t size, size_t length, WorkerConnection *wc)
 {
     assert(client_connection);
-    if (client_connection->has_info_flag()) {
+    /*
+    if (client_connection->is_report_enabled()) {
         assert(0);
-        /*loomcomm::ClientMessage cmsg;
+        loomcomm::ClientMessage cmsg;
         cmsg.set_type(loomcomm::ClientMessage_Type_INFO);
         loomcomm::Info *info = cmsg.mutable_info();
         info->set_id(task.get_id());
@@ -95,8 +96,8 @@ void Server::on_task_finished(loom::Id id, size_t size, size_t length, WorkerCon
         SendBuffer *buffer = new SendBuffer;
         buffer->add(cmsg);
 
-        client_connection->send_buffer(buffer);*/
-    }
+        client_connection->send_buffer(buffer);
+    }*/
     task_manager.on_task_finished(id, size, length, wc);
 }
 

tests/client/cv_test.py

@@ -37,7 +37,7 @@ def test_cv_iris(loom_env):
                               [(chunk, "testdata"), (model, "model")])
             predict.append(task)
 
-        results = loom_env.submit(p, predict)
+        results = loom_env.submit(p, predict, report="cv")
 
         assert len(results) == CHUNKS
         for line in results: