Initial commit

.gitignore

+_build
+*~
+*.autosave
+*.a
+*.core
+*.moc
+*.o
+*.obj
+*.orig
+*.rej
+*.so
+*.so.*
+*_pch.h.cpp
+*_resource.rc
+
+# qtcreator generated files
+*.pro.user*
+
+# xemacs temporary files
+*.flc
+
+# Vim temporary files
+.*.swp
+
+# Python byte code
+*.pyc
+
+tests/client/build

CMakeLists.txt

+project(loom-rt)
+cmake_minimum_required(VERSION 2.8)
+enable_testing()
+
+add_subdirectory(src)
+add_subdirectory(tests)
+
+add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND} --verbose)

src/CMakeLists.txt

+add_subdirectory(libloom)
+add_subdirectory(worker)
+add_subdirectory(server)

src/client/__init__.py

+
+from .client import Client  # noqa
+from .plan import Plan  # noqa

src/client/client.py

+from loomcomm_pb2 import Register, Data
+
+import socket
+from connection import Connection
+from plan import Task
+
+LOOM_PROTOCOL_VERSION = 1
+
+
+class Client(object):
+
+    def __init__(self, address, port):
+        self.server_address = address
+        self.server_port = port
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        s.connect((address, port))
+        self.connection = Connection(s)
+
+        msg = Register()
+        msg.type = Register.REGISTER_CLIENT
+        msg.protocol_version = LOOM_PROTOCOL_VERSION
+        self._send_message(msg)
+
+    def submit(self, plan, results):
+
+        msg = plan.create_message()
+
+        if isinstance(results, Task):
+            single_result = True
+            msg.result_ids.extend((results.id,))
+            expected = 1
+        else:
+            single_result = False
+            r = set(results)
+            msg.result_ids.extend(r.id for r in r)
+            expected = len(r)
+
+        self._send_message(msg)
+
+        data = {}
+        while expected != len(data):
+            msg = self.connection.receive_message()
+            msg_data = Data()
+            msg_data.ParseFromString(msg)
+            data[msg_data.id] = self.connection.read_data(msg_data.size)
+        if single_result:
+            return data[results.id]
+        else:
+            return [data[task.id] for task in results]
+
+    def _send_message(self, message):
+        data = message.SerializeToString()
+        self.connection.send_message(data)

src/client/connection.py

+
+import struct
+
+u32 = struct.Struct("<I")
+
+
+class Connection(object):
+
+    def __init__(self, socket):
+        self.socket = socket
+        self.data = ""
+
+    def receive_message(self):
+        while True:
+            size = len(self.data)
+            if size > 4:
+                msg_size = u32.unpack(self.data[:4])[0]
+                msg_size += 4
+                if size >= msg_size:
+                    message = self.data[4:msg_size]
+                    self.data = self.data[msg_size:]
+                    return message
+            new_data = self.socket.recv(65536)
+            if not new_data:
+                raise Exception("Connection to server lost")
+            self.data += new_data
+
+    def read_data(self, data_size):
+        result = ""
+        while True:
+            change = min(data_size, len(self.data))
+            result += self.data[:change]
+            self.data = self.data[change:]
+            data_size -= change
+            if data_size == 0:
+                return result
+            self.data = self.socket.recv(65536)
+
+    def send_message(self, data):
+        data = u32.pack(len(data)) + data
+        self.socket.sendall(data)

src/client/loomcomm_pb2.py

+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: loomcomm.proto
+
+import sys
+_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf import descriptor_pb2
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='loomcomm.proto',
+  package='loomcomm',
+  serialized_pb=_b('\n\x0eloomcomm.proto\x12\x08loomcomm\"\x9f\x01\n\x08Register\x12\x18\n\x10protocol_version\x18\x01 \x02(\x05\x12%\n\x04type\x18\x02 \x02(\x0e\x32\x17.loomcomm.Register.Type\x12\x0c\n\x04port\x18\x03 \x01(\x05\x12\x12\n\ntask_types\x18\x04 \x03(\t\"0\n\x04Type\x12\x13\n\x0fREGISTER_WORKER\x10\x01\x12\x13\n\x0fREGISTER_CLIENT\x10\x02\"&\n\rServerMessage\"\x15\n\x04Type\x12\r\n\tSTART_JOB\x10\x01\"\xb1\x01\n\rWorkerCommand\x12*\n\x04type\x18\x01 \x02(\x0e\x32\x1c.loomcomm.WorkerCommand.Type\x12\n\n\x02id\x18\x02 \x01(\x05\x12\x11\n\ttask_type\x18\x03 \x01(\x05\x12\x13\n\x0btask_config\x18\x04 \x01(\t\x12\x13\n\x0btask_inputs\x18\x05 \x03(\x05\x12\x0f\n\x07\x61\x64\x64ress\x18\n \x01(\t\"\x1a\n\x04Type\x12\x08\n\x04TASK\x10\x01\x12\x08\n\x04SEND\x10\x02\"\x1c\n\x0eWorkerResponse\x12\n\n\x02id\x18\x02 \x01(\x05\"\x18\n\x08\x41nnounce\x12\x0c\n\x04port\x18\x01 \x02(\x05\" \n\x04\x44\x61ta\x12\n\n\x02id\x18\x01 \x02(\x05\x12\x0c\n\x04size\x18\x02 \x02(\x04\x42\x02H\x03')
+)
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+
+
+_REGISTER_TYPE = _descriptor.EnumDescriptor(
+  name='Type',
+  full_name='loomcomm.Register.Type',
+  filename=None,
+  file=DESCRIPTOR,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='REGISTER_WORKER', index=0, number=1,
+      options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='REGISTER_CLIENT', index=1, number=2,
+      options=None,
+      type=None),
+  ],
+  containing_type=None,
+  options=None,
+  serialized_start=140,
+  serialized_end=188,
+)
+_sym_db.RegisterEnumDescriptor(_REGISTER_TYPE)
+
+_SERVERMESSAGE_TYPE = _descriptor.EnumDescriptor(
+  name='Type',
+  full_name='loomcomm.ServerMessage.Type',
+  filename=None,
+  file=DESCRIPTOR,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='START_JOB', index=0, number=1,
+      options=None,
+      type=None),
+  ],
+  containing_type=None,
+  options=None,
+  serialized_start=207,
+  serialized_end=228,
+)
+_sym_db.RegisterEnumDescriptor(_SERVERMESSAGE_TYPE)
+
+_WORKERCOMMAND_TYPE = _descriptor.EnumDescriptor(
+  name='Type',
+  full_name='loomcomm.WorkerCommand.Type',
+  filename=None,
+  file=DESCRIPTOR,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='TASK', index=0, number=1,
+      options=None,
+      type=None),
+    _descriptor.EnumValueDescriptor(
+      name='SEND', index=1, number=2,
+      options=None,
+      type=None),
+  ],
+  containing_type=None,
+  options=None,
+  serialized_start=382,
+  serialized_end=408,
+)
+_sym_db.RegisterEnumDescriptor(_WORKERCOMMAND_TYPE)
+
+
+_REGISTER = _descriptor.Descriptor(
+  name='Register',
+  full_name='loomcomm.Register',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='protocol_version', full_name='loomcomm.Register.protocol_version', index=0,
+      number=1, type=5, cpp_type=1, label=2,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='type', full_name='loomcomm.Register.type', index=1,
+      number=2, type=14, cpp_type=8, label=2,
+      has_default_value=False, default_value=1,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='port', full_name='loomcomm.Register.port', index=2,
+      number=3, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='task_types', full_name='loomcomm.Register.task_types', index=3,
+      number=4, type=9, cpp_type=9, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+    _REGISTER_TYPE,
+  ],
+  options=None,
+  is_extendable=False,
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=29,
+  serialized_end=188,
+)
+
+
+_SERVERMESSAGE = _descriptor.Descriptor(
+  name='ServerMessage',
+  full_name='loomcomm.ServerMessage',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+    _SERVERMESSAGE_TYPE,
+  ],
+  options=None,
+  is_extendable=False,
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=190,
+  serialized_end=228,
+)
+
+
+_WORKERCOMMAND = _descriptor.Descriptor(
+  name='WorkerCommand',
+  full_name='loomcomm.WorkerCommand',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='type', full_name='loomcomm.WorkerCommand.type', index=0,
+      number=1, type=14, cpp_type=8, label=2,
+      has_default_value=False, default_value=1,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='id', full_name='loomcomm.WorkerCommand.id', index=1,
+      number=2, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='task_type', full_name='loomcomm.WorkerCommand.task_type', index=2,
+      number=3, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='task_config', full_name='loomcomm.WorkerCommand.task_config', index=3,
+      number=4, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='task_inputs', full_name='loomcomm.WorkerCommand.task_inputs', index=4,
+      number=5, type=5, cpp_type=1, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='address', full_name='loomcomm.WorkerCommand.address', index=5,
+      number=10, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+    _WORKERCOMMAND_TYPE,
+  ],
+  options=None,
+  is_extendable=False,
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=231,
+  serialized_end=408,
+)
+
+
+_WORKERRESPONSE = _descriptor.Descriptor(
+  name='WorkerResponse',
+  full_name='loomcomm.WorkerResponse',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='id', full_name='loomcomm.WorkerResponse.id', index=0,
+      number=2, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=410,
+  serialized_end=438,
+)
+
+
+_ANNOUNCE = _descriptor.Descriptor(
+  name='Announce',
+  full_name='loomcomm.Announce',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='port', full_name='loomcomm.Announce.port', index=0,
+      number=1, type=5, cpp_type=1, label=2,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=440,
+  serialized_end=464,
+)
+
+
+_DATA = _descriptor.Descriptor(
+  name='Data',
+  full_name='loomcomm.Data',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='id', full_name='loomcomm.Data.id', index=0,
+      number=1, type=5, cpp_type=1, label=2,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='size', full_name='loomcomm.Data.size', index=1,
+      number=2, type=4, cpp_type=4, label=2,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=466,
+  serialized_end=498,
+)
+
+_REGISTER.fields_by_name['type'].enum_type = _REGISTER_TYPE
+_REGISTER_TYPE.containing_type = _REGISTER
+_SERVERMESSAGE_TYPE.containing_type = _SERVERMESSAGE
+_WORKERCOMMAND.fields_by_name['type'].enum_type = _WORKERCOMMAND_TYPE
+_WORKERCOMMAND_TYPE.containing_type = _WORKERCOMMAND
+DESCRIPTOR.message_types_by_name['Register'] = _REGISTER
+DESCRIPTOR.message_types_by_name['ServerMessage'] = _SERVERMESSAGE
+DESCRIPTOR.message_types_by_name['WorkerCommand'] = _WORKERCOMMAND
+DESCRIPTOR.message_types_by_name['WorkerResponse'] = _WORKERRESPONSE
+DESCRIPTOR.message_types_by_name['Announce'] = _ANNOUNCE
+DESCRIPTOR.message_types_by_name['Data'] = _DATA
+
+Register = _reflection.GeneratedProtocolMessageType('Register', (_message.Message,), dict(
+  DESCRIPTOR = _REGISTER,
+  __module__ = 'loomcomm_pb2'
+  # @@protoc_insertion_point(class_scope:loomcomm.Register)
+  ))
+_sym_db.RegisterMessage(Register)
+
+ServerMessage = _reflection.GeneratedProtocolMessageType('ServerMessage', (_message.Message,), dict(
+  DESCRIPTOR = _SERVERMESSAGE,
+  __module__ = 'loomcomm_pb2'
+  # @@protoc_insertion_point(class_scope:loomcomm.ServerMessage)
+  ))
+_sym_db.RegisterMessage(ServerMessage)
+
+WorkerCommand = _reflection.GeneratedProtocolMessageType('WorkerCommand', (_message.Message,), dict(
+  DESCRIPTOR = _WORKERCOMMAND,
+  __module__ = 'loomcomm_pb2'
+  # @@protoc_insertion_point(class_scope:loomcomm.WorkerCommand)
+  ))
+_sym_db.RegisterMessage(WorkerCommand)
+
+WorkerResponse = _reflection.GeneratedProtocolMessageType('WorkerResponse', (_message.Message,), dict(
+  DESCRIPTOR = _WORKERRESPONSE,
+  __module__ = 'loomcomm_pb2'
+  # @@protoc_insertion_point(class_scope:loomcomm.WorkerResponse)
+  ))
+_sym_db.RegisterMessage(WorkerResponse)
+
+Announce = _reflection.GeneratedProtocolMessageType('Announce', (_message.Message,), dict(
+  DESCRIPTOR = _ANNOUNCE,
+  __module__ = 'loomcomm_pb2'
+  # @@protoc_insertion_point(class_scope:loomcomm.Announce)
+  ))
+_sym_db.RegisterMessage(Announce)
+
+Data = _reflection.GeneratedProtocolMessageType('Data', (_message.Message,), dict(
+  DESCRIPTOR = _DATA,
+  __module__ = 'loomcomm_pb2'
+  # @@protoc_insertion_point(class_scope:loomcomm.Data)
+  ))
+_sym_db.RegisterMessage(Data)
+
+
+DESCRIPTOR.has_options = True
+DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('H\003'))
+# @@protoc_insertion_point(module_scope)

src/client/loomplan_pb2.py

+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: loomplan.proto
+
+import sys
+_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf import descriptor_pb2
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='loomplan.proto',
+  package='loomplan',
+  serialized_pb=_b('\n\x0eloomplan.proto\x12\x08loomplan\"<\n\x04Task\x12\x11\n\ttask_type\x18\x01 \x02(\x05\x12\x0e\n\x06\x63onfig\x18\x02 \x02(\x0c\x12\x11\n\tinput_ids\x18\x03 \x03(\x05\"M\n\x04Plan\x12\x12\n\ntask_types\x18\x01 \x03(\t\x12\x1d\n\x05tasks\x18\x02 \x03(\x0b\x32\x0e.loomplan.Task\x12\x12\n\nresult_ids\x18\x03 \x03(\x05\x42\x02H\x03')
+)
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+
+
+
+_TASK = _descriptor.Descriptor(
+  name='Task',
+  full_name='loomplan.Task',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='task_type', full_name='loomplan.Task.task_type', index=0,
+      number=1, type=5, cpp_type=1, label=2,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='config', full_name='loomplan.Task.config', index=1,
+      number=2, type=12, cpp_type=9, label=2,
+      has_default_value=False, default_value=_b(""),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+    _descriptor.FieldDescriptor(
+      name='input_ids', full_name='loomplan.Task.input_ids', index=2,
+      number=3, type=5, cpp_type=1, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  options=None,
+  is_extendable=False,
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=28,
+  serialized_end=88,
+)
+
+
+_PLAN = _descriptor.Descriptor(
+  name='Plan',
+  full_name='loomplan.Plan',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,