Commit a3b5327b authored by Stanislav Bohm's avatar Stanislav Bohm

Initial commit

parents
_build
*~
*.autosave
*.a
*.core
*.moc
*.o
*.obj
*.orig
*.rej
*.so
*.so.*
*_pch.h.cpp
*_resource.rc
# qtcreator generated files
*.pro.user*
# xemacs temporary files
*.flc
# Vim temporary files
.*.swp
# Python byte code
*.pyc
tests/client/build
project(loom-rt)
cmake_minimum_required(VERSION 2.8)
enable_testing()
add_subdirectory(src)
add_subdirectory(tests)
add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND} --verbose)
add_subdirectory(libloom)
add_subdirectory(worker)
add_subdirectory(server)
from .client import Client # noqa
from .plan import Plan # noqa
from loomcomm_pb2 import Register, Data
import socket
from connection import Connection
from plan import Task
LOOM_PROTOCOL_VERSION = 1
class Client(object):
def __init__(self, address, port):
self.server_address = address
self.server_port = port
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((address, port))
self.connection = Connection(s)
msg = Register()
msg.type = Register.REGISTER_CLIENT
msg.protocol_version = LOOM_PROTOCOL_VERSION
self._send_message(msg)
def submit(self, plan, results):
msg = plan.create_message()
if isinstance(results, Task):
single_result = True
msg.result_ids.extend((results.id,))
expected = 1
else:
single_result = False
r = set(results)
msg.result_ids.extend(r.id for r in r)
expected = len(r)
self._send_message(msg)
data = {}
while expected != len(data):
msg = self.connection.receive_message()
msg_data = Data()
msg_data.ParseFromString(msg)
data[msg_data.id] = self.connection.read_data(msg_data.size)
if single_result:
return data[results.id]
else:
return [data[task.id] for task in results]
def _send_message(self, message):
data = message.SerializeToString()
self.connection.send_message(data)
import struct
u32 = struct.Struct("<I")
class Connection(object):
def __init__(self, socket):
self.socket = socket
self.data = ""
def receive_message(self):
while True:
size = len(self.data)
if size > 4:
msg_size = u32.unpack(self.data[:4])[0]
msg_size += 4
if size >= msg_size:
message = self.data[4:msg_size]
self.data = self.data[msg_size:]
return message
new_data = self.socket.recv(65536)
if not new_data:
raise Exception("Connection to server lost")
self.data += new_data
def read_data(self, data_size):
result = ""
while True:
change = min(data_size, len(self.data))
result += self.data[:change]
self.data = self.data[change:]
data_size -= change
if data_size == 0:
return result
self.data = self.socket.recv(65536)
def send_message(self, data):
data = u32.pack(len(data)) + data
self.socket.sendall(data)
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: loomcomm.proto
import sys
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
from google.protobuf import descriptor as _descriptor
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
from google.protobuf import descriptor_pb2
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor.FileDescriptor(
name='loomcomm.proto',
package='loomcomm',
serialized_pb=_b('\n\x0eloomcomm.proto\x12\x08loomcomm\"\x9f\x01\n\x08Register\x12\x18\n\x10protocol_version\x18\x01 \x02(\x05\x12%\n\x04type\x18\x02 \x02(\x0e\x32\x17.loomcomm.Register.Type\x12\x0c\n\x04port\x18\x03 \x01(\x05\x12\x12\n\ntask_types\x18\x04 \x03(\t\"0\n\x04Type\x12\x13\n\x0fREGISTER_WORKER\x10\x01\x12\x13\n\x0fREGISTER_CLIENT\x10\x02\"&\n\rServerMessage\"\x15\n\x04Type\x12\r\n\tSTART_JOB\x10\x01\"\xb1\x01\n\rWorkerCommand\x12*\n\x04type\x18\x01 \x02(\x0e\x32\x1c.loomcomm.WorkerCommand.Type\x12\n\n\x02id\x18\x02 \x01(\x05\x12\x11\n\ttask_type\x18\x03 \x01(\x05\x12\x13\n\x0btask_config\x18\x04 \x01(\t\x12\x13\n\x0btask_inputs\x18\x05 \x03(\x05\x12\x0f\n\x07\x61\x64\x64ress\x18\n \x01(\t\"\x1a\n\x04Type\x12\x08\n\x04TASK\x10\x01\x12\x08\n\x04SEND\x10\x02\"\x1c\n\x0eWorkerResponse\x12\n\n\x02id\x18\x02 \x01(\x05\"\x18\n\x08\x41nnounce\x12\x0c\n\x04port\x18\x01 \x02(\x05\" \n\x04\x44\x61ta\x12\n\n\x02id\x18\x01 \x02(\x05\x12\x0c\n\x04size\x18\x02 \x02(\x04\x42\x02H\x03')
)
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
_REGISTER_TYPE = _descriptor.EnumDescriptor(
name='Type',
full_name='loomcomm.Register.Type',
filename=None,
file=DESCRIPTOR,
values=[
_descriptor.EnumValueDescriptor(
name='REGISTER_WORKER', index=0, number=1,
options=None,
type=None),
_descriptor.EnumValueDescriptor(
name='REGISTER_CLIENT', index=1, number=2,
options=None,
type=None),
],
containing_type=None,
options=None,
serialized_start=140,
serialized_end=188,
)
_sym_db.RegisterEnumDescriptor(_REGISTER_TYPE)
_SERVERMESSAGE_TYPE = _descriptor.EnumDescriptor(
name='Type',
full_name='loomcomm.ServerMessage.Type',
filename=None,
file=DESCRIPTOR,
values=[
_descriptor.EnumValueDescriptor(
name='START_JOB', index=0, number=1,
options=None,
type=None),
],
containing_type=None,
options=None,
serialized_start=207,
serialized_end=228,
)
_sym_db.RegisterEnumDescriptor(_SERVERMESSAGE_TYPE)
_WORKERCOMMAND_TYPE = _descriptor.EnumDescriptor(
name='Type',
full_name='loomcomm.WorkerCommand.Type',
filename=None,
file=DESCRIPTOR,
values=[
_descriptor.EnumValueDescriptor(
name='TASK', index=0, number=1,
options=None,
type=None),
_descriptor.EnumValueDescriptor(
name='SEND', index=1, number=2,
options=None,
type=None),
],
containing_type=None,
options=None,
serialized_start=382,
serialized_end=408,
)
_sym_db.RegisterEnumDescriptor(_WORKERCOMMAND_TYPE)
_REGISTER = _descriptor.Descriptor(
name='Register',
full_name='loomcomm.Register',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='protocol_version', full_name='loomcomm.Register.protocol_version', index=0,
number=1, type=5, cpp_type=1, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='type', full_name='loomcomm.Register.type', index=1,
number=2, type=14, cpp_type=8, label=2,
has_default_value=False, default_value=1,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='port', full_name='loomcomm.Register.port', index=2,
number=3, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='task_types', full_name='loomcomm.Register.task_types', index=3,
number=4, type=9, cpp_type=9, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
],
extensions=[
],
nested_types=[],
enum_types=[
_REGISTER_TYPE,
],
options=None,
is_extendable=False,
extension_ranges=[],
oneofs=[
],
serialized_start=29,
serialized_end=188,
)
_SERVERMESSAGE = _descriptor.Descriptor(
name='ServerMessage',
full_name='loomcomm.ServerMessage',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
],
extensions=[
],
nested_types=[],
enum_types=[
_SERVERMESSAGE_TYPE,
],
options=None,
is_extendable=False,
extension_ranges=[],
oneofs=[
],
serialized_start=190,
serialized_end=228,
)
_WORKERCOMMAND = _descriptor.Descriptor(
name='WorkerCommand',
full_name='loomcomm.WorkerCommand',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='type', full_name='loomcomm.WorkerCommand.type', index=0,
number=1, type=14, cpp_type=8, label=2,
has_default_value=False, default_value=1,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='id', full_name='loomcomm.WorkerCommand.id', index=1,
number=2, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='task_type', full_name='loomcomm.WorkerCommand.task_type', index=2,
number=3, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='task_config', full_name='loomcomm.WorkerCommand.task_config', index=3,
number=4, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='task_inputs', full_name='loomcomm.WorkerCommand.task_inputs', index=4,
number=5, type=5, cpp_type=1, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='address', full_name='loomcomm.WorkerCommand.address', index=5,
number=10, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
],
extensions=[
],
nested_types=[],
enum_types=[
_WORKERCOMMAND_TYPE,
],
options=None,
is_extendable=False,
extension_ranges=[],
oneofs=[
],
serialized_start=231,
serialized_end=408,
)
_WORKERRESPONSE = _descriptor.Descriptor(
name='WorkerResponse',
full_name='loomcomm.WorkerResponse',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='id', full_name='loomcomm.WorkerResponse.id', index=0,
number=2, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
extension_ranges=[],
oneofs=[
],
serialized_start=410,
serialized_end=438,
)
_ANNOUNCE = _descriptor.Descriptor(
name='Announce',
full_name='loomcomm.Announce',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='port', full_name='loomcomm.Announce.port', index=0,
number=1, type=5, cpp_type=1, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
extension_ranges=[],
oneofs=[
],
serialized_start=440,
serialized_end=464,
)
_DATA = _descriptor.Descriptor(
name='Data',
full_name='loomcomm.Data',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='id', full_name='loomcomm.Data.id', index=0,
number=1, type=5, cpp_type=1, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='size', full_name='loomcomm.Data.size', index=1,
number=2, type=4, cpp_type=4, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
extension_ranges=[],
oneofs=[
],
serialized_start=466,
serialized_end=498,
)
_REGISTER.fields_by_name['type'].enum_type = _REGISTER_TYPE
_REGISTER_TYPE.containing_type = _REGISTER
_SERVERMESSAGE_TYPE.containing_type = _SERVERMESSAGE
_WORKERCOMMAND.fields_by_name['type'].enum_type = _WORKERCOMMAND_TYPE
_WORKERCOMMAND_TYPE.containing_type = _WORKERCOMMAND
DESCRIPTOR.message_types_by_name['Register'] = _REGISTER
DESCRIPTOR.message_types_by_name['ServerMessage'] = _SERVERMESSAGE
DESCRIPTOR.message_types_by_name['WorkerCommand'] = _WORKERCOMMAND
DESCRIPTOR.message_types_by_name['WorkerResponse'] = _WORKERRESPONSE
DESCRIPTOR.message_types_by_name['Announce'] = _ANNOUNCE
DESCRIPTOR.message_types_by_name['Data'] = _DATA
Register = _reflection.GeneratedProtocolMessageType('Register', (_message.Message,), dict(
DESCRIPTOR = _REGISTER,
__module__ = 'loomcomm_pb2'
# @@protoc_insertion_point(class_scope:loomcomm.Register)
))
_sym_db.RegisterMessage(Register)
ServerMessage = _reflection.GeneratedProtocolMessageType('ServerMessage', (_message.Message,), dict(
DESCRIPTOR = _SERVERMESSAGE,
__module__ = 'loomcomm_pb2'
# @@protoc_insertion_point(class_scope:loomcomm.ServerMessage)
))
_sym_db.RegisterMessage(ServerMessage)
WorkerCommand = _reflection.GeneratedProtocolMessageType('WorkerCommand', (_message.Message,), dict(
DESCRIPTOR = _WORKERCOMMAND,
__module__ = 'loomcomm_pb2'
# @@protoc_insertion_point(class_scope:loomcomm.WorkerCommand)
))
_sym_db.RegisterMessage(WorkerCommand)
WorkerResponse = _reflection.GeneratedProtocolMessageType('WorkerResponse', (_message.Message,), dict(
DESCRIPTOR = _WORKERRESPONSE,
__module__ = 'loomcomm_pb2'
# @@protoc_insertion_point(class_scope:loomcomm.WorkerResponse)
))
_sym_db.RegisterMessage(WorkerResponse)
Announce = _reflection.GeneratedProtocolMessageType('Announce', (_message.Message,), dict(
DESCRIPTOR = _ANNOUNCE,
__module__ = 'loomcomm_pb2'
# @@protoc_insertion_point(class_scope:loomcomm.Announce)
))
_sym_db.RegisterMessage(Announce)
Data = _reflection.GeneratedProtocolMessageType('Data', (_message.Message,), dict(
DESCRIPTOR = _DATA,
__module__ = 'loomcomm_pb2'
# @@protoc_insertion_point(class_scope:loomcomm.Data)
))
_sym_db.RegisterMessage(Data)
DESCRIPTOR.has_options = True
DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('H\003'))
# @@protoc_insertion_point(module_scope)
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: loomplan.proto
import sys
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
from google.protobuf import descriptor as _descriptor
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
from google.protobuf import descriptor_pb2
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor.FileDescriptor(
name='loomplan.proto',
package='loomplan',
serialized_pb=_b('\n\x0eloomplan.proto\x12\x08loomplan\"<\n\x04Task\x12\x11\n\ttask_type\x18\x01 \x02(\x05\x12\x0e\n\x06\x63onfig\x18\x02 \x02(\x0c\x12\x11\n\tinput_ids\x18\x03 \x03(\x05\"M\n\x04Plan\x12\x12\n\ntask_types\x18\x01 \x03(\t\x12\x1d\n\x05tasks\x18\x02 \x03(\x0b\x32\x0e.loomplan.Task\x12\x12\n\nresult_ids\x18\x03 \x03(\x05\x42\x02H\x03')
)
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
_TASK = _descriptor.Descriptor(
name='Task',
full_name='loomplan.Task',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='task_type', full_name='loomplan.Task.task_type', index=0,
number=1, type=5, cpp_type=1, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='config', full_name='loomplan.Task.config', index=1,
number=2, type=12, cpp_type=9, label=2,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='input_ids', full_name='loomplan.Task.input_ids', index=2,
number=3, type=5, cpp_type=1, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,