Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
ADAS
HyperLoom
Commits
f424553a
Commit
f424553a
authored
Aug 02, 2016
by
Stanislav Bohm
Browse files
ENH: Symbols for data types
parent
c1055fc6
Changes
25
Hide whitespace changes
Inline
Side-by-side
src/client/client.py
View file @
f424553a
...
...
@@ -26,6 +26,13 @@ class Client(object):
def
__init__
(
self
,
address
,
port
,
info
=
False
):
self
.
server_address
=
address
self
.
server_port
=
port
self
.
dictionary_symbols
=
None
self
.
dictionary_map
=
None
self
.
array_id
=
None
self
.
rawdata_id
=
None
s
=
socket
.
socket
(
socket
.
AF_INET
,
socket
.
SOCK_STREAM
)
s
.
connect
((
address
,
port
))
self
.
connection
=
Connection
(
s
)
...
...
@@ -69,6 +76,13 @@ class Client(object):
self
.
add_info
(
cmsg
.
info
)
elif
cmsg
.
type
==
ClientMessage
.
ERROR
:
self
.
process_error
(
cmsg
)
elif
cmsg
.
type
==
ClientMessage
.
DICTIONARY
:
self
.
dictionary_symbols
=
cmsg
.
symbols
self
.
dictionary_map
=
{}
for
i
,
s
in
enumerate
(
self
.
dictionary_symbols
):
self
.
dictionary_map
[
s
]
=
i
self
.
array_id
=
self
.
dictionary_map
[
"loom/array"
]
self
.
rawdata_id
=
self
.
dictionary_map
[
"loom/data"
]
if
single_result
:
return
data
[
results
.
id
]
...
...
@@ -87,9 +101,9 @@ class Client(object):
msg_data
=
Data
()
msg_data
.
ParseFromString
(
self
.
connection
.
receive_message
())
type_id
=
msg_data
.
type_id
if
type_id
==
300
:
# Data
if
type_id
==
self
.
rawdata_id
:
return
self
.
connection
.
read_data
(
msg_data
.
size
)
if
type_id
==
400
:
# Array
if
type_id
==
self
.
array_id
:
return
[
self
.
_receive_data
()
for
i
in
xrange
(
msg_data
.
length
)]
assert
0
...
...
src/client/loomcomm_pb2.py
View file @
f424553a
...
...
@@ -18,7 +18,7 @@ _sym_db = _symbol_database.Default()
DESCRIPTOR
=
_descriptor
.
FileDescriptor
(
name
=
'loomcomm.proto'
,
package
=
'loomcomm'
,
serialized_pb
=
_b
(
'
\n\x0e
loomcomm.proto
\x12\x08
loomcomm
\"\x
bb
\x01\n\x08
Register
\x12\x18\n\x10
protocol_version
\x18\x01
\x02
(
\x05\x12
%
\n\x04
type
\x18\x02
\x02
(
\x0e\x32\x17
.loomcomm.Register.Type
\x12\x0c\n\x04
port
\x18\x03
\x01
(
\x05\x12\x12\n\n
task_types
\x18\x04
\x03
(
\t\x12\x0c\n\x04\x63
pus
\x18\x0
5
\x01
(
\x05\x12\x0c\n\x04
info
\x18\n
\x01
(
\x08\"
0
\n\x04
Type
\x12\x13\n\x0f
REGISTER_WORKER
\x10\x01\x12\x13\n\x0f
REGISTER_CLIENT
\x10\x02\"
&
\n\r
ServerMessage
\"\x15\n\x04
Type
\x12\r\n\t
START_JOB
\x10\x01\"\xf1\x01\n\r
WorkerCommand
\x12
*
\n\x04
type
\x18\x01
\x02
(
\x0e\x32\x1c
.loomcomm.WorkerCommand.Type
\x12\n\n\x02
id
\x18\x02
\x01
(
\x05\x12\x11\n\t
task_type
\x18\x03
\x01
(
\x05\x12\x13\n\x0b
task_config
\x18\x04
\x01
(
\t\x12\x13\n\x0b
task_inputs
\x18\x05
\x03
(
\x05\x12\x0f\n\x07\x61\x64\x64
ress
\x18\n
\x01
(
\t\x12\x11\n\t
with_size
\x18\x0b
\x01
(
\x08\x12\x0f\n\x07
symbols
\x18\x64
\x03
(
\t\"
6
\n\x04
Type
\x12\x08\n\x04
TASK
\x10\x01\x12\x08\n\x04
SEND
\x10\x02\x12\n\n\x06
REMOVE
\x10\x03\x12\x0e\n\n
DICTIONARY
\x10\x04\"\x9a\x01\n\x0e
WorkerResponse
\x12
+
\n\x04
type
\x18\x01
\x02
(
\x0e\x32\x1d
.loomcomm.WorkerResponse.Type
\x12\n\n\x02
id
\x18\x02
\x02
(
\x05\x12\x0c\n\x04
size
\x18\x03
\x01
(
\x04\x12\x0e\n\x06
length
\x18\x04
\x01
(
\x04\x12\x11\n\t
error_msg
\x18\x64
\x01
(
\t\"\x1e\n\x04
Type
\x12\n\n\x06\x46
INISH
\x10\x01\x12\n\n\x06\x46\x41
ILED
\x10\x02\"\x18\n\x08\x41
nnounce
\x12\x0c\n\x04
port
\x18\x01
\x02
(
\x05\"
-
\n\x0c\x44\x61
taPrologue
\x12\n\n\x02
id
\x18\x01
\x02
(
\x05\x12\x11\n\t
data_size
\x18\x03
\x01
(
\x04\"
Y
\n\x04\x44\x61
ta
\x12\x0f\n\x07
type_id
\x18\x01
\x02
(
\x05\x12\x0c\n\x04
size
\x18\x02
\x02
(
\x04\x12\x0e\n\x06
length
\x18\x03
\x01
(
\x04\x12\x10\n\x08\x61
rg0_u64
\x18\x08
\x01
(
\x04\x12\x10\n\x08\x61
rg1_u64
\x18\t
\x01
(
\x04\"\"\n\x04
Info
\x12\n\n\x02
id
\x18\x01
\x02
(
\x05\x12\x0e\n\x06
worker
\x18\x02
\x02
(
\t\"
6
\n\x05\x45
rror
\x12\n\n\x02
id
\x18\x01
\x02
(
\x05\x12\x0e\n\x06
worker
\x18\x02
\x02
(
\t\x12\x11\n\t
error_msg
\x18\x03
\x02
(
\t\"\x
c6
\x01\n\r
ClientMessage
\x12
*
\n\x04
type
\x18\x01
\x02
(
\x0e\x32\x1c
.loomcomm.ClientMessage.Type
\x12
$
\n\x04\x64\x61
ta
\x18\x02
\x01
(
\x0b\x32\x16
.loomcomm.DataPrologue
\x12\x1c\n\x04
info
\x18\x03
\x01
(
\x0b\x32\x0e
.loomcomm.Info
\x12\x1e\n\x05\x65
rror
\x18\x04
\x01
(
\x0b\x32\x0f
.loomcomm.Error
\
"
%
\n\x04
Type
\x12\x08\n\x04\x44\x41
TA
\x10\x01\x12\x08\n\x04
INFO
\x10\x02\x12\t\n\x05\x45
RROR
\x10\x03\x42\x02
H
\x03
'
)
serialized_pb
=
_b
(
'
\n\x0e
loomcomm.proto
\x12\x08
loomcomm
\"\x
cf
\x01\n\x08
Register
\x12\x18\n\x10
protocol_version
\x18\x01
\x02
(
\x05\x12
%
\n\x04
type
\x18\x02
\x02
(
\x0e\x32\x17
.loomcomm.Register.Type
\x12\x0c\n\x04
port
\x18\x03
\x01
(
\x05\x12\x12\n\n
task_types
\x18\x04
\x03
(
\t\x12\x12\n\n
data_types
\x18\x05
\x03
(
\t\x12\x0c\n\x04\x63
pus
\x18\x0
6
\x01
(
\x05\x12\x0c\n\x04
info
\x18\n
\x01
(
\x08\"
0
\n\x04
Type
\x12\x13\n\x0f
REGISTER_WORKER
\x10\x01\x12\x13\n\x0f
REGISTER_CLIENT
\x10\x02\"
&
\n\r
ServerMessage
\"\x15\n\x04
Type
\x12\r\n\t
START_JOB
\x10\x01\"\xf1\x01\n\r
WorkerCommand
\x12
*
\n\x04
type
\x18\x01
\x02
(
\x0e\x32\x1c
.loomcomm.WorkerCommand.Type
\x12\n\n\x02
id
\x18\x02
\x01
(
\x05\x12\x11\n\t
task_type
\x18\x03
\x01
(
\x05\x12\x13\n\x0b
task_config
\x18\x04
\x01
(
\t\x12\x13\n\x0b
task_inputs
\x18\x05
\x03
(
\x05\x12\x0f\n\x07\x61\x64\x64
ress
\x18\n
\x01
(
\t\x12\x11\n\t
with_size
\x18\x0b
\x01
(
\x08\x12\x0f\n\x07
symbols
\x18\x64
\x03
(
\t\"
6
\n\x04
Type
\x12\x08\n\x04
TASK
\x10\x01\x12\x08\n\x04
SEND
\x10\x02\x12\n\n\x06
REMOVE
\x10\x03\x12\x0e\n\n
DICTIONARY
\x10\x04\"\x9a\x01\n\x0e
WorkerResponse
\x12
+
\n\x04
type
\x18\x01
\x02
(
\x0e\x32\x1d
.loomcomm.WorkerResponse.Type
\x12\n\n\x02
id
\x18\x02
\x02
(
\x05\x12\x0c\n\x04
size
\x18\x03
\x01
(
\x04\x12\x0e\n\x06
length
\x18\x04
\x01
(
\x04\x12\x11\n\t
error_msg
\x18\x64
\x01
(
\t\"\x1e\n\x04
Type
\x12\n\n\x06\x46
INISH
\x10\x01\x12\n\n\x06\x46\x41
ILED
\x10\x02\"\x18\n\x08\x41
nnounce
\x12\x0c\n\x04
port
\x18\x01
\x02
(
\x05\"
-
\n\x0c\x44\x61
taPrologue
\x12\n\n\x02
id
\x18\x01
\x02
(
\x05\x12\x11\n\t
data_size
\x18\x03
\x01
(
\x04\"
Y
\n\x04\x44\x61
ta
\x12\x0f\n\x07
type_id
\x18\x01
\x02
(
\x05\x12\x0c\n\x04
size
\x18\x02
\x02
(
\x04\x12\x0e\n\x06
length
\x18\x03
\x01
(
\x04\x12\x10\n\x08\x61
rg0_u64
\x18\x08
\x01
(
\x04\x12\x10\n\x08\x61
rg1_u64
\x18\t
\x01
(
\x04\"\"\n\x04
Info
\x12\n\n\x02
id
\x18\x01
\x02
(
\x05\x12\x0e\n\x06
worker
\x18\x02
\x02
(
\t\"
6
\n\x05\x45
rror
\x12\n\n\x02
id
\x18\x01
\x02
(
\x05\x12\x0e\n\x06
worker
\x18\x02
\x02
(
\t\x12\x11\n\t
error_msg
\x18\x03
\x02
(
\t\"\x
e7
\x01\n\r
ClientMessage
\x12
*
\n\x04
type
\x18\x01
\x02
(
\x0e\x32\x1c
.loomcomm.ClientMessage.Type
\x12
$
\n\x04\x64\x61
ta
\x18\x02
\x01
(
\x0b\x32\x16
.loomcomm.DataPrologue
\x12\x1c\n\x04
info
\x18\x03
\x01
(
\x0b\x32\x0e
.loomcomm.Info
\x12\x1e\n\x05\x65
rror
\x18\x04
\x01
(
\x0b\x32\x0f
.loomcomm.Error
\
x12\x0f\n\x07
symbols
\x18\x05
\x03
(
\t\"
5
\n\x04
Type
\x12\x08\n\x04\x44\x41
TA
\x10\x01\x12\x08\n\x04
INFO
\x10\x02\x12\t\n\x05\x45
RROR
\x10\x03\
x12\x0e\n\n
DICTIONARY
\x10\x04\
x42\x02
H
\x03
'
)
)
_sym_db
.
RegisterFileDescriptor
(
DESCRIPTOR
)
...
...
@@ -41,8 +41,8 @@ _REGISTER_TYPE = _descriptor.EnumDescriptor(
],
containing_type
=
None
,
options
=
None
,
serialized_start
=
1
6
8
,
serialized_end
=
2
1
6
,
serialized_start
=
1
8
8
,
serialized_end
=
2
3
6
,
)
_sym_db
.
RegisterEnumDescriptor
(
_REGISTER_TYPE
)
...
...
@@ -59,8 +59,8 @@ _SERVERMESSAGE_TYPE = _descriptor.EnumDescriptor(
],
containing_type
=
None
,
options
=
None
,
serialized_start
=
2
3
5
,
serialized_end
=
2
5
6
,
serialized_start
=
2
5
5
,
serialized_end
=
2
7
6
,
)
_sym_db
.
RegisterEnumDescriptor
(
_SERVERMESSAGE_TYPE
)
...
...
@@ -89,8 +89,8 @@ _WORKERCOMMAND_TYPE = _descriptor.EnumDescriptor(
],
containing_type
=
None
,
options
=
None
,
serialized_start
=
4
4
6
,
serialized_end
=
5
0
0
,
serialized_start
=
4
6
6
,
serialized_end
=
5
2
0
,
)
_sym_db
.
RegisterEnumDescriptor
(
_WORKERCOMMAND_TYPE
)
...
...
@@ -111,8 +111,8 @@ _WORKERRESPONSE_TYPE = _descriptor.EnumDescriptor(
],
containing_type
=
None
,
options
=
None
,
serialized_start
=
6
2
7
,
serialized_end
=
6
5
7
,
serialized_start
=
6
4
7
,
serialized_end
=
6
7
7
,
)
_sym_db
.
RegisterEnumDescriptor
(
_WORKERRESPONSE_TYPE
)
...
...
@@ -134,11 +134,15 @@ _CLIENTMESSAGE_TYPE = _descriptor.EnumDescriptor(
name
=
'ERROR'
,
index
=
2
,
number
=
3
,
options
=
None
,
type
=
None
),
_descriptor
.
EnumValueDescriptor
(
name
=
'DICTIONARY'
,
index
=
3
,
number
=
4
,
options
=
None
,
type
=
None
),
],
containing_type
=
None
,
options
=
None
,
serialized_start
=
1
077
,
serialized_end
=
11
14
,
serialized_start
=
1
114
,
serialized_end
=
11
67
,
)
_sym_db
.
RegisterEnumDescriptor
(
_CLIENTMESSAGE_TYPE
)
...
...
@@ -179,14 +183,21 @@ _REGISTER = _descriptor.Descriptor(
is_extension
=
False
,
extension_scope
=
None
,
options
=
None
),
_descriptor
.
FieldDescriptor
(
name
=
'cpus'
,
full_name
=
'loomcomm.Register.cpus'
,
index
=
4
,
number
=
5
,
type
=
5
,
cpp_type
=
1
,
label
=
1
,
name
=
'data_types'
,
full_name
=
'loomcomm.Register.data_types'
,
index
=
4
,
number
=
5
,
type
=
9
,
cpp_type
=
9
,
label
=
3
,
has_default_value
=
False
,
default_value
=
[],
message_type
=
None
,
enum_type
=
None
,
containing_type
=
None
,
is_extension
=
False
,
extension_scope
=
None
,
options
=
None
),
_descriptor
.
FieldDescriptor
(
name
=
'cpus'
,
full_name
=
'loomcomm.Register.cpus'
,
index
=
5
,
number
=
6
,
type
=
5
,
cpp_type
=
1
,
label
=
1
,
has_default_value
=
False
,
default_value
=
0
,
message_type
=
None
,
enum_type
=
None
,
containing_type
=
None
,
is_extension
=
False
,
extension_scope
=
None
,
options
=
None
),
_descriptor
.
FieldDescriptor
(
name
=
'info'
,
full_name
=
'loomcomm.Register.info'
,
index
=
5
,
name
=
'info'
,
full_name
=
'loomcomm.Register.info'
,
index
=
6
,
number
=
10
,
type
=
8
,
cpp_type
=
7
,
label
=
1
,
has_default_value
=
False
,
default_value
=
False
,
message_type
=
None
,
enum_type
=
None
,
containing_type
=
None
,
...
...
@@ -205,7 +216,7 @@ _REGISTER = _descriptor.Descriptor(
oneofs
=
[
],
serialized_start
=
29
,
serialized_end
=
2
1
6
,
serialized_end
=
2
3
6
,
)
...
...
@@ -228,8 +239,8 @@ _SERVERMESSAGE = _descriptor.Descriptor(
extension_ranges
=
[],
oneofs
=
[
],
serialized_start
=
2
1
8
,
serialized_end
=
2
5
6
,
serialized_start
=
2
3
8
,
serialized_end
=
2
7
6
,
)
...
...
@@ -308,8 +319,8 @@ _WORKERCOMMAND = _descriptor.Descriptor(
extension_ranges
=
[],
oneofs
=
[
],
serialized_start
=
2
5
9
,
serialized_end
=
5
0
0
,
serialized_start
=
2
7
9
,
serialized_end
=
5
2
0
,
)
...
...
@@ -367,8 +378,8 @@ _WORKERRESPONSE = _descriptor.Descriptor(
extension_ranges
=
[],
oneofs
=
[
],
serialized_start
=
5
0
3
,
serialized_end
=
6
5
7
,
serialized_start
=
5
2
3
,
serialized_end
=
6
7
7
,
)
...
...
@@ -397,8 +408,8 @@ _ANNOUNCE = _descriptor.Descriptor(
extension_ranges
=
[],
oneofs
=
[
],
serialized_start
=
6
5
9
,
serialized_end
=
68
3
,
serialized_start
=
6
7
9
,
serialized_end
=
70
3
,
)
...
...
@@ -434,8 +445,8 @@ _DATAPROLOGUE = _descriptor.Descriptor(
extension_ranges
=
[],
oneofs
=
[
],
serialized_start
=
68
5
,
serialized_end
=
7
3
0
,
serialized_start
=
70
5
,
serialized_end
=
7
5
0
,
)
...
...
@@ -492,8 +503,8 @@ _DATA = _descriptor.Descriptor(
extension_ranges
=
[],
oneofs
=
[
],
serialized_start
=
7
3
2
,
serialized_end
=
8
2
1
,
serialized_start
=
7
5
2
,
serialized_end
=
8
4
1
,
)
...
...
@@ -529,8 +540,8 @@ _INFO = _descriptor.Descriptor(
extension_ranges
=
[],
oneofs
=
[
],
serialized_start
=
8
2
3
,
serialized_end
=
8
5
7
,
serialized_start
=
8
4
3
,
serialized_end
=
8
7
7
,
)
...
...
@@ -573,8 +584,8 @@ _ERROR = _descriptor.Descriptor(
extension_ranges
=
[],
oneofs
=
[
],
serialized_start
=
8
5
9
,
serialized_end
=
9
1
3
,
serialized_start
=
8
7
9
,
serialized_end
=
9
3
3
,
)
...
...
@@ -613,6 +624,13 @@ _CLIENTMESSAGE = _descriptor.Descriptor(
message_type
=
None
,
enum_type
=
None
,
containing_type
=
None
,
is_extension
=
False
,
extension_scope
=
None
,
options
=
None
),
_descriptor
.
FieldDescriptor
(
name
=
'symbols'
,
full_name
=
'loomcomm.ClientMessage.symbols'
,
index
=
4
,
number
=
5
,
type
=
9
,
cpp_type
=
9
,
label
=
3
,
has_default_value
=
False
,
default_value
=
[],
message_type
=
None
,
enum_type
=
None
,
containing_type
=
None
,
is_extension
=
False
,
extension_scope
=
None
,
options
=
None
),
],
extensions
=
[
],
...
...
@@ -625,8 +643,8 @@ _CLIENTMESSAGE = _descriptor.Descriptor(
extension_ranges
=
[],
oneofs
=
[
],
serialized_start
=
9
1
6
,
serialized_end
=
11
14
,
serialized_start
=
9
3
6
,
serialized_end
=
11
67
,
)
_REGISTER
.
fields_by_name
[
'type'
].
enum_type
=
_REGISTER_TYPE
...
...
src/libloom/data.cpp
View file @
f424553a
...
...
@@ -26,7 +26,7 @@ std::shared_ptr<Data> Data::get_slice(size_t from, size_t to)
void
Data
::
serialize
(
Worker
&
worker
,
SendBuffer
&
buffer
,
std
::
shared_ptr
<
Data
>
&
data_ptr
)
{
loomcomm
::
Data
msg
;
msg
.
set_type_id
(
get_type_id
(
));
msg
.
set_type_id
(
worker
.
get_dictionary
().
lookup_symbol
(
get_type_name
()
));
msg
.
set_size
(
get_size
());
auto
length
=
get_length
();
if
(
length
)
{
...
...
src/libloom/data.h
View file @
f424553a
...
...
@@ -20,7 +20,7 @@ class Data
public:
virtual
~
Data
();
virtual
in
t
get_type_
id
()
=
0
;
virtual
std
::
str
in
g
get_type_
name
()
const
=
0
;
virtual
size_t
get_size
()
=
0
;
virtual
std
::
string
get_info
()
=
0
;
virtual
size_t
get_length
();
...
...
src/libloom/data/array.cpp
View file @
f424553a
...
...
@@ -76,6 +76,11 @@ void Array::serialize_data(Worker &worker, SendBuffer &buffer, std::shared_ptr<D
}
}
std
::
string
Array
::
get_type_name
()
const
{
return
ArrayUnpacker
::
get_type_name
();
}
ArrayUnpacker
::~
ArrayUnpacker
()
{
...
...
src/libloom/data/array.h
View file @
f424553a
...
...
@@ -6,16 +6,10 @@
namespace
loom
{
class
Array
:
public
Data
{
public:
static
const
int
TYPE_ID
=
400
;
public:
Array
(
size_t
length
,
std
::
unique_ptr
<
std
::
shared_ptr
<
Data
>
[]
>
items
);
~
Array
();
int
get_type_id
()
{
return
TYPE_ID
;
}
size_t
get_length
()
{
return
length
;
}
...
...
@@ -27,7 +21,8 @@ public:
std
::
shared_ptr
<
Data
>&
get_ref_at_index
(
size_t
index
);
void
serialize_data
(
Worker
&
worker
,
SendBuffer
&
buffer
,
std
::
shared_ptr
<
Data
>
&
data_ptr
);
void
serialize_data
(
Worker
&
worker
,
SendBuffer
&
buffer
,
std
::
shared_ptr
<
Data
>
&
data_ptr
);
std
::
string
get_type_name
()
const
;
private:
size_t
length
;
...
...
@@ -45,6 +40,10 @@ public:
void
on_data_chunk
(
const
char
*
data
,
size_t
size
);
bool
on_data_finish
(
Connection
&
connection
);
static
const
char
*
get_type_name
()
{
return
"loom/array"
;
}
protected:
void
finish
();
...
...
src/libloom/data/externfile.cpp
View file @
f424553a
...
...
@@ -11,6 +11,11 @@
using
namespace
loom
;
std
::
string
ExternFile
::
get_type_name
()
const
{
return
"loom/file"
;
}
ExternFile
::
ExternFile
(
const
std
::
string
&
filename
)
:
data
(
nullptr
),
filename
(
filename
)
{
...
...
src/libloom/data/externfile.h
View file @
f424553a
...
...
@@ -9,15 +9,10 @@ namespace loom {
class
ExternFile
:
public
Data
{
public:
static
const
int
TYPE_ID
=
301
;
std
::
string
get_type_name
()
const
;
ExternFile
(
const
std
::
string
&
filename
);
~
ExternFile
();
int
get_type_id
()
{
return
TYPE_ID
;
}
size_t
get_size
()
{
return
size
;
}
...
...
@@ -35,6 +30,7 @@ public:
std
::
string
get_filename
()
const
;
private:
void
open
();
...
...
src/libloom/data/index.cpp
View file @
f424553a
...
...
@@ -19,6 +19,11 @@ Index::~Index()
llog
->
debug
(
"Disposing index"
);
}
std
::
string
Index
::
get_type_name
()
const
{
return
IndexUnpacker
::
get_type_name
();
}
size_t
Index
::
get_length
()
{
return
length
;
...
...
src/libloom/data/index.h
View file @
f424553a
...
...
@@ -13,8 +13,6 @@ class Worker;
class
Index
:
public
Data
{
public:
static
const
int
TYPE_ID
=
500
;
Index
(
Worker
&
worker
,
std
::
shared_ptr
<
Data
>
&
data
,
size_t
length
,
...
...
@@ -22,10 +20,7 @@ public:
~
Index
();
int
get_type_id
()
{
return
TYPE_ID
;
}
std
::
string
get_type_name
()
const
;
size_t
get_length
();
size_t
get_size
();
std
::
string
get_info
();
...
...
@@ -34,6 +29,7 @@ public:
void
serialize_data
(
Worker
&
worker
,
SendBuffer
&
buffer
,
std
::
shared_ptr
<
Data
>
&
data_ptr
);
private:
Worker
&
worker
;
std
::
shared_ptr
<
Data
>
data
;
...
...
@@ -52,6 +48,10 @@ public:
void
on_data_chunk
(
const
char
*
data
,
size_t
size
);
bool
on_data_finish
(
Connection
&
connection
);
static
const
char
*
get_type_name
()
{
return
"loom/index"
;
}
protected:
void
finish_data
();
...
...
src/libloom/data/rawdata.cpp
View file @
f424553a
...
...
@@ -37,7 +37,10 @@ RawData::~RawData()
}
}
std
::
string
RawData
::
get_type_name
()
const
{
return
RawDataUnpacker
::
get_type_name
();
}
/*char* RawData::init_memonly(size_t size)
{
...
...
@@ -103,7 +106,7 @@ std::string RawData::get_filename() const
void
RawData
::
open
(
Worker
&
worker
)
{
if
(
size
==
0
)
{
return
;
return
;
}
assert
(
!
filename
.
empty
());
int
fd
=
::
open
(
filename
.
c_str
(),
O_RDONLY
,
S_IRUSR
|
S_IWUSR
);
...
...
@@ -135,7 +138,7 @@ void RawData::map(int fd, bool write)
std
::
string
RawData
::
get_info
()
{
return
"RawData"
;
return
"RawData"
;
}
void
RawData
::
serialize_data
(
Worker
&
worker
,
SendBuffer
&
buffer
,
std
::
shared_ptr
<
Data
>
&
data_ptr
)
...
...
src/libloom/data/rawdata.h
View file @
f424553a
...
...
@@ -6,15 +6,12 @@
namespace
loom
{
class
RawData
:
public
Data
{
public:
static
const
int
TYPE_ID
=
300
;
public:
RawData
();
~
RawData
();
int
get_type_id
()
{
return
TYPE_ID
;
}
std
::
string
get_type_name
()
const
;
size_t
get_size
()
{
return
size
;
...
...
@@ -63,6 +60,11 @@ public:
bool
init
(
Worker
&
worker
,
Connection
&
connection
,
const
loomcomm
::
Data
&
msg
);
void
on_data_chunk
(
const
char
*
data
,
size_t
size
);
bool
on_data_finish
(
Connection
&
connection
);
static
const
char
*
get_type_name
()
{
return
"loom/data"
;
}
protected:
char
*
pointer
=
nullptr
;
};
...
...
src/libloom/loomcomm.pb.cc
View file @
f424553a
...
...
@@ -102,6 +102,7 @@ const int Register::kProtocolVersionFieldNumber;
const
int
Register
::
kTypeFieldNumber
;
const
int
Register
::
kPortFieldNumber
;
const
int
Register
::
kTaskTypesFieldNumber
;
const
int
Register
::
kDataTypesFieldNumber
;
const
int
Register
::
kCpusFieldNumber
;
const
int
Register
::
kInfoFieldNumber
;
#endif // !_MSC_VER
...
...
@@ -178,16 +179,18 @@ void Register::Clear() {
::memset(&first, 0, n); \
} while (0)
if
(
_has_bits_
[
0
/
32
]
&
55
)
{
ZR_
(
port_
,
info
_
);
if
(
_has_bits_
[
0
/
32
]
&
103
)
{
ZR_
(
port_
,
cpus
_
);
protocol_version_
=
0
;
type_
=
1
;
info_
=
false
;
}
#undef OFFSET_OF_FIELD_
#undef ZR_
task_types_
.
Clear
();
data_types_
.
Clear
();
::
memset
(
_has_bits_
,
0
,
sizeof
(
_has_bits_
));
mutable_unknown_fields
()
->
clear
();
}
...
...
@@ -266,13 +269,27 @@ bool Register::MergePartialFromCodedStream(
goto
handle_unusual
;
}
if
(
input
->
ExpectTag
(
34
))
goto
parse_task_types
;
if
(
input
->
ExpectTag
(
4
0
))
goto
parse_
cpu
s
;
if
(
input
->
ExpectTag
(
4
2
))
goto
parse_
data_type
s
;
break
;
}
//
optional int32 cpu
s = 5;
//
repeated string data_type
s = 5;
case
5
:
{
if
(
tag
==
40
)
{
if
(
tag
==
42
)
{
parse_data_types:
DO_
(
::
google
::
protobuf
::
internal
::
WireFormatLite
::
ReadString
(
input
,
this
->
add_data_types
()));
}
else
{
goto
handle_unusual
;
}
if
(
input
->
ExpectTag
(
42
))
goto
parse_data_types
;
if
(
input
->
ExpectTag
(
48
))
goto
parse_cpus
;
break
;
}
// optional int32 cpus = 6;
case
6
:
{
if
(
tag
==
48
)
{
parse_cpus:
DO_
((
::
google
::
protobuf
::
internal
::
WireFormatLite
::
ReadPrimitive
<
::
google
::
protobuf
::
int32
,
::
google
::
protobuf
::
internal
::
WireFormatLite
::
TYPE_INT32
>
(
...
...
@@ -347,9 +364,15 @@ void Register::SerializeWithCachedSizes(
4
,
this
->
task_types
(
i
),
output
);
}
// optional int32 cpus = 5;
// repeated string data_types = 5;
for
(
int
i
=
0
;
i
<
this
->
data_types_size
();
i
++
)
{
::
google
::
protobuf
::
internal
::
WireFormatLite
::
WriteString
(
5
,
this
->
data_types
(
i
),
output
);
}
// optional int32 cpus = 6;
if
(
has_cpus
())
{
::
google
::
protobuf
::
internal
::
WireFormatLite
::
WriteInt32
(
5
,
this
->
cpus
(),
output
);
::
google
::
protobuf
::
internal
::
WireFormatLite
::
WriteInt32
(
6
,
this
->
cpus
(),
output
);
}
// optional bool info = 10;
...
...
@@ -386,7 +409,7 @@ int Register::ByteSize() const {
this
->
port
());
}
// optional int32 cpus =
5
;
// optional int32 cpus =
6
;
if
(
has_cpus
())
{
total_size
+=
1
+
::
google
::
protobuf
::
internal
::
WireFormatLite
::
Int32Size
(
...
...
@@ -406,6 +429,13 @@ int Register::ByteSize() const {
this
->
task_types
(
i
));
}
// repeated string data_types = 5;
total_size
+=
1
*
this
->
data_types_size
();
for
(
int
i
=
0
;
i
<
this
->
data_types_size
();
i
++
)
{
total_size
+=
::
google
::
protobuf
::
internal
::
WireFormatLite
::
StringSize
(
this
->
data_types
(
i
));
}
total_size
+=
unknown_fields
().
size
();
GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN
();
...
...
@@ -422,6 +452,7 @@ void Register::CheckTypeAndMergeFrom(
void
Register
::
MergeFrom
(
const
Register
&
from
)
{
GOOGLE_CHECK_NE
(
&
from
,
this
);
task_types_
.
MergeFrom
(
from
.
task_types_
);
data_types_
.
MergeFrom
(
from
.
data_types_
);
if
(
from
.
_has_bits_
[
0
/
32
]
&
(
0xffu
<<
(
0
%
32
)))
{
if
(
from
.
has_protocol_version
())
{
set_protocol_version
(
from
.
protocol_version
());
...
...
@@ -460,6 +491,7 @@ void Register::Swap(Register* other) {
std
::
swap
(
type_
,
other
->
type_
);
std
::
swap
(
port_
,
other
->
port_
);
task_types_
.
Swap
(
&
other
->
task_types_
);
data_types_
.
Swap
(
&
other
->
data_types_
);
std
::
swap
(
cpus_
,
other
->
cpus_
);
std
::
swap
(
info_
,
other
->
info_
);
std
::
swap
(
_has_bits_
[
0
],
other
->
_has_bits_
[
0
]);
...
...
@@ -2781,6 +2813,7 @@ bool ClientMessage_Type_IsValid(int value) {
case
1
:
case
2
:
case
3
:
case
4
:
return
true
;
default:
return
false
;
...
...
@@ -2791,6 +2824,7 @@ bool ClientMessage_Type_IsValid(int value) {
const
ClientMessage_Type
ClientMessage
::
DATA
;
const
ClientMessage_Type
ClientMessage
::
INFO
;
const
ClientMessage_Type
ClientMessage
::
ERROR
;
const
ClientMessage_Type
ClientMessage
::
DICTIONARY
;
const
ClientMessage_Type
ClientMessage
::
Type_MIN
;
const
ClientMessage_Type
ClientMessage
::
Type_MAX
;
const
int
ClientMessage
::
Type_ARRAYSIZE
;
...
...
@@ -2800,6 +2834,7 @@ const int ClientMessage::kTypeFieldNumber;
const
int
ClientMessage
::
kDataFieldNumber
;
const
int
ClientMessage
::
kInfoFieldNumber
;
const
int
ClientMessage
::
kErrorFieldNumber
;
const
int
ClientMessage
::
kSymbolsFieldNumber
;
#endif // !_MSC_VER
ClientMessage
::
ClientMessage
()
...
...
@@ -2837,6 +2872,7 @@ ClientMessage::ClientMessage(const ClientMessage& from)
}
void
ClientMessage
::
SharedCtor
()
{
::
google
::
protobuf
::
internal
::
GetEmptyString
();