Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
H
HyperLoom
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Service Desk
Milestones
Merge Requests
0
Merge Requests
0
Operations
Operations
Incidents
Packages & Registries
Packages & Registries
Container Registry
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
ADAS
HyperLoom
Commits
7e2afba4
Commit
7e2afba4
authored
Jun 28, 2016
by
Stanislav Bohm
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
ENH: Introduced SendBuffer class
parent
a3b5327b
Changes
12
Hide whitespace changes
Inline
Side-by-side
Showing
12 changed files
with
168 additions
and
194 deletions
+168
-194
src/libloom/CMakeLists.txt
src/libloom/CMakeLists.txt
+2
-2
src/libloom/connection.cpp
src/libloom/connection.cpp
+13
-21
src/libloom/connection.h
src/libloom/connection.h
+4
-2
src/libloom/fileservice.cpp
src/libloom/fileservice.cpp
+0
-43
src/libloom/fileservice.h
src/libloom/fileservice.h
+0
-48
src/libloom/interconnect.cpp
src/libloom/interconnect.cpp
+11
-33
src/libloom/interconnect.h
src/libloom/interconnect.h
+5
-11
src/libloom/sendbuffer.cpp
src/libloom/sendbuffer.cpp
+31
-0
src/libloom/sendbuffer.h
src/libloom/sendbuffer.h
+60
-0
src/server/clientconn.h
src/server/clientconn.h
+9
-1
src/server/resendtask.cpp
src/server/resendtask.cpp
+21
-27
src/server/resendtask.h
src/server/resendtask.h
+12
-6
No files found.
src/libloom/CMakeLists.txt
View file @
7e2afba4
...
...
@@ -15,8 +15,8 @@ add_library(libloom
interconnect.cpp
task.cpp
task.h
fileservice.cpp
fileservice.h
sendbuffer.h
sendbuffer.cpp
loomcomm.pb.h
loomcomm.pb.cc
loomplan.pb.h
...
...
src/libloom/connection.cpp
View file @
7e2afba4
#include "connection.h"
#include "utils.h"
#include
"log.h" // DEBUG
#include
<string.h>
using
namespace
loom
;
struct
SendBuffer
{
uv_write_t
write_req
;
std
::
unique_ptr
<
char
[]
>
data
;
};
Connection
::
Connection
(
ConnectionCallback
*
callback
,
uv_loop_t
*
loop
)
:
callback
(
callback
),
state
(
ConnectionNew
),
...
...
@@ -121,28 +116,25 @@ void Connection::connect(std::string host, int port)
UV_CHECK
(
uv_tcp_connect
(
connect
,
&
socket
,
(
const
struct
sockaddr
*
)
&
dest
,
_on_connection
));
}
void
Connection
::
_on_
fbb_
write
(
uv_write_t
*
write_req
,
int
status
)
void
Connection
::
_on_write
(
uv_write_t
*
write_req
,
int
status
)
{
UV_CHECK
(
status
);
SendBuffer
*
buffer
=
static_cast
<
SendBuffer
*>
(
write_req
->
data
);
delete
buffer
;
buffer
->
on_finish
(
status
)
;
}
void
Connection
::
send_message
(
google
::
protobuf
::
MessageLite
&
message
)
{
SendBuffer
*
buffer
=
new
SendBuffer
;
buffer
->
write_req
.
data
=
buffer
;
uint32_t
size
=
message
.
ByteSize
();
buffer
->
data
=
std
::
make_unique
<
char
[]
>
(
size
+
sizeof
(
size
));
uv_buf_t
buf
;
buf
.
base
=
buffer
->
data
.
get
();
buf
.
len
=
size
+
sizeof
(
size
);
uint32_t
*
size_ptr
=
reinterpret_cast
<
uint32_t
*>
(
buf
.
base
);
*
size_ptr
=
size
;
message
.
SerializeToArray
(
buf
.
base
+
sizeof
(
size
),
size
);
UV_CHECK
(
uv_write
(
&
buffer
->
write_req
,
(
uv_stream_t
*
)
&
socket
,
&
buf
,
1
,
_on_fbb_write
));
SendBuffer
*
buffer
=
new
SendBuffer
();
buffer
->
add
(
message
);
send_buffer
(
buffer
);
}
void
Connection
::
send_buffer
(
SendBuffer
*
buffer
)
{
uv_buf_t
*
bufs
=
buffer
->
get_uv_bufs
();
size_t
count
=
buffer
->
get_uv_bufs_count
();
UV_CHECK
(
uv_write
(
&
buffer
->
request
,
(
uv_stream_t
*
)
&
socket
,
bufs
,
count
,
_on_write
));
}
void
Connection
::
_on_read
(
uv_stream_t
*
stream
,
ssize_t
nread
,
const
uv_buf_t
*
buf
)
...
...
src/libloom/connection.h
View file @
7e2afba4
#ifndef LOOM_CONNECTION_H
#define LOOM_CONNECTION_H
#include "loomcomm.pb.h"
#include "utils.h"
#include "sendbuffer.h"
#include <uv.h>
#include <string>
#include <functional>
#include <memory>
#include <assert.h>
namespace
loom
{
...
...
@@ -63,6 +64,7 @@ public:
void
connect
(
std
::
string
host
,
int
port
);
void
send_message
(
::
google
::
protobuf
::
MessageLite
&
message
);
void
send_buffer
(
SendBuffer
*
buffer
);
void
close
();
void
close_and_discard_remaining_data
();
...
...
@@ -83,7 +85,7 @@ protected:
private:
static
void
_on_connection
(
uv_connect_t
*
connect
,
int
status
);
static
void
_on_
fbb_
write
(
uv_write_t
*
write_req
,
int
status
);
static
void
_on_write
(
uv_write_t
*
write_req
,
int
status
);
static
void
_on_read
(
uv_stream_t
*
stream
,
ssize_t
nread
,
const
uv_buf_t
*
buf
);
static
void
_on_close
(
uv_handle_t
*
handle
);
static
void
_buf_alloc
(
uv_handle_t
*
handle
,
size_t
suggested_size
,
uv_buf_t
*
buf
);
...
...
src/libloom/fileservice.cpp
deleted
100644 → 0
View file @
a3b5327b
//#include "fileservice.h"
//#include "worker.h"
//#include "utils.h"
//#include "log.h"
//#include <sys/stat.h>
//using namespace loom;
//FileService::FileService(FileServiceCallback &callback,
// const std::string &filename)
// : callback(callback), filename(filename)
//{
// request1.data = this;
// request2.data = this;
//}
//void FileService::start()
//{
// llog->debug("Opening file service: {}", filename);
// uv_loop_t *loop = callback.get_worker().get_loop();
// UV_CHECK(uv_fs_open(
// loop, &request1, filename.c_str(), O_WRONLY, 0, _on_open));
//}
//void FileService::_on_open(uv_fs_t *request)
//{
// FileService *service = static_cast<FileService*>(request->data);
// uv_buf_t buf = service->input_data->get_uv_buf(worker);
// uv_loop_t *loop = service->callback.get_worker().get_loop();
// UV_CHECK(uv_fs_write(loop,
// &service->request1,
// request->file,
// &buf, 1, 0, _on_write));
//}
//void FileService::_on_write(uv_fs_t *request)
//{
// FileService *service = static_cast<FileService*>(request->data);
// service->input_data.reset();
// service->callback.on_finish();
//}
src/libloom/fileservice.h
deleted
100644 → 0
View file @
a3b5327b
//#ifndef LIBLOOM_FILESERVICE_H
//#define LIBLOOM_FILESERVICE_H
//#include <uv.h>
//#include <string>
//#include <memory>
//namespace loom {
//class Worker;
//class Data;
//class FileServiceCallback
//{
//public:
// virtual ~FileServiceCallback() {}
// virtual void on_finish() = 0;
// virtual Worker& get_worker() = 0;
//};
//class FileService
//{
//public:
// FileService(FileServiceCallback &callback, const std::string &filename);
// void set_input_data(std::shared_ptr<Data> &data) {
// input_data = data;
// }
// void start();
//protected:
// FileServiceCallback &callback;
// std::string filename;
// std::shared_ptr<Data> input_data;
// uv_fs_t request1;
// uv_fs_t request2;
// static void _on_open(uv_fs_t *request);
// static void _on_write(uv_fs_t *request);
//};
//}
//#endif
src/libloom/interconnect.cpp
View file @
7e2afba4
...
...
@@ -17,7 +17,7 @@ InterConnection::InterConnection(Worker &worker)
InterConnection
::~
InterConnection
()
{
assert
(
send_records
.
size
()
==
0
);
}
void
InterConnection
::
on_connection
()
...
...
@@ -28,9 +28,10 @@ void InterConnection::on_connection()
msg
.
set_port
(
worker
.
get_listen_port
());
send_message
(
msg
);
for
(
auto
&
record
:
send_recor
ds
)
{
_send
(
*
record
);
for
(
auto
&
buffer
:
early_sen
ds
)
{
connection
.
send_buffer
(
buffer
.
release
()
);
}
early_sends
.
clear
();
}
void
InterConnection
::
on_close
()
...
...
@@ -78,52 +79,29 @@ void InterConnection::on_data_finish()
void
InterConnection
::
send
(
std
::
shared_ptr
<
Data
>
&
data
)
{
loomcomm
::
Data
msg
;
msg
.
set_id
(
data
->
get_id
());
msg
.
set_size
(
data
->
get_size
());
size_t
msg_size
=
msg
.
ByteSize
();
//std::cout << "MSG =" << msg_size << std::endl;
auto
msg_data
=
std
::
make_unique
<
char
[]
>
(
msg_size
+
sizeof
(
uint32_t
));
uint32_t
*
size_ptr
=
reinterpret_cast
<
uint32_t
*>
(
msg_data
.
get
());
*
size_ptr
=
msg_size
;
msg
.
SerializeToArray
(
msg_data
.
get
()
+
sizeof
(
uint32_t
),
msg_size
);
auto
record
=
std
::
make_unique
<
SendRecord
>
();
record
->
request
.
data
=
this
;
record
->
raw_message
=
std
::
move
(
msg_data
);
record
->
raw_message_size
=
msg_size
+
sizeof
(
uint32_t
);
record
->
data
=
data
;
send_records
.
push_back
(
std
::
move
(
record
));
SendBuffer
*
buffer
=
new
SendBuffer
();
buffer
->
add
(
msg
);
buffer
->
add
(
data
,
data
->
get_data
(
worker
),
data
->
get_size
());
Connection
::
State
state
=
connection
.
get_state
();
assert
(
state
==
Connection
::
ConnectionOpen
||
state
==
Connection
::
ConnectionConnecting
);
if
(
state
==
Connection
::
ConnectionOpen
)
{
_send
(
*
send_records
[
send_records
.
size
()
-
1
].
get
());
connection
.
send_buffer
(
buffer
);
}
else
{
early_sends
.
push_back
(
std
::
unique_ptr
<
SendBuffer
>
(
buffer
));
}
}
void
InterConnection
::
_send
(
InterConnection
::
SendRecord
&
record
)
void
InterConnection
::
send
(
std
::
unique_ptr
<
SendBuffer
>
buffer
)
{
llog
->
debug
(
"Sending data id={} size={}"
,
record
.
data
->
get_id
(),
record
.
data
->
get_size
());
uv_buf_t
bufs
[
2
];
bufs
[
0
].
base
=
record
.
raw_message
.
get
();
bufs
[
0
].
len
=
record
.
raw_message_size
;
bufs
[
1
]
=
record
.
data
->
get_uv_buf
(
worker
);
connection
.
send
(
&
record
.
request
,
bufs
,
2
,
_on_write
);
}
void
InterConnection
::
_on_write
(
uv_write_t
*
write_req
,
int
status
)
{
UV_CHECK
(
status
);
InterConnection
*
connection
=
static_cast
<
InterConnection
*>
(
write_req
->
data
);
assert
(
connection
->
send_records
.
size
());
connection
->
send_records
.
erase
(
connection
->
send_records
.
begin
());
}
std
::
string
InterConnection
::
make_address
(
const
std
::
string
&
host
,
int
port
)
...
...
src/libloom/interconnect.h
View file @
7e2afba4
...
...
@@ -11,6 +11,7 @@ namespace loom {
class
Worker
;
class
DataBuilder
;
class
SendBuffer
;
class
InterConnection
:
public
SimpleConnectionCallback
{
...
...
@@ -19,6 +20,7 @@ public:
~
InterConnection
();
void
send
(
std
::
shared_ptr
<
Data
>
&
data
);
void
send
(
std
::
unique_ptr
<
SendBuffer
>
buffer
);
void
accept
(
uv_tcp_t
*
listen_socket
)
{
connection
.
accept
(
listen_socket
);
}
...
...
@@ -36,14 +38,7 @@ public:
protected:
struct
SendRecord
{
uv_write_t
request
;
std
::
unique_ptr
<
char
[]
>
raw_message
;
size_t
raw_message_size
;
std
::
shared_ptr
<
Data
>
data
;
};
void
_send
(
SendRecord
&
record
);
void
_send
(
SendBuffer
&
buffer
);
void
on_message
(
const
char
*
buffer
,
size_t
size
);
void
on_data_chunk
(
const
char
*
buffer
,
size_t
size
);
...
...
@@ -54,12 +49,11 @@ protected:
Worker
&
worker
;
std
::
string
address
;
std
::
vector
<
std
::
unique_ptr
<
SendRecord
>>
send_records
;
std
::
unique_ptr
<
DataBuilder
>
data_builder
;
static
void
_on_write
(
uv_write_t
*
write_req
,
int
status
);
static
std
::
string
make_address
(
const
std
::
string
&
host
,
int
port
);
std
::
vector
<
std
::
unique_ptr
<
SendBuffer
>>
early_sends
;
};
}
...
...
src/libloom/sendbuffer.cpp
0 → 100644
View file @
7e2afba4
#include "sendbuffer.h"
#include "utils.h"
#include "loomcomm.pb.h"
using
namespace
loom
;
SendBuffer
::~
SendBuffer
()
{
}
void
SendBuffer
::
on_finish
(
int
status
)
{
UV_CHECK
(
status
);
delete
this
;
}
void
SendBuffer
::
add
(
google
::
protobuf
::
MessageLite
&
message
)
{
uint32_t
size
=
message
.
ByteSize
();
auto
data
=
std
::
make_unique
<
char
[]
>
(
size
+
sizeof
(
size
));
uint32_t
*
size_ptr
=
reinterpret_cast
<
uint32_t
*>
(
data
.
get
());
*
size_ptr
=
size
;
message
.
SerializeToArray
(
data
.
get
()
+
sizeof
(
size
),
size
);
add
(
std
::
move
(
data
),
size
+
sizeof
(
size
));
}
src/libloom/sendbuffer.h
0 → 100644
View file @
7e2afba4
#ifndef LIBLOOM_SENDBUFFER_H
#define LIBLOOM_SENDBUFFER_H
#include <uv.h>
#include <memory>
#include <vector>
namespace
google
{
namespace
protobuf
{
class
MessageLite
;
}
}
namespace
loom
{
class
Data
;
class
SendBuffer
{
public:
SendBuffer
()
{
request
.
data
=
this
;
}
virtual
~
SendBuffer
();
virtual
void
on_finish
(
int
status
);
uv_buf_t
*
get_uv_bufs
()
{
return
&
bufs
[
0
];
}
size_t
get_uv_bufs_count
()
{
return
bufs
.
size
();
}
void
add
(
std
::
unique_ptr
<
char
[]
>
data
,
size_t
size
)
{
bufs
.
emplace_back
(
uv_buf_t
{
data
.
get
(),
size
});
raw_memory
.
push_back
(
std
::
move
(
data
));
}
void
add
(
std
::
shared_ptr
<
Data
>
&
data
,
char
*
data_ptr
,
size_t
size
)
{
bufs
.
emplace_back
(
uv_buf_t
{
data_ptr
,
size
});
data_vector
.
push_back
(
data
);
}
void
add
(
::
google
::
protobuf
::
MessageLite
&
message
);
uv_write_t
request
;
protected:
std
::
vector
<
uv_buf_t
>
bufs
;
std
::
vector
<
std
::
unique_ptr
<
char
[]
>>
raw_memory
;
std
::
vector
<
std
::
shared_ptr
<
Data
>>
data_vector
;
};
}
#endif // LIBLOOM_SENDBUFFER_H
src/server/clientconn.h
View file @
7e2afba4
...
...
@@ -3,6 +3,10 @@
#include "libloom/connection.h"
namespace
loom
{
class
SendBuffer
;
}
class
Server
;
class
ClientConnection
:
public
loom
::
ConnectionCallback
{
...
...
@@ -13,8 +17,12 @@ public:
void
on_message
(
const
char
*
buffer
,
size_t
size
);
void
on_close
();
void
send
(
uv_write_t
*
request
,
uv_buf_t
bufs
[],
unsigned
int
nbufs
,
uv_write_cb
cb
)
{
/*
void send(uv_write_t *request, uv_buf_t bufs[], unsigned int nbufs, uv_write_cb cb) {
connection->send(request, bufs, nbufs, cb);
}*/
void
send_buffer
(
loom
::
SendBuffer
*
buffer
)
{
connection
->
send_buffer
(
buffer
);
}
protected:
...
...
src/server/resendtask.cpp
View file @
7e2afba4
#include "resendtask.h"
#include "server.h"
#include "libloom/loomcomm.pb.h"
#include "libloom/log.h"
using
namespace
loom
;
ResendTask
::
ResendTask
(
Server
&
server
,
std
::
unique_ptr
<
Task
>
task
)
:
TaskInstance
(
server
.
get_dummy_worker
(),
std
::
move
(
task
)),
server
(
server
)
:
TaskInstance
(
server
.
get_dummy_worker
(),
std
::
move
(
task
)),
server
(
server
),
buffer
(
*
this
)
{
request
.
data
=
this
;
}
void
ResendTask
::
start
(
DataVector
&
input_data
)
{
assert
(
input_data
.
size
()
==
1
);
data
=
*
input_data
[
0
];
auto
data
=
*
input_data
[
0
];
llog
->
debug
(
"Resending data id={} to client"
,
data
->
get_id
());
loomcomm
::
Data
msg
;
msg
.
set_id
(
data
->
get_id
());
msg
.
set_size
(
data
->
get_size
());
llog
->
debug
(
"Resending data id={} to client"
,
data
->
get_id
());
buffer
.
add
(
msg
);
buffer
.
add
(
data
,
data
->
get_data
(
worker
),
data
->
get_size
());
auto
size
=
msg
.
ByteSize
();
message_data
=
std
::
make_unique
<
char
[]
>
(
size
+
sizeof
(
uint32_t
));
uint32_t
*
size_ptr
=
(
uint32_t
*
)
message_data
.
get
();
*
size_ptr
=
size
;
msg
.
SerializePartialToArray
(
message_data
.
get
()
+
sizeof
(
uint32_t
),
size
);
message_size
=
size
;
uv_buf_t
bufs
[
2
];
bufs
[
0
].
base
=
message_data
.
get
();
bufs
[
0
].
len
=
message_size
+
sizeof
(
uint32_t
);
bufs
[
1
]
=
data
->
get_uv_buf
(
worker
);
auto
&
connection
=
server
.
get_client_connection
();
connection
.
send
(
&
request
,
bufs
,
2
,
_on_write
);
}
void
ResendTask
::
_on_write
(
uv_write_t
*
write_req
,
int
status
)
{
UV_CHECK
(
status
);
ResendTask
*
task
=
static_cast
<
ResendTask
*>
(
write_req
->
data
);
llog
->
debug
(
"Resend task id={} finished"
,
task
->
get_id
());
task
->
finish_without_data
();
connection
.
send_buffer
(
&
buffer
);
}
ResendTaskFactory
::
ResendTaskFactory
(
Server
&
server
)
...
...
@@ -58,3 +40,15 @@ std::unique_ptr<TaskInstance> ResendTaskFactory::make_instance(Worker &worker, s
{
return
std
::
make_unique
<
ResendTask
>
(
server
,
std
::
move
(
task
));
}
ResendTask
::
_SendBuffer
::
_SendBuffer
(
ResendTask
&
task
)
:
task
(
task
)
{
}
void
ResendTask
::
_SendBuffer
::
on_finish
(
int
status
)
{
UV_CHECK
(
status
);
llog
->
debug
(
"Resend task id={} finished"
,
task
.
get_id
());
task
.
finish_without_data
();
}
src/server/resendtask.h
View file @
7e2afba4
...
...
@@ -3,6 +3,7 @@
#include "libloom/taskinstance.h"
#include "libloom/taskfactory.h"
#include "libloom/sendbuffer.h"
namespace
loom
{
class
Data
;
...
...
@@ -13,16 +14,21 @@ class Server;
class
ResendTask
:
public
loom
::
TaskInstance
{
public:
class
_SendBuffer
:
public
loom
::
SendBuffer
{
public:
_SendBuffer
(
ResendTask
&
task
);
~
_SendBuffer
()
{}
void
on_finish
(
int
status
);
protected:
ResendTask
&
task
;
};
ResendTask
(
Server
&
server
,
std
::
unique_ptr
<
loom
::
Task
>
task
);
void
start
(
loom
::
DataVector
&
input_data
);
protected:
Server
&
server
;
std
::
shared_ptr
<
loom
::
Data
>
data
;
uv_write_t
request
;
std
::
unique_ptr
<
char
[]
>
message_data
;
size_t
message_size
;
Server
&
server
;
_SendBuffer
buffer
;
static
void
_on_write
(
uv_write_t
*
write_req
,
int
status
);
};
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment