Commit 82a58ff1 authored by Stanislav Bohm's avatar Stanislav Bohm

RF: libloomnet -> libloom

parent 04a91a7d
add_subdirectory(libloomnet)
add_subdirectory(libloom)
add_subdirectory(libloomw)
add_subdirectory(worker)
add_subdirectory(server)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -g -Wall")
add_library(libloomnet
add_library(libloom
compat.h
socket.cpp
socket.h
......@@ -13,8 +13,8 @@ add_library(libloomnet
pbutils.cpp
)
target_include_directories(libloomnet PUBLIC ${PROJECT_SOURCE_DIR}/src)
install(TARGETS libloomnet
target_include_directories(libloom PUBLIC ${PROJECT_SOURCE_DIR}/src)
install(TARGETS libloom
ARCHIVE DESTINATION lib)
install(
DIRECTORY ${CMAKE_SOURCE_DIR}/src/libloomnet
......
......@@ -3,7 +3,7 @@
#include "log.h"
using namespace loom::net;
using namespace loom::base;
void Listener::start(uv_loop_t *loop, int port, const std::function<void()> &callback)
{
......
......@@ -6,7 +6,7 @@
#include <uv.h>
namespace loom {
namespace net {
namespace base {
class Listener
{
......
......@@ -6,26 +6,26 @@
#include <stdlib.h>
#include <limits.h>
using namespace loom::net;
using namespace loom::base;
namespace loom {
namespace net {
namespace base {
std::shared_ptr<spdlog::logger> log = spdlog::stdout_logger_mt("net", true);
}}
void loom::net::report_uv_error(int error_code, int line_number, const char *filename)
void loom::base::report_uv_error(int error_code, int line_number, const char *filename)
{
log->critical("libuv fail: {} ({}:{})", uv_strerror(error_code), filename, line_number);
exit(1);
}
void loom::net::log_errno_abort(const char *tmp)
void loom::base::log_errno_abort(const char *tmp)
{
log->critical("{}: {}", tmp, strerror(errno));
abort();
}
void loom::net::log_errno_abort(const char *tmp, const char *tmp2)
void loom::base::log_errno_abort(const char *tmp, const char *tmp2)
{
log->critical("{}: {} ({})", tmp, strerror(errno), tmp2);
abort();
......
......@@ -5,7 +5,7 @@
#include "spdlog/spdlog.h"
namespace loom {
namespace net {
namespace base {
#define likely(x) __builtin_expect((x),1)
#define unlikely(x) __builtin_expect((x),0)
......@@ -13,7 +13,7 @@ namespace net {
#define UV_CHECK(call) \
{ int _uv_r = (call); \
if (unlikely(_uv_r)) { \
loom::net::report_uv_error(_uv_r, __LINE__, __FILE__); \
loom::base::report_uv_error(_uv_r, __LINE__, __FILE__); \
} \
}
......
......@@ -3,17 +3,17 @@
#include "compat.h"
std::unique_ptr<loom::net::SendBufferItem> loom::net::message_to_item(google::protobuf::MessageLite &msg)
std::unique_ptr<loom::base::SendBufferItem> loom::base::message_to_item(google::protobuf::MessageLite &msg)
{
auto size = msg.ByteSize();
auto item = std::make_unique<loom::net::MemItemWithSz>(size);
auto item = std::make_unique<loom::base::MemItemWithSz>(size);
msg.SerializeToArray(item->get_ptr(), size);
return item;
}
void loom::net::send_message(loom::net::Socket &socket, google::protobuf::MessageLite &msg)
void loom::base::send_message(loom::base::Socket &socket, google::protobuf::MessageLite &msg)
{
auto buffer = std::make_unique<loom::net::SendBuffer>();
auto buffer = std::make_unique<loom::base::SendBuffer>();
buffer->add(message_to_item(msg));
socket.send(std::move(buffer));
}
......@@ -11,12 +11,12 @@ namespace google {
}
namespace loom {
namespace net {
namespace base {
std::unique_ptr<loom::net::SendBufferItem>
std::unique_ptr<loom::base::SendBufferItem>
message_to_item(::google::protobuf::MessageLite &msg);
void send_message(loom::net::Socket &socket, ::google::protobuf::MessageLite &msg);
void send_message(loom::base::Socket &socket, ::google::protobuf::MessageLite &msg);
}}
......
......@@ -2,9 +2,9 @@
#include "compat.h"
using namespace loom::net;
using namespace loom::base;
std::vector<uv_buf_t> loom::net::SendBuffer::get_bufs()
std::vector<uv_buf_t> loom::base::SendBuffer::get_bufs()
{
std::vector<uv_buf_t> bufs;
bufs.reserve(items.size());
......@@ -14,12 +14,12 @@ std::vector<uv_buf_t> loom::net::SendBuffer::get_bufs()
return bufs;
}
loom::net::MemItem::MemItem(size_t size) : size(size)
loom::base::MemItem::MemItem(size_t size) : size(size)
{
mem = std::make_unique<char[]>(size);
}
loom::net::MemItem::~MemItem()
loom::base::MemItem::~MemItem()
{
}
......
......@@ -6,7 +6,7 @@
#include <vector>
namespace loom {
namespace net {
namespace base {
class SendBufferItem {
......
#include "socket.h"
#include "log.h"
#include "libloomnet/compat.h"
#include "libloom/compat.h"
#include <assert.h>
using namespace loom::net;
using namespace loom::base;
Socket::Socket(uv_loop_t *loop)
: state(State::New), stream_mode(false), stream_remaining(0)
......
......@@ -8,7 +8,7 @@
#include <vector>
namespace loom {
namespace net {
namespace base {
class Socket {
......
......@@ -5,7 +5,7 @@
#include "loomcomm.pb.h"
#include "libloomnet/sendbuffer.h"
#include "libloom/sendbuffer.h"
#include <uv.h>
#include <string>
......@@ -41,7 +41,7 @@ public:
virtual std::shared_ptr<Data> get_slice(size_t from, size_t to);
/** Serialize object into send buffer */
virtual size_t serialize(Worker &worker, loom::net::SendBuffer &buffer, std::shared_ptr<Data> &data_ptr) = 0;
virtual size_t serialize(Worker &worker, loom::base::SendBuffer &buffer, std::shared_ptr<Data> &data_ptr) = 0;
/** Get pointer to raw data, returns nullptr when it is not possible */
virtual const char *get_raw_data() const;
......@@ -57,7 +57,7 @@ protected:
};
class DataBufferItem : public loom::net::SendBufferItem {
class DataBufferItem : public loom::base::SendBufferItem {
public:
DataBufferItem(std::shared_ptr<Data> &data, const char *mem, size_t size);
uv_buf_t get_buf();
......
#include "array.h"
#include "libloomnet/compat.h"
#include "libloom/compat.h"
#include "../worker.h"
#include "../log.h"
......@@ -74,9 +74,9 @@ std::shared_ptr<Data> Array::get_at_index(size_t index)
return items[index];
}
size_t Array::serialize(Worker &worker, loom::net::SendBuffer &buffer, std::shared_ptr<Data> &data_ptr)
size_t Array::serialize(Worker &worker, loom::base::SendBuffer &buffer, std::shared_ptr<Data> &data_ptr)
{
auto types = std::make_unique<net::MemItemWithSz>(sizeof(loom::Id) * length);
auto types = std::make_unique<base::MemItemWithSz>(sizeof(loom::Id) * length);
loom::Id *ts = reinterpret_cast<loom::Id*>(types->get_ptr());
for (size_t i = 0; i < length; i++) {
ts[i] = items[i]->get_type_id(worker);
......
......@@ -24,7 +24,7 @@ public:
std::shared_ptr<Data>& get_ref_at_index(size_t index);
std::string get_type_name() const;
size_t serialize(Worker &worker, loom::net::SendBuffer &buffer, std::shared_ptr<Data> &data_ptr);
size_t serialize(Worker &worker, loom::base::SendBuffer &buffer, std::shared_ptr<Data> &data_ptr);
protected:
......
......@@ -34,7 +34,7 @@ std::string loom::ExternFile::get_info()
return "<ExternFile '" + filename + "'>";
}
size_t ExternFile::serialize(Worker &worker, loom::net::SendBuffer &buffer, std::shared_ptr<Data> &data_ptr)
size_t ExternFile::serialize(Worker &worker, loom::base::SendBuffer &buffer, std::shared_ptr<Data> &data_ptr)
{
assert(0); // TODO
}
......@@ -49,12 +49,12 @@ void ExternFile::open()
llog->debug("Opening extern file {}", filename);
int fd = ::open(filename.c_str(), O_RDONLY, S_IRUSR | S_IWUSR);
if (fd < 0) {
net::log_errno_abort("open");
base::log_errno_abort("open");
}
size = lseek(fd, (size_t)0, SEEK_END);
if (size == (size_t) -1) {
net::log_errno_abort("lseek");
base::log_errno_abort("lseek");
}
lseek(fd, 0, SEEK_SET);
......@@ -75,6 +75,6 @@ void ExternFile::map(int fd, bool write)
data = (char*) mmap(0, size, flags, MAP_SHARED, fd, 0);
if (data == MAP_FAILED) {
llog->critical("Cannot mmap '{}' size={}", filename, size);
net::log_errno_abort("mmap");
base::log_errno_abort("mmap");
}
}
......@@ -28,7 +28,7 @@ public:
std::string get_info();
std::string get_filename() const;
size_t serialize(Worker &worker, loom::net::SendBuffer &buffer, std::shared_ptr<Data> &data_ptr);
size_t serialize(Worker &worker, loom::base::SendBuffer &buffer, std::shared_ptr<Data> &data_ptr);
protected:
......
......@@ -80,7 +80,7 @@ std::shared_ptr<Data> Index::get_slice(size_t from, size_t to)
return data;
}
size_t Index::serialize(Worker &worker, loom::net::SendBuffer &buffer, std::shared_ptr<Data> &data_ptr)
size_t Index::serialize(Worker &worker, loom::base::SendBuffer &buffer, std::shared_ptr<Data> &data_ptr)
{
llog->critical("Index::serialize_data");
abort();
......
......@@ -27,7 +27,7 @@ public:
std::string get_info();
std::shared_ptr<Data> get_at_index(size_t index);
std::shared_ptr<Data> get_slice(size_t from, size_t to);
size_t serialize(Worker &worker, loom::net::SendBuffer &buffer, std::shared_ptr<Data> &data_ptr);
size_t serialize(Worker &worker, loom::base::SendBuffer &buffer, std::shared_ptr<Data> &data_ptr);
protected:
......
......@@ -3,7 +3,7 @@
#include "../log.h"
#include "../utils.h"
#include "../worker.h"
#include "libloomnet/sendbuffer.h"
#include "libloom/sendbuffer.h"
#include <sstream>
#include <assert.h>
......@@ -151,9 +151,9 @@ void RawData::init_from_mem(const std::string &work_dir, const void *ptr, size_t
memcpy(mem, ptr, size);
}
size_t RawData::serialize(Worker &worker, loom::net::SendBuffer &buffer, std::shared_ptr<Data> &data_ptr)
size_t RawData::serialize(Worker &worker, loom::base::SendBuffer &buffer, std::shared_ptr<Data> &data_ptr)
{
buffer.add(std::make_unique<net::SizeBufferItem>(size));
buffer.add(std::make_unique<base::SizeBufferItem>(size));
buffer.add(std::make_unique<DataBufferItem>(data_ptr, get_raw_data(), size));
return 1;
}
......
......@@ -39,7 +39,7 @@ public:
void assign_filename(const std::string &work_dir);
std::string get_filename() const;
size_t serialize(Worker &worker, loom::net::SendBuffer &buffer, std::shared_ptr<Data> &data_ptr);
size_t serialize(Worker &worker, loom::base::SendBuffer &buffer, std::shared_ptr<Data> &data_ptr);
protected:
......
......@@ -2,8 +2,8 @@
#include "worker.h"
#include "loomcomm.pb.h"
#include "log.h"
#include "libloomnet/pbutils.h"
#include "libloomnet/sendbuffer.h"
#include "libloom/pbutils.h"
#include "libloom/sendbuffer.h"
#include <sstream>
......@@ -117,7 +117,7 @@ void InterConnection::on_stream_data(const char *buffer, size_t size, size_t rem
void InterConnection::send(Id id, std::shared_ptr<Data> &data)
{
auto buffer = std::make_unique<loom::net::SendBuffer>();
auto buffer = std::make_unique<loom::base::SendBuffer>();
size_t n_messages = data->serialize(worker, *buffer, data);
......@@ -125,12 +125,12 @@ void InterConnection::send(Id id, std::shared_ptr<Data> &data)
msg.set_id(id);
msg.set_type_id(data->get_type_id(worker));
msg.set_n_messages(n_messages);
buffer->insert(0, loom::net::message_to_item(msg));
buffer->insert(0, loom::base::message_to_item(msg));
auto state = socket.get_state();
assert(state == loom::net::Socket::State::Open ||
state == loom::net::Socket::State::Connecting);
if (state == loom::net::Socket::State::Open) {
assert(state == loom::base::Socket::State::Open ||
state == loom::base::Socket::State::Connecting);
if (state == loom::base::Socket::State::Open) {
socket.send(std::move(buffer));
} else {
early_sends.push_back(std::move(buffer));
......
#ifndef LOOM_INTERCONNECT_H
#define LOOM_INTERCONNECT_H
#include "libloomnet/socket.h"
#include "libloomnet/listener.h"
#include "libloom/socket.h"
#include "libloom/listener.h"
#include "data.h"
#include "unpacking.h"
......@@ -36,7 +36,7 @@ public:
socket.connect(address, port);
}
void accept(loom::net::Listener &listener) {
void accept(loom::base::Listener &listener) {
listener.accept(socket);
}
......@@ -52,7 +52,7 @@ protected:
void on_stream_data(const char *buffer, size_t size, size_t remaining);
void on_connect();
net::Socket socket;
base::Socket socket;
Worker &worker;
std::string address;
......@@ -61,7 +61,7 @@ protected:
static std::string make_address(const std::string &host, int port);
std::vector<std::unique_ptr<net::SendBuffer>> early_sends;
std::vector<std::unique_ptr<base::SendBuffer>> early_sends;
};
}
......
......@@ -3,7 +3,7 @@
#include "data.h"
#include "task.h"
#include "libloomnet/compat.h"
#include "libloom/compat.h"
#include<uv.h>
......
#include "arraytasks.h"
#include "libloomnet/compat.h"
#include "libloom/compat.h"
#include "libloomw/data/array.h"
using namespace loom;
......
......@@ -2,7 +2,7 @@
#include "python.h"
#include "../data/rawdata.h"
#include "../log.h"
#include "libloomnet/compat.h"
#include "libloom/compat.h"
#include "../worker.h"
#include "python_wrapper.h"
......
#include "rawdatatasks.h"
#include "libloomnet/compat.h"
#include "libloom/compat.h"
#include "libloomw/data/rawdata.h"
#include "libloomw/data/index.h"
#include "libloomw/data/externfile.h"
......
......@@ -4,7 +4,7 @@
#include "data.h"
#include "taskinstance.h"
#include "worker.h"
#include "libloomnet/log.h"
#include "libloom/log.h"
#include<uv.h>
......
......@@ -2,10 +2,10 @@
#define LOOM_UTILS_H
#include <sys/types.h>
#include <libloomnet/log.h>
#include <libloom/log.h>
#include "libloomnet/socket.h"
#include "libloomnet/sendbuffer.h"
#include "libloom/socket.h"
#include "libloom/sendbuffer.h"
namespace loom {
......
......@@ -15,8 +15,8 @@
#include "tasks/runtask.h"
#include "tasks/python.h"
#include "libloomnet/sendbuffer.h"
#include "libloomnet/pbutils.h"
#include "libloom/sendbuffer.h"
#include "libloom/pbutils.h"
#include <stdlib.h>
#include <sstream>
......@@ -183,7 +183,7 @@ void Worker::register_worker()
msg.add_data_types(pair.first);
}
loom::net::send_message(server_conn, msg);
loom::base::send_message(server_conn, msg);
}
void Worker::new_task(std::unique_ptr<Task> task)
......
......@@ -7,7 +7,7 @@
#include "taskfactory.h"
#include "dictionary.h"
#include "libloomnet/listener.h"
#include "libloom/listener.h"
#include <uv.h>
......@@ -138,7 +138,7 @@ private:
std::unordered_map<DataTypeId, UnpackFactoryFn> unpack_ffs;
net::Socket server_conn;
base::Socket server_conn;
std::unordered_map<std::string, std::unique_ptr<InterConnection>> connections;
std::vector<std::unique_ptr<InterConnection>> nonregistered_connections;
......@@ -147,7 +147,7 @@ private:
std::string server_address;
int server_port;
net::Listener listener;
base::Listener listener;
std::vector<std::unique_ptr<TaskFactory>> unregistered_task_factories;
std::unordered_map<std::string, UnpackFactoryFn> unregistered_unpack_ffs;
......
......@@ -29,7 +29,7 @@ add_executable(loom-server
$<TARGET_OBJECTS:loom-server-lib>
main.cpp)
target_link_libraries(loom-server libloomnet libloomw ${LIBUV_LIBRARY} pthread)
target_link_libraries(loom-server libloom libloomw ${LIBUV_LIBRARY} pthread)
target_link_libraries(loom-server ${PROTOBUF_LIBRARIES})
install (TARGETS loom-server DESTINATION bin)
......@@ -6,14 +6,14 @@
#include "libloomw/loomplan.pb.h"
#include "libloomw/loomcomm.pb.h"
#include "libloomw/log.h"
#include "libloomnet/compat.h"
#include "libloomnet/pbutils.h"
#include "libloom/compat.h"
#include "libloom/pbutils.h"
using namespace loom;
ClientConnection::ClientConnection(Server &server,
std::unique_ptr<loom::net::Socket> socket)
std::unique_ptr<loom::base::Socket> socket)
: server(server), socket(std::move(socket))
{
this->socket->set_on_message([this](const char *buffer, size_t size) {
......@@ -35,7 +35,7 @@ ClientConnection::ClientConnection(Server &server,
std::string *s = cmsg.add_symbols();
*s = symbol;
}
loom::net::send_message(*this->socket, cmsg);
loom::base::send_message(*this->socket, cmsg);
// End of send dictionary
}
......
#ifndef LOOM_SERVER_CLIENTCONN_H
#define LOOM_SERVER_CLIENTCONN_H
#include "libloomnet/pbutils.h"
#include "libloomnet/socket.h"
#include "libloom/pbutils.h"
#include "libloom/socket.h"
namespace loom {
......@@ -15,21 +15,21 @@ class Server;
class ClientConnection {
public:
ClientConnection(Server &server,
std::unique_ptr<loom::net::Socket> socket);
std::unique_ptr<loom::base::Socket> socket);
void on_message(const char *buffer, size_t size);
void send(std::unique_ptr<loom::net::SendBuffer> buffer) {
void send(std::unique_ptr<loom::base::SendBuffer> buffer) {
socket->send(std::move(buffer));
}
void send_message(google::protobuf::MessageLite &msg) {
loom::net::send_message(*socket, msg);
loom::base::send_message(*socket, msg);
}
protected:
Server &server;
std::unique_ptr<loom::net::Socket> socket;
std::unique_ptr<loom::base::Socket> socket;
};
......
......@@ -2,8 +2,8 @@
#include "dummyworker.h"
#include "server.h"
#include <libloomnet/compat.h>
#include <libloomnet/pbutils.h>
#include <libloom/compat.h>
#include <libloom/pbutils.h>
#include <libloomw/utils.h>
#include <libloomw/log.h>
......@@ -60,7 +60,7 @@ DWConnection::~DWConnection()
}
void DWConnection::accept(loom::net::Listener &listener)
void DWConnection::accept(loom::base::Listener &listener)
{
listener.accept(socket);
}
......@@ -74,7 +74,7 @@ void DWConnection::on_message(const char *buffer, size_t size)
}
if (remaining_messages) {
auto item = std::make_unique<net::MemItemWithSz>(size);
auto item = std::make_unique<base::MemItemWithSz>(size);
memcpy(item->get_ptr(), buffer, size);
send_buffer->add(std::move(item));
......@@ -89,7 +89,7 @@ void DWConnection::on_message(const char *buffer, size_t size)
}
assert(!send_buffer);
send_buffer = std::make_unique<loom::net::SendBuffer>();
send_buffer = std::make_unique<loom::base::SendBuffer>();
loomcomm::DataHeader msg;
assert(msg.ParseFromArray(buffer, size));
......@@ -104,5 +104,5 @@ void DWConnection::on_message(const char *buffer, size_t size)
loomcomm::ClientMessage cmsg;
cmsg.set_type(loomcomm::ClientMessage_Type_DATA);
*cmsg.mutable_data() = msg;
send_buffer->add(net::message_to_item(cmsg));
send_buffer->add(base::message_to_item(cmsg));
}
......@@ -6,9 +6,9 @@
#include <memory>
#include <vector>
#include <libloomnet/listener.h>
#include <libloomnet/socket.h>
#include <libloomnet/sendbuffer.h>
#include <libloom/listener.h>
#include <libloom/socket.h>
#include <libloom/sendbuffer.h>
#include <libloomw/types.h>
class Server;
......@@ -39,7 +39,7 @@ protected:
std::vector<std::unique_ptr<DWConnection>> connections;
loom::net::Listener listener;
loom::base::Listener listener;
};
class DWConnection
......@@ -52,7 +52,7 @@ public:
return socket.get_peername();
}
void accept(loom::net::Listener &listener);
void accept(loom::base::Listener &listener);
protected:
......@@ -60,8 +60,8 @@ protected:
void on_message(const char *buffer, size_t size);
DummyWorker &worker;
loom::net::Socket socket;
std::unique_ptr<loom::net::SendBuffer> send_buffer;
loom::base::Socket socket;
std::unique_ptr<loom::base::SendBuffer> send_buffer;
size_t remaining_messages;
bool registered;
};
......
......@@ -3,7 +3,7 @@
#include "clientconn.h"
#include "server.h"
#include "libloomnet/compat.h"
#include "libloom/compat.h"
#include "libloomw/log.h"
#include "libloomw/loomcomm.pb.h"
......@@ -13,7 +13,7 @@ using namespace loom;
FreshConnection::FreshConnection(Server &server) :
server(server),
socket(std::make_unique<net::Socket>(server.get_loop()))
socket(std::make_unique<base::Socket>(server.get_loop()))
{
socket->set_on_close([this](){
llog->error("Connection closed without registration");
......@@ -25,7 +25,7 @@ FreshConnection::FreshConnection(Server &server) :
});
}
void FreshConnection::accept(loom::net::Listener &listener) {
void FreshConnection::accept(loom::base::Listener &listener) {
listener.accept(*socket);
llog->debug("New connection to server ({})", socket->get_peername());
}
......
#ifndef LOOM_SERVER_FRESHCONN
#define LOOM_SERVER_FRESHCONN
#include "libloomnet/socket.h"
#include "libloomnet/listener.h"
#include "libloom/socket.h"
#include "libloom/listener.h"
class Server;
......@@ -18,13 +18,13 @@ public:
return server;
}
void accept(loom::net::Listener &listener);
void accept(loom::base::Listener &listener);