ENH: New scheduler based on lp_solve

CMakeLists.txt

@@ -16,6 +16,17 @@ mark_as_advanced(LIBUV_INCLUDE_DIR LIBUV_LIBRARY)
 find_package(Protobuf REQUIRED)
 include_directories(${PROTOBUF_INCLUDE_DIRS})
 
+find_library(LPSOLVE_LIBRARY
+        NAMES lpsolve55)
+mark_as_advanced(LPSOLVE_LIBRARY)
+
+find_library(COLAMD_LIBRARY
+        NAMES colamd)
+mark_as_advanced(COLAMD_LIBRARY)
+
+	#PATHS ${LPSOLVE_LIBRARIES} ${LPSOLVE_LIB_TRIALPATH}
+	#PATH_SUFFIXES lp_solve)
+
 add_subdirectory(src)
 add_subdirectory(tests)
 

src/server/CMakeLists.txt

@@ -19,7 +19,9 @@ add_library(loom-server-lib OBJECT
                plannode.cpp
                plannode.h
                plan.h
-               plan.cpp)
+               plan.cpp
+               solver.h
+               solver.cpp)
 
 target_include_directories(loom-server-lib PUBLIC ${PROJECT_SOURCE_DIR}/src)
 
@@ -28,6 +30,7 @@ add_executable(loom-server
                main.cpp)
 
 target_link_libraries(loom-server libloom ${LIBUV_LIBRARY} pthread)
+target_link_libraries(loom-server ${LPSOLVE_LIBRARY} ${COLAMD_LIBRARY} dl)
 target_link_libraries(loom-server ${PROTOBUF_LIBRARIES})
 
 install (TARGETS loom-server DESTINATION bin)

src/server/compstate.h

@@ -9,6 +9,11 @@ class Server;
 using TaskDistribution = std::unordered_map<WorkerConnection*, std::vector<loom::Id>>;
 
 class ComputationState {
+    struct WorkerInfo {
+        WorkerInfo() : n_tasks(0) {}
+        size_t index;
+        int n_tasks;
+    };
 public:
     ComputationState(Server &server);
 
@@ -20,11 +25,7 @@ public:
     TaskDistribution compute_initial_distribution();
     TaskDistribution compute_distribution();
 
-    TaskState& get_state(loom::Id id) {
-        auto it = states.find(id);
-        assert(it != states.end());
-        return it->second;
-    }
+    TaskState& get_state(loom::Id id);
 
     TaskState* get_state_ptr(loom::Id id) {
         auto it = states.find(id);
@@ -48,7 +49,7 @@ public:
         return plan.get_node(id);
     }
 
-    void add_ready_nodes(std::vector<loom::Id> &ids);
+    void add_ready_nodes(const std::vector<loom::Id> &ids);
     void set_task_finished(loom::Id id, size_t size, size_t length, WorkerConnection *wc);
 
     const Plan& get_plan() const {
@@ -81,6 +82,10 @@ private:
     void expand_dget(const PlanNode &node);
     void make_expansion(std::vector<std::string> &configs, const PlanNode &node1,
                         const PlanNode &node2, loom::Id id_base1, loom::Id id_base2);
+    void collect_requirements_for_node(WorkerConnection *wc,
+                                       const PlanNode &node,
+                                       std::unordered_set<loom::Id> &nonlocals);
+    size_t task_transfer_cost(const PlanNode &node);
 };
 
 

src/server/solver.cpp

+#include "solver.h"
+
+#include <libloom/log.h>
+#include <libloom/utils.h>
+
+#include <lpsolve/lp_lib.h>
+
+#define LP_CHECK(call) \
+{ \
+    if (unlikely((!call))) { \
+    report_lp_error(__LINE__, __FILE__); \
+    } \
+    }
+
+static void report_lp_error(int line, const char* filename)
+{
+    loom::llog->critical("lp_solve error at {}:{}", filename, line);
+    exit(1);
+}
+
+Solver::Solver(size_t variables)
+    : lp_solver(nullptr)
+{
+    assert(variables > 0);
+    assert(sizeof(REAL) == sizeof(double));
+
+    lprec *lp;
+    lp = make_lp(0, variables);
+    LP_CHECK(lp);
+
+    for (size_t i = 1; i <= variables; i++) {
+        set_binary(lp, i, TRUE);
+    }
+
+    set_add_rowmode(lp, TRUE);
+
+
+    lp_solver = lp;
+    this->variables = variables;
+}
+
+Solver::~Solver()
+{
+    lprec *lp = static_cast<lprec*>(lp_solver);
+    if (lp) {
+        delete_lp(lp);
+    }
+}
+
+void Solver::add_constraint_lq(int index1, int index2)
+{
+    if (index1 == index2) {
+        return;
+    }
+    double v[2] = { 1.0, -1.0 };
+    int i[2];
+    i[0] = index1;
+    i[1] = index2;
+    lprec *lp = static_cast<lprec*>(lp_solver);
+    LP_CHECK(add_constraintex(lp, 2, v, i, LE, 0.0));
+}
+
+void Solver::add_constraint_eq(const std::vector<int> &indices, const std::vector<double> &values, double rhs)
+{
+    lprec *lp = static_cast<lprec*>(lp_solver);
+    size_t size = indices.size();
+    assert(values.size() >= size);
+    LP_CHECK(add_constraintex(
+                 lp, size,
+                 const_cast<double*>(&values[0]),
+             const_cast<int*>(&indices[0]),
+            EQ, rhs));
+}
+
+void Solver::add_constraint_lq(const std::vector<int> &indices, const std::vector<double> &values, double rhs)
+{
+    lprec *lp = static_cast<lprec*>(lp_solver);
+    size_t size = indices.size();
+    assert(values.size() >= size);
+    LP_CHECK(add_constraintex(lp, size,
+                              const_cast<double*>(&values[0]),
+             const_cast<int*>(&indices[0]),
+            LE, rhs));
+}
+
+void Solver::set_objective_fn(const std::vector<double> &values)
+{
+    assert(values.size() == variables + 1);
+    lprec *lp = static_cast<lprec*>(lp_solver);
+    set_obj_fn(lp, const_cast<double*>(&values[0]));
+}
+
+std::vector<double> Solver::solve()
+{
+    lprec *lp = static_cast<lprec*>(lp_solver);
+    set_add_rowmode(lp, FALSE);
+    //write_LP(lp, stdout);
+    set_verbose(lp, IMPORTANT);
+
+    set_presolve(lp, /* PRESOLVE_ROWS | */ PRESOLVE_REDUCEMIP + PRESOLVE_KNAPSACK + PRESOLVE_COLS + PRESOLVE_LINDEP, get_presolveloops(lp));
+
+    int ret = ::solve(lp);    
+    assert(ret == OPTIMAL || ret == SUBOPTIMAL);
+
+    std::vector<double> result(variables);
+    get_variables(lp, &result[0]);
+
+    /*for (size_t i = 0; i < variables; i++) {
+        loom::llog->alert("{}: {}", i + 1, result[i]);
+    }*/
+    return result;
+}

src/server/solver.h

+#ifndef LOOM_SERVER_SOLVER_H
+#define LOOM_SERVER_SOLVER_H
+
+#include <vector>
+
+class Solver
+{
+public:
+    Solver(std::size_t variables);
+    ~Solver();
+
+    void add_constraint_lq(int index1, int index2);
+    void add_constraint_eq(const std::vector<int> &indices, const std::vector<double> &values, double rhs);
+    void add_constraint_lq(const std::vector<int> &indices, const std::vector<double> &values, double rhs);
+    void set_objective_fn(const std::vector<double> &values);
+    std::vector<double> solve();
+protected:
+    /* lp_solve heavily pollutes namespace with macros,
+     * so we do not want to include the library publicaly
+     * lp_solver is in fact lprec internaly casted to the
+     * the right type */
+    void *lp_solver;
+    std::size_t variables;
+};
+
+#endif // LOOM_SERVER_SOLVER_H

src/server/taskmanager.cpp

@@ -49,11 +49,11 @@ void TaskManager::start_task(WorkerConnection *wc, Id task_id)
     for (loom::Id id : node.get_inputs()) {
         TaskState state = cstate.get_state_or_create(id);
         TaskState::WStatus st = state.get_worker_status(wc);
-        if (st == TaskState::NONE) {
+        if (st == TaskState::S_NONE) {
             WorkerConnection *owner = state.get_first_owner();
             assert(owner);
             owner->send_data(id, wc->get_address(), false);
-            state.set_worker_status(wc, TaskState::OWNER);
+            state.set_worker_status(wc, TaskState::S_OWNER);
         }
     }
     wc->send_task(node);
@@ -89,10 +89,15 @@ void TaskManager::on_task_finished(loom::Id id, size_t size, size_t length, Work
         llog->debug("Job id={} finished (size={}, length={})", id, size, length);
     }
 
-    for (loom::Id id : node.get_inputs()) {
-        TaskState *state = cstate.get_state_ptr(id);
-        if (state && state->dec_ref_counter()) {
-            remove_state(*state);
+    // Remove duplicates
+    std::vector<loom::Id> inputs = node.get_inputs();
+    std::sort(inputs.begin(), inputs.end());
+    inputs.erase(std::unique(inputs.begin(), inputs.end()), inputs.end());
+
+    for (loom::Id id : inputs) {
+        TaskState &state = cstate.get_state(id);
+        if (state.dec_ref_counter()) {
+            remove_state(state);
         }
     }
 

src/server/taskstate.h

@@ -14,10 +14,10 @@ class TaskState {
 
 public:
     enum WStatus {
-        NONE,
-        READY,
-        RUNNING,
-        OWNER,
+        S_NONE,
+        S_READY,
+        S_RUNNING,
+        S_OWNER,
     };
 
     TaskState(const PlanNode &node);
@@ -61,14 +61,14 @@ public:
     WStatus get_worker_status(WorkerConnection *wc) {
         auto i = workers.find(wc);
         if (i == workers.end()) {
-            return NONE;
+            return S_NONE;
         }
         return i->second;
     }
 
     WorkerConnection *get_first_owner() {
         for (auto &p : workers) {
-            if (p.second == OWNER) {
+            if (p.second == S_OWNER) {
                 return p.first;
             }
         }
@@ -81,7 +81,7 @@ public:
 
     template<typename F> void foreach_owner(F f) {
         for(auto &pair : workers) {
-            if (pair.second == OWNER) {
+            if (pair.second == S_OWNER) {
                 f(pair.first);
             }
         }
@@ -95,9 +95,4 @@ private:
     size_t length;
 };
 
-struct WorkerInfo {
-    WorkerInfo() : n_tasks(0) {}
-    int n_tasks;
-};
-
 #endif // LOOM_SERVER_TASKSTATE_H

tests/server/CMakeLists.txt

@@ -9,6 +9,7 @@ add_executable(server-test
                main.cpp)
 
 target_link_libraries(server-test Catch libloom ${LIBUV_LIBRARY} pthread)
+target_link_libraries(server-test ${LPSOLVE_LIBRARY} ${COLAMD_LIBRARY} dl)
 target_link_libraries(server-test ${PROTOBUF_LIBRARIES})
 
 target_include_directories(server-test PUBLIC ${PROJECT_SOURCE_DIR})