Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
H
HyperLoom
Project overview
Project overview
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
Packages
Packages
Container Registry
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Commits
Issue Boards
Open sidebar
ADAS
HyperLoom
Commits
a3a1fc21
Commit
a3a1fc21
authored
Oct 06, 2016
by
Stanislav Bohm
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
ENH: New scheduler based on lp_solve
parent
bb6a3621
Changes
10
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
10 changed files
with
838 additions
and
229 deletions
+838
-229
CMakeLists.txt
CMakeLists.txt
+11
-0
src/server/CMakeLists.txt
src/server/CMakeLists.txt
+4
-1
src/server/compstate.cpp
src/server/compstate.cpp
+322
-182
src/server/compstate.h
src/server/compstate.h
+11
-6
src/server/solver.cpp
src/server/solver.cpp
+112
-0
src/server/solver.h
src/server/solver.h
+26
-0
src/server/taskmanager.cpp
src/server/taskmanager.cpp
+11
-6
src/server/taskstate.h
src/server/taskstate.h
+7
-12
tests/server/CMakeLists.txt
tests/server/CMakeLists.txt
+1
-0
tests/server/scheduler.cpp
tests/server/scheduler.cpp
+333
-22
No files found.
CMakeLists.txt
View file @
a3a1fc21
...
...
@@ -16,6 +16,17 @@ mark_as_advanced(LIBUV_INCLUDE_DIR LIBUV_LIBRARY)
find_package
(
Protobuf REQUIRED
)
include_directories
(
${
PROTOBUF_INCLUDE_DIRS
}
)
find_library
(
LPSOLVE_LIBRARY
NAMES lpsolve55
)
mark_as_advanced
(
LPSOLVE_LIBRARY
)
find_library
(
COLAMD_LIBRARY
NAMES colamd
)
mark_as_advanced
(
COLAMD_LIBRARY
)
#PATHS ${LPSOLVE_LIBRARIES} ${LPSOLVE_LIB_TRIALPATH}
#PATH_SUFFIXES lp_solve)
add_subdirectory
(
src
)
add_subdirectory
(
tests
)
...
...
src/server/CMakeLists.txt
View file @
a3a1fc21
...
...
@@ -19,7 +19,9 @@ add_library(loom-server-lib OBJECT
plannode.cpp
plannode.h
plan.h
plan.cpp
)
plan.cpp
solver.h
solver.cpp
)
target_include_directories
(
loom-server-lib PUBLIC
${
PROJECT_SOURCE_DIR
}
/src
)
...
...
@@ -28,6 +30,7 @@ add_executable(loom-server
main.cpp
)
target_link_libraries
(
loom-server libloom
${
LIBUV_LIBRARY
}
pthread
)
target_link_libraries
(
loom-server
${
LPSOLVE_LIBRARY
}
${
COLAMD_LIBRARY
}
dl
)
target_link_libraries
(
loom-server
${
PROTOBUF_LIBRARIES
}
)
install
(
TARGETS loom-server DESTINATION bin
)
src/server/compstate.cpp
View file @
a3a1fc21
This diff is collapsed.
Click to expand it.
src/server/compstate.h
View file @
a3a1fc21
...
...
@@ -9,6 +9,11 @@ class Server;
using
TaskDistribution
=
std
::
unordered_map
<
WorkerConnection
*
,
std
::
vector
<
loom
::
Id
>>
;
class
ComputationState
{
struct
WorkerInfo
{
WorkerInfo
()
:
n_tasks
(
0
)
{}
size_t
index
;
int
n_tasks
;
};
public:
ComputationState
(
Server
&
server
);
...
...
@@ -20,11 +25,7 @@ public:
TaskDistribution
compute_initial_distribution
();
TaskDistribution
compute_distribution
();
TaskState
&
get_state
(
loom
::
Id
id
)
{
auto
it
=
states
.
find
(
id
);
assert
(
it
!=
states
.
end
());
return
it
->
second
;
}
TaskState
&
get_state
(
loom
::
Id
id
);
TaskState
*
get_state_ptr
(
loom
::
Id
id
)
{
auto
it
=
states
.
find
(
id
);
...
...
@@ -48,7 +49,7 @@ public:
return
plan
.
get_node
(
id
);
}
void
add_ready_nodes
(
std
::
vector
<
loom
::
Id
>
&
ids
);
void
add_ready_nodes
(
const
std
::
vector
<
loom
::
Id
>
&
ids
);
void
set_task_finished
(
loom
::
Id
id
,
size_t
size
,
size_t
length
,
WorkerConnection
*
wc
);
const
Plan
&
get_plan
()
const
{
...
...
@@ -81,6 +82,10 @@ private:
void
expand_dget
(
const
PlanNode
&
node
);
void
make_expansion
(
std
::
vector
<
std
::
string
>
&
configs
,
const
PlanNode
&
node1
,
const
PlanNode
&
node2
,
loom
::
Id
id_base1
,
loom
::
Id
id_base2
);
void
collect_requirements_for_node
(
WorkerConnection
*
wc
,
const
PlanNode
&
node
,
std
::
unordered_set
<
loom
::
Id
>
&
nonlocals
);
size_t
task_transfer_cost
(
const
PlanNode
&
node
);
};
...
...
src/server/solver.cpp
0 → 100644
View file @
a3a1fc21
#include "solver.h"
#include <libloom/log.h>
#include <libloom/utils.h>
#include <lpsolve/lp_lib.h>
#define LP_CHECK(call) \
{ \
if (unlikely((!call))) { \
report_lp_error(__LINE__, __FILE__); \
} \
}
static
void
report_lp_error
(
int
line
,
const
char
*
filename
)
{
loom
::
llog
->
critical
(
"lp_solve error at {}:{}"
,
filename
,
line
);
exit
(
1
);
}
Solver
::
Solver
(
size_t
variables
)
:
lp_solver
(
nullptr
)
{
assert
(
variables
>
0
);
assert
(
sizeof
(
REAL
)
==
sizeof
(
double
));
lprec
*
lp
;
lp
=
make_lp
(
0
,
variables
);
LP_CHECK
(
lp
);
for
(
size_t
i
=
1
;
i
<=
variables
;
i
++
)
{
set_binary
(
lp
,
i
,
TRUE
);
}
set_add_rowmode
(
lp
,
TRUE
);
lp_solver
=
lp
;
this
->
variables
=
variables
;
}
Solver
::~
Solver
()
{
lprec
*
lp
=
static_cast
<
lprec
*>
(
lp_solver
);
if
(
lp
)
{
delete_lp
(
lp
);
}
}
void
Solver
::
add_constraint_lq
(
int
index1
,
int
index2
)
{
if
(
index1
==
index2
)
{
return
;
}
double
v
[
2
]
=
{
1.0
,
-
1.0
};
int
i
[
2
];
i
[
0
]
=
index1
;
i
[
1
]
=
index2
;
lprec
*
lp
=
static_cast
<
lprec
*>
(
lp_solver
);
LP_CHECK
(
add_constraintex
(
lp
,
2
,
v
,
i
,
LE
,
0.0
));
}
void
Solver
::
add_constraint_eq
(
const
std
::
vector
<
int
>
&
indices
,
const
std
::
vector
<
double
>
&
values
,
double
rhs
)
{
lprec
*
lp
=
static_cast
<
lprec
*>
(
lp_solver
);
size_t
size
=
indices
.
size
();
assert
(
values
.
size
()
>=
size
);
LP_CHECK
(
add_constraintex
(
lp
,
size
,
const_cast
<
double
*>
(
&
values
[
0
]),
const_cast
<
int
*>
(
&
indices
[
0
]),
EQ
,
rhs
));
}
void
Solver
::
add_constraint_lq
(
const
std
::
vector
<
int
>
&
indices
,
const
std
::
vector
<
double
>
&
values
,
double
rhs
)
{
lprec
*
lp
=
static_cast
<
lprec
*>
(
lp_solver
);
size_t
size
=
indices
.
size
();
assert
(
values
.
size
()
>=
size
);
LP_CHECK
(
add_constraintex
(
lp
,
size
,
const_cast
<
double
*>
(
&
values
[
0
]),
const_cast
<
int
*>
(
&
indices
[
0
]),
LE
,
rhs
));
}
void
Solver
::
set_objective_fn
(
const
std
::
vector
<
double
>
&
values
)
{
assert
(
values
.
size
()
==
variables
+
1
);
lprec
*
lp
=
static_cast
<
lprec
*>
(
lp_solver
);
set_obj_fn
(
lp
,
const_cast
<
double
*>
(
&
values
[
0
]));
}
std
::
vector
<
double
>
Solver
::
solve
()
{
lprec
*
lp
=
static_cast
<
lprec
*>
(
lp_solver
);
set_add_rowmode
(
lp
,
FALSE
);
//write_LP(lp, stdout);
set_verbose
(
lp
,
IMPORTANT
);
set_presolve
(
lp
,
/* PRESOLVE_ROWS | */
PRESOLVE_REDUCEMIP
+
PRESOLVE_KNAPSACK
+
PRESOLVE_COLS
+
PRESOLVE_LINDEP
,
get_presolveloops
(
lp
));
int
ret
=
::
solve
(
lp
);
assert
(
ret
==
OPTIMAL
||
ret
==
SUBOPTIMAL
);
std
::
vector
<
double
>
result
(
variables
);
get_variables
(
lp
,
&
result
[
0
]);
/*for (size_t i = 0; i < variables; i++) {
loom::llog->alert("{}: {}", i + 1, result[i]);
}*/
return
result
;
}
src/server/solver.h
0 → 100644
View file @
a3a1fc21
#ifndef LOOM_SERVER_SOLVER_H
#define LOOM_SERVER_SOLVER_H
#include <vector>
class
Solver
{
public:
Solver
(
std
::
size_t
variables
);
~
Solver
();
void
add_constraint_lq
(
int
index1
,
int
index2
);
void
add_constraint_eq
(
const
std
::
vector
<
int
>
&
indices
,
const
std
::
vector
<
double
>
&
values
,
double
rhs
);
void
add_constraint_lq
(
const
std
::
vector
<
int
>
&
indices
,
const
std
::
vector
<
double
>
&
values
,
double
rhs
);
void
set_objective_fn
(
const
std
::
vector
<
double
>
&
values
);
std
::
vector
<
double
>
solve
();
protected:
/* lp_solve heavily pollutes namespace with macros,
* so we do not want to include the library publicaly
* lp_solver is in fact lprec internaly casted to the
* the right type */
void
*
lp_solver
;
std
::
size_t
variables
;
};
#endif // LOOM_SERVER_SOLVER_H
src/server/taskmanager.cpp
View file @
a3a1fc21
...
...
@@ -49,11 +49,11 @@ void TaskManager::start_task(WorkerConnection *wc, Id task_id)
for
(
loom
::
Id
id
:
node
.
get_inputs
())
{
TaskState
state
=
cstate
.
get_state_or_create
(
id
);
TaskState
::
WStatus
st
=
state
.
get_worker_status
(
wc
);
if
(
st
==
TaskState
::
NONE
)
{
if
(
st
==
TaskState
::
S_
NONE
)
{
WorkerConnection
*
owner
=
state
.
get_first_owner
();
assert
(
owner
);
owner
->
send_data
(
id
,
wc
->
get_address
(),
false
);
state
.
set_worker_status
(
wc
,
TaskState
::
OWNER
);
state
.
set_worker_status
(
wc
,
TaskState
::
S_
OWNER
);
}
}
wc
->
send_task
(
node
);
...
...
@@ -89,10 +89,15 @@ void TaskManager::on_task_finished(loom::Id id, size_t size, size_t length, Work
llog
->
debug
(
"Job id={} finished (size={}, length={})"
,
id
,
size
,
length
);
}
for
(
loom
::
Id
id
:
node
.
get_inputs
())
{
TaskState
*
state
=
cstate
.
get_state_ptr
(
id
);
if
(
state
&&
state
->
dec_ref_counter
())
{
remove_state
(
*
state
);
// Remove duplicates
std
::
vector
<
loom
::
Id
>
inputs
=
node
.
get_inputs
();
std
::
sort
(
inputs
.
begin
(),
inputs
.
end
());
inputs
.
erase
(
std
::
unique
(
inputs
.
begin
(),
inputs
.
end
()),
inputs
.
end
());
for
(
loom
::
Id
id
:
inputs
)
{
TaskState
&
state
=
cstate
.
get_state
(
id
);
if
(
state
.
dec_ref_counter
())
{
remove_state
(
state
);
}
}
...
...
src/server/taskstate.h
View file @
a3a1fc21
...
...
@@ -14,10 +14,10 @@ class TaskState {
public:
enum
WStatus
{
NONE
,
READY
,
RUNNING
,
OWNER
,
S_
NONE
,
S_
READY
,
S_
RUNNING
,
S_
OWNER
,
};
TaskState
(
const
PlanNode
&
node
);
...
...
@@ -61,14 +61,14 @@ public:
WStatus
get_worker_status
(
WorkerConnection
*
wc
)
{
auto
i
=
workers
.
find
(
wc
);
if
(
i
==
workers
.
end
())
{
return
NONE
;
return
S_
NONE
;
}
return
i
->
second
;
}
WorkerConnection
*
get_first_owner
()
{
for
(
auto
&
p
:
workers
)
{
if
(
p
.
second
==
OWNER
)
{
if
(
p
.
second
==
S_
OWNER
)
{
return
p
.
first
;
}
}
...
...
@@ -81,7 +81,7 @@ public:
template
<
typename
F
>
void
foreach_owner
(
F
f
)
{
for
(
auto
&
pair
:
workers
)
{
if
(
pair
.
second
==
OWNER
)
{
if
(
pair
.
second
==
S_
OWNER
)
{
f
(
pair
.
first
);
}
}
...
...
@@ -95,9 +95,4 @@ private:
size_t
length
;
};
struct
WorkerInfo
{
WorkerInfo
()
:
n_tasks
(
0
)
{}
int
n_tasks
;
};
#endif // LOOM_SERVER_TASKSTATE_H
tests/server/CMakeLists.txt
View file @
a3a1fc21
...
...
@@ -9,6 +9,7 @@ add_executable(server-test
main.cpp
)
target_link_libraries
(
server-test Catch libloom
${
LIBUV_LIBRARY
}
pthread
)
target_link_libraries
(
server-test
${
LPSOLVE_LIBRARY
}
${
COLAMD_LIBRARY
}
dl
)
target_link_libraries
(
server-test
${
PROTOBUF_LIBRARIES
}
)
target_include_directories
(
server-test PUBLIC
${
PROJECT_SOURCE_DIR
}
)
...
...
tests/server/scheduler.cpp
View file @
a3a1fc21
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment