diff --git a/python/loom/lore/html.py b/python/loom/lore/html.py
index df9460d9d85e76adf14e0d26989916c253092de6..78064512fc2ed91a8577f2e649b998a56feda21c 100644
--- a/python/loom/lore/html.py
+++ b/python/loom/lore/html.py
@@ -3,7 +3,7 @@ from bokeh.charts import save, output_file, BoxPlot
from bokeh.layouts import column, gridplot
from bokeh.palettes import all_palettes
from bokeh.plotting import figure
-from bokeh.models.widgets import Panel, Tabs
+from bokeh.models.widgets import Panel, Tabs, Div
from bokeh.models.widgets import DataTable, TableColumn
from bokeh.models import ColumnDataSource
@@ -20,8 +20,6 @@ def create_colors(count):
def create_timelines(report):
- print("\tTimeline")
-
worker_timelines, group_names = report.get_timelines()
worker_timelines = list(worker_timelines.items())
worker_timelines.sort()
@@ -34,7 +32,9 @@ def create_timelines(report):
else:
y_labels.append("Worker {}.{}".format(worker_id, i))
- f = figure(plot_width=1000, plot_height=1000, y_range=y_labels, webgl=True)
+ f = figure(plot_width=1000, plot_height=1000, y_range=y_labels,
+ x_range=[0, report.end_time],
+ webgl=True)
line_id = 1
@@ -59,6 +59,7 @@ def create_monitoring(report):
result = []
for worker in report.worker_list:
f = figure(plot_width=1000, plot_height=130,
+ x_range=[0, report.end_time],
title="Worker {}".format(worker.address))
f.line(worker.monitoring.time, worker.monitoring.cpu,
color="blue", legend="CPU %")
@@ -72,6 +73,7 @@ def create_transfer(report):
result = []
for worker in report.worker_list:
f = figure(plot_width=1000, plot_height=130,
+ x_range=[0, report.end_time],
title="Worker {}".format(worker.address))
f.line(worker.sends.time, worker.sends.data_size.cumsum(),
color="green", legend="send")
@@ -84,10 +86,11 @@ def create_transfer(report):
def create_ctransfer(report):
sends = report.all_sends()
f = figure(plot_width=1000, plot_height=500,
+ x_range=[0, report.end_time],
title="Cumulative transfers")
sends = sends.join(report.task_frame["group"], on="id")
names = report.group_names
- f = figure(plot_width=1000, plot_height=500)
+ f = figure(plot_width=1000, plot_height=500, x_range=[0, report.end_time])
for color, name in zip(create_colors(len(names)), names):
frame = sends[sends["group"] == name]
frame = frame.sort_values("time")
@@ -103,6 +106,7 @@ def create_ctime(report):
colors = create_colors(len(names))
f1 = figure(plot_width=1000, plot_height=400,
+ x_range=[0, report.end_time],
title="Cumulative time of finished tasks")
for name, color in zip(names, colors):
frame = ds[ds["group"] == name]
@@ -110,6 +114,7 @@ def create_ctime(report):
legend=name, color=color, line_width=2)
f2 = figure(plot_width=1000, plot_height=400,
+ x_range=[0, report.end_time],
title="Number of finished tasks")
for name, color in zip(names, colors):
@@ -139,9 +144,9 @@ def create_task_summary(report):
def create_pending_tasks(report):
- print("\tPending tasks")
ds = report.get_pending_tasks()
- f1 = figure(plot_width=1000, plot_height=400)
+ f1 = figure(plot_width=1000, plot_height=400,
+ x_range=[0, report.end_time])
names = report.group_names
for color, name in zip(create_colors(len(names)), names):
frame = ds[ds["group"] == name]
@@ -154,20 +159,19 @@ def create_pending_tasks(report):
def create_scheduling_time(report):
- print("\tScheduling time")
ds = report.scheduler_times
duration = ds["end_time"] - ds["start_time"]
df = pd.DataFrame({"time": ds["end_time"], "duration": duration})
df["label"] = 0
- f1 = figure(plot_width=1000, plot_height=400)
+ f1 = figure(plot_width=1000, plot_height=400,
+ x_range=[0, report.end_time])
f1.line(df["time"], df["duration"].cumsum())
f2 = BoxPlot(df, values="duration", label="label")
return column([f1, f2])
def create_worker_load(report):
- print("\tWorker load")
data = []
for task in report.task_frame.itertuples():
data.append((task.start_time, 1, task.worker))
@@ -177,6 +181,7 @@ def create_worker_load(report):
result = []
for worker in report.worker_list:
f = figure(plot_width=1000, plot_height=130,
+ x_range=[0, report.end_time],
title="Worker {}".format(worker.address))
data = frame[frame.worker == worker.worker_id]
f.line(data.time, data.change.cumsum(),
@@ -191,41 +196,46 @@ def create_task_durations(report):
return f
-def create_html(report, filename):
+def level_index(level):
+ return ("brief", "normal", "full").index(level)
+
+
+def create_html(report, filename, full):
output_file(filename)
- print("Task plots")
- tasks = Tabs(tabs=[
- Panel(child=create_ctime(report), title="Task progress"),
- Panel(child=create_task_summary(report), title="Task summary"),
- Panel(child=create_task_durations(report), title="Task durations"),
- ])
-
- print("Monitoring plots")
- monitoring = Tabs(tabs=[
- Panel(child=create_monitoring(report), title="CPU & Memory usage"),
- ])
-
- print("Scheduling plots")
- scheduling = Tabs(tabs=[
- Panel(child=create_timelines(report), title="Timeline"),
- Panel(child=create_pending_tasks(report), title="Pending tasks"),
- Panel(child=create_scheduling_time(report), title="Scheduling time"),
- Panel(child=create_worker_load(report), title="Worker load"),
- ])
-
- print("Communication plots")
- comm = Tabs(tabs=[
- Panel(child=create_ctransfer(report), title="Transfer per tasks"),
- Panel(child=create_transfer(report), title="Transfer per nodes"),
- ])
-
- main = Tabs(tabs=[
- Panel(child=tasks, title="Tasks"),
- Panel(child=monitoring, title="Monitoring"),
- Panel(child=scheduling, title="Scheduling"),
- Panel(child=comm, title="Communication"),
- ])
-
- print("Saving result")
+ structure = (
+ ("Tasks",
+ [("Task progress", create_ctime, False),
+ ("Task summary", create_task_summary, False),
+ ("Task durations", create_task_durations, False)]),
+ ("Monitoring",
+ [("CPU & Memory usage", create_monitoring, False)]),
+ ("Scheduling",
+ [("Timeline", create_timelines, True),
+ ("Pending tasks", create_pending_tasks, False),
+ ("Scheduling time", create_scheduling_time, False),
+ ("Worker load", create_worker_load, False)]),
+ ("Communication",
+ [("Transfer per tasks", create_ctransfer, False),
+ ("Transfer per nodes", create_transfer, False)])
+ )
+
+ tabs = []
+ for name, subtabs in structure:
+ print("Tab:", name)
+ tabs2 = []
+ for name2, fn, full_only in subtabs:
+ if full_only and not full:
+ print(" - {} ... SKIPPED".format(name2))
+ f = Div(text="""Chart is disabled.
+ Use parameter <strong>--full</strong>
+ to enabled this graph.""")
+ else:
+ print(" - {} ...".format(name2))
+ f = fn(report)
+ tabs2.append(Panel(child=f, title=name2))
+ tabs.append(Panel(child=Tabs(tabs=tabs2), title=name))
+
+ print("Saving results ...")
+ main = Tabs(tabs=tabs)
save(main)
diff --git a/python/loom/lore/lore.py b/python/loom/lore/lore.py
index f5982bf222141a255d258722b01dfe9b83a864d9..c629929e929a68c0494dab26ff085b9a38735bb5 100644
--- a/python/loom/lore/lore.py
+++ b/python/loom/lore/lore.py
@@ -15,6 +15,8 @@ def parse_args():
metavar="TRACE_PATH",
type=str,
help="Path to report")
+ parser.add_argument("--full", action="store_true",
+ help="full report")
return parser.parse_args()
@@ -25,7 +27,7 @@ def main():
sys.stderr.write("Directory '{}' not found\n".format(args.trace_path))
report = Report(args.trace_path)
- html.create_html(report, "output.html")
+ html.create_html(report, "output.html", args.full)
if __name__ == "__main__":
diff --git a/python/loom/lore/report.py b/python/loom/lore/report.py
index d8a0650432d51c2a492a3ce94ed9db90924819a2..f07501741a23eca133944bd90074dd376dd73263 100644
--- a/python/loom/lore/report.py
+++ b/python/loom/lore/report.py
@@ -100,6 +100,11 @@ class Report:
group_names.sort()
self.group_names = group_names
+ self.end_time = max(task_frame["start_time"].max(),
+ task_frame["end_time"].max(),
+ max(w.monitoring["time"].max()
+ for w in self.workers.values()))
+
@property
def worker_list(self):
result = list(self.workers.values())