Skip to content
Snippets Groups Projects
Commit 8c4343a9 authored by Ada Böhm's avatar Ada Böhm
Browse files

ENH: Small improvements of loom.lore

parent 68fb647f
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
python/loom/lore/html.py
+ 55
− 45
View file @ 8c4343a9
......@@ -3,7 +3,7 @@ from bokeh.charts import save, output_file, BoxPlot
from bokeh.layouts import column, gridplot
from bokeh.palettes import all_palettes
from bokeh.plotting import figure
from bokeh.models.widgets import Panel, Tabs
from bokeh.models.widgets import Panel, Tabs, Div
from bokeh.models.widgets import DataTable, TableColumn
from bokeh.models import ColumnDataSource
......@@ -20,8 +20,6 @@ def create_colors(count):
def create_timelines(report):
print("\tTimeline")
worker_timelines, group_names = report.get_timelines()
worker_timelines = list(worker_timelines.items())
worker_timelines.sort()
......@@ -34,7 +32,9 @@ def create_timelines(report):
else:
y_labels.append("Worker {}.{}".format(worker_id, i))
f = figure(plot_width=1000, plot_height=1000, y_range=y_labels, webgl=True)
f = figure(plot_width=1000, plot_height=1000, y_range=y_labels,
x_range=[0, report.end_time],
webgl=True)
line_id = 1
......@@ -59,6 +59,7 @@ def create_monitoring(report):
result = []
for worker in report.worker_list:
f = figure(plot_width=1000, plot_height=130,
x_range=[0, report.end_time],
title="Worker {}".format(worker.address))
f.line(worker.monitoring.time, worker.monitoring.cpu,
color="blue", legend="CPU %")
......@@ -72,6 +73,7 @@ def create_transfer(report):
result = []
for worker in report.worker_list:
f = figure(plot_width=1000, plot_height=130,
x_range=[0, report.end_time],
title="Worker {}".format(worker.address))
f.line(worker.sends.time, worker.sends.data_size.cumsum(),
color="green", legend="send")
......@@ -84,10 +86,11 @@ def create_transfer(report):
def create_ctransfer(report):
sends = report.all_sends()
f = figure(plot_width=1000, plot_height=500,
x_range=[0, report.end_time],
title="Cumulative transfers")
sends = sends.join(report.task_frame["group"], on="id")
names = report.group_names
f = figure(plot_width=1000, plot_height=500)
f = figure(plot_width=1000, plot_height=500, x_range=[0, report.end_time])
for color, name in zip(create_colors(len(names)), names):
frame = sends[sends["group"] == name]
frame = frame.sort_values("time")
......@@ -103,6 +106,7 @@ def create_ctime(report):
colors = create_colors(len(names))
f1 = figure(plot_width=1000, plot_height=400,
x_range=[0, report.end_time],
title="Cumulative time of finished tasks")
for name, color in zip(names, colors):
frame = ds[ds["group"] == name]
......@@ -110,6 +114,7 @@ def create_ctime(report):
legend=name, color=color, line_width=2)
f2 = figure(plot_width=1000, plot_height=400,
x_range=[0, report.end_time],
title="Number of finished tasks")
for name, color in zip(names, colors):
......@@ -139,9 +144,9 @@ def create_task_summary(report):
def create_pending_tasks(report):
print("\tPending tasks")
ds = report.get_pending_tasks()
f1 = figure(plot_width=1000, plot_height=400)
f1 = figure(plot_width=1000, plot_height=400,
x_range=[0, report.end_time])
names = report.group_names
for color, name in zip(create_colors(len(names)), names):
frame = ds[ds["group"] == name]
......@@ -154,20 +159,19 @@ def create_pending_tasks(report):
def create_scheduling_time(report):
print("\tScheduling time")
ds = report.scheduler_times
duration = ds["end_time"] - ds["start_time"]
df = pd.DataFrame({"time": ds["end_time"], "duration": duration})
df["label"] = 0
f1 = figure(plot_width=1000, plot_height=400)
f1 = figure(plot_width=1000, plot_height=400,
x_range=[0, report.end_time])
f1.line(df["time"], df["duration"].cumsum())
f2 = BoxPlot(df, values="duration", label="label")
return column([f1, f2])
def create_worker_load(report):
print("\tWorker load")
data = []
for task in report.task_frame.itertuples():
data.append((task.start_time, 1, task.worker))
......@@ -177,6 +181,7 @@ def create_worker_load(report):
result = []
for worker in report.worker_list:
f = figure(plot_width=1000, plot_height=130,
x_range=[0, report.end_time],
title="Worker {}".format(worker.address))
data = frame[frame.worker == worker.worker_id]
f.line(data.time, data.change.cumsum(),
......@@ -191,41 +196,46 @@ def create_task_durations(report):
return f
def create_html(report, filename):
def level_index(level):
return ("brief", "normal", "full").index(level)
def create_html(report, filename, full):
output_file(filename)
print("Task plots")
tasks = Tabs(tabs=[
Panel(child=create_ctime(report), title="Task progress"),
Panel(child=create_task_summary(report), title="Task summary"),
Panel(child=create_task_durations(report), title="Task durations"),
])
print("Monitoring plots")
monitoring = Tabs(tabs=[
Panel(child=create_monitoring(report), title="CPU & Memory usage"),
])
print("Scheduling plots")
scheduling = Tabs(tabs=[
Panel(child=create_timelines(report), title="Timeline"),
Panel(child=create_pending_tasks(report), title="Pending tasks"),
Panel(child=create_scheduling_time(report), title="Scheduling time"),
Panel(child=create_worker_load(report), title="Worker load"),
])
print("Communication plots")
comm = Tabs(tabs=[
Panel(child=create_ctransfer(report), title="Transfer per tasks"),
Panel(child=create_transfer(report), title="Transfer per nodes"),
])
main = Tabs(tabs=[
Panel(child=tasks, title="Tasks"),
Panel(child=monitoring, title="Monitoring"),
Panel(child=scheduling, title="Scheduling"),
Panel(child=comm, title="Communication"),
])
print("Saving result")
structure = (
("Tasks",
[("Task progress", create_ctime, False),
("Task summary", create_task_summary, False),
("Task durations", create_task_durations, False)]),
("Monitoring",
[("CPU & Memory usage", create_monitoring, False)]),
("Scheduling",
[("Timeline", create_timelines, True),
("Pending tasks", create_pending_tasks, False),
("Scheduling time", create_scheduling_time, False),
("Worker load", create_worker_load, False)]),
("Communication",
[("Transfer per tasks", create_ctransfer, False),
("Transfer per nodes", create_transfer, False)])
)
tabs = []
for name, subtabs in structure:
print("Tab:", name)
tabs2 = []
for name2, fn, full_only in subtabs:
if full_only and not full:
print(" - {} ... SKIPPED".format(name2))
f = Div(text="""Chart is disabled.
Use parameter <strong>--full</strong>
to enabled this graph.""")
else:
print(" - {} ...".format(name2))
f = fn(report)
tabs2.append(Panel(child=f, title=name2))
tabs.append(Panel(child=Tabs(tabs=tabs2), title=name))
print("Saving results ...")
main = Tabs(tabs=tabs)
save(main)
python/loom/lore/lore.py
+ 3
− 1
View file @ 8c4343a9
......@@ -15,6 +15,8 @@ def parse_args():
metavar="TRACE_PATH",
type=str,
help="Path to report")
parser.add_argument("--full", action="store_true",
help="full report")
return parser.parse_args()
......@@ -25,7 +27,7 @@ def main():
sys.stderr.write("Directory '{}' not found\n".format(args.trace_path))
report = Report(args.trace_path)
html.create_html(report, "output.html")
html.create_html(report, "output.html", args.full)
if __name__ == "__main__":
......
python/loom/lore/report.py
+ 5
− 0
View file @ 8c4343a9
......@@ -100,6 +100,11 @@ class Report:
group_names.sort()
self.group_names = group_names
self.end_time = max(task_frame["start_time"].max(),
task_frame["end_time"].max(),
max(w.monitoring["time"].max()
for w in self.workers.values()))
@property
def worker_list(self):
result = list(self.workers.values())
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment