From 1d2f382037396ac164e50d9cf398ffc2864de84d Mon Sep 17 00:00:00 2001
From: Unknown <petr.bainar@hotmail.com>
Date: Tue, 23 Jan 2018 15:35:06 +0100
Subject: [PATCH] formatStatistics: implementing csv parse
---
.../core/BenchmarkJobManager.java | 84 +++++++++++++++++++
.../core/ResultFileJob.java | 14 ++++
.../core/ResultFileTask.java | 41 +++++++++
3 files changed, 139 insertions(+)
create mode 100644 haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/core/ResultFileJob.java
create mode 100644 haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/core/ResultFileTask.java
diff --git a/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/core/BenchmarkJobManager.java b/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/core/BenchmarkJobManager.java
index bb403212..5230e846 100644
--- a/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/core/BenchmarkJobManager.java
+++ b/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/core/BenchmarkJobManager.java
@@ -1,13 +1,20 @@
package cz.it4i.fiji.haas_spim_benchmark.core;
+import java.io.BufferedReader;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
+import java.nio.file.Files;
import java.nio.file.InvalidPathException;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.text.NumberFormat;
+import java.text.ParseException;
import java.util.Arrays;
import java.util.Collection;
+import java.util.LinkedList;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.function.Predicate;
import java.util.stream.Collectors;
@@ -68,6 +75,9 @@ public class BenchmarkJobManager {
public void downloadStatistics(Progress progress) throws IOException {
JobInfo ji = jobInfo;
ji.downloadData(BenchmarkJobManager.downloadStatistics(), progress, true);
+ Path resultFile = ji.getDirectory().resolve(Constants.BENCHMARK_RESULT_FILE);
+ if (resultFile != null)
+ BenchmarkJobManager.formatResultFile(resultFile);
}
public List<String> getOutput(List<JobSynchronizableFile> files) {
@@ -218,6 +228,80 @@ public class BenchmarkJobManager {
return null;
}
}
+
+ private static void formatResultFile(Path filename) throws FileNotFoundException {
+
+ LinkedList<ResultFileTask> identifiedTasks = new LinkedList<ResultFileTask>();
+
+ try {
+ String line = null;
+ final String separator = ";";
+
+ ResultFileTask processedTask = null;
+ LinkedList<String> id = new LinkedList<String>();
+ LinkedList<Double> memoryUsed = new LinkedList<Double>();
+ LinkedList<Integer> wallTime = new LinkedList<Integer>();
+ LinkedList<Integer> cpuPercentage = new LinkedList<Integer>();
+
+ BufferedReader reader = Files.newBufferedReader(filename);
+ while (null != (line = reader.readLine())) {
+
+ line = line.trim();
+ if (line.isEmpty()) {
+ continue;
+ }
+
+ String[] columns = line.split(separator);
+
+ if (columns[0].equals("Task name")) {
+
+ if (null != processedTask ) {
+ for (int i = 0; i < id.size(); i++) {
+ processedTask.jobs.add(new ResultFileJob(id.get(i), memoryUsed.get(i), wallTime.get(i), cpuPercentage.get(i)));
+ }
+ identifiedTasks.add(processedTask);
+ }
+
+ processedTask = new ResultFileTask(columns[1]);
+
+ id = new LinkedList<String>();
+ memoryUsed = new LinkedList<Double>();
+ wallTime = new LinkedList<Integer>();
+ cpuPercentage = new LinkedList<Integer>();
+
+ } else if (columns[0].equals("job ids")) {
+ for (int i = 1; i < columns.length; i++) {
+ id.add(columns[i]);
+ }
+ } else if (columns[0].equals("resources_used.mem")) {
+ for (int i = 1; i < columns.length; i++) {
+ Number number = NumberFormat.getInstance(Locale.GERMANY).parse(columns[i]);
+ memoryUsed.add(number.doubleValue());
+ }
+ } else if (columns[0].equals("resources_used.walltime")) {
+ for (int i = 1; i < columns.length; i++) {
+ wallTime.add(Integer.parseInt(columns[i]));
+ }
+ } else if (columns[0].equals("resources_used.cpupercent")) {
+ for (int i = 1; i < columns.length; i++) {
+ cpuPercentage.add(Integer.parseInt(columns[i]));
+ }
+ }
+ }
+
+ if (null != processedTask ) {
+ for (int i = 1; i < id.size(); i++) {
+ processedTask.jobs.add(new ResultFileJob(id.get(i), memoryUsed.get(i), wallTime.get(i), cpuPercentage.get(i)));
+ }
+ identifiedTasks.add(processedTask);
+ }
+
+ } catch (IOException e) {
+ log.error(e.getMessage(), e);
+ } catch (ParseException e) {
+ log.error(e.getMessage(), e);
+ }
+ }
private static Settings constructSettingsFromParams(BenchmarkSPIMParameters params) {
// TODO Auto-generated method stub
diff --git a/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/core/ResultFileJob.java b/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/core/ResultFileJob.java
new file mode 100644
index 00000000..5a17fcad
--- /dev/null
+++ b/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/core/ResultFileJob.java
@@ -0,0 +1,14 @@
+package cz.it4i.fiji.haas_spim_benchmark.core;
+
+public class ResultFileJob {
+ String id;
+ double memoryUsed;
+ int wallTime;
+ int cpuPercentage;
+ public ResultFileJob(String id, double memoryUsed, int wallTime, int cpuPercentage) {
+ this.id = id;
+ this.memoryUsed = memoryUsed;
+ this.wallTime = wallTime;
+ this.cpuPercentage = cpuPercentage;
+ }
+}
\ No newline at end of file
diff --git a/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/core/ResultFileTask.java b/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/core/ResultFileTask.java
new file mode 100644
index 00000000..e52d0f8b
--- /dev/null
+++ b/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/core/ResultFileTask.java
@@ -0,0 +1,41 @@
+package cz.it4i.fiji.haas_spim_benchmark.core;
+
+import java.util.LinkedList;
+
+public class ResultFileTask {
+ String name;
+ LinkedList<ResultFileJob> jobs;
+
+ public ResultFileTask(String name) {
+ this.name = name;
+ this.jobs = new LinkedList<ResultFileJob>();
+ }
+
+ public int getJobCount() {
+ return this.jobs.size();
+ }
+
+ public double getAverageMemoryUsage() {
+ double memoryUsageTotal = 0;
+ for (ResultFileJob job : this.jobs ) {
+ memoryUsageTotal += job.memoryUsed;
+ }
+ return (double)memoryUsageTotal / this.jobs.size();
+ }
+
+ public double getAverageWallTime() {
+ double wallTimeTotal = 0;
+ for (ResultFileJob job : this.jobs ) {
+ wallTimeTotal += job.wallTime;
+ }
+ return (double)wallTimeTotal / this.jobs.size();
+ }
+
+ public double getAverageCpuPercentage() {
+ int cpuPercentageTotal = 0;
+ for (ResultFileJob job : this.jobs ) {
+ cpuPercentageTotal += job.cpuPercentage;
+ }
+ return (double)cpuPercentageTotal / this.jobs.size();
+ }
+}
\ No newline at end of file
--
GitLab