diff --git a/.gitignore b/.gitignore
index e00ec70b1681e8e921195d51802b1a297943e2fc..af76913d69121f6761533b4e5df9ad850f488ba2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,3 +2,7 @@
.classpath
.settings/
target/
+
+*.bench
+
+benchmark-collector/build\.sh
diff --git a/benchmark-collector/.gitignore b/benchmark-collector/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..ae3c1726048cd06b9a143e0376ed46dd9b9a8d53
--- /dev/null
+++ b/benchmark-collector/.gitignore
@@ -0,0 +1 @@
+/bin/
diff --git a/benchmark-collector/pom.xml b/benchmark-collector/pom.xml
new file mode 100644
index 0000000000000000000000000000000000000000..beadd8d8f330c1d6550259ecd32826110b2f817b
--- /dev/null
+++ b/benchmark-collector/pom.xml
@@ -0,0 +1,42 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>it4i-fiji</groupId>
+ <artifactId>benchmark-collector</artifactId>
+ <version>0.0.1-SNAPSHOT</version>
+ <build>
+ <sourceDirectory>src</sourceDirectory>
+ <plugins>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.7.0</version>
+ <configuration>
+ <source>1.8</source>
+ <target>1.8</target>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>3.1.0</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <transformers>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass>it4i.fiji.benchmark_collector.Run</mainClass>
+ </transformer>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+
+ </build>
+</project>
\ No newline at end of file
diff --git a/benchmark-collector/src/it4i/fiji/benchmark_collector/Run.java b/benchmark-collector/src/it4i/fiji/benchmark_collector/Run.java
new file mode 100644
index 0000000000000000000000000000000000000000..6b70fb909f8640b77ecba16829a66fcd2571b489
--- /dev/null
+++ b/benchmark-collector/src/it4i/fiji/benchmark_collector/Run.java
@@ -0,0 +1,55 @@
+package it4i.fiji.benchmark_collector;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+public class Run {
+ public static void main(String[] args) throws IOException {
+
+ try (DirectoryStream<Path> stream = Files.newDirectoryStream(Paths.get(args[0]), "snakejob.*.*.sh.e*")) {
+ Pattern p = Pattern.compile("snakejob[\\.]([^\\.]+)[\\.][0-9]+[\\.]sh[\\.]e([0-9]*)");
+ Map<String, List<String>> taskName2Jobsid = new HashMap<>();
+ for (Path file : stream) {
+ String fileName = file.getFileName().toString();
+ Matcher m = p.matcher(fileName);
+ m.matches();
+ String taskName = m.group(1);
+ String jobId = m.group(2);
+ List<String> jobIds = taskName2Jobsid.computeIfAbsent(taskName, k -> new ArrayList<String>());
+ jobIds.add(jobId);
+ }
+ try (BufferedWriter pw = Files.newBufferedWriter(Paths.get(args[1]))) {
+ for (Entry<String, List<String>> entry : taskName2Jobsid.entrySet()) {
+ pw.write("Task name: " + entry.getKey() + "\n");
+ pw.write("Jobs count: " + entry.getValue().size() + "\n");
+ pw.write("Jobs: " + entry.getValue().stream().collect(Collectors.joining(", ")) + "\n");
+ for (String jobId : entry.getValue()) {
+ Process proc = new ProcessBuilder("qstat", "-xf", jobId).start();
+ try (BufferedReader br = new BufferedReader(new InputStreamReader(proc.getInputStream()))) {
+ String line;
+ while (null != (line = br.readLine())) {
+ pw.write(line);
+ pw.newLine();
+ }
+ }
+ }
+
+ }
+ }
+ }
+ }
+}
diff --git a/benchmark-collector/src/it4i/fiji/benchmark_collector/package-info.java b/benchmark-collector/src/it4i/fiji/benchmark_collector/package-info.java
new file mode 100644
index 0000000000000000000000000000000000000000..724dfa88663e9029db099778eac3f92cccfe9a76
--- /dev/null
+++ b/benchmark-collector/src/it4i/fiji/benchmark_collector/package-info.java
@@ -0,0 +1,8 @@
+/**
+ *
+ */
+/**
+ * @author koz01
+ *
+ */
+package it4i.fiji.benchmark_collector;
\ No newline at end of file
diff --git a/benchmark-collector/src/it4i/fiji/benchmark_extractor/Extractor.java b/benchmark-collector/src/it4i/fiji/benchmark_extractor/Extractor.java
new file mode 100644
index 0000000000000000000000000000000000000000..9b9917692d3446e1bb74947a48629294981a455e
--- /dev/null
+++ b/benchmark-collector/src/it4i/fiji/benchmark_extractor/Extractor.java
@@ -0,0 +1,181 @@
+package it4i.fiji.benchmark_extractor;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.text.NumberFormat;
+import java.time.Duration;
+import java.time.LocalTime;
+import java.time.format.DateTimeFormatter;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+
+public class Extractor {
+
+ private static Logger log = Logger.getLogger(it4i.fiji.benchmark_extractor.Extractor.class.getName());
+
+ private Set<String> valuesToExport;
+ private Path inputFile;
+ private OutputStream out;
+
+ private Map<String, String> valueWithTypes;
+
+ public Extractor(Path inputFile, Set<String> valuesToExport, Map<String, String> valuesWithTypes, OutputStream out) {
+ this.inputFile = inputFile;
+ this.valuesToExport = valuesToExport;
+ this.valueWithTypes = valuesWithTypes;
+ this.out = out;
+ }
+
+ public void doExtraction() {
+ PrintWriter pw = new PrintWriter(out);
+ try (BufferedReader br = Files.newBufferedReader(inputFile)) {
+ String line;
+ P_Collector collector = new EmptyP_Collector();
+ P_ValueCollector valueCollector = new P_ValueCollector();
+ while (null != (line = br.readLine())) {
+ line = line.trim();
+ if (line.isEmpty()) {
+ continue;
+ }
+ if (line.startsWith("Task name:")) {
+ valueCollector.flush(collector);
+ collector.writeToOuput(pw);
+ collector = new P_Collector(line.split(":")[1].trim());
+ } else if (line.startsWith("Job Id:")) {
+ collector.addJob(line.split(":")[1].trim());
+ } else {
+ if (line.contains(" = ")) {
+ String[] tokens = line.split(" = ");
+ valueCollector.startNew(collector, tokens[0], tokens[1]);
+ } else {
+ valueCollector.append(line);
+ }
+ }
+ }
+ } catch (IOException e) {
+ log.log(Level.SEVERE, e.getMessage(), e);
+ } finally {
+ pw.flush();
+ }
+ }
+
+ private class P_Collector {
+ private List<String> ids = new LinkedList<>();
+ private Map<String, List<String>> values4Job = new HashMap<>();
+
+ private String taskName;
+
+ public P_Collector(String taskName) {
+ super();
+ this.taskName = taskName;
+ }
+
+ public void addJob(String jobId) {
+ ids.add(jobId);
+ }
+
+ public void addValue(String key, String value) {
+ List<String> values = values4Job.computeIfAbsent(key, this::factoryForMap);
+ values.add(value);
+ }
+
+ public void writeToOuput(PrintWriter out) {
+ addNullForMissingValues();
+ out.printf("Task name;%s\n", taskName);
+ out.printf("jobs #;%d\n", ids.size());
+ out.printf("job ids;%s\n", String.join(";", ids));
+ for (String key : valuesToExport) {
+ out.printf("%s;%s\n", key, String.join(";", convert(key,values4Job.get(key))));
+ }
+ }
+
+ private List<String> convert(String key, List<String> list) {
+ if(!valueWithTypes.containsKey(key)) {
+ return list;
+ }
+ Function<String, String> conversion = getConversion(valueWithTypes.get(key));
+ return list.stream().map(conversion).collect(Collectors.toList());
+ }
+
+ private Function<String, String> getConversion(String format) {
+ Locale l =new Locale("cs");
+ NumberFormat nf = NumberFormat.getNumberInstance(l);
+ switch(format) {
+ case "kb":
+ return str->nf.format(Double.parseDouble(str.replace("kb", ""))/1024.);
+ case "tm":
+ return str-> Duration.between(LocalTime.of(0, 0, 0), LocalTime.parse(str, DateTimeFormatter.ofPattern("H:m:s"))).getSeconds() + "";
+ }
+
+ return str->str;
+
+ }
+
+ private void addNullForMissingValues() {
+ for (String key : valuesToExport) {
+ List<String> values = values4Job.computeIfAbsent(key, this::factoryForMap);
+ if (values.size() < ids.size()) {
+ values.add(null);
+ }
+ }
+
+ }
+
+ private List<String> factoryForMap(String key) {
+ return new LinkedList<>();
+ }
+ }
+
+ private class EmptyP_Collector extends P_Collector {
+
+ public EmptyP_Collector() {
+ super(null);
+ }
+
+ @Override
+ public void addValue(String key, String value) {
+ }
+
+ @Override
+ public void writeToOuput(PrintWriter out) {
+ }
+
+ }
+
+ private class P_ValueCollector {
+ private String key;
+ private StringBuilder value = new StringBuilder();
+
+ public void flush(P_Collector collector) {
+ if (key != null) {
+ collector.addValue(key, value.toString());
+ key = null;
+ value.setLength(0);
+ }
+ }
+
+ public void append(String line) {
+ if (key != null) {
+ this.value.append(line);
+ }
+ }
+
+ public void startNew(P_Collector collector, String key, String value) {
+ flush(collector);
+ this.key = key;
+ this.value.append(value);
+ }
+ }
+}
diff --git a/benchmark-collector/src/it4i/fiji/benchmark_extractor/Run.java b/benchmark-collector/src/it4i/fiji/benchmark_extractor/Run.java
new file mode 100644
index 0000000000000000000000000000000000000000..7ab44bcb2e6cc6b6437b99e49c2c14109ac0c612
--- /dev/null
+++ b/benchmark-collector/src/it4i/fiji/benchmark_extractor/Run.java
@@ -0,0 +1,37 @@
+package it4i.fiji.benchmark_extractor;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.LinkedHashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class Run {
+
+ public static void main(String[] args) {
+ Path outputFile = Paths.get(args[0]);
+ List<String> arguments = getAsSublist(args,1, args.length);
+ Set<String> valuesToExport = new LinkedHashSet<>(arguments
+ .stream()
+ .map(str->str.contains(":")?str.split(":")[0]:str)
+ .collect(Collectors.toList()));
+ Map<String,String> valuesWithTypes = arguments
+ .stream()
+ .filter(str->str.contains(":"))
+ .map(str->str.split(":"))
+ .collect(Collectors.toMap(pair->pair[0], pair->pair[1]));
+ new Extractor(outputFile, valuesToExport,valuesWithTypes,System.out).doExtraction();
+ }
+
+ private static List<String> getAsSublist(String[] args, int start, int end) {
+ List<String> result = new LinkedList<>();
+ for(int i = start; i <end; i++ ) {
+ result.add(args[i]);
+ }
+ return result;
+ }
+
+}
diff --git a/benchmark-collector/src/it4i/fiji/benchmark_extractor/package-info.java b/benchmark-collector/src/it4i/fiji/benchmark_extractor/package-info.java
new file mode 100644
index 0000000000000000000000000000000000000000..753616504c00108c141df478c7a58b7725225f73
--- /dev/null
+++ b/benchmark-collector/src/it4i/fiji/benchmark_extractor/package-info.java
@@ -0,0 +1,8 @@
+/**
+ *
+ */
+/**
+ * @author koz01
+ *
+ */
+package it4i.fiji.benchmark_extractor;
\ No newline at end of file