benchmark collector

.gitignore

@@ -2,3 +2,7 @@
 .classpath
 .settings/
 target/
+
+*.bench
+
+benchmark-collector/build\.sh

benchmark-collector/.gitignore

+/bin/

benchmark-collector/pom.xml

+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<groupId>it4i-fiji</groupId>
+	<artifactId>benchmark-collector</artifactId>
+	<version>0.0.1-SNAPSHOT</version>
+	<build>
+		<sourceDirectory>src</sourceDirectory>
+		<plugins>
+			<plugin>
+				<artifactId>maven-compiler-plugin</artifactId>
+				<version>3.7.0</version>
+				<configuration>
+					<source>1.8</source>
+					<target>1.8</target>
+				</configuration>
+			</plugin>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-shade-plugin</artifactId>
+				<version>3.1.0</version>
+				<executions>
+					<execution>
+						<phase>package</phase>
+						<goals>
+							<goal>shade</goal>
+						</goals>
+						<configuration>
+							<transformers>
+								<transformer
+									implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+									<mainClass>it4i.fiji.benchmark_collector.Run</mainClass>
+								</transformer>
+							</transformers>
+						</configuration>
+					</execution>
+				</executions>
+			</plugin>
+		</plugins>
+
+	</build>
+</project>
\ No newline at end of file

benchmark-collector/src/it4i/fiji/benchmark_collector/Run.java

+package it4i.fiji.benchmark_collector;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+public class Run {
+	public static void main(String[] args) throws IOException {
+
+		try (DirectoryStream<Path> stream = Files.newDirectoryStream(Paths.get(args[0]), "snakejob.*.*.sh.e*")) {
+			Pattern p = Pattern.compile("snakejob[\\.]([^\\.]+)[\\.][0-9]+[\\.]sh[\\.]e([0-9]*)");
+			Map<String, List<String>> taskName2Jobsid = new HashMap<>();
+			for (Path file : stream) {
+				String fileName = file.getFileName().toString();
+				Matcher m = p.matcher(fileName);
+				m.matches();
+				String taskName = m.group(1);
+				String jobId = m.group(2);
+				List<String> jobIds = taskName2Jobsid.computeIfAbsent(taskName, k -> new ArrayList<String>());
+				jobIds.add(jobId);
+			}
+			try (BufferedWriter pw = Files.newBufferedWriter(Paths.get(args[1]))) {
+				for (Entry<String, List<String>> entry : taskName2Jobsid.entrySet()) {
+					pw.write("Task name: " + entry.getKey() + "\n");
+					pw.write("Jobs count: " + entry.getValue().size() + "\n");
+					pw.write("Jobs: " + entry.getValue().stream().collect(Collectors.joining(", ")) + "\n");
+					for (String jobId : entry.getValue()) {
+						Process proc = new ProcessBuilder("qstat", "-xf", jobId).start();
+						try (BufferedReader br = new BufferedReader(new InputStreamReader(proc.getInputStream()))) {
+							String line;
+							while (null != (line = br.readLine())) {
+								pw.write(line);
+								pw.newLine();
+							}
+						}
+					}
+
+				}
+			}
+		}
+	}
+}

benchmark-collector/src/it4i/fiji/benchmark_collector/package-info.java

+/**
+ * 
+ */
+/**
+ * @author koz01
+ *
+ */
+package it4i.fiji.benchmark_collector;
\ No newline at end of file

benchmark-collector/src/it4i/fiji/benchmark_extractor/Extractor.java

+package it4i.fiji.benchmark_extractor;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.text.NumberFormat;
+import java.time.Duration;
+import java.time.LocalTime;
+import java.time.format.DateTimeFormatter;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+
+public class Extractor {
+
+	private static Logger log = Logger.getLogger(it4i.fiji.benchmark_extractor.Extractor.class.getName());
+
+	private Set<String> valuesToExport;
+	private Path inputFile;
+	private OutputStream out;
+
+	private Map<String, String> valueWithTypes;
+
+	public Extractor(Path inputFile, Set<String> valuesToExport, Map<String, String> valuesWithTypes, OutputStream out) {
+		this.inputFile = inputFile;
+		this.valuesToExport = valuesToExport;
+		this.valueWithTypes = valuesWithTypes;
+		this.out = out;
+	}
+
+	public void doExtraction() {
+		PrintWriter pw = new PrintWriter(out);
+		try (BufferedReader br = Files.newBufferedReader(inputFile)) {
+			String line;
+			P_Collector collector = new EmptyP_Collector();
+			P_ValueCollector valueCollector = new P_ValueCollector();
+			while (null != (line = br.readLine())) {
+				line = line.trim();
+				if (line.isEmpty()) {
+					continue;
+				}
+				if (line.startsWith("Task name:")) {
+					valueCollector.flush(collector);
+					collector.writeToOuput(pw);
+					collector = new P_Collector(line.split(":")[1].trim());
+				} else if (line.startsWith("Job Id:")) {
+					collector.addJob(line.split(":")[1].trim());
+				} else {
+					if (line.contains(" = ")) {
+						String[] tokens = line.split(" = ");
+						valueCollector.startNew(collector, tokens[0], tokens[1]);
+					} else {
+						valueCollector.append(line);
+					}
+				}
+			}
+		} catch (IOException e) {
+			log.log(Level.SEVERE, e.getMessage(), e);
+		} finally {
+			pw.flush();
+		}
+	}
+
+	private class P_Collector {
+		private List<String> ids = new LinkedList<>();
+		private Map<String, List<String>> values4Job = new HashMap<>();
+
+		private String taskName;
+
+		public P_Collector(String taskName) {
+			super();
+			this.taskName = taskName;
+		}
+
+		public void addJob(String jobId) {
+			ids.add(jobId);
+		}
+
+		public void addValue(String key, String value) {
+			List<String> values = values4Job.computeIfAbsent(key, this::factoryForMap);
+			values.add(value);
+		}
+
+		public void writeToOuput(PrintWriter out) {
+			addNullForMissingValues();
+			out.printf("Task name;%s\n", taskName);
+			out.printf("jobs #;%d\n", ids.size());
+			out.printf("job ids;%s\n", String.join(";", ids));
+			for (String key : valuesToExport) {
+				out.printf("%s;%s\n", key, String.join(";", convert(key,values4Job.get(key))));
+			}
+		}
+
+		private List<String> convert(String key, List<String> list) {
+			if(!valueWithTypes.containsKey(key)) {
+				return list;
+			}
+			Function<String, String> conversion = getConversion(valueWithTypes.get(key));
+			return list.stream().map(conversion).collect(Collectors.toList());
+		}
+
+		private Function<String, String> getConversion(String format) {
+			Locale l =new Locale("cs");
+			NumberFormat nf = NumberFormat.getNumberInstance(l);
+			switch(format) {
+			case "kb":
+				return str->nf.format(Double.parseDouble(str.replace("kb", ""))/1024.);
+			case "tm":
+				return str-> Duration.between(LocalTime.of(0, 0, 0), LocalTime.parse(str, DateTimeFormatter.ofPattern("H:m:s"))).getSeconds() + "";
+			}
+			
+			return str->str;
+
+		}
+
+		private void addNullForMissingValues() {
+			for (String key : valuesToExport) {
+				List<String> values = values4Job.computeIfAbsent(key, this::factoryForMap);
+				if (values.size() < ids.size()) {
+					values.add(null);
+				}
+			}
+
+		}
+
+		private List<String> factoryForMap(String key) {
+			return new LinkedList<>();
+		}
+	}
+
+	private class EmptyP_Collector extends P_Collector {
+
+		public EmptyP_Collector() {
+			super(null);
+		}
+
+		@Override
+		public void addValue(String key, String value) {
+		}
+
+		@Override
+		public void writeToOuput(PrintWriter out) {
+		}
+
+	}
+
+	private class P_ValueCollector {
+		private String key;
+		private StringBuilder value = new StringBuilder();
+
+		public void flush(P_Collector collector) {
+			if (key != null) {
+				collector.addValue(key, value.toString());
+				key = null;
+				value.setLength(0);
+			}
+		}
+
+		public void append(String line) {
+			if (key != null) {
+				this.value.append(line);
+			}
+		}
+
+		public void startNew(P_Collector collector, String key, String value) {
+			flush(collector);
+			this.key = key;
+			this.value.append(value);
+		}
+	}
+}

benchmark-collector/src/it4i/fiji/benchmark_extractor/Run.java

+package it4i.fiji.benchmark_extractor;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.LinkedHashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class Run {
+
+	public static void main(String[] args) {
+		Path outputFile = Paths.get(args[0]);
+		List<String> arguments = getAsSublist(args,1, args.length);
+		Set<String> valuesToExport = new LinkedHashSet<>(arguments
+				                                         .stream()
+				                                         .map(str->str.contains(":")?str.split(":")[0]:str)
+				                                         .collect(Collectors.toList()));
+		Map<String,String> valuesWithTypes = arguments
+				                             .stream()
+				                             .filter(str->str.contains(":"))
+				                             .map(str->str.split(":"))
+				                             .collect(Collectors.toMap(pair->pair[0], pair->pair[1]));
+		new Extractor(outputFile, valuesToExport,valuesWithTypes,System.out).doExtraction();
+	}
+
+	private static List<String> getAsSublist(String[] args, int start, int end) {
+		List<String> result = new LinkedList<>();
+		for(int i = start; i <end; i++ ) {
+			result.add(args[i]);
+		}
+		return result;
+	}
+
+}

benchmark-collector/src/it4i/fiji/benchmark_extractor/package-info.java

+/**
+ * 
+ */
+/**
+ * @author koz01
+ *
+ */
+package it4i.fiji.benchmark_extractor;
\ No newline at end of file