Skip to content
Snippets Groups Projects
Commit 7e880516 authored by Jan Kožusznik's avatar Jan Kožusznik
Browse files

store outfiles to properties

parent 8964d603
No related branches found
No related tags found
No related merge requests found
package cz.it4i.fiji.haas;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Collection;
......@@ -9,6 +10,8 @@ import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.yaml.snakeyaml.Yaml;
import cz.it4i.fiji.haas.JobManager.JobInfo;
import cz.it4i.fiji.haas.JobManager.JobSynchronizableFile;
import cz.it4i.fiji.haas_java_client.HaaSClient;
......@@ -32,10 +35,12 @@ public class BenchmarkJobManager {
return indexJob(jobInfo);
}
public void startJob(long jobId) {
public void startJob(long jobId) throws IOException {
JobInfo jobInfo = jobs.get(jobId);
jobInfo.uploadFilesByName(() -> Arrays.asList(CONFIG_YAML).stream());
String outputName = getOutputName(jobInfo.openLocalFile(CONFIG_YAML));
jobInfo.submit();
jobInfo.setProperty("spim.outputFilenamePattern", outputName);
}
......@@ -66,6 +71,21 @@ public class BenchmarkJobManager {
return jobInfo.getId();
}
@SuppressWarnings("rawtypes")
private String getOutputName(InputStream openLocalFile) throws IOException {
try(InputStream is = openLocalFile){
Yaml yaml = new Yaml();
Map map = yaml.load(is);
String result = (String) ((Map)map.get("common")).get("hdf5_xml_filename");
if(result == null) {
throw new IllegalArgumentException("hdf5_xml_filename not found");
}
return result;
}
}
......
......@@ -23,7 +23,6 @@ import cz.it4i.fiji.haas_java_client.HaaSClient.UploadingFile;
import cz.it4i.fiji.haas_java_client.JobInfo;
import cz.it4i.fiji.haas_java_client.JobState;
import cz.it4i.fiji.haas_java_client.ProgressNotifier;
import cz.it4i.fiji.haas_java_client.SynchronizableFileType;
import net.imagej.updater.util.Progress;
public class Job {
......@@ -176,27 +175,56 @@ public class Job {
return jobInfo.getEndTime();
}
public Iterable<String> getOutput(Iterable<JobSynchronizableFile> output) {
HaaSClient.SynchronizableFiles taskFileOffset = new HaaSClient.SynchronizableFiles();
long taskId = (Long) getJobInfo().getTasks().toArray()[0];
output.forEach(file->taskFileOffset.addFile(taskId, file.getType(), file.getOffset()));
return haasClientSupplier.get().downloadPartsOfJobFiles(jobId, taskFileOffset).stream().map(f -> f.getContent())
.collect(Collectors.toList());
}
public InputStream openLocalFile(String name) throws IOException {
return Files.newInputStream(jobDir.resolve(name));
}
public void setProperty(String name, String value) throws IOException {
Properties prop = loadPropertiesIfExists();
prop.setProperty(name, value);
}
private synchronized void saveJobinfo() throws IOException {
Properties prop = loadPropertiesIfExists();
if (needsDownload != null) {
prop.setProperty(JOB_HAS_DATA_TO_DOWNLOAD_PROPERTY, needsDownload.toString());
}
prop.setProperty(JOB_NAME, name);
storeProperties(prop);
}
private void storeProperties(Properties prop) throws IOException {
try (OutputStream ow = Files.newOutputStream(jobDir.resolve(JOB_INFO_FILE),
StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE)) {
Properties prop = new Properties();
if (needsDownload != null) {
prop.setProperty(JOB_HAS_DATA_TO_DOWNLOAD_PROPERTY, needsDownload.toString());
}
prop.setProperty(JOB_NAME, name);
prop.store(ow, null);
}
}
private synchronized void loadJobInfo() throws IOException {
try (InputStream is = Files.newInputStream(jobDir.resolve(JOB_INFO_FILE))) {
Properties prop = new Properties();
prop.load(is);
if (prop.containsKey(JOB_HAS_DATA_TO_DOWNLOAD_PROPERTY)) {
needsDownload = Boolean.parseBoolean(prop.getProperty(JOB_HAS_DATA_TO_DOWNLOAD_PROPERTY));
name = prop.getProperty(JOB_NAME);
Properties prop = loadPropertiesIfExists();
if (prop.containsKey(JOB_HAS_DATA_TO_DOWNLOAD_PROPERTY)) {
needsDownload = Boolean.parseBoolean(prop.getProperty(JOB_HAS_DATA_TO_DOWNLOAD_PROPERTY));
name = prop.getProperty(JOB_NAME);
}
}
private Properties loadPropertiesIfExists() throws IOException {
Properties prop = new Properties();
if(Files.exists(jobDir.resolve(JOB_INFO_FILE))) {
try (InputStream is = Files.newInputStream(jobDir.resolve(JOB_INFO_FILE))) {
prop.load(is);
}
}
return prop;
}
private JobInfo getJobInfo() {
......@@ -258,12 +286,4 @@ public class Job {
}
public Iterable<String> getOutput(Iterable<JobSynchronizableFile> output) {
HaaSClient.SynchronizableFiles taskFileOffset = new HaaSClient.SynchronizableFiles();
long taskId = (Long) getJobInfo().getTasks().toArray()[0];
output.forEach(file->taskFileOffset.addFile(taskId, file.getType(), file.getOffset()));
return haasClientSupplier.get().downloadPartsOfJobFiles(jobId, taskFileOffset).stream().map(f -> f.getContent())
.collect(Collectors.toList());
}
}
package cz.it4i.fiji.haas;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Calendar;
......@@ -189,6 +190,15 @@ public class JobManager {
return time != null ? time.getTime().toString() : "N/A";
}
public InputStream openLocalFile(String name) throws IOException {
return job.openLocalFile(name);
}
public void setProperty(String name, String value) throws IOException {
job.setProperty(name, value);
}
}
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment