From c6bc586172f849ddaf3ea1a7a74bc4e3dfb71595 Mon Sep 17 00:00:00 2001
From: Jan Kozusznik <jan@kozusznik.cz>
Date: Mon, 15 Jan 2018 14:32:21 +0100
Subject: [PATCH] Refactoring for benchmark running
---
.../it4i/fiji/haas/BenchmarkJobManager.java | 28 ++++++++----------
.../src/main/java/cz/it4i/fiji/haas/Job.java | 29 +++++++++++++++----
.../java/cz/it4i/fiji/haas/JobManager.java | 29 +++++++++++++++++--
3 files changed, 62 insertions(+), 24 deletions(-)
diff --git a/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/BenchmarkJobManager.java b/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/BenchmarkJobManager.java
index c255015e..69a3fd25 100644
--- a/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/BenchmarkJobManager.java
+++ b/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/BenchmarkJobManager.java
@@ -3,6 +3,8 @@ package cz.it4i.fiji.haas;
import java.io.IOException;
import java.nio.file.Path;
import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
import cz.it4i.fiji.haas.JobManager.JobInfo;
import cz.it4i.fiji.haas_java_client.HaaSClient;
@@ -12,37 +14,31 @@ import net.imagej.updater.util.Progress;
public class BenchmarkJobManager {
private JobManager jobManager;
private Progress progress;
- private Path workDirectory;
+ private Map<JobInfo, Path> uploadedFiles = new HashMap<>();
-
public BenchmarkJobManager(Path workDirectory, Progress progress) throws IOException {
- this.workDirectory = workDirectory;
jobManager = new JobManager(workDirectory, TestingConstants.getSettings(3, 6));
this.progress = progress;
}
- public JobInfo startJob() throws IOException {
-
- JobInfo jobInfo = jobManager.startJob(Arrays.asList(getUploadingFile()).stream(), progress);
- jobInfo.waitForStart();
- if (jobInfo.getState() != JobState.Running) {
- throw new IllegalStateException("start of job: " + jobInfo + " failed");
- }
+ public JobInfo createJob() throws IOException {
+ JobInfo jobInfo = jobManager.createJob(progress);
+ Path file = jobInfo.storeDataInWorkdirectory(getUploadingFile());
+ uploadedFiles.put(jobInfo, file);
return jobInfo;
}
- private HaaSClient.UploadingFile getUploadingFile() {
- return new UploadingFileFromResource("", "config.yaml");
+ public void startJob(JobInfo jobInfo) {
+ jobInfo.uploadFiles(Arrays.asList(HaaSClient.getUploadingFile(uploadedFiles.get(jobInfo))).stream());
+ jobInfo.submit();
}
public JobState getState(long jobId) {
return jobManager.getState(jobId);
}
-
-
- private byte[] updateConfigFile(byte[] data) throws IOException {
- return data;
+ private HaaSClient.UploadingFile getUploadingFile() {
+ return new UploadingFileFromResource("", "config.yaml");
}
}
diff --git a/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/Job.java b/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/Job.java
index 31f71bc7..bdfb5698 100644
--- a/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/Job.java
+++ b/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/Job.java
@@ -44,6 +44,7 @@ public class Job {
private Boolean needsDownload;
private JobInfo jobInfo;
private Long jobId;
+ private ProgressNotifier notifier;
final private Progress dummy = new Progress() {
@@ -72,12 +73,11 @@ public class Job {
}
};
- public Job(Path basePath, Stream<UploadingFile> files, Supplier<HaaSClient> haasClientSupplier, Progress progress)
- throws IOException {
+ public Job(Path basePath, Supplier<HaaSClient> haasClientSupplier, Progress progress) throws IOException {
this(haasClientSupplier);
HaaSClient client = this.haasClientSupplier.get();
- long id = client.start(files, "TestOutRedirect", Collections.emptyList(),
- new P_ProgressNotifierAdapter(progress));
+ long id = client.createJob("TestOutRedirect", Collections.emptyList(),
+ notifier = new P_ProgressNotifierAdapter(progress));
jobDir = basePath.resolve("" + id);
Files.createDirectory(jobDir);
updateState();
@@ -89,6 +89,16 @@ public class Job {
loadJobInfo();
}
+ public void uploadFiles(Stream<UploadingFile> files) {
+ HaaSClient client = this.haasClientSupplier.get();
+ client.uploadFiles(jobId, files, notifier);
+ }
+
+ public void submit() {
+ HaaSClient client = this.haasClientSupplier.get();
+ client.submitJob(jobId, notifier);
+ }
+
private Job(Supplier<HaaSClient> haasClientSupplier) {
this.haasClientSupplier = haasClientSupplier;
}
@@ -116,6 +126,14 @@ public class Job {
public void download() {
download(dummy);
}
+
+ public Path storeDataInWorkdirectory(UploadingFile uploadingFile) throws IOException {
+ Path result;
+ try(InputStream is = uploadingFile.getInputStream()) {
+ Files.copy(is, result = jobDir.resolve(uploadingFile.getName()));
+ }
+ return result;
+ }
synchronized public void download(Progress progress) {
if (!needsDownload()) {
@@ -129,7 +147,7 @@ public class Job {
log.error(e);
}
}
-
+
public JobState getState() {
return state;
}
@@ -218,5 +236,4 @@ public class Job {
}
}
-
}
diff --git a/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/JobManager.java b/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/JobManager.java
index 27b3d8ca..1af11efc 100644
--- a/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/JobManager.java
+++ b/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/JobManager.java
@@ -45,9 +45,9 @@ public class JobManager {
}
- public JobInfo startJob(Stream<UploadingFile> files, Progress progress) throws IOException {
+ public JobInfo createJob(Progress progress) throws IOException {
Job job;
- jobs.add(job = new Job(workDirectory, files, this::getHaasClient, progress));
+ jobs.add(job = new Job(workDirectory, this::getHaasClient, progress));
return new JobInfo(job) {
@Override
public JobState getState() {
@@ -60,6 +60,14 @@ public class JobManager {
}
};
}
+
+ public JobInfo startJob(Stream<UploadingFile> files, Progress progress) throws IOException {
+ JobInfo result = createJob(progress);
+ result.uploadFiles(files);
+ result.submit();
+ return result;
+ }
+
public Iterable<JobInfo> getJobsNeedingDownload() {
return () -> jobs.stream().filter(j -> j.needsDownload()).map(j -> new JobInfo(j)).iterator();
@@ -95,6 +103,16 @@ public class JobManager {
this.job = job;
}
+
+ public void uploadFiles(Stream<UploadingFile> files) {
+ job.uploadFiles(files);
+ }
+
+ public void submit() {
+ job.submit();
+ }
+
+
public Long getId() {
return job.getJobId();
}
@@ -142,6 +160,13 @@ public class JobManager {
return time != null ? time.getTime().toString() : "N/A";
}
+
+ public Path storeDataInWorkdirectory(UploadingFile uploadingFile) throws IOException {
+ return job.storeDataInWorkdirectory(uploadingFile);
+ }
+
}
+
+
}
--
GitLab