diff --git a/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/Job.java b/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/Job.java
index d192e5cf33de74b6b8a9bd8da0ff4e3a98e3d094..1135b3672141dfb185422b2d35db8b6826d68ef4 100644
--- a/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/Job.java
+++ b/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/Job.java
@@ -21,29 +21,27 @@ import org.slf4j.LoggerFactory;
import cz.it4i.fiji.haas.JobManager.JobManager4Job;
import cz.it4i.fiji.haas.JobManager.JobSynchronizableFile;
import cz.it4i.fiji.haas.data_transfer.Synchronization;
-import cz.it4i.fiji.haas_java_client.DummyProgressNotifier;
import cz.it4i.fiji.haas_java_client.HaaSClient;
import cz.it4i.fiji.haas_java_client.HaaSClient.UploadingFile;
import cz.it4i.fiji.haas_java_client.HaaSFileTransfer;
import cz.it4i.fiji.haas_java_client.JobInfo;
import cz.it4i.fiji.haas_java_client.JobState;
import cz.it4i.fiji.haas_java_client.ProgressNotifier;
+import cz.it4i.fiji.haas_java_client.TransferFileProgressForHaaSClient;
import net.imagej.updater.util.Progress;
public class Job {
private static final String JOB_NAME = "job.name";
-
+
private static final String JOB_NEEDS_UPLOAD = "job.needs_upload";
-
- private static final String JOB_INFO_FILENAME = ".jobinfo";
+ private static final String JOB_INFO_FILENAME = ".jobinfo";
public static boolean isJobPath(Path p) {
return isValidPath(p);
}
-
private static Logger log = LoggerFactory.getLogger(cz.it4i.fiji.haas.Job.class);
private Path jobDir;
@@ -77,14 +75,12 @@ public class Job {
resumeUpload();
}
-
-
private Job(JobManager4Job jobManager, Supplier<HaaSClient> haasClientSupplier) {
this.haasClientSupplier = haasClientSupplier;
this.jobManager = jobManager;
}
- public void startUploadData() {
+ public void startUploadData() {
setProperty(JOB_NEEDS_UPLOAD, true);
try {
this.synchronization.startUpload();
@@ -93,8 +89,8 @@ public class Job {
throw new RuntimeException(e);
}
}
-
- public void stopUploadData() {
+
+ public void stopUploadData() {
setProperty(JOB_NEEDS_UPLOAD, false);
try {
this.synchronization.stopUpload();
@@ -103,15 +99,46 @@ public class Job {
throw new RuntimeException(e);
}
}
-
+
+ public void startDownload(Predicate<String> predicate, Progress notifier) throws IOException {
+ Collection<String> files = getHaaSClient().getChangedFiles(jobId).stream().filter(predicate)
+ .collect(Collectors.toList());
+ synchronization.startDownload(files);
+ }
+
public boolean isUploading() {
return Boolean.parseBoolean(getProperty(JOB_NEEDS_UPLOAD));
}
- public void uploadFile(String file, Progress notifier) {
- Iterable<UploadingFile> uploadingFiles = Arrays.asList(file).stream()
- .map((String name) -> HaaSClient.getUploadingFile(jobDir.resolve(name))).collect(Collectors.toList());
- uploadFiles(uploadingFiles, notifier);
+ public void uploadFile(String file, ProgressNotifier notifier) {
+ uploadFiles(Arrays.asList(file), notifier);
+ }
+
+ public void uploadFiles(Collection<String> filesNames, ProgressNotifier notifier) {
+ Collection<UploadingFile> files = filesNames.stream()
+ .map(file -> HaaSClient.getUploadingFile(jobDir.resolve(file))).collect(Collectors.toList());
+ List<Long> totalSizes = files.stream().map(f -> {
+ try {
+ return f.getLength();
+ } catch (IOException e1) {
+ throw new RuntimeException(e1);
+ }
+ }).collect(Collectors.toList());
+ long totalSize = totalSizes.stream().mapToLong(l -> l.longValue()).sum();
+ TransferFileProgressForHaaSClient progress = new TransferFileProgressForHaaSClient(totalSize, notifier);
+
+ HaaSClient client = getHaaSClient();
+ try (HaaSFileTransfer transfer = client.startFileTransfer(getId(), progress)) {
+ int index = 0;
+ for (UploadingFile file : files) {
+ String item;
+ progress.startNewFile(totalSizes.get(index));
+ notifier.addItem(item = "Uploading file: " + file.getName());
+ transfer.upload(file);
+ notifier.itemDone(item);
+ index++;
+ }
+ }
}
public void submit() {
@@ -126,10 +153,6 @@ public class Job {
return jobId;
}
- public void download(Progress notifier) {
- download(x -> true, notifier);
- }
-
public Path storeDataInWorkdirectory(UploadingFile uploadingFile) throws IOException {
Path result;
try (InputStream is = uploadingFile.getInputStream()) {
@@ -138,12 +161,22 @@ public class Job {
return result;
}
- synchronized public void download(Predicate<String> predicate, Progress notifier) {
- try (HaaSFileTransfer fileTransfer = getHaaSClient().startFileTransfer(jobId,
- new P_ProgressNotifierAdapter(notifier))) {
- fileTransfer.download(
- getHaaSClient().getChangedFiles(jobId).stream().filter(predicate).collect(Collectors.toList()),
- jobDir);
+ synchronized public void download(Predicate<String> predicate, ProgressNotifier notifier) {
+ List<String> files = getHaaSClient().getChangedFiles(jobId).stream().filter(predicate).collect(Collectors.toList());
+ try (HaaSFileTransfer transfer = haasClientSupplier.get().startFileTransfer(getId(), HaaSClient.DUMMY_TRANSFER_FILE_PROGRESS)) {
+ List<Long> fileSizes = transfer.obtainSize(files);
+ final long totalFileSize = fileSizes.stream().mapToLong(i -> i.longValue()).sum();
+ TransferFileProgressForHaaSClient progress = new TransferFileProgressForHaaSClient(totalFileSize, notifier);
+ transfer.setProgress(progress);
+ int idx = 0;
+ for (String fileName : files) {
+ String item;
+ progress.addItem(item = fileName);
+ progress.startNewFile(fileSizes.get(idx));
+ transfer.download(fileName, jobDir);
+ progress.itemDone(item);
+ idx++;
+ }
}
}
@@ -212,31 +245,49 @@ public class Job {
return result;
}
+ public Collection<String> getChangedFiles() {
+ return getHaaSClient().getChangedFiles(getId());
+ }
+
+ public void cancelJob() {
+ getHaaSClient().cancelJob(jobId);
+ }
+
+ public List<Long> getFileSizes(List<String> names) {
+
+ try (HaaSFileTransfer transfer = getHaaSClient().startFileTransfer(getId(),
+ HaaSClient.DUMMY_TRANSFER_FILE_PROGRESS)) {
+ return transfer.obtainSize(names);
+ }
+ }
+
+ public List<String> getFileContents(List<String> logs) {
+ try (HaaSFileTransfer transfer = getHaaSClient().startFileTransfer(getId(),
+ HaaSClient.DUMMY_TRANSFER_FILE_PROGRESS)) {
+ return transfer.getContent(logs);
+ }
+ }
+
private void setJobDirectory(Path jobDirectory) {
this.jobDir = jobDirectory;
try {
- this.synchronization = new Synchronization(()->haasClientSupplier.get().startFileTransfer(getId(), new DummyProgressNotifier()), jobDir, Executors.newFixedThreadPool(2), ()-> {
- setProperty(JOB_NEEDS_UPLOAD, false);
- });
+ this.synchronization = new Synchronization(
+ () -> haasClientSupplier.get().startFileTransfer(getId(), HaaSClient.DUMMY_TRANSFER_FILE_PROGRESS),
+ jobDir, Executors.newFixedThreadPool(2), () -> {
+ setProperty(JOB_NEEDS_UPLOAD, false);
+ });
} catch (IOException e) {
log.error(e.getMessage(), e);
throw new RuntimeException(e);
}
}
-
+
private synchronized void resumeUpload() {
- if(Boolean.parseBoolean(getProperty(JOB_NEEDS_UPLOAD))) {
+ if (Boolean.parseBoolean(getProperty(JOB_NEEDS_UPLOAD))) {
synchronization.resumeUpload();
}
}
- private void uploadFiles(Iterable<UploadingFile> files, Progress notifier) {
- HaaSClient client = getHaaSClient();
- try (HaaSFileTransfer transfer = client.startFileTransfer(getId(), new P_ProgressNotifierAdapter(notifier))) {
- transfer.upload(files);
- }
- }
-
private void setName(String name) {
setProperty(JOB_NAME, name);
}
@@ -270,58 +321,4 @@ public class Job {
return Long.parseLong(path.getFileName().toString());
}
- private class P_ProgressNotifierAdapter implements ProgressNotifier {
- private Progress progress;
-
- public P_ProgressNotifierAdapter(Progress progress) {
- this.progress = progress;
- }
-
- public void setTitle(String title) {
- progress.setTitle(title);
- }
-
- public void setCount(int count, int total) {
- progress.setCount(count, total);
- }
-
- public void addItem(Object item) {
- progress.addItem(item);
- }
-
- public void setItemCount(int count, int total) {
- progress.setItemCount(count, total);
- }
-
- public void itemDone(Object item) {
- progress.itemDone(item);
- }
-
- public void done() {
- progress.done();
- }
-
- }
-
- public Collection<String> getChangedFiles() {
- return getHaaSClient().getChangedFiles(getId());
- }
-
- public void cancelJob() {
- getHaaSClient().cancelJob(jobId);
- }
-
- public List<Long> getFileSizes(List<String> names) {
-
- try (HaaSFileTransfer transfer = getHaaSClient().startFileTransfer(getId(), new DummyProgressNotifier())) {
- return transfer.obtainSize(names);
- }
- }
-
- public List<String> getFileContents(List<String> logs) {
- try (HaaSFileTransfer transfer = getHaaSClient().startFileTransfer(getId(), new DummyProgressNotifier())) {
- return transfer.getContent(logs);
- }
- }
-
}
diff --git a/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/JobManager.java b/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/JobManager.java
index 77116f511670dace631b3456f271921bf16b6c09..267c2662beafa85882f5da50b65b6606f65be5ca 100644
--- a/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/JobManager.java
+++ b/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/JobManager.java
@@ -5,7 +5,6 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collection;
import java.util.Collections;
-import java.util.Iterator;
import java.util.LinkedList;
import org.slf4j.Logger;
@@ -15,7 +14,6 @@ import cz.it4i.fiji.haas_java_client.HaaSClient;
import cz.it4i.fiji.haas_java_client.JobState;
import cz.it4i.fiji.haas_java_client.Settings;
import cz.it4i.fiji.haas_java_client.SynchronizableFileType;
-import net.imagej.updater.util.Progress;
public class JobManager {
@@ -72,12 +70,12 @@ public class JobManager {
return Collections.unmodifiableCollection(jobs);
}
- public void downloadJob(Long id, Progress notifier) {
- Iterator<Job> job = jobs.stream().filter(j -> j.getId() == id).iterator();
- assert job.hasNext();
- job.next().download(notifier);
-
- }
+// public void downloadJob(Long id, Progress notifier) {
+// Iterator<Job> job = jobs.stream().filter(j -> j.getId() == id).iterator();
+// assert job.hasNext();
+// job.next().download(notifier);
+//
+// }
public JobState getState(long id) {
return getHaasClient().obtainJobInfo(id).getState();
diff --git a/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/data_transfer/Synchronization.java b/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/data_transfer/Synchronization.java
index 1b09f99ff8c2977cde181794fe1a5abbf689e6ab..30680c25424af2f615099275cc56488dbd1e7faf 100644
--- a/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/data_transfer/Synchronization.java
+++ b/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/data_transfer/Synchronization.java
@@ -5,6 +5,7 @@ import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Queue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
@@ -16,12 +17,18 @@ import org.slf4j.LoggerFactory;
import cz.it4i.fiji.haas_java_client.HaaSClient.UploadingFile;
import cz.it4i.fiji.haas_java_client.HaaSFileTransfer;
+import cz.it4i.fiji.haas_java_client.UploadingFileImpl;
public class Synchronization {
public static final Logger log = LoggerFactory.getLogger(cz.it4i.fiji.haas.data_transfer.Synchronization.class);
- private static final String FILE_INDEX_FILENAME = ".toUploadFiles";
+ private static final String FILE_INDEX_TO_UPLOAD_FILENAME = ".toUploadFiles";
+
+ private static final String FILE_INDEX_TO_DOWNLOAD_FILENAME = ".toDownloadFiles";
+
+ private static final String FILE_INDEX_DOWNLOADED_FILENAME = ".downloaded";
+
private Supplier<HaaSFileTransfer> fileTransferSupplier;
@@ -29,7 +36,11 @@ public class Synchronization {
private Queue<Path> toUpload = new LinkedBlockingQueue<>();
- private FileIndex fileRepository;
+ private FileIndex filesToUpload;
+
+ private FileIndex filesToDownload;
+
+ private FileIndex filesDownloaded;
private SimpleThreadRunner runnerForUpload;
@@ -41,40 +52,43 @@ public class Synchronization {
ExecutorService service, Runnable uploadFinishedNotifier ) throws IOException {
this.fileTransferSupplier = fileTransferSupplier;
this.workingDirectory = workingDirectory;
- this.fileRepository = new FileIndex(workingDirectory.resolve(FILE_INDEX_FILENAME));
+ this.filesToUpload = new FileIndex(workingDirectory.resolve(FILE_INDEX_TO_UPLOAD_FILENAME));
+ this.filesToDownload = new FileIndex(workingDirectory.resolve(FILE_INDEX_TO_DOWNLOAD_FILENAME));
+ this.filesDownloaded = new FileIndex(workingDirectory.resolve(FILE_INDEX_DOWNLOADED_FILENAME));
this.runnerForUpload = new SimpleThreadRunner(service);
this.uploadFinishedNotifier = uploadFinishedNotifier;
}
public synchronized void startUpload() throws IOException {
startUploadFinished = false;
- fileRepository.clear();
+ filesToUpload.clear();
try(DirectoryStream<Path> ds = Files.newDirectoryStream(workingDirectory,this::isNotHidden)) {
for (Path file : ds) {
- fileRepository.insert(file);
+ filesToUpload.insert(file);
toUpload.add(file);
runnerForUpload.runIfNotRunning(this::doUpload);
}
} finally {
startUploadFinished = true;
- fileRepository.storeToFile();
+ filesToUpload.storeToFile();
}
}
public void stopUpload() throws IOException {
toUpload.clear();
- fileRepository.clear();
+ filesToUpload.clear();
}
public void resumeUpload() {
- fileRepository.fillQueue(toUpload);
+ filesToUpload.fillQueue(toUpload);
if(!toUpload.isEmpty()) {
runnerForUpload.runIfNotRunning(this::doUpload);
}
}
- public void startDownload() {
+ public synchronized void startDownload(Collection<String> files) throws IOException {
+ filesToDownload.clear();
}
@@ -89,7 +103,7 @@ public class Synchronization {
Path p = toUpload.poll();
UploadingFile uf = createUploadingFile(p);
log.info("upload: " + p);
- tr.upload(Arrays.asList(uf));
+ tr.upload(uf);
fileUploaded(p);
log.info("uploaded: " + p);
reRun.set(false);
@@ -105,8 +119,8 @@ public class Synchronization {
private void fileUploaded(Path p) {
try {
- fileRepository.uploaded(p);
- fileRepository.storeToFile();
+ filesToUpload.uploaded(p);
+ filesToUpload.storeToFile();
} catch (IOException e) {
log.error(e.getMessage(), e);
}
diff --git a/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/ui/ProgressDialog.java b/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/ui/ProgressDialog.java
index c86acb066ac80fa35b9db30573361a743a3e55ba..c545ae0b5f17fe770decb8d8f13ae56f02f272f7 100644
--- a/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/ui/ProgressDialog.java
+++ b/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/ui/ProgressDialog.java
@@ -291,10 +291,4 @@ public class ProgressDialog extends JDialog implements Progress {
itemLatestUpdate = System.currentTimeMillis();
return false;
}
-
- private static void main(final String[] args) {
- final ProgressDialog dialog = new ProgressDialog(null, "Hello");
- dialog.addItem("Bello");
- dialog.setVisible(true);
- }
}
diff --git a/haas-java-client/src/main/java/cz/it4i/fiji/haas_java_client/DummyProgressNotifier.java b/haas-java-client/src/main/java/cz/it4i/fiji/haas_java_client/DummyProgressNotifier.java
deleted file mode 100644
index 927d4c048226546d76acd64d99fb3953b886ce70..0000000000000000000000000000000000000000
--- a/haas-java-client/src/main/java/cz/it4i/fiji/haas_java_client/DummyProgressNotifier.java
+++ /dev/null
@@ -1,29 +0,0 @@
-package cz.it4i.fiji.haas_java_client;
-
-public class DummyProgressNotifier implements ProgressNotifier {
-
- @Override
- public void setCount(int count, int total) {
- }
-
- @Override
- public void addItem(Object item) {
- }
-
- @Override
- public void setItemCount(int count, int total) {
-
- }
-
- @Override
- public void itemDone(Object item) {
- }
-
- @Override
- public void done() {
- }
-
- @Override
- public void setTitle(String title) {
- }
-}
diff --git a/haas-java-client/src/main/java/cz/it4i/fiji/haas_java_client/HaaSClient.java b/haas-java-client/src/main/java/cz/it4i/fiji/haas_java_client/HaaSClient.java
index 098b34b7aebb132c6d74b759fceb86ad1a880f7a..d49c73da4e421a944c86cdbeb38e082dd02bd9e5 100644
--- a/haas-java-client/src/main/java/cz/it4i/fiji/haas_java_client/HaaSClient.java
+++ b/haas-java-client/src/main/java/cz/it4i/fiji/haas_java_client/HaaSClient.java
@@ -12,11 +12,9 @@ import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
-import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.stream.Collectors;
-import java.util.stream.StreamSupport;
import javax.xml.rpc.ServiceException;
@@ -44,22 +42,45 @@ import cz.it4i.fiji.haas_java_client.proxy.TaskSpecificationExt;
import cz.it4i.fiji.haas_java_client.proxy.UserAndLimitationManagementWsLocator;
import cz.it4i.fiji.haas_java_client.proxy.UserAndLimitationManagementWsSoap;
import cz.it4i.fiji.scpclient.ScpClient;
+import cz.it4i.fiji.scpclient.TransferFileProgress;
public class HaaSClient {
- public static List<Long> getSizes(List<String> asList, ScpClient scpClient, ProgressNotifier notifier)
- throws JSchException, IOException {
- List<Long> result = new LinkedList<>();
+ public static final TransferFileProgress DUMMY_TRANSFER_FILE_PROGRESS = new TransferFileProgress() {
+
+ @Override
+ public void dataTransfered(long bytesTransfered) {
+ // TODO Auto-generated method stub
+
+ }
+ };
- String item;
- notifier.addItem(item = "Checking sizes");
- for (String lfile : asList) {
- result.add(scpClient.size(lfile));
- notifier.setItemCount(result.size(), asList.size());
+ public static ProgressNotifier DUMMY_PROGRESS_NOTIFIER = new ProgressNotifier() {
+
+ @Override
+ public void setTitle(String title) {
}
- notifier.itemDone(item);
- return result;
- }
+
+ @Override
+ public void setItemCount(int count, int total) {
+ }
+
+ @Override
+ public void setCount(int count, int total) {
+ }
+
+ @Override
+ public void itemDone(Object item) {
+ }
+
+ @Override
+ public void done() {
+ }
+
+ @Override
+ public void addItem(Object item) {
+ }
+ };
public static UploadingFile getUploadingFile(Path file) {
return new UploadingFile() {
@@ -147,6 +168,20 @@ public class HaaSClient {
private static Logger log = LoggerFactory.getLogger(cz.it4i.fiji.haas_java_client.HaaSClient.class);
+ final static private Map<JobStateExt, JobState> WS_STATE2STATE;
+
+ static {
+ Map<JobStateExt, JobState> map = new HashMap<JobStateExt, JobState>();
+ map.put(JobStateExt.Canceled, JobState.Canceled);
+ map.put(JobStateExt.Configuring, JobState.Configuring);
+ map.put(JobStateExt.Failed, JobState.Failed);
+ map.put(JobStateExt.Finished, JobState.Finished);
+ map.put(JobStateExt.Queued, JobState.Queued);
+ map.put(JobStateExt.Running, JobState.Running);
+ map.put(JobStateExt.Submitted, JobState.Submitted);
+ WS_STATE2STATE = Collections.unmodifiableMap(map);
+ }
+
private String sessionID;
private UserAndLimitationManagementWsSoap userAndLimitationManagement;
@@ -165,49 +200,9 @@ public class HaaSClient {
private Map<Long, P_FileTransferPool> filetransferPoolMap = new HashMap<>();
- public static ProgressNotifier DUMMY_NOTIFIER = new ProgressNotifier() {
-
- @Override
- public void setTitle(String title) {
- }
-
- @Override
- public void setItemCount(int count, int total) {
- }
-
- @Override
- public void setCount(int count, int total) {
- }
-
- @Override
- public void itemDone(Object item) {
- }
-
- @Override
- public void done() {
- }
-
- @Override
- public void addItem(Object item) {
- }
- };
-
+
private Settings settings;
- final static private Map<JobStateExt, JobState> WS_STATE2STATE;
-
- static {
- Map<JobStateExt, JobState> map = new HashMap<JobStateExt, JobState>();
- map.put(JobStateExt.Canceled, JobState.Canceled);
- map.put(JobStateExt.Configuring, JobState.Configuring);
- map.put(JobStateExt.Failed, JobState.Failed);
- map.put(JobStateExt.Finished, JobState.Finished);
- map.put(JobStateExt.Queued, JobState.Queued);
- map.put(JobStateExt.Running, JobState.Running);
- map.put(JobStateExt.Submitted, JobState.Submitted);
- WS_STATE2STATE = Collections.unmodifiableMap(map);
- }
-
public HaaSClient(Settings settings) {
this.settings = settings;
this.templateId = settings.getTemplateId();
@@ -216,28 +211,6 @@ public class HaaSClient {
this.projectId = settings.getProjectId();
}
- public long start(Iterable<Path> files, String name, Collection<Entry<String, String>> templateParameters) {
- Iterable<UploadingFile> uploadingFiles = StreamSupport.stream(files.spliterator(), false)
- .map(HaaSClient::getUploadingFile).collect(Collectors.toList());
- return start(uploadingFiles, name, templateParameters, DUMMY_NOTIFIER);
- }
-
- public long start(Iterable<UploadingFile> files, String name, Collection<Entry<String, String>> templateParameters,
- ProgressNotifier notifier) {
- notifier.setTitle("Starting job");
- try {
- long jobId = doCreateJob(name, templateParameters);
- try (HaaSFileTransfer transfer = startFileTransfer(jobId, notifier)) {
- transfer.upload(files);
- }
- doSubmitJob(jobId);
- return jobId;
- } catch (ServiceException | IOException e) {
- throw new HaaSClientException(e);
- }
-
- }
-
public long createJob(String name, Collection<Entry<String, String>> templateParameters) {
try {
return doCreateJob(name, templateParameters);
@@ -246,13 +219,12 @@ public class HaaSClient {
}
}
- public HaaSFileTransfer startFileTransfer(long jobId, ProgressNotifier notifier) {
+ public HaaSFileTransfer startFileTransfer(long jobId, TransferFileProgress notifier) {
try {
return getFileTransferMethod(jobId, notifier);
} catch (RemoteException | ServiceException | UnsupportedEncodingException | JSchException e) {
throw new HaaSClientException(e);
}
-
}
public void submitJob(long jobId) {
@@ -322,12 +294,12 @@ public class HaaSClient {
}
}
- private HaaSFileTransferImp getFileTransferMethod(long jobId, ProgressNotifier notifier)
+ private HaaSFileTransferImp getFileTransferMethod(long jobId, TransferFileProgress progress)
throws RemoteException, UnsupportedEncodingException, ServiceException, JSchException {
P_FileTransferPool pool = filetransferPoolMap.computeIfAbsent(jobId, id -> new P_FileTransferPool(id));
FileTransferMethodExt ft = pool.obtain();
try {
- return new HaaSFileTransferImp(ft, getScpClient(ft), notifier) {
+ return new HaaSFileTransferImp(ft, getScpClient(ft), progress) {
public void close() {
super.close();
try {
diff --git a/haas-java-client/src/main/java/cz/it4i/fiji/haas_java_client/HaaSFileTransfer.java b/haas-java-client/src/main/java/cz/it4i/fiji/haas_java_client/HaaSFileTransfer.java
index c6c8a4b9b31142c0b5041b1ccf1882bae1a82b4d..e3c751a3c89bbf44a9135d91b70f775dac3133db 100644
--- a/haas-java-client/src/main/java/cz/it4i/fiji/haas_java_client/HaaSFileTransfer.java
+++ b/haas-java-client/src/main/java/cz/it4i/fiji/haas_java_client/HaaSFileTransfer.java
@@ -5,6 +5,7 @@ import java.nio.file.Path;
import java.util.List;
import cz.it4i.fiji.haas_java_client.HaaSClient.UploadingFile;
+import cz.it4i.fiji.scpclient.TransferFileProgress;
public interface HaaSFileTransfer extends Closeable {
@@ -12,11 +13,13 @@ public interface HaaSFileTransfer extends Closeable {
@Override
void close();
- void upload(Iterable<UploadingFile> files);
+ void upload(UploadingFile files);
- void download(Iterable<String> files, Path workDIrectory);
+ void download(String files, Path workDirectory);
List<Long> obtainSize(List<String> files);
List<String> getContent(List<String> logs);
+
+ void setProgress(TransferFileProgress progress);
}
diff --git a/haas-java-client/src/main/java/cz/it4i/fiji/haas_java_client/HaaSFileTransferImp.java b/haas-java-client/src/main/java/cz/it4i/fiji/haas_java_client/HaaSFileTransferImp.java
index d103a5c60ab1cf37f669ba1df03f8c4cf41f54e2..ae970984461097d6c39fb106a9084dc04d508a80 100644
--- a/haas-java-client/src/main/java/cz/it4i/fiji/haas_java_client/HaaSFileTransferImp.java
+++ b/haas-java-client/src/main/java/cz/it4i/fiji/haas_java_client/HaaSFileTransferImp.java
@@ -7,17 +7,16 @@ import java.nio.file.Path;
import java.util.LinkedList;
import java.util.List;
import java.util.stream.Collectors;
-import java.util.stream.StreamSupport;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.jcraft.jsch.JSchException;
-import cz.it4i.fiji.haas_java_client.HaaSClient.P_ProgressNotifierDecorator4Size;
import cz.it4i.fiji.haas_java_client.HaaSClient.UploadingFile;
import cz.it4i.fiji.haas_java_client.proxy.FileTransferMethodExt;
import cz.it4i.fiji.scpclient.ScpClient;
+import cz.it4i.fiji.scpclient.TransferFileProgress;
class HaaSFileTransferImp implements HaaSFileTransfer {
@@ -26,12 +25,12 @@ class HaaSFileTransferImp implements HaaSFileTransfer {
private FileTransferMethodExt ft;
private ScpClient scpClient;
- private ProgressNotifier notifier;
-
- public HaaSFileTransferImp(FileTransferMethodExt ft, ScpClient scpClient, ProgressNotifier notifier) {
+ private TransferFileProgress progress;
+
+ public HaaSFileTransferImp(FileTransferMethodExt ft, ScpClient scpClient, TransferFileProgress progress) {
this.ft = ft;
this.scpClient = scpClient;
- this.notifier = notifier;
+ this.progress = progress;
}
@Override
@@ -40,46 +39,47 @@ class HaaSFileTransferImp implements HaaSFileTransfer {
}
@Override
- public void upload(Iterable<UploadingFile> files) {
- List<Long> totalSizes = StreamSupport.stream(files.spliterator(), false).map(f -> {
- try {
- return f.getLength();
- } catch (IOException e1) {
- throw new RuntimeException(e1);
- }
- }).collect(Collectors.toList());
- long totalSize = totalSizes.stream().mapToLong(l -> l.longValue()).sum();
- TransferFileProgressForHaaSClient progress = new TransferFileProgressForHaaSClient(totalSize, notifier);
- int index = 0;
- for (UploadingFile file : files) {
- String item;
- progress.startNewFile(totalSizes.get(index));
- notifier.addItem(item = "Uploading file: " + file.getName());
- String destFile = "'" + ft.getSharedBasepath() + "/" + file.getName() + "'";
- try (InputStream is = file.getInputStream()) {
- boolean result = scpClient.upload(is, destFile, file.getLength(), file.getLastTime(), progress);
- notifier.itemDone(item);
- if (!result) {
- throw new HaaSClientException("Uploading of " + file + " to " + destFile + " failed");
- }
- } catch (JSchException | IOException e) {
- throw new HaaSClientException();
+ public void upload(UploadingFile file) {
+ String destFile = "'" + ft.getSharedBasepath() + "/" + file.getName() + "'";
+ try (InputStream is = file.getInputStream()) {
+ boolean result = scpClient.upload(is, destFile, file.getLength(), file.getLastTime(), progress);
+ if (!result) {
+ throw new HaaSClientException("Uploading of " + file + " to " + destFile + " failed");
}
- index++;
+ } catch (JSchException | IOException e) {
+ throw new HaaSClientException();
}
-
}
+ @Override
+ public void download(String fileName, Path workDirectory) {
+ try {
+ fileName = fileName.replaceFirst("/", "");
+ Path rFile = workDirectory.resolve(fileName);
+ String fileToDownload = "'" + ft.getSharedBasepath() + "/" + fileName + "'";
+ scpClient.download(fileToDownload, rFile, progress);
+ } catch (JSchException | IOException e) {
+ throw new HaaSClientException(e);
+ }
+ }
+
+ @Override
+ public void setProgress(TransferFileProgress progress) {
+ this.progress = progress;
+ }
+
+ /*
@Override
public void download(Iterable<String> files, Path workDirectory) {
List<Long> fileSizes;
try {
fileSizes = HaaSClient.getSizes(StreamSupport.stream(files.spliterator(), false)
.map(filename -> "'" + ft.getSharedBasepath() + "/" + filename + "'").collect(Collectors.toList()),
- scpClient, new P_ProgressNotifierDecorator4Size(notifier));
+ scpClient);
final long totalFileSize = fileSizes.stream().mapToLong(i -> i.longValue()).sum();
- TransferFileProgressForHaaSClient progress = new TransferFileProgressForHaaSClient(totalFileSize, notifier);
+ TransferFileProgressForHaaSClient progress = new TransferFileProgressForHaaSClient(totalFileSize,
+ HaaSClient.DUMMY_PROGRESS_NOTIFIER);
int idx = 0;
for (String fileName : files) {
fileName = fileName.replaceFirst("/", "");
@@ -96,13 +96,13 @@ class HaaSFileTransferImp implements HaaSFileTransfer {
throw new HaaSClientException(e);
}
}
+ */
@Override
public List<Long> obtainSize(List<String> files) {
try {
- return HaaSClient.getSizes(files.stream()
- .map(filename -> "'" + ft.getSharedBasepath() + "/" + filename + "'").collect(Collectors.toList()),
- scpClient, notifier);
+ return getSizes(files.stream()
+ .map(filename -> "'" + ft.getSharedBasepath() + "/" + filename + "'").collect(Collectors.toList()));
} catch (JSchException | IOException e) {
throw new HaaSClientException(e);
}
@@ -113,26 +113,13 @@ class HaaSFileTransferImp implements HaaSFileTransfer {
@Override
public List<String> getContent(List<String> files) {
List<String> result = new LinkedList<>();
- List<Long> fileSizes;
try {
- fileSizes = HaaSClient.getSizes(StreamSupport.stream(files.spliterator(), false)
- .map(filename -> "'" + ft.getSharedBasepath() + "/" + filename + "'").collect(Collectors.toList()),
- scpClient, new P_ProgressNotifierDecorator4Size(notifier));
-
- final long totalFileSize = fileSizes.stream().mapToLong(i -> i.longValue()).sum();
- TransferFileProgressForHaaSClient progress = new TransferFileProgressForHaaSClient(totalFileSize, notifier);
- int idx = 0;
for (String fileName : files) {
fileName = replaceIfFirstFirst(fileName, "/", "");
try (ByteArrayOutputStream os = new ByteArrayOutputStream()) {
String fileToDownload = "'" + ft.getSharedBasepath() + "/" + fileName + "'";
- String item;
- progress.addItem(item = fileName);
- progress.startNewFile(fileSizes.get(idx));
scpClient.download(fileToDownload, os, progress);
os.flush();
- progress.itemDone(item);
- idx++;
result.add(os.toString());
}
}
@@ -148,5 +135,13 @@ class HaaSFileTransferImp implements HaaSFileTransfer {
}
return fileName;
}
+
+ private List<Long> getSizes(List<String> asList) throws JSchException, IOException {
+ List<Long> result = new LinkedList<>();
+ for (String lfile : asList) {
+ result.add(scpClient.size(lfile));
+ }
+ return result;
+ }
}
diff --git a/haas-java-client/src/main/java/cz/it4i/fiji/haas_java_client/TransferFileProgressForHaaSClient.java b/haas-java-client/src/main/java/cz/it4i/fiji/haas_java_client/TransferFileProgressForHaaSClient.java
index 40fec855c342e8740f64aded4faf2f3c795a4b1c..d1b250c95c93a5bdd7e200d45000ad953abe380f 100644
--- a/haas-java-client/src/main/java/cz/it4i/fiji/haas_java_client/TransferFileProgressForHaaSClient.java
+++ b/haas-java-client/src/main/java/cz/it4i/fiji/haas_java_client/TransferFileProgressForHaaSClient.java
@@ -2,7 +2,7 @@ package cz.it4i.fiji.haas_java_client;
import cz.it4i.fiji.scpclient.TransferFileProgress;
-class TransferFileProgressForHaaSClient implements TransferFileProgress {
+public class TransferFileProgressForHaaSClient implements TransferFileProgress {
private long totalSize;
private long totalTransfered;
@@ -35,6 +35,14 @@ class TransferFileProgressForHaaSClient implements TransferFileProgress {
}
+ public void addItem(String item) {
+ notifier.addItem(item);
+ }
+
+ public void itemDone(String item) {
+ notifier.itemDone(item);
+ }
+
private static int[] normalizaSizes(long part, long total) {
int[] result = new int[2];
if(total > Integer.MAX_VALUE) {
@@ -49,12 +57,4 @@ class TransferFileProgressForHaaSClient implements TransferFileProgress {
}
return result;
}
-
- public void addItem(String item) {
- notifier.addItem(item);
- }
-
- public void itemDone(String item) {
- notifier.itemDone(item);
- }
}
diff --git a/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/data_transfer/UploadingFileImpl.java b/haas-java-client/src/main/java/cz/it4i/fiji/haas_java_client/UploadingFileImpl.java
similarity index 90%
rename from haas-imagej-client/src/main/java/cz/it4i/fiji/haas/data_transfer/UploadingFileImpl.java
rename to haas-java-client/src/main/java/cz/it4i/fiji/haas_java_client/UploadingFileImpl.java
index ac3a403a66992187c64d1e6b2ff7b14e2518adc1..9c921e5a3f8317d9ecf7472c9816b2db9561ac11 100644
--- a/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/data_transfer/UploadingFileImpl.java
+++ b/haas-java-client/src/main/java/cz/it4i/fiji/haas_java_client/UploadingFileImpl.java
@@ -1,4 +1,4 @@
-package cz.it4i.fiji.haas.data_transfer;
+package cz.it4i.fiji.haas_java_client;
import java.io.IOException;
import java.io.InputStream;
@@ -12,7 +12,7 @@ import cz.it4i.fiji.haas_java_client.HaaSClient.UploadingFile;
public class UploadingFileImpl implements UploadingFile {
- public static final Logger log = LoggerFactory.getLogger(cz.it4i.fiji.haas.data_transfer.UploadingFileImpl.class);
+ public static final Logger log = LoggerFactory.getLogger(cz.it4i.fiji.haas_java_client.UploadingFileImpl.class);
private final Path path;
diff --git a/haas-java-client/src/test/java/cz/it4i/fiji/haas_java_client/TestConcurentAccessToHaaSFileTransfer.java b/haas-java-client/src/test/java/cz/it4i/fiji/haas_java_client/TestConcurentAccessToHaaSFileTransfer.java
index cdeedef7f7bcb4a58bdd249520d82c5dfa6eae2b..8646f9816af46e211d499ce6ed1704fe44ab0653 100644
--- a/haas-java-client/src/test/java/cz/it4i/fiji/haas_java_client/TestConcurentAccessToHaaSFileTransfer.java
+++ b/haas-java-client/src/test/java/cz/it4i/fiji/haas_java_client/TestConcurentAccessToHaaSFileTransfer.java
@@ -14,8 +14,8 @@ public class TestConcurentAccessToHaaSFileTransfer {
public static void main(String[] args) throws ServiceException, IOException {
HaaSClient client = new HaaSClient(TestingConstants.getSettings(1l, 600, 7l, "OPEN-12-20"));
- HaaSFileTransfer tr1 = client.startFileTransfer(250, new DummyProgressNotifier());
- HaaSFileTransfer tr2 = client.startFileTransfer(249, new DummyProgressNotifier());
+ HaaSFileTransfer tr1 = client.startFileTransfer(250, HaaSClient.DUMMY_TRANSFER_FILE_PROGRESS);
+ HaaSFileTransfer tr2 = client.startFileTransfer(249, HaaSClient.DUMMY_TRANSFER_FILE_PROGRESS);
log.info("config.yaml - size:" + tr1.obtainSize(Arrays.asList("config.yaml")));
tr1.close();
log.info("config.yaml - size:" + tr2.obtainSize(Arrays.asList("config.yaml")));
diff --git a/haas-java-client/src/test/java/cz/it4i/fiji/haas_java_client/TestHaaSJavaClient.java b/haas-java-client/src/test/java/cz/it4i/fiji/haas_java_client/TestHaaSJavaClient.java
index c31d5294bfa064a6fe2eabec712f86508947b89a..886a34da546abb8359aee67c0289610c62b82291 100644
--- a/haas-java-client/src/test/java/cz/it4i/fiji/haas_java_client/TestHaaSJavaClient.java
+++ b/haas-java-client/src/test/java/cz/it4i/fiji/haas_java_client/TestHaaSJavaClient.java
@@ -4,7 +4,6 @@ import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
-import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
@@ -25,8 +24,11 @@ public class TestHaaSJavaClient {
params.put("inputParam", "someStringParam");
Path baseDir = Paths.get("/home/koz01/aaa");
HaaSClient client = new HaaSClient(TestingConstants.getSettings(1l, 600, 7l, "DD-17-31"));
- long jobId = client.start(Arrays.asList(Paths.get("/home/koz01/aaa/vecmath.jar")), "TestOutRedirect",
- params.entrySet());
+ long jobId = client.createJob("TestOutRedirect", params.entrySet());
+ try (HaaSFileTransfer tr = client.startFileTransfer(jobId, HaaSClient.DUMMY_TRANSFER_FILE_PROGRESS)) {
+ tr.upload(new UploadingFileImpl(Paths.get("/home/koz01/aaa/vecmath.jar")));
+ }
+ client.submitJob(jobId);
Path workDir = baseDir.resolve("" + jobId);
if (!Files.isDirectory(workDir)) {
Files.createDirectories(workDir);
@@ -45,8 +47,9 @@ public class TestHaaSJavaClient {
}
client.downloadPartsOfJobFiles(jobId, taskFileOffset).forEach(jfc -> showJFC(jfc));
if (info.getState() == JobState.Finished) {
- try (HaaSFileTransfer fileTransfer = client.startFileTransfer(jobId, HaaSClient.DUMMY_NOTIFIER)) {
- fileTransfer.download(client.getChangedFiles(jobId), workDir);
+ try (HaaSFileTransfer fileTransfer = client.startFileTransfer(jobId,
+ HaaSClient.DUMMY_TRANSFER_FILE_PROGRESS)) {
+ client.getChangedFiles(jobId).forEach(file -> fileTransfer.download(file, workDir));
}
}
log.info("JobId :" + jobId + ", state" + info.getState());
diff --git a/haas-java-client/src/test/java/cz/it4i/fiji/haas_java_client/TestHaaSJavaClientWithSPIM.java b/haas-java-client/src/test/java/cz/it4i/fiji/haas_java_client/TestHaaSJavaClientWithSPIM.java
index 99b4d85370b717e3aa67e0c138b677f2522b3734..c4afe21c9197219255e1885608c99eb5fff9266c 100644
--- a/haas-java-client/src/test/java/cz/it4i/fiji/haas_java_client/TestHaaSJavaClientWithSPIM.java
+++ b/haas-java-client/src/test/java/cz/it4i/fiji/haas_java_client/TestHaaSJavaClientWithSPIM.java
@@ -5,6 +5,7 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Collections;
+import java.util.stream.StreamSupport;
import javax.xml.rpc.ServiceException;
@@ -22,8 +23,15 @@ public class TestHaaSJavaClientWithSPIM {
HaaSClient client = new HaaSClient(TestingConstants.getSettings(2, 9600, 6l, "DD-17-31"));
Path baseDir = Paths.get("/home/koz01/Work/vyzkumnik/fiji/work/aaa");
- long jobId = client.start(getAllFiles(baseDir.resolve("spim-data")), "TestOutRedirect",
+ long jobId = client.createJob( "TestOutRedirect",
Collections.emptyList());
+
+ try(HaaSFileTransfer tr = client.startFileTransfer(jobId, HaaSClient.DUMMY_TRANSFER_FILE_PROGRESS)) {
+ StreamSupport.stream(getAllFiles(baseDir.resolve("spim-data")).spliterator(), false)
+ .map(UploadingFileImpl::new).forEach(f -> tr.upload(f));
+ }
+ client.submitJob(jobId);
+
Path workDir = baseDir.resolve("" + jobId);
if (!Files.isDirectory(workDir)) {
Files.createDirectories(workDir);
@@ -45,8 +53,8 @@ public class TestHaaSJavaClientWithSPIM {
}
client.downloadPartsOfJobFiles(jobId, taskFileOffset).forEach(jfc -> showJFC(jfc));
if (info.getState() == JobState.Finished) {
- try (HaaSFileTransfer fileTransfer = client.startFileTransfer(jobId, HaaSClient.DUMMY_NOTIFIER)) {
- fileTransfer.download(client.getChangedFiles(jobId), workDir);
+ try (HaaSFileTransfer fileTransfer = client.startFileTransfer(jobId, HaaSClient.DUMMY_TRANSFER_FILE_PROGRESS)) {
+ client.getChangedFiles(jobId).forEach(file -> fileTransfer.download(file, workDir));
}
}
diff --git a/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/core/BenchmarkJobManager.java b/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/core/BenchmarkJobManager.java
index 1fcdc5692f9a8e07262dfe0afea34ab05a380c8a..2dd3c31e21aa46301aa35b8a2197eb00dfc555c5 100644
--- a/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/core/BenchmarkJobManager.java
+++ b/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/core/BenchmarkJobManager.java
@@ -42,6 +42,7 @@ import cz.it4i.fiji.haas.JobManager;
import cz.it4i.fiji.haas.UploadingFileFromResource;
import cz.it4i.fiji.haas_java_client.HaaSClient;
import cz.it4i.fiji.haas_java_client.JobState;
+import cz.it4i.fiji.haas_java_client.ProgressNotifier;
import cz.it4i.fiji.haas_java_client.Settings;
import cz.it4i.fiji.haas_java_client.SynchronizableFileType;
import net.imagej.updater.util.Progress;
@@ -74,7 +75,7 @@ public class BenchmarkJobManager {
}
public synchronized void startJob(Progress progress) throws IOException {
- job.uploadFile(Constants.CONFIG_YAML, progress);
+ job.uploadFile(Constants.CONFIG_YAML, new P_ProgressNotifierAdapter(progress));
String outputName = getOutputName(job.openLocalFile(Constants.CONFIG_YAML));
verifiedState = null;
verifiedStateProcessed = false;
@@ -87,15 +88,15 @@ public class BenchmarkJobManager {
public JobState getState() {
return getStateAsync(r -> r.run()).getNow(JobState.Unknown);
}
-
+
public void startUpload() {
job.startUploadData();
}
-
+
public void stopUpload() {
job.stopUploadData();
}
-
+
public boolean isUploading() {
return job.isUploading();
}
@@ -114,16 +115,16 @@ public class BenchmarkJobManager {
public void downloadData(Progress progress) throws IOException {
if (job.getState() == JobState.Finished) {
String filePattern = job.getProperty(SPIM_OUTPUT_FILENAME_PATTERN);
- job.download(downloadFinishedData(filePattern), progress);
+ job.download(downloadFinishedData(filePattern), new P_ProgressNotifierAdapter(progress));
} else if (job.getState() == JobState.Failed || job.getState() == JobState.Canceled) {
- job.download(downloadFailedData(), progress);
+ job.download(downloadFailedData(), new P_ProgressNotifierAdapter(progress));
}
setDownloaded(true);
}
public void downloadStatistics(Progress progress) throws IOException {
- job.download(BenchmarkJobManager.downloadStatistics(), progress);
+ job.download(BenchmarkJobManager.downloadStatistics(), new P_ProgressNotifierAdapter(progress));
Path resultFile = job.getDirectory().resolve(BENCHMARK_RESULT_FILE);
if (resultFile != null)
BenchmarkJobManager.formatResultFile(resultFile);
@@ -431,77 +432,6 @@ public class BenchmarkJobManager {
return jobManager.getJobs().stream().map(this::convertJob).collect(Collectors.toList());
}
- private HaaSClient.UploadingFile getUploadingFile() {
- return new UploadingFileFromResource("", Constants.CONFIG_YAML);
- }
-
- private BenchmarkJob convertJob(Job job) {
- return new BenchmarkJob(job);
- }
-
- private String getOutputName(InputStream openLocalFile) throws IOException {
- try (InputStream is = openLocalFile) {
- Yaml yaml = new Yaml();
-
- Map<String, Map<String, String>> map = yaml.load(is);
- String result = map.get("common").get("hdf5_xml_filename");
- if (result == null) {
- throw new IllegalArgumentException("hdf5_xml_filename not found");
- }
- if (result.charAt(0) == '"' || result.charAt(0) == '\'') {
- if (result.charAt(result.length() - 1) != result.charAt(0)) {
- throw new IllegalArgumentException(result);
- }
- result = result.substring(1, result.length() - 1);
- }
-
- return result;
- }
-
- }
-
- private static Predicate<String> downloadFinishedData(String filePattern) {
- return name -> {
- Path path = getPathSafely(name);
- if (path == null)
- return false;
-
- String fileName = path.getFileName().toString();
- return fileName.startsWith(filePattern) && fileName.endsWith("h5") || fileName.equals(filePattern + ".xml")
- || fileName.equals(Constants.BENCHMARK_RESULT_FILE);
- };
- }
-
- private static Predicate<String> downloadStatistics() {
- return name -> {
- Path path = getPathSafely(name);
- if (path == null)
- return false;
-
- String fileName = path.getFileName().toString();
- return fileName.equals(Constants.BENCHMARK_RESULT_FILE);
- };
- }
-
- private static Predicate<String> downloadFailedData() {
- return name -> {
- Path path = getPathSafely(name);
- if (path == null)
- return false;
- return path.getFileName().toString().startsWith("snakejob.")
- || path.getParent() != null && path.getParent().getFileName() != null
- && path.getParent().getFileName().toString().equals("logs");
- };
- }
-
- private static Path getPathSafely(String name) {
- try {
- return Paths.get(name);
- } catch (InvalidPathException ex) {
- return null;
- }
- }
-
public static void formatResultFile(Path filename) throws FileNotFoundException {
List<ResultFileTask> identifiedTasks = new LinkedList<ResultFileTask>();
@@ -603,6 +533,77 @@ public class BenchmarkJobManager {
}
}
+ private HaaSClient.UploadingFile getUploadingFile() {
+ return new UploadingFileFromResource("", Constants.CONFIG_YAML);
+ }
+
+ private BenchmarkJob convertJob(Job job) {
+ return new BenchmarkJob(job);
+ }
+
+ private String getOutputName(InputStream openLocalFile) throws IOException {
+ try (InputStream is = openLocalFile) {
+ Yaml yaml = new Yaml();
+
+ Map<String, Map<String, String>> map = yaml.load(is);
+ String result = map.get("common").get("hdf5_xml_filename");
+ if (result == null) {
+ throw new IllegalArgumentException("hdf5_xml_filename not found");
+ }
+ if (result.charAt(0) == '"' || result.charAt(0) == '\'') {
+ if (result.charAt(result.length() - 1) != result.charAt(0)) {
+ throw new IllegalArgumentException(result);
+ }
+ result = result.substring(1, result.length() - 1);
+ }
+
+ return result;
+ }
+
+ }
+
+ private static Predicate<String> downloadFinishedData(String filePattern) {
+ return name -> {
+ Path path = getPathSafely(name);
+ if (path == null)
+ return false;
+
+ String fileName = path.getFileName().toString();
+ return fileName.startsWith(filePattern) && fileName.endsWith("h5") || fileName.equals(filePattern + ".xml")
+ || fileName.equals(Constants.BENCHMARK_RESULT_FILE);
+ };
+ }
+
+ private static Predicate<String> downloadStatistics() {
+ return name -> {
+ Path path = getPathSafely(name);
+ if (path == null)
+ return false;
+
+ String fileName = path.getFileName().toString();
+ return fileName.equals(Constants.BENCHMARK_RESULT_FILE);
+ };
+ }
+
+ private static Predicate<String> downloadFailedData() {
+ return name -> {
+ Path path = getPathSafely(name);
+ if (path == null)
+ return false;
+ return path.getFileName().toString().startsWith("snakejob.")
+ || path.getParent() != null && path.getParent().getFileName() != null
+ && path.getParent().getFileName().toString().equals("logs");
+ };
+ }
+
+ private static Path getPathSafely(String name) {
+ try {
+ return Paths.get(name);
+ } catch (InvalidPathException ex) {
+ return null;
+ }
+ }
+
private static Settings constructSettingsFromParams(BenchmarkSPIMParameters params) {
return new Settings() {
@@ -653,4 +654,37 @@ public class BenchmarkJobManager {
};
}
+ private class P_ProgressNotifierAdapter implements ProgressNotifier {
+ private Progress progress;
+
+ public P_ProgressNotifierAdapter(Progress progress) {
+ this.progress = progress;
+ }
+
+ public void setTitle(String title) {
+ progress.setTitle(title);
+ }
+
+ public void setCount(int count, int total) {
+ progress.setCount(count, total);
+ }
+
+ public void addItem(Object item) {
+ progress.addItem(item);
+ }
+
+ public void setItemCount(int count, int total) {
+ progress.setItemCount(count, total);
+ }
+
+ public void itemDone(Object item) {
+ progress.itemDone(item);
+ }
+
+ public void done() {
+ progress.done();
+ }
+
+ }
+
}