diff --git a/haas-imagej-client/pom.xml b/haas-imagej-client/pom.xml index 5e9a13ad436e25fd35884cde28d1e98b65ea4c46..93bb30ca3b75300d021a081a811e593900b10d3f 100644 --- a/haas-imagej-client/pom.xml +++ b/haas-imagej-client/pom.xml @@ -73,7 +73,7 @@ <dependency> <groupId>cz.it4i.fiji</groupId> <artifactId>haas-java-client</artifactId> - <version>0.0.1-SNAPSHOT</version> + <version>0.0.2-SNAPSHOT</version> </dependency> <!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-api --> <dependency> diff --git a/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/Job.java b/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/Job.java index 1b0eea1e19d7e24ff87de7bf18f653122ffdc0e7..330bb892da3866d92dae5e2f887329059628ad0d 100644 --- a/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/Job.java +++ b/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/Job.java @@ -5,12 +5,14 @@ import java.io.InputStream; import java.io.InterruptedIOException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Arrays; import java.util.Calendar; import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -29,13 +31,24 @@ import cz.it4i.fiji.haas_java_client.ProgressNotifier; import cz.it4i.fiji.haas_java_client.TransferFileProgressForHaaSClient; import cz.it4i.fiji.haas_java_client.UploadingFile; import cz.it4i.fiji.scpclient.TransferFileProgress; + /*** - * TASK - napojit na UI + * TASK - napojit na UI + * * @author koz01 * */ public class Job { + public static boolean isValidJobPath(Path path) { + try { + getJobId(path); + } catch (NumberFormatException e) { + return false; + } + return Files.isRegularFile(path.resolve(JOB_INFO_FILENAME)); + } + private static final String JOB_NAME = "job.name"; private static final String JOB_NEEDS_UPLOAD = "job.needs_upload"; @@ -43,21 +56,18 @@ public class Job { private static final String JOB_INFO_FILENAME = ".jobinfo"; private static final String JOB_NEEDS_DOWNLOAD = "job.needs_download"; - + private static final String JOB_CAN_BE_DOWNLOADED = "job.can_be_downloaded"; - + private static final String JOB_IS_DOWNLOADED = "job.downloaded"; - + private static final String JOB_IS_UPLOADED = "job.uploaded"; - public static boolean isValidJobPath(Path path) { - try { - getJobId(path); - } catch (NumberFormatException e) { - return false; - } - return Files.isRegularFile(path.resolve(JOB_INFO_FILENAME)); - } + private static final String JOB_OUTPUT_DIRECTORY_PATH = "job.output_directory_path"; + + private static final String JOB_INPUT_DIRECTORY_PATH = "job.input_directory_path"; + + private static final String JOB_USE_DEMO_DATA = "job.use_demo_data"; private static Logger log = LoggerFactory.getLogger(cz.it4i.fiji.haas.Job.class); @@ -66,41 +76,50 @@ public class Job { private final Supplier<HaaSClient> haasClientSupplier; private JobInfo jobInfo; - + private Long jobId; - + private PropertyHolder propertyHolder; - + private final JobManager4Job jobManager; - + private Synchronization synchronization; + + private Path inputDirectory; + + private Path outputDirectory; + + private boolean useDemoData; + - - public Job(JobManager4Job jobManager, String name, Path basePath, Supplier<HaaSClient> haasClientSupplier) + + public Job(JobManager4Job jobManager, String name, Path basePath, Supplier<HaaSClient> haasClientSupplier, + Function<Path, Path> inputDirectoryProvider, Function<Path, Path> outputDirectoryProvider) throws IOException { this(jobManager, haasClientSupplier); HaaSClient client = getHaaSClient(); long id = client.createJob(name, Collections.emptyList()); - setJobDirectory(basePath.resolve("" + id)); + setJobDirectory(basePath.resolve("" + id), inputDirectoryProvider, outputDirectoryProvider); propertyHolder = new PropertyHolder(jobDir.resolve(JOB_INFO_FILENAME)); - Files.createDirectory(jobDir); + Files.createDirectory(this.jobDir); + storeInputOutputDirectory(); setName(name); - + } public Job(JobManager4Job jobManager, Path jobDirectory, Supplier<HaaSClient> haasClientSupplier) { this(jobManager, haasClientSupplier); - setJobDirectory(jobDirectory); - propertyHolder = new PropertyHolder(jobDir.resolve(JOB_INFO_FILENAME)); + propertyHolder = new PropertyHolder(jobDirectory.resolve(JOB_INFO_FILENAME)); + useDemoData = getSafeBoolean(propertyHolder.getValue(JOB_USE_DEMO_DATA)); + setJobDirectory(jobDirectory, jd -> useDemoData ? null : getDataDirectory(JOB_INPUT_DIRECTORY_PATH, jd), + jd -> getDataDirectory(JOB_OUTPUT_DIRECTORY_PATH, jd)); } - private Job(JobManager4Job jobManager, Supplier<HaaSClient> haasClientSupplier) { this.haasClientSupplier = haasClientSupplier; this.jobManager = jobManager; } - public void startUploadData() { setProperty(JOB_NEEDS_UPLOAD, true); try { @@ -127,7 +146,7 @@ public class Job { .collect(Collectors.toList()); synchronization.startDownload(files); } - + public void stopDownloadData() { setProperty(JOB_NEEDS_DOWNLOAD, false); try { @@ -137,14 +156,12 @@ public class Job { throw new RuntimeException(e); } } - + public synchronized void resumeUpload() { if (needsUpload()) { synchronization.resumeUpload(); } } - - public synchronized void resumeDownload() { if (needsDownload()) { @@ -152,11 +169,10 @@ public class Job { } } - public boolean canBeDownloaded() { return Boolean.parseBoolean(getProperty(JOB_CAN_BE_DOWNLOADED)); } - + public void setUploaded(boolean b) { setProperty(JOB_IS_UPLOADED, b); } @@ -164,23 +180,23 @@ public class Job { public void setDownloaded(boolean b) { setProperty(JOB_IS_DOWNLOADED, b); } - + public boolean isUploaded() { - return getSafeBoolean(getProperty(JOB_IS_UPLOADED)); + return getSafeBoolean(getProperty(JOB_IS_UPLOADED)); } public boolean isDownloaded() { return getSafeBoolean(getProperty(JOB_IS_DOWNLOADED)); } - + public boolean needsDownload() { return Boolean.parseBoolean(getProperty(JOB_NEEDS_DOWNLOAD)); } - + public boolean needsUpload() { return Boolean.parseBoolean(getProperty(JOB_NEEDS_UPLOAD)); } - + public void uploadFile(String file, ProgressNotifier notifier) { uploadFiles(Arrays.asList(file), notifier); } @@ -224,14 +240,16 @@ public class Job { setCanBeDownloaded(true); } - - synchronized public long getId() { if (jobId == null) { jobId = getJobId(jobDir); } return jobId; } + + public boolean isUseDemoData() { + return useDemoData; + } public Path storeDataInWorkdirectory(UploadingFile uploadingFile) throws IOException { Path result; @@ -242,8 +260,10 @@ public class Job { } synchronized public void download(Predicate<String> predicate, ProgressNotifier notifier) { - List<String> files = getHaaSClient().getChangedFiles(jobId).stream().filter(predicate).collect(Collectors.toList()); - try (HaaSFileTransfer transfer = haasClientSupplier.get().startFileTransfer(getId(), HaaSClient.DUMMY_TRANSFER_FILE_PROGRESS)) { + List<String> files = getHaaSClient().getChangedFiles(jobId).stream().filter(predicate) + .collect(Collectors.toList()); + try (HaaSFileTransfer transfer = haasClientSupplier.get().startFileTransfer(getId(), + HaaSClient.DUMMY_TRANSFER_FILE_PROGRESS)) { List<Long> fileSizes; try { fileSizes = transfer.obtainSize(files); @@ -318,9 +338,9 @@ public class Job { return jobDir; } - public boolean remove() { + public boolean delete() { boolean result; - if ((result = jobManager.remove(this)) && Files.isDirectory(jobDir)) { + if ((result = jobManager.deleteJob(this)) && Files.isDirectory(jobDir)) { List<Path> pathsToDelete; try { pathsToDelete = Files.walk(jobDir).sorted(Comparator.reverseOrder()).collect(Collectors.toList()); @@ -369,41 +389,65 @@ public class Job { public void setUploadNotifier(ProgressNotifier notifier) { synchronization.setUploadNotifier(notifier); } - + public void close() { synchronization.close(); } + public Path getInputDirectory() { + return inputDirectory; + } + + private void storeInputOutputDirectory() { + if (inputDirectory == null) { + useDemoData = true; + propertyHolder.setValue(JOB_USE_DEMO_DATA, "" + useDemoData); + } else { + storeDataDirectory(JOB_INPUT_DIRECTORY_PATH, inputDirectory); + } + storeDataDirectory(JOB_OUTPUT_DIRECTORY_PATH, outputDirectory); + } + + private void storeDataDirectory(String directoryPropertyName, Path directory) { + if (!jobDir.equals(directory)) { + propertyHolder.setValue(directoryPropertyName, directory.toString()); + } + } + + private Path getDataDirectory(String typeOfDirectory, Path jobDirectory) { + String directory = propertyHolder.getValue(typeOfDirectory); + return directory != null ? Paths.get(directory) : jobDirectory; + } + private boolean getSafeBoolean(String value) { return value != null ? Boolean.parseBoolean(value) : false; } - private void setJobDirectory(Path jobDirectory) { + private void setJobDirectory(Path jobDirectory, Function<Path, Path> inputDirectoryProvider, + Function<Path, Path> outputDirectoryProvider) { this.jobDir = jobDirectory; + try { this.synchronization = new Synchronization(() -> startFileTransfer(HaaSClient.DUMMY_TRANSFER_FILE_PROGRESS), - jobDir, () -> { + jobDir, this.inputDirectory = inputDirectoryProvider.apply(jobDir), + this.outputDirectory = outputDirectoryProvider.apply(jobDir), () -> { setProperty(JOB_NEEDS_UPLOAD, false); setUploaded(true); }, () -> { setDownloaded(true); setProperty(JOB_NEEDS_DOWNLOAD, false); setCanBeDownloaded(false); - }); + }, p -> jobManager.canUpload(Job.this, p)); } catch (IOException e) { log.error(e.getMessage(), e); throw new RuntimeException(e); } } - - - private HaaSFileTransfer startFileTransfer( TransferFileProgress progress) { + private HaaSFileTransfer startFileTransfer(TransferFileProgress progress) { return haasClientSupplier.get().startFileTransfer(getId(), progress); } - - private void setName(String name) { setProperty(JOB_NAME, name); } @@ -426,7 +470,7 @@ public class Job { private static long getJobId(Path path) { return Long.parseLong(path.getFileName().toString()); } - + private void setCanBeDownloaded(boolean b) { setProperty(JOB_CAN_BE_DOWNLOADED, b); } diff --git a/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/JobManager.java b/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/JobManager.java index 862562e515eac0d1d48d6e81e87a5557f20324eb..902780ceb5812c3d3f42d58cf3140c662ae47357 100644 --- a/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/JobManager.java +++ b/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/JobManager.java @@ -7,6 +7,8 @@ import java.nio.file.Path; import java.util.Collection; import java.util.Collections; import java.util.LinkedList; +import java.util.function.BiPredicate; +import java.util.function.Function; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -18,11 +20,15 @@ import cz.it4i.fiji.haas_java_client.SynchronizableFileType; public class JobManager implements Closeable { interface JobManager4Job { - boolean remove(Job job); + boolean deleteJob(Job job); + + boolean canUpload(Job j, Path p); } private static Logger log = LoggerFactory.getLogger(cz.it4i.fiji.haas.JobManager.class); + private static final BiPredicate<Job, Path> DUMMY_UPLOAD_FILTER = (X, Y) -> true; + private final Path workDirectory; private Collection<Job> jobs; @@ -31,12 +37,20 @@ public class JobManager implements Closeable { private final Settings settings; + private BiPredicate<Job, Path> uploadFilter = DUMMY_UPLOAD_FILTER; + private final JobManager4Job remover = new JobManager4Job() { @Override - public boolean remove(Job job) { + public boolean deleteJob(Job job) { + haasClient.deleteJob(job.getId()); return jobs.remove(job); } + + @Override + public boolean canUpload(Job j, Path p) { + return uploadFilter.test(j, p); + } }; public JobManager(Path workDirectory, Settings settings) { @@ -44,10 +58,12 @@ public class JobManager implements Closeable { this.settings = settings; } - public Job createJob() throws IOException { + public Job createJob(Function<Path, Path> inputDirectoryProvider, Function<Path, Path> outputDirectoryProvider) + throws IOException { Job result; initJobsIfNecessary(); - jobs.add(result = new Job(remover, settings.getJobName(), workDirectory, this::getHaasClient)); + jobs.add(result = new Job(remover, settings.getJobName(), workDirectory, this::getHaasClient, + inputDirectoryProvider, outputDirectoryProvider)); return result; } @@ -61,6 +77,10 @@ public class JobManager implements Closeable { jobs.forEach(job -> job.close()); } + public void setUploadFilter(BiPredicate<Job, Path> filter) { + uploadFilter = filter != null ? filter : DUMMY_UPLOAD_FILTER; + } + private HaaSClient getHaasClient() { if (haasClient == null) { haasClient = new HaaSClient(settings); diff --git a/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/data_transfer/Synchronization.java b/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/data_transfer/Synchronization.java index 5e17ff9db7f13120247a1f4c460ca06886732abb..f2f2181efb8e24e8db9ebd754ab58384fd78b9bc 100644 --- a/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/data_transfer/Synchronization.java +++ b/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/data_transfer/Synchronization.java @@ -6,12 +6,13 @@ import java.io.InterruptedIOException; import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Collection; import java.util.Collections; import java.util.LinkedList; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.function.Function; +import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -24,20 +25,22 @@ import cz.it4i.fiji.haas_java_client.ProgressNotifier; import cz.it4i.fiji.haas_java_client.UploadingFile; import cz.it4i.fiji.haas_java_client.UploadingFileImpl; -public class Synchronization implements Closeable{ +public class Synchronization implements Closeable { public static final Logger log = LoggerFactory.getLogger(cz.it4i.fiji.haas.data_transfer.Synchronization.class); - + private static final String FILE_INDEX_TO_UPLOAD_FILENAME = ".toUploadFiles"; - + private static final String FILE_INDEX_TO_DOWNLOAD_FILENAME = ".toDownloadFiles"; - + private static final String FILE_INDEX_DOWNLOADED_FILENAME = ".downloaded"; - + private final Path workingDirectory; - - private final Function<String,Path> pathResolver; - + + private final Path inputDirectory; + + private final Path outputDirectory; + private final PersistentIndex<Path> filesDownloaded; private final PersistentSynchronizationProcess<Path> uploadProcess; @@ -45,31 +48,36 @@ public class Synchronization implements Closeable{ private final P_PersistentDownloadProcess downloadProcess; private final ExecutorService service; - - public Synchronization(Supplier<HaaSFileTransfer> fileTransferSupplier, Path workingDirectory, - Runnable uploadFinishedNotifier, Runnable downloadFinishedNotifier) throws IOException { - this.service = Executors.newFixedThreadPool(2); + + private final Predicate<Path> uploadFilter; + + public Synchronization(Supplier<HaaSFileTransfer> fileTransferSupplier, Path workingDirectory, Path inputDirectory, + Path outputDirectory, Runnable uploadFinishedNotifier, Runnable downloadFinishedNotifier, Predicate<Path> uploadFilter) + throws IOException { this.workingDirectory = workingDirectory; - this.pathResolver = name -> workingDirectory.resolve(name); + this.inputDirectory = inputDirectory; + this.outputDirectory = outputDirectory; + this.service = Executors.newFixedThreadPool(2); this.filesDownloaded = new PersistentIndex<>(workingDirectory.resolve(FILE_INDEX_DOWNLOADED_FILENAME), - pathResolver); + name -> Paths.get(name)); this.uploadProcess = createUploadProcess(fileTransferSupplier, service, uploadFinishedNotifier); this.downloadProcess = createDownloadProcess(fileTransferSupplier, service, downloadFinishedNotifier); + this.uploadFilter = uploadFilter; } public synchronized void setUploadNotifier(ProgressNotifier notifier) { uploadProcess.setNotifier(notifier); } - + public void setDownloadNotifier(ProgressNotifier notifier) { downloadProcess.setNotifier(notifier); - + } public synchronized void startUpload() throws IOException { uploadProcess.start(); } - + public void stopUpload() throws IOException { uploadProcess.stop(); } @@ -77,16 +85,16 @@ public class Synchronization implements Closeable{ public void resumeUpload() { uploadProcess.resume(); } - + public synchronized void startDownload(Collection<String> files) throws IOException { this.downloadProcess.setItems(files); this.downloadProcess.start(); } - + public synchronized void stopDownload() throws IOException { this.downloadProcess.stop(); } - + public synchronized void resumeDownload() { this.downloadProcess.resume(); } @@ -99,23 +107,24 @@ public class Synchronization implements Closeable{ } private boolean canUpload(Path file) { - - return !file.getFileName().toString().matches("[.][^.]+") && !filesDownloaded.contains(file); + return uploadFilter.test(file) && !file.getFileName().toString().matches("[.][^.]+") + && !filesDownloaded.contains(file); } private PersistentSynchronizationProcess<Path> createUploadProcess(Supplier<HaaSFileTransfer> fileTransferSupplier, ExecutorService service, Runnable uploadFinishedNotifier) throws IOException { return new PersistentSynchronizationProcess<Path>(service, fileTransferSupplier, uploadFinishedNotifier, - workingDirectory.resolve(FILE_INDEX_TO_UPLOAD_FILENAME), pathResolver) { + workingDirectory.resolve(FILE_INDEX_TO_UPLOAD_FILENAME), name -> inputDirectory.resolve(name)) { @Override protected Iterable<Path> getItems() throws IOException { - try(DirectoryStream<Path> ds = Files.newDirectoryStream(workingDirectory,Synchronization.this::canUpload)) { - return StreamSupport.stream(ds.spliterator(), false).collect(Collectors.toList()); + try (DirectoryStream<Path> ds = Files.newDirectoryStream(inputDirectory, + Synchronization.this::canUpload)) { + return StreamSupport.stream(ds.spliterator(), false).collect(Collectors.toList()); } - + } - + @Override protected void processItem(HaaSFileTransfer tr, Path p) throws InterruptedIOException { UploadingFile uf = new UploadingFileImpl(p); @@ -124,29 +133,28 @@ public class Synchronization implements Closeable{ @Override protected long getTotalSize(Iterable<Path> items, HaaSFileTransfer tr) { - return StreamSupport.stream(items.spliterator(), false).map(p->{ + return StreamSupport.stream(items.spliterator(), false).map(p -> { try { return Files.size(p); } catch (IOException e) { log.error(e.getMessage(), e); - return 0; + return 0; } - }).collect(Collectors.summingLong(val->val.longValue())); + }).collect(Collectors.summingLong(val -> val.longValue())); } }; } - private P_PersistentDownloadProcess createDownloadProcess( - Supplier<HaaSFileTransfer> fileTransferSupplier, ExecutorService service, - Runnable uploadFinishedNotifier) throws IOException { - + private P_PersistentDownloadProcess createDownloadProcess(Supplier<HaaSFileTransfer> fileTransferSupplier, + ExecutorService service, Runnable uploadFinishedNotifier) throws IOException { + return new P_PersistentDownloadProcess(service, fileTransferSupplier, uploadFinishedNotifier); } - - private class P_PersistentDownloadProcess extends PersistentSynchronizationProcess<String>{ + + private class P_PersistentDownloadProcess extends PersistentSynchronizationProcess<String> { private Collection<String> items = Collections.emptyList(); - + public P_PersistentDownloadProcess(ExecutorService service, Supplier<HaaSFileTransfer> fileTransferSupplier, Runnable processFinishedNotifier) throws IOException { super(service, fileTransferSupplier, processFinishedNotifier, @@ -156,7 +164,7 @@ public class Synchronization implements Closeable{ private synchronized void setItems(Collection<String> items) { this.items = new LinkedList<>(items); } - + @Override protected synchronized Iterable<String> getItems() throws IOException { return items; @@ -164,19 +172,20 @@ public class Synchronization implements Closeable{ @Override protected void processItem(HaaSFileTransfer tr, String file) throws InterruptedIOException { - filesDownloaded.insert(workingDirectory.resolve(file)); + filesDownloaded.insert(outputDirectory.resolve(file)); try { filesDownloaded.storeToWorkingFile(); } catch (IOException e) { log.error(e.getMessage(), e); } - tr.download(file, workingDirectory); + tr.download(file, outputDirectory); } @Override protected long getTotalSize(Iterable<String> items, HaaSFileTransfer tr) throws InterruptedIOException { - return tr.obtainSize( StreamSupport.stream(items.spliterator(), false).collect(Collectors.toList())).stream().collect(Collectors.summingLong(val->val)); + return tr.obtainSize(StreamSupport.stream(items.spliterator(), false).collect(Collectors.toList())).stream() + .collect(Collectors.summingLong(val -> val)); } - + } } diff --git a/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/ui/JavaFXRoutines.java b/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/ui/JavaFXRoutines.java index c33528cde87f4dad3d174482c44b0698620a7924..f3c67a341f10bb75bb1d652f74de0263f03487d6 100644 --- a/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/ui/JavaFXRoutines.java +++ b/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/ui/JavaFXRoutines.java @@ -28,7 +28,7 @@ public interface JavaFXRoutines { void accept(TableCell<?, ?> cell, B value, boolean empty); } - private TableCellUpdater<S, T> updater; + private final TableCellUpdater<S, T> updater; public TableCellAdapter(TableCellUpdater<S, T> updater) { this.updater = updater; @@ -36,14 +36,18 @@ public interface JavaFXRoutines { @Override protected void updateItem(T item, boolean empty) { - updater.accept(this, item, empty); + if(empty) { + this.setText(""); + } else { + updater.accept(this, item, empty); + } } } static public class FutureValueUpdater<S, T, U extends CompletableFuture<T>> implements TableCellUpdater<S, U> { - private TableCellUpdater<S, T> inner; - private Executor executor; + private final TableCellUpdater<S, T> inner; + private final Executor executor; public FutureValueUpdater(TableCellUpdater<S, T> inner, Executor exec) { this.inner = inner; diff --git a/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/ui/ObservableValueRegistry.java b/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/ui/ObservableValueRegistry.java index 54af7f42dc79c816fa6f4716f2e9c7fe2aae54be..e55921801fb2a36ca79c4e5e1723bc6df02f3317 100644 --- a/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/ui/ObservableValueRegistry.java +++ b/haas-imagej-client/src/main/java/cz/it4i/fiji/haas/ui/ObservableValueRegistry.java @@ -30,16 +30,16 @@ public abstract class ObservableValueRegistry<K, V extends UpdatableObservableVa }; } - public V addIfAbsent(K key) { + public synchronized V addIfAbsent(K key) { V uov = map.computeIfAbsent(key, k -> constructObservableValue(k)); return uov; } - public V get(K key) { + public synchronized V get(K key) { return map.get(key); } - public Collection<V> getAllItems() { + public synchronized Collection<V> getAllItems() { return map.values().stream().map(val -> val).collect(Collectors.toList()); } diff --git a/haas-java-client/src/main/java/cz/it4i/fiji/haas_java_client/HaaSClient.java b/haas-java-client/src/main/java/cz/it4i/fiji/haas_java_client/HaaSClient.java index 518dab5ef745370b63a19285db8844db6dfaf9ef..b6190091364b7b3f129078a78d98ae0479c0228e 100644 --- a/haas-java-client/src/main/java/cz/it4i/fiji/haas_java_client/HaaSClient.java +++ b/haas-java-client/src/main/java/cz/it4i/fiji/haas_java_client/HaaSClient.java @@ -298,6 +298,14 @@ public class HaaSClient { } } + public void deleteJob(long id) { + try { + getJobManagement().deleteJob(id, getSessionID()); + } catch (RemoteException | ServiceException e) { + throw new HaaSClientException(e); + } + } + private HaaSFileTransferImp getFileTransferMethod(long jobId, TransferFileProgress progress) throws RemoteException, UnsupportedEncodingException, ServiceException, JSchException { P_FileTransferPool pool = filetransferPoolMap.computeIfAbsent(jobId, id -> new P_FileTransferPool(id)); @@ -327,7 +335,7 @@ public class HaaSClient { private long doCreateJob(String name, int numberOfNodes, Collection<Entry<String, String>> templateParameters) throws RemoteException, ServiceException { Collection<TaskSpecificationExt> taskSpec = Arrays - .asList(createTaskSpecification(name + ": ", templateId, numberOfNodes, templateParameters)); + .asList(createTaskSpecification(name + "-task", templateId, numberOfNodes, templateParameters)); JobSpecificationExt jobSpecification = createJobSpecification(name, numberOfNodes, taskSpec); SubmittedJobInfoExt job = getJobManagement().createJob(jobSpecification, getSessionID()); return job.getId(); diff --git a/haas-java-client/src/main/java/cz/it4i/fiji/haas_java_client/JobState.java b/haas-java-client/src/main/java/cz/it4i/fiji/haas_java_client/JobState.java index 374481004f6c13fb6f4dcccee151dc97b4cb0ab3..3468aeda2d3b641547c1e0fc975d91af04a3837d 100644 --- a/haas-java-client/src/main/java/cz/it4i/fiji/haas_java_client/JobState.java +++ b/haas-java-client/src/main/java/cz/it4i/fiji/haas_java_client/JobState.java @@ -8,5 +8,6 @@ public enum JobState { Running, Finished, Failed, - Canceled; + Canceled, + Disposed; } diff --git a/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/core/BenchmarkJobManager.java b/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/core/BenchmarkJobManager.java index 17237edabd25cf28118fe0b64015f27dc179c967..e565d1d65a5c53e1bcee290802eb4d8bb731e336 100644 --- a/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/core/BenchmarkJobManager.java +++ b/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/core/BenchmarkJobManager.java @@ -25,9 +25,11 @@ import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Scanner; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executor; +import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -40,7 +42,6 @@ import cz.it4i.fiji.haas.HaaSOutputHolder; import cz.it4i.fiji.haas.HaaSOutputHolderImpl; import cz.it4i.fiji.haas.Job; import cz.it4i.fiji.haas.JobManager; -import cz.it4i.fiji.haas.UploadingFileFromResource; import cz.it4i.fiji.haas_java_client.JobState; import cz.it4i.fiji.haas_java_client.ProgressNotifier; import cz.it4i.fiji.haas_java_client.Settings; @@ -48,9 +49,8 @@ import cz.it4i.fiji.haas_java_client.SynchronizableFileType; import cz.it4i.fiji.haas_java_client.UploadingFile; import net.imagej.updater.util.Progress; -public class BenchmarkJobManager implements Closeable{ +public class BenchmarkJobManager implements Closeable { - private static Logger log = LoggerFactory .getLogger(cz.it4i.fiji.haas_spim_benchmark.core.BenchmarkJobManager.class); @@ -59,7 +59,11 @@ public class BenchmarkJobManager implements Closeable{ public final class BenchmarkJob implements HaaSOutputHolder { private final Job job; - + + public boolean isUseDemoData() { + return job.isUseDemoData(); + } + private final List<Task> tasks; private final List<BenchmarkError> nonTaskSpecificErrors; private final SPIMComputationAccessor computationAccessor; @@ -84,14 +88,14 @@ public class BenchmarkJobManager implements Closeable{ } public synchronized void startJob(Progress progress) throws IOException { - job.uploadFile(Constants.CONFIG_YAML, new P_ProgressNotifierAdapter(progress)); + job.uploadFile(Constants.CONFIG_YAML, new P_ProgressNotifierAdapter(progress)); String outputName = getOutputName(job.openLocalFile(Constants.CONFIG_YAML)); verifiedState = null; verifiedStateProcessed = false; running = null; job.submit(); job.setProperty(SPIM_OUTPUT_FILENAME_PATTERN, outputName); - + } public JobState getState() { @@ -106,7 +110,6 @@ public class BenchmarkJobManager implements Closeable{ job.stopUploadData(); } - public synchronized CompletableFuture<JobState> getStateAsync(Executor executor) { if (running != null) { return running; @@ -121,12 +124,12 @@ public class BenchmarkJobManager implements Closeable{ public void startDownload() throws IOException { if (job.getState() == JobState.Finished) { String filePattern = job.getProperty(SPIM_OUTPUT_FILENAME_PATTERN); - job.startDownload(downloadFinishedData(filePattern) ); + job.startDownload(downloadFinishedData(filePattern)); } else if (job.getState() == JobState.Failed || job.getState() == JobState.Canceled) { job.startDownload(downloadFailedData()); } } - + public boolean canBeDownloaded() { return job.canBeDownloaded(); } @@ -205,8 +208,8 @@ public class BenchmarkJobManager implements Closeable{ return computationAccessor.getActualOutput(Arrays.asList(SynchronizableFileType.StandardErrorFile)).get(0); } - public boolean remove() { - return job.remove(); + public boolean delete() { + return job.delete(); } public void cancelJob() { @@ -223,11 +226,47 @@ public class BenchmarkJobManager implements Closeable{ job.resumeUpload(); } + public void setDownloaded(Boolean val) { + job.setDownloaded(val); + } + + public void setUploaded(boolean b) { + job.setUploaded(b); + } + + public boolean isDownloaded() { + return job.isDownloaded(); + } + + public boolean isUploaded() { + return job.isUploaded(); + } + + public void stopDownload() { + job.stopDownloadData(); + } + + public boolean needsDownload() { + return job.needsDownload(); + } + + public boolean needsUpload() { + return job.needsUpload(); + } + @Override public String toString() { return "" + getId(); } + public void storeDataInWorkdirectory(UploadingFile file) throws IOException { + job.storeDataInWorkdirectory(file); + } + + public Path getInputDirectory() { + return job.getInputDirectory(); + } + private ProgressNotifier convertTo(Progress progress) { return progress == null ? null : new P_ProgressNotifierAdapter(progress); } @@ -385,7 +424,8 @@ public class BenchmarkJobManager implements Closeable{ private SPIMComputationAccessor getComputationAccessor() { SPIMComputationAccessor result = new SPIMComputationAccessor() { - private final HaaSOutputHolder outputOfSnakemake = new HaaSOutputHolderImpl(list -> job.getOutput(list)); + private final HaaSOutputHolder outputOfSnakemake = new HaaSOutputHolderImpl( + list -> job.getOutput(list)); @Override public List<String> getActualOutput(List<SynchronizableFileType> content) { @@ -421,42 +461,16 @@ public class BenchmarkJobManager implements Closeable{ return Stream.concat(nonTaskSpecificErrors.stream(), taskSpecificErrors).collect(Collectors.toList()); } - public void setDownloaded(Boolean val) { - job.setDownloaded(val); - } - - public void setUploaded(boolean b) { - job.setUploaded(b); - } - - public boolean isDownloaded() { - return job.isDownloaded(); - } - - public boolean isUploaded() { - return job.isUploaded(); - } - - public void stopDownload() { - job.stopDownloadData(); - } - - public boolean needsDownload() { - return job.needsDownload(); - } - - public boolean needsUpload() { - return job.needsUpload(); - } } public BenchmarkJobManager(BenchmarkSPIMParameters params) throws IOException { jobManager = new JobManager(params.workingDirectory(), constructSettingsFromParams(params)); + jobManager.setUploadFilter(this::canUpload); } - public BenchmarkJob createJob() throws IOException { - Job job = jobManager.createJob(); - job.storeDataInWorkdirectory(getUploadingFile()); + public BenchmarkJob createJob(Function<Path, Path> inputDirectoryProvider, + Function<Path, Path> outputDirectoryProvider) throws IOException { + Job job = jobManager.createJob(inputDirectoryProvider, outputDirectoryProvider); return convertJob(job); } @@ -570,8 +584,8 @@ public class BenchmarkJobManager implements Closeable{ jobManager.close(); } - private UploadingFile getUploadingFile() { - return new UploadingFileFromResource("", Constants.CONFIG_YAML); + private boolean canUpload(Job job, Path p) { + return job.getInputDirectory() == null || !p.equals(job.getInputDirectory().resolve(Constants.CONFIG_YAML)); } private BenchmarkJob convertJob(Job job) { @@ -581,9 +595,9 @@ public class BenchmarkJobManager implements Closeable{ private String getOutputName(InputStream openLocalFile) throws IOException { try (InputStream is = openLocalFile) { Yaml yaml = new Yaml(); - Map<String, Map<String, String>> map = yaml.load(is); - String result = map.get("common").get("hdf5_xml_filename"); + String result = Optional.ofNullable(map).map(m -> m.get("common")).map(m -> m.get("hdf5_xml_filename")) + .orElse(null); if (result == null) { throw new IllegalArgumentException("hdf5_xml_filename not found"); } @@ -688,6 +702,11 @@ public class BenchmarkJobManager implements Closeable{ public long getClusterNodeType() { return Constants.HAAS_CLUSTER_NODE_TYPE; } + + @Override + public int getNumberOfCoresPerNode() { + return Constants.CORES_PER_NODE; + } }; } diff --git a/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/core/Constants.java b/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/core/Constants.java index dd3ce0507d0be10acf7170cce0f1360630e85b94..57dc02e866f93af3c1321bfddba4e3e80c6010dd 100644 --- a/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/core/Constants.java +++ b/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/core/Constants.java @@ -10,7 +10,7 @@ public interface Constants { String HAAS_JOB_NAME = "HaaSSPIMBenchmark"; int HAAS_CLUSTER_NODE_TYPE = 6; int HAAS_TEMPLATE_ID = 4; - String HAAS_PROJECT_ID = "DD-17-31"; + String HAAS_PROJECT_ID = "OPEN-12-20"; int HAAS_TIMEOUT = 9600; //Walltime in seconds final String NEW_LINE_SEPARATOR = "\n"; @@ -52,5 +52,6 @@ public interface Constants { String STATISTICS_SUMMARY_FILENAME = "summary.csv"; String SUMMARY_FILE_HEADER = "Task;AvgMemoryUsage;AvgWallTime;MaxWallTime;TotalTime;JobCount"; String DONE_TASK = "done"; + int CORES_PER_NODE = 24; } diff --git a/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/core/TaskComputation.java b/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/core/TaskComputation.java index 4aacf63016dd8d92ee92a6c8fcbbd9dd09ae74e1..c1a9b3ec9e3c43591fbf2104239f491488a2f3d7 100644 --- a/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/core/TaskComputation.java +++ b/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/core/TaskComputation.java @@ -185,7 +185,8 @@ public class TaskComputation { // Should the state be queued, try to find out whether a log file exists if (state == JobState.Queued) { - if (null != logs && !logs.stream().anyMatch(logFile -> computationAccessor.fileExists(logFile))) { + if (!taskDescription.equals(Constants.DONE_TASK) && null != logs + && !logs.stream().anyMatch(logFile -> computationAccessor.fileExists(logFile))) { return; // No log file exists yet } state = JobState.Running; diff --git a/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/ui/BenchmarkSPIMController.java b/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/ui/BenchmarkSPIMController.java index 17a2429b65e1067afe55265869f85020ba36b037..cf34023108706defbe30d1456a720d107a8837c1 100644 --- a/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/ui/BenchmarkSPIMController.java +++ b/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/ui/BenchmarkSPIMController.java @@ -1,8 +1,12 @@ package cz.it4i.fiji.haas_spim_benchmark.ui; +import static cz.it4i.fiji.haas_spim_benchmark.core.Constants.CONFIG_YAML; + import java.awt.Desktop; import java.awt.Window; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.EnumSet; import java.util.HashSet; import java.util.LinkedList; @@ -22,6 +26,7 @@ import javax.swing.WindowConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import cz.it4i.fiji.haas.UploadingFileFromResource; import cz.it4i.fiji.haas.ui.CloseableControl; import cz.it4i.fiji.haas.ui.DummyProgress; import cz.it4i.fiji.haas.ui.InitiableControl; @@ -30,6 +35,7 @@ import cz.it4i.fiji.haas.ui.ModalDialogs; import cz.it4i.fiji.haas.ui.ProgressDialog; import cz.it4i.fiji.haas.ui.TableViewContextMenu; import cz.it4i.fiji.haas_java_client.JobState; +import cz.it4i.fiji.haas_java_client.UploadingFile; import cz.it4i.fiji.haas_spim_benchmark.core.BenchmarkJobManager; import cz.it4i.fiji.haas_spim_benchmark.core.BenchmarkJobManager.BenchmarkJob; import cz.it4i.fiji.haas_spim_benchmark.core.Constants; @@ -38,9 +44,13 @@ import cz.it4i.fiji.haas_spim_benchmark.core.ObservableBenchmarkJob; import cz.it4i.fiji.haas_spim_benchmark.core.ObservableBenchmarkJob.TransferProgress; import javafx.beans.value.ObservableValue; import javafx.fxml.FXML; +import javafx.scene.control.Alert; +import javafx.scene.control.Alert.AlertType; +import javafx.scene.control.ButtonType; import javafx.scene.control.TableColumn; import javafx.scene.control.TableView; import javafx.scene.layout.BorderPane; +import javafx.scene.layout.Region; import net.imagej.updater.util.Progress; //FIXME: fix Exception during context menu request on task with N/A state @@ -50,21 +60,21 @@ public class BenchmarkSPIMController extends BorderPane implements CloseableCont private TableView<ObservableBenchmarkJob> jobs; private final BenchmarkJobManager manager; - + private final ExecutorService executorServiceJobState = Executors.newWorkStealingPool(); - + private final Executor executorServiceFX = new FXFrameExecutorService(); - + private Window root; private ExecutorService executorServiceUI; - + private ExecutorService executorServiceWS; - + private Timer timer; - + private ObservableBenchmarkJobRegistry registry; - + private static Logger log = LoggerFactory .getLogger(cz.it4i.fiji.haas_spim_benchmark.ui.BenchmarkSPIMController.class); @@ -102,7 +112,7 @@ public class BenchmarkSPIMController extends BorderPane implements CloseableCont private void initMenu() { TableViewContextMenu<ObservableBenchmarkJob> menu = new TableViewContextMenu<>(jobs); - menu.addItem("Create job", x -> executeWSCallAsync("Creating job", p -> manager.createJob()), j -> true); + menu.addItem("Create job", x -> askForCreateJob(), j -> true); menu.addItem("Start job", job -> executeWSCallAsync("Starting job", p -> { job.getValue().startJob(p); job.getValue().update(); @@ -112,7 +122,8 @@ public class BenchmarkSPIMController extends BorderPane implements CloseableCont menu.addItem("Cancel job", job -> executeWSCallAsync("Canceling job", p -> { job.getValue().cancelJob(); job.getValue().update(); - }), job -> JavaFXRoutines.notNullValue(job, j -> j.getState() == JobState.Running || j.getState() == JobState.Queued )); + }), job -> JavaFXRoutines.notNullValue(job, + j -> j.getState() == JobState.Running || j.getState() == JobState.Queued)); menu.addItem("Execution details", job -> { try { @@ -126,16 +137,16 @@ public class BenchmarkSPIMController extends BorderPane implements CloseableCont menu.addItem("Upload data", job -> executeWSCallAsync("Uploading data", p -> job.getValue().startUpload()), job -> executeWSCallAsync("Stop uploading data", p -> job.getValue().stopUpload()), - job -> JavaFXRoutines.notNullValue(job, j -> !EnumSet.of(JobState.Running).contains(j.getState())), + job -> JavaFXRoutines.notNullValue(job, + j -> !j.isUseDemoData() && !EnumSet.of(JobState.Running, JobState.Disposed).contains(j.getState())), job -> job.getUploadProgress().isWorking()); - - menu.addItem("Download result", job -> executeWSCallAsync("Downloading data", p -> job.getValue().startDownload()), job -> executeWSCallAsync("Stop downloading data", p -> job.getValue().stopDownload()), - job -> JavaFXRoutines.notNullValue(job, - j -> EnumSet.of(JobState.Failed, JobState.Finished, JobState.Canceled).contains(j.getState()) - && j.canBeDownloaded()), + job -> JavaFXRoutines + .notNullValue(job, + j -> EnumSet.of(JobState.Failed, JobState.Finished, JobState.Canceled) + .contains(j.getState()) && j.canBeDownloaded()), job -> job.getDownloadProgress().isWorking()); menu.addItem("Download statistics", @@ -146,6 +157,71 @@ public class BenchmarkSPIMController extends BorderPane implements CloseableCont job -> JavaFXRoutines.notNullValue(job, j -> j.getState().equals(JobState.Failed))); menu.addItem("Open working directory", j -> open(j.getValue()), x -> JavaFXRoutines.notNullValue(x, j -> true)); + + menu.addItem("Delete", j -> deleteJob(j.getValue()), x -> JavaFXRoutines.notNullValue(x, j -> true)); + } + + private void deleteJob(BenchmarkJob bj) { + bj.delete(); + registry.update(); + } + + private void askForCreateJob() { + NewJobWindow newJobWindow = new NewJobWindow(null); + ModalDialogs.doModal(newJobWindow, WindowConstants.DISPOSE_ON_CLOSE); + newJobWindow.setCreatePressedNotifier(() -> executeWSCallAsync("Creating job", false, + new P_JobAction() { + @Override + public void doAction(Progress p) throws IOException { + BenchmarkJob job = doCreateJob(wd -> newJobWindow.getInputDirectory(wd), wd -> newJobWindow.getOutputDirectory(wd)); + if (job.isUseDemoData()) { + job.storeDataInWorkdirectory(getConfigYamlFile()); + } else if (Files.exists(job.getInputDirectory().resolve(CONFIG_YAML))) { + executorServiceFX.execute(new Runnable() { + + @Override + public void run() { + Alert al = new Alert(AlertType.CONFIRMATION, + "Main file \"" + CONFIG_YAML + "\" found in input directory \"" + + job.getInputDirectory() + + "\". Would you like to copy it into job subdirectory \"" + + job.getDirectory() + "\"?", + ButtonType.YES, ButtonType.NO); + al.setHeaderText(null); + al.setTitle("Copy " + CONFIG_YAML + "?"); + al.getDialogPane().setMinHeight(Region.USE_PREF_SIZE); + if (al.showAndWait().get() == ButtonType.YES) { + try { + Files.copy(job.getInputDirectory().resolve(CONFIG_YAML), job.getDirectory().resolve(CONFIG_YAML)); + } catch (IOException e) { + log.error(e.getMessage(), e); + } + } + } + }); + + } + } + } + )); + + + } + + private UploadingFile getConfigYamlFile() { + return new UploadingFileFromResource("", Constants.CONFIG_YAML); + } + + private BenchmarkJob doCreateJob(Function<Path,Path> inputProvider, Function<Path,Path> outputProvider) throws IOException { + BenchmarkJob bj = manager.createJob(inputProvider, outputProvider); + ObservableBenchmarkJob obj = registry.addIfAbsent(bj); + addJobToItems(obj); + jobs.refresh(); + return bj; + } + + private synchronized void addJobToItems(ObservableBenchmarkJob obj) { + jobs.getItems().add(obj); } private void open(BenchmarkJob j) { @@ -174,8 +250,9 @@ public class BenchmarkSPIMController extends BorderPane implements CloseableCont } return null; }, x -> { - if (update) + if (update) { updateJobs(); + } }); } @@ -201,11 +278,11 @@ public class BenchmarkSPIMController extends BorderPane implements CloseableCont } registry.update(); Set<ObservableValue<BenchmarkJob>> actual = new HashSet<>(this.jobs.getItems()); - + executorServiceFX.execute(() -> { for (ObservableBenchmarkJob value : registry.getAllItems()) { if (!actual.contains(value)) { - this.jobs.getItems().add(value); + addJobToItems(value); } } }); @@ -242,7 +319,6 @@ public class BenchmarkSPIMController extends BorderPane implements CloseableCont private void remove(BenchmarkJob bj) { jobs.getItems().remove(registry.get(bj)); - bj.remove(); } private void setCellValueFactory(int index, Function<BenchmarkJob, String> mapper) { @@ -252,12 +328,10 @@ public class BenchmarkSPIMController extends BorderPane implements CloseableCont @SuppressWarnings("unchecked") private void setCellValueFactoryCompletable(int index, Function<BenchmarkJob, CompletableFuture<String>> mapper) { JavaFXRoutines.setCellValueFactory(jobs, index, mapper); - ((TableColumn<ObservableBenchmarkJob, CompletableFuture<String>>) jobs.getColumns().get(index)) - .setCellFactory( - column -> new JavaFXRoutines.TableCellAdapter<ObservableBenchmarkJob, CompletableFuture<String>>( - new JavaFXRoutines.FutureValueUpdater<ObservableBenchmarkJob, String, CompletableFuture<String>>( - new JavaFXRoutines.StringValueUpdater<ObservableBenchmarkJob>(), - executorServiceFX))); + ((TableColumn<ObservableBenchmarkJob, CompletableFuture<String>>) jobs.getColumns().get(index)).setCellFactory( + column -> new JavaFXRoutines.TableCellAdapter<ObservableBenchmarkJob, CompletableFuture<String>>( + new JavaFXRoutines.FutureValueUpdater<ObservableBenchmarkJob, String, CompletableFuture<String>>( + new JavaFXRoutines.StringValueUpdater<ObservableBenchmarkJob>(), executorServiceFX))); } private interface P_JobAction { diff --git a/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/ui/NewJobController.java b/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/ui/NewJobController.java new file mode 100644 index 0000000000000000000000000000000000000000..f896bbbe7ce3ffe19bcfb074724a3269e1c70e38 --- /dev/null +++ b/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/ui/NewJobController.java @@ -0,0 +1,183 @@ +package cz.it4i.fiji.haas_spim_benchmark.ui; + +import java.awt.Window; +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import cz.it4i.fiji.haas.ui.CloseableControl; +import cz.it4i.fiji.haas.ui.FXFrame; +import cz.it4i.fiji.haas.ui.InitiableControl; +import cz.it4i.fiji.haas.ui.JavaFXRoutines; +import javafx.beans.value.ObservableValue; +import javafx.fxml.FXML; +import javafx.scene.Parent; +import javafx.scene.control.Alert; +import javafx.scene.control.Alert.AlertType; +import javafx.scene.control.Button; +import javafx.scene.control.RadioButton; +import javafx.scene.control.TextField; +import javafx.scene.control.Toggle; +import javafx.scene.control.ToggleGroup; +import javafx.scene.layout.BorderPane; +import javafx.stage.DirectoryChooser; + +public class NewJobController extends BorderPane implements CloseableControl, InitiableControl { + + public enum DataLocation { + DEMONSTRATION_ON_SERVER, WORK_DIRECTORY, CUSTOM_DIRECTORY + } + + private static final Runnable EMPTY_NOTIFIER = () -> { + }; + + @SuppressWarnings("unused") + private static Logger log = LoggerFactory.getLogger(cz.it4i.fiji.haas_spim_benchmark.ui.NewJobController.class); + + @FXML + private Button bt_create; + + @FXML + private ToggleGroup tg_inputDataLocation; + + @FXML + private ToggleGroup tg_outputDataLocation; + + @FXML + private RadioButton rb_ownInput; + + @FXML + private RadioButton rb_ownOutput; + + @FXML + private TextField et_inputDirectory; + + @FXML + private TextField et_outputDirectory; + + private DataLocation inputDataLocation; + + private DataLocation outputDataLocation; + + private FXFrame<?> ownerWindow; + + private Runnable createPressedNotifier; + + @FXML + private Button bt_selectInput; + + @FXML + private Button bt_selectOutput; + + public NewJobController() { + JavaFXRoutines.initRootAndController("NewJobView.fxml", this); + getStylesheets().add(getClass().getResource("NewJobView.css").toExternalForm()); + bt_create.setOnMouseClicked(X -> createPressed()); + tg_inputDataLocation.selectedToggleProperty().addListener((v, old, n) -> selected(v, old, n, rb_ownInput)); + tg_outputDataLocation.selectedToggleProperty().addListener((v, o, n) -> selected(v, o, n, rb_ownOutput)); + initSelectButton(et_inputDirectory, bt_selectInput); + initSelectButton(et_outputDirectory, bt_selectOutput); + } + + @Override + public void close() { + } + + @Override + public void init(Window parameter) { + ownerWindow = (FXFrame<?>) parameter; + } + + public Path getInputDirectory(Path workingDirectory) { + return getDirectory(inputDataLocation, et_inputDirectory.getText(), workingDirectory); + } + + public Path getOutputDirectory(Path workingDirectory) { + return getDirectory(outputDataLocation, et_outputDirectory.getText(), workingDirectory); + } + + public void setCreatePressedNotifier(Runnable createPressedNotifier) { + if (createPressedNotifier != null) { + this.createPressedNotifier = createPressedNotifier; + } else { + this.createPressedNotifier = EMPTY_NOTIFIER; + } + } + + private void initSelectButton(TextField textField, Button button) { + button.setOnAction(x -> { + Path p = Paths.get(textField.getText()); + DirectoryChooser dch = new DirectoryChooser(); + if (Files.exists(p)) { + dch.setInitialDirectory(p.toAbsolutePath().toFile()); + } + File result = dch.showDialog(ownerWindow.getFxPanel().getScene().getWindow()); + if (result != null) { + textField.setText(result.toString()); + } + }); + } + + private Path getDirectory(DataLocation dataLocation, String selectedDirectory, Path workingDirectory) { + switch (dataLocation) { + case DEMONSTRATION_ON_SERVER: + return null; + case WORK_DIRECTORY: + return workingDirectory; + case CUSTOM_DIRECTORY: + return Paths.get(selectedDirectory).toAbsolutePath(); + default: + throw new UnsupportedOperationException("Not support " + dataLocation); + } + } + + private void createPressed() { + obtainValues(); + if (checkDirectoryLocationIfNeeded()) { + ownerWindow.setVisible(false); + ownerWindow.dispose(); + createPressedNotifier.run(); + } + } + + private boolean checkDirectoryLocationIfNeeded() { + return checkDataLocationValue(inputDataLocation, et_inputDirectory.getText(), "input") + && checkDataLocationValue(outputDataLocation, et_outputDirectory.getText(), "output"); + + } + + private boolean checkDataLocationValue(DataLocation dataLocation, String directory, String type) { + Path directoryPath = Paths.get(directory); + if (dataLocation == DataLocation.CUSTOM_DIRECTORY && (!Files.exists(directoryPath) || directory.isEmpty())) { + Alert alert = new Alert(AlertType.WARNING); + alert.setTitle("Invalid input provided"); + alert.setHeaderText(null); + String message = !directory.isEmpty() ? "Directory %s for %s not exists" + : "Directory for %2$s is not selected."; + alert.setContentText(String.format(message, directoryPath.toAbsolutePath(), type)); + alert.showAndWait(); + return false; + } + return true; + } + + private void obtainValues() { + inputDataLocation = obtainDataLocation(tg_inputDataLocation); + outputDataLocation = obtainDataLocation(tg_outputDataLocation); + } + + private DataLocation obtainDataLocation(ToggleGroup group) { + int backawardOrderOfSelected = group.getToggles().size() + - group.getToggles().indexOf(group.getSelectedToggle()); + return DataLocation.values()[DataLocation.values().length - backawardOrderOfSelected]; + } + + private void selected(ObservableValue<? extends Toggle> v, Toggle o, Toggle n, Parent disableIfNotSelected) { + disableIfNotSelected.getChildrenUnmodifiable().forEach(node -> node.setDisable(n != disableIfNotSelected)); + } + +} diff --git a/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/ui/NewJobWindow.java b/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/ui/NewJobWindow.java new file mode 100644 index 0000000000000000000000000000000000000000..7d62083e06f1ff9712c663c13e2f1a9098805f64 --- /dev/null +++ b/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/ui/NewJobWindow.java @@ -0,0 +1,34 @@ +package cz.it4i.fiji.haas_spim_benchmark.ui; + +import java.awt.Window; +import java.nio.file.Path; + +import cz.it4i.fiji.haas.ui.FXFrame; + +public class NewJobWindow extends FXFrame<NewJobController>{ + + private static final long serialVersionUID = 1L; + + + + public NewJobWindow(Window parentWindow) { + super(parentWindow,()->{ + return new NewJobController(); + + }); + setTitle("Create job"); + } + + public Path getInputDirectory(Path workingDirectory) { + return getFxPanel().getControl().getInputDirectory(workingDirectory); + } + + public Path getOutputDirectory(Path workingDirectory) { + return getFxPanel().getControl().getOutputDirectory(workingDirectory); + } + + + public void setCreatePressedNotifier(Runnable runnable) { + getFxPanel().getControl().setCreatePressedNotifier(runnable); + } +} \ No newline at end of file diff --git a/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/ui/BenchmarkSPIM.fxml b/haas-spim-benchmark/src/main/resources/cz/it4i/fiji/haas_spim_benchmark/ui/BenchmarkSPIM.fxml similarity index 100% rename from haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/ui/BenchmarkSPIM.fxml rename to haas-spim-benchmark/src/main/resources/cz/it4i/fiji/haas_spim_benchmark/ui/BenchmarkSPIM.fxml diff --git a/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/ui/JobDetail.fxml b/haas-spim-benchmark/src/main/resources/cz/it4i/fiji/haas_spim_benchmark/ui/JobDetail.fxml similarity index 100% rename from haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/ui/JobDetail.fxml rename to haas-spim-benchmark/src/main/resources/cz/it4i/fiji/haas_spim_benchmark/ui/JobDetail.fxml diff --git a/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/ui/LogView.fxml b/haas-spim-benchmark/src/main/resources/cz/it4i/fiji/haas_spim_benchmark/ui/LogView.fxml similarity index 100% rename from haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/ui/LogView.fxml rename to haas-spim-benchmark/src/main/resources/cz/it4i/fiji/haas_spim_benchmark/ui/LogView.fxml diff --git a/haas-spim-benchmark/src/main/resources/cz/it4i/fiji/haas_spim_benchmark/ui/NewJobView.css b/haas-spim-benchmark/src/main/resources/cz/it4i/fiji/haas_spim_benchmark/ui/NewJobView.css new file mode 100644 index 0000000000000000000000000000000000000000..65d090c4224334c3948f7fa8749306f4dbf5fc8d --- /dev/null +++ b/haas-spim-benchmark/src/main/resources/cz/it4i/fiji/haas_spim_benchmark/ui/NewJobView.css @@ -0,0 +1,4 @@ +{ + -fx-padding: 3 3 3 3; + +} \ No newline at end of file diff --git a/haas-spim-benchmark/src/main/resources/cz/it4i/fiji/haas_spim_benchmark/ui/NewJobView.fxml b/haas-spim-benchmark/src/main/resources/cz/it4i/fiji/haas_spim_benchmark/ui/NewJobView.fxml new file mode 100644 index 0000000000000000000000000000000000000000..344e6488da521cba151f50e777b62333b2890d88 --- /dev/null +++ b/haas-spim-benchmark/src/main/resources/cz/it4i/fiji/haas_spim_benchmark/ui/NewJobView.fxml @@ -0,0 +1,105 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<?import javafx.geometry.Insets?> +<?import javafx.scene.control.Button?> +<?import javafx.scene.control.Label?> +<?import javafx.scene.control.RadioButton?> +<?import javafx.scene.control.TextField?> +<?import javafx.scene.control.TitledPane?> +<?import javafx.scene.control.ToggleGroup?> +<?import javafx.scene.layout.AnchorPane?> +<?import javafx.scene.layout.BorderPane?> +<?import javafx.scene.layout.HBox?> +<?import javafx.scene.layout.VBox?> + +<fx:root maxHeight="-Infinity" maxWidth="-Infinity" minHeight="-Infinity" minWidth="-Infinity" type="BorderPane" xmlns="http://javafx.com/javafx/8.0.65" xmlns:fx="http://javafx.com/fxml/1" fx:controller="cz.it4i.fiji.haas_spim_benchmark.ui.NewJobController"> + <center> + <VBox BorderPane.alignment="CENTER"> + <children> + <TitledPane animated="false" collapsible="false" text="Input"> + <content> + <AnchorPane> + <children> + <VBox> + <children> + <RadioButton mnemonicParsing="false" selected="true"> + <toggleGroup> + <ToggleGroup fx:id="tg_inputDataLocation" /> + </toggleGroup> + <graphic> + <Label maxHeight="1.7976931348623157E308" text="Demonstration data on IT4I cluster" /> + </graphic> + </RadioButton> + <RadioButton maxHeight="1.7976931348623157E308" mnemonicParsing="false" toggleGroup="$tg_inputDataLocation"> + <graphic> + <Label maxHeight="1.7976931348623157E308" text="Job subdirectory" /> + </graphic> + </RadioButton> + <RadioButton fx:id="rb_ownInput" mnemonicParsing="false" toggleGroup="$tg_inputDataLocation"> + <graphic> + <HBox disable="true"> + <children> + <TextField fx:id="et_inputDirectory" prefWidth="350.0" /> + <Button fx:id="bt_selectInput" mnemonicParsing="false" text="Select" /> + </children> + </HBox> + </graphic> + </RadioButton> + </children> + </VBox> + </children> + </AnchorPane> + </content> + </TitledPane> + <TitledPane animated="false" collapsible="false" text="Output"> + <content> + <AnchorPane minHeight="0.0" minWidth="0.0"> + <children> + <VBox> + <children> + <HBox> + <children> + <RadioButton maxHeight="1.7976931348623157E308" mnemonicParsing="false" selected="true"> + <toggleGroup> + <ToggleGroup fx:id="tg_outputDataLocation" /> + </toggleGroup> + <graphic> + <Label maxHeight="1.7976931348623157E308" text="Job subdirectory" /> + </graphic> + </RadioButton> + </children> + </HBox> + <HBox> + <children> + <RadioButton fx:id="rb_ownOutput" mnemonicParsing="false" toggleGroup="$tg_outputDataLocation"> + <graphic> + <HBox disable="true"> + <children> + <TextField fx:id="et_outputDirectory" prefWidth="350.0" /> + <Button fx:id="bt_selectOutput" mnemonicParsing="false" text="Select" /> + </children> + </HBox> + </graphic> + </RadioButton> + </children> + </HBox> + </children> + </VBox> + </children> + </AnchorPane> + </content> + </TitledPane> + </children> + </VBox> + </center> + <bottom> + <BorderPane BorderPane.alignment="CENTER"> + <right> + <Button fx:id="bt_create" mnemonicParsing="false" prefHeight="22.0" prefWidth="71.0" text="Create" BorderPane.alignment="CENTER"> + <BorderPane.margin> + <Insets right="3.0" /> + </BorderPane.margin></Button> + </right> + </BorderPane> + </bottom> +</fx:root> diff --git a/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/ui/RemoteFilesInfo.fxml b/haas-spim-benchmark/src/main/resources/cz/it4i/fiji/haas_spim_benchmark/ui/RemoteFilesInfo.fxml similarity index 100% rename from haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/ui/RemoteFilesInfo.fxml rename to haas-spim-benchmark/src/main/resources/cz/it4i/fiji/haas_spim_benchmark/ui/RemoteFilesInfo.fxml diff --git a/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/ui/SPIMPipelineProgressView.fxml b/haas-spim-benchmark/src/main/resources/cz/it4i/fiji/haas_spim_benchmark/ui/SPIMPipelineProgressView.fxml similarity index 100% rename from haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/ui/SPIMPipelineProgressView.fxml rename to haas-spim-benchmark/src/main/resources/cz/it4i/fiji/haas_spim_benchmark/ui/SPIMPipelineProgressView.fxml diff --git a/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/ui/TaskComputationView.fxml b/haas-spim-benchmark/src/main/resources/cz/it4i/fiji/haas_spim_benchmark/ui/TaskComputationView.fxml similarity index 100% rename from haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/ui/TaskComputationView.fxml rename to haas-spim-benchmark/src/main/resources/cz/it4i/fiji/haas_spim_benchmark/ui/TaskComputationView.fxml diff --git a/haas-spim-benchmark/src/main/resources/cz/it4i/fiji/haas_spim_benchmark/ui/package-info.java b/haas-spim-benchmark/src/main/resources/cz/it4i/fiji/haas_spim_benchmark/ui/package-info.java new file mode 100644 index 0000000000000000000000000000000000000000..6dfa85ade2c7142fecfd098fd7beb11247a7b690 --- /dev/null +++ b/haas-spim-benchmark/src/main/resources/cz/it4i/fiji/haas_spim_benchmark/ui/package-info.java @@ -0,0 +1,8 @@ +/** + * + */ +/** + * @author koz01 + * + */ +package cz.it4i.fiji.haas_spim_benchmark.ui; \ No newline at end of file diff --git a/haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/ui/style.css b/haas-spim-benchmark/src/main/resources/cz/it4i/fiji/haas_spim_benchmark/ui/style.css similarity index 100% rename from haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/ui/style.css rename to haas-spim-benchmark/src/main/resources/cz/it4i/fiji/haas_spim_benchmark/ui/style.css diff --git a/haas-spim-benchmark/src/test/java/cz/it4i/fiji/haas/NewJobWindowShow.java b/haas-spim-benchmark/src/test/java/cz/it4i/fiji/haas/NewJobWindowShow.java new file mode 100644 index 0000000000000000000000000000000000000000..525b110ddbb9e0078993902b8c67bbb0e88cf03e --- /dev/null +++ b/haas-spim-benchmark/src/test/java/cz/it4i/fiji/haas/NewJobWindowShow.java @@ -0,0 +1,11 @@ +package cz.it4i.fiji.haas; + +import java.io.IOException; + +import cz.it4i.fiji.haas_spim_benchmark.ui.NewJobWindow; + +public class NewJobWindowShow { + public static void main(String[] args) throws IOException { + new NewJobWindow(null).setVisible(true); + } +} diff --git a/haas-spim-benchmark/src/test/java/cz/it4i/fiji/haas/RunBenchmark.java b/haas-spim-benchmark/src/test/java/cz/it4i/fiji/haas/RunBenchmark.java index 6a5fbd4be91416c77fad569cb74753ad4a8ded8d..42c914a7275466e6ab55db05f3571db0e33fc608 100644 --- a/haas-spim-benchmark/src/test/java/cz/it4i/fiji/haas/RunBenchmark.java +++ b/haas-spim-benchmark/src/test/java/cz/it4i/fiji/haas/RunBenchmark.java @@ -20,7 +20,7 @@ public class RunBenchmark { public static class CreateJob { public static void main(String[] args) throws IOException { BenchmarkJobManager benchmarkJobManager = new BenchmarkJobManager(getBenchmarkSPIMParameters()); - BenchmarkJob ji = benchmarkJobManager.createJob(); + BenchmarkJob ji = benchmarkJobManager.createJob(jd -> jd, jd -> jd); log.info("job: " + ji.getId() + " created."); } } @@ -42,8 +42,6 @@ public class RunBenchmark { } } - - private static BenchmarkSPIMParameters getBenchmarkSPIMParameters() throws IOException { Path p = Paths.get("/tmp/benchmark"); if (!Files.exists(p)) { diff --git a/java-scpclient/src/main/java/cz/it4i/fiji/scpclient/ScpClient.java b/java-scpclient/src/main/java/cz/it4i/fiji/scpclient/ScpClient.java index b29e82e106e83b7cdd547e1458ad02a658c1835a..0d3acb4d9132b7ee0afaf80729de0d7ddaa627b2 100644 --- a/java-scpclient/src/main/java/cz/it4i/fiji/scpclient/ScpClient.java +++ b/java-scpclient/src/main/java/cz/it4i/fiji/scpclient/ScpClient.java @@ -34,9 +34,9 @@ public class ScpClient implements Closeable { private String hostName; private String username; - private JSch jsch = new JSch(); + private final JSch jsch = new JSch(); private Session session; - private TransferFileProgress dummyProgress = new TransferFileProgress() { + private final TransferFileProgress dummyProgress = new TransferFileProgress() { @Override public void dataTransfered(long bytesTransfered) { @@ -123,7 +123,7 @@ public class ScpClient implements Closeable { } if (buf[0] == ' ') break; - filesize = filesize * 10L + (long) (buf[0] - '0'); + filesize = filesize * 10L + buf[0] - '0'; } @SuppressWarnings("unused") @@ -295,7 +295,7 @@ public class ScpClient implements Closeable { } if (buf[0] == ' ') break; - filesize = filesize * 10L + (long) (buf[0] - '0'); + filesize = filesize * 10L + buf[0] - '0'; } return filesize;