Skip to content
Snippets Groups Projects
Commit 9f4c0aa9 authored by Jan Kožusznik's avatar Jan Kožusznik
Browse files

feature: ask for copy config.yaml and avoid upload it with input data

parent 0b516afe
No related branches found
No related tags found
No related merge requests found
......@@ -437,7 +437,7 @@ public class Job {
setDownloaded(true);
setProperty(JOB_NEEDS_DOWNLOAD, false);
setCanBeDownloaded(false);
});
}, p -> jobManager.canUpload(Job.this, p));
} catch (IOException e) {
log.error(e.getMessage(), e);
throw new RuntimeException(e);
......
......@@ -7,6 +7,7 @@ import java.nio.file.Path;
import java.util.Collection;
import java.util.Collections;
import java.util.LinkedList;
import java.util.function.BiPredicate;
import java.util.function.Function;
import org.slf4j.Logger;
......@@ -20,10 +21,14 @@ public class JobManager implements Closeable {
interface JobManager4Job {
boolean deleteJob(Job job);
boolean canUpload(Job j, Path p);
}
private static Logger log = LoggerFactory.getLogger(cz.it4i.fiji.haas.JobManager.class);
private static final BiPredicate<Job, Path> DUMMY_UPLOAD_FILTER = (X, Y) -> true;
private final Path workDirectory;
private Collection<Job> jobs;
......@@ -32,6 +37,8 @@ public class JobManager implements Closeable {
private final Settings settings;
private BiPredicate<Job, Path> uploadFilter = DUMMY_UPLOAD_FILTER;
private final JobManager4Job remover = new JobManager4Job() {
@Override
......@@ -39,6 +46,11 @@ public class JobManager implements Closeable {
haasClient.deleteJob(job.getId());
return jobs.remove(job);
}
@Override
public boolean canUpload(Job j, Path p) {
return uploadFilter.test(j, p);
}
};
public JobManager(Path workDirectory, Settings settings) {
......@@ -65,6 +77,10 @@ public class JobManager implements Closeable {
jobs.forEach(job -> job.close());
}
public void setUploadFilter(BiPredicate<Job, Path> filter) {
uploadFilter = filter != null ? filter : DUMMY_UPLOAD_FILTER;
}
private HaaSClient getHaasClient() {
if (haasClient == null) {
haasClient = new HaaSClient(settings);
......
......@@ -12,6 +12,7 @@ import java.util.Collections;
import java.util.LinkedList;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.function.Predicate;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
......@@ -48,14 +49,10 @@ public class Synchronization implements Closeable {
private final ExecutorService service;
// public Synchronization(Supplier<HaaSFileTransfer> fileTransferSupplier, Path workingDirectory,
// Runnable uploadFinishedNotifier, Runnable downloadFinishedNotifier) throws IOException {
// this(fileTransferSupplier, workingDirectory, workingDirectory, workingDirectory, uploadFinishedNotifier,
// downloadFinishedNotifier);
// }
private final Predicate<Path> uploadFilter;
public Synchronization(Supplier<HaaSFileTransfer> fileTransferSupplier, Path workingDirectory, Path inputDirectory,
Path outputDirectory, Runnable uploadFinishedNotifier, Runnable downloadFinishedNotifier)
Path outputDirectory, Runnable uploadFinishedNotifier, Runnable downloadFinishedNotifier, Predicate<Path> uploadFilter)
throws IOException {
this.workingDirectory = workingDirectory;
this.inputDirectory = inputDirectory;
......@@ -65,6 +62,7 @@ public class Synchronization implements Closeable {
name -> Paths.get(name));
this.uploadProcess = createUploadProcess(fileTransferSupplier, service, uploadFinishedNotifier);
this.downloadProcess = createDownloadProcess(fileTransferSupplier, service, downloadFinishedNotifier);
this.uploadFilter = uploadFilter;
}
public synchronized void setUploadNotifier(ProgressNotifier notifier) {
......@@ -109,8 +107,8 @@ public class Synchronization implements Closeable {
}
private boolean canUpload(Path file) {
return !file.getFileName().toString().matches("[.][^.]+") && !filesDownloaded.contains(file);
return uploadFilter.test(file) && !file.getFileName().toString().matches("[.][^.]+")
&& !filesDownloaded.contains(file);
}
private PersistentSynchronizationProcess<Path> createUploadProcess(Supplier<HaaSFileTransfer> fileTransferSupplier,
......
......@@ -25,6 +25,7 @@ import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Scanner;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Executor;
......@@ -252,13 +253,12 @@ public class BenchmarkJobManager implements Closeable {
public boolean needsUpload() {
return job.needsUpload();
}
@Override
public String toString() {
return "" + getId();
}
public void storeDataInWorkdirectory(UploadingFile file) throws IOException {
job.storeDataInWorkdirectory(file);
}
......@@ -460,11 +460,12 @@ public class BenchmarkJobManager implements Closeable {
Stream<BenchmarkError> taskSpecificErrors = tasks.stream().flatMap(s -> s.getErrors().stream());
return Stream.concat(nonTaskSpecificErrors.stream(), taskSpecificErrors).collect(Collectors.toList());
}
}
public BenchmarkJobManager(BenchmarkSPIMParameters params) throws IOException {
jobManager = new JobManager(params.workingDirectory(), constructSettingsFromParams(params));
jobManager.setUploadFilter(this::canUpload);
}
public BenchmarkJob createJob(Function<Path, Path> inputDirectoryProvider,
......@@ -583,7 +584,10 @@ public class BenchmarkJobManager implements Closeable {
jobManager.close();
}
private boolean canUpload(Job job, Path p) {
return job.getInputDirectory() == null || !p.equals(job.getInputDirectory().resolve(Constants.CONFIG_YAML));
}
private BenchmarkJob convertJob(Job job) {
return new BenchmarkJob(job);
}
......@@ -591,9 +595,9 @@ public class BenchmarkJobManager implements Closeable {
private String getOutputName(InputStream openLocalFile) throws IOException {
try (InputStream is = openLocalFile) {
Yaml yaml = new Yaml();
Map<String, Map<String, String>> map = yaml.load(is);
String result = map.get("common").get("hdf5_xml_filename");
String result = Optional.ofNullable(map).map(m -> m.get("common")).map(m -> m.get("hdf5_xml_filename"))
.orElse(null);
if (result == null) {
throw new IllegalArgumentException("hdf5_xml_filename not found");
}
......
package cz.it4i.fiji.haas_spim_benchmark.ui;
import static cz.it4i.fiji.haas_spim_benchmark.core.Constants.CONFIG_YAML;
import java.awt.Desktop;
import java.awt.Window;
import java.io.IOException;
......@@ -21,7 +23,6 @@ import java.util.function.Function;
import javax.swing.WindowConstants;
import org.apache.commons.lang3.NotImplementedException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -43,9 +44,13 @@ import cz.it4i.fiji.haas_spim_benchmark.core.ObservableBenchmarkJob;
import cz.it4i.fiji.haas_spim_benchmark.core.ObservableBenchmarkJob.TransferProgress;
import javafx.beans.value.ObservableValue;
import javafx.fxml.FXML;
import javafx.scene.control.Alert;
import javafx.scene.control.Alert.AlertType;
import javafx.scene.control.ButtonType;
import javafx.scene.control.TableColumn;
import javafx.scene.control.TableView;
import javafx.scene.layout.BorderPane;
import javafx.scene.layout.Region;
import net.imagej.updater.util.Progress;
//FIXME: fix Exception during context menu request on task with N/A state
......@@ -171,8 +176,30 @@ public class BenchmarkSPIMController extends BorderPane implements CloseableCont
BenchmarkJob job = doCreateJob(wd -> newJobWindow.getInputDirectory(wd), wd -> newJobWindow.getOutputDirectory(wd));
if (job.isUseDemoData()) {
job.storeDataInWorkdirectory(getConfigYamlFile());
} else if (Files.exists(job.getInputDirectory().resolve(Constants.CONFIG_YAML))) {
throw new NotImplementedException("");
} else if (Files.exists(job.getInputDirectory().resolve(CONFIG_YAML))) {
executorServiceFX.execute(new Runnable() {
@Override
public void run() {
Alert al = new Alert(AlertType.CONFIRMATION,
"Main file \"" + CONFIG_YAML + "\" found in input directory \""
+ job.getInputDirectory()
+ "\". Would you like to copy it into job subdirectory \""
+ job.getDirectory() + "\"?",
ButtonType.YES, ButtonType.NO);
al.setHeaderText(null);
al.setTitle("Copy " + CONFIG_YAML + "?");
al.getDialogPane().setMinHeight(Region.USE_PREF_SIZE);
if (al.showAndWait().get() == ButtonType.YES) {
try {
Files.copy(job.getInputDirectory().resolve(CONFIG_YAML), job.getDirectory().resolve(CONFIG_YAML));
} catch (IOException e) {
log.error(e.getMessage(), e);
}
}
}
});
}
}
}
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment