Skip to content
Snippets Groups Projects
Commit 8444af16 authored by Jan Kožusznik's avatar Jan Kožusznik
Browse files

Merge branch 'iss1007-propertyHolder'

parents 474ff59a 36d156a0
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 143 additions and 144 deletions
haas-imagej-client/src/main/java/cz/it4i/fiji/haas/Job.java
+ 20
− 84
View file @ 8444af16
......@@ -2,15 +2,11 @@ package cz.it4i.fiji.haas;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.Calendar;
import java.util.Collections;
import java.util.EnumSet;
import java.util.List;
import java.util.Properties;
import java.util.function.Predicate;
import java.util.function.Supplier;
import java.util.stream.Collectors;
......@@ -29,8 +25,6 @@ import net.imagej.updater.util.Progress;
public class Job {
private static final String JOB_HAS_DATA_TO_DOWNLOAD_PROPERTY = "job.needDownload";
private static final String JOB_NAME = "job.name";
public static boolean isJobPath(Path p) {
......@@ -47,27 +41,31 @@ public class Job {
private Supplier<HaaSClient> haasClientSupplier;
// private JobState state;
private Boolean needsDownload;
//private Boolean needsDownload;
private JobInfo jobInfo;
private Long jobId;
private String name;
private PropertyHolder propertyHolder;
public Job(String name, Path basePath, Supplier<HaaSClient> haasClientSupplier) throws IOException {
this(haasClientSupplier);
HaaSClient client = this.haasClientSupplier.get();
long id = client.createJob(name, Collections.emptyList());
jobDir = basePath.resolve("" + id);
this.name = name;
propertyHolder = new PropertyHolder(jobDir.resolve(JOB_INFO_FILE));
setName(name);
Files.createDirectory(jobDir);
updateNeedsDownload();
}
public Job(Path p, Supplier<HaaSClient> haasClientSupplier) throws IOException {
public void setName(String name) {
setProperty(JOB_NAME, name);
}
public Job(Path p, Supplier<HaaSClient> haasClientSupplier) {
this(haasClientSupplier);
jobDir = p;
loadJobInfo();
updateNeedsDownload();
propertyHolder = new PropertyHolder(jobDir.resolve(JOB_INFO_FILE));
}
public void uploadFiles(Iterable<UploadingFile> files, Progress notifier) {
......@@ -87,13 +85,11 @@ public class Job {
client.submitJob(jobId);
}
private Job(Supplier<HaaSClient> haasClientSupplier) throws IOException {
private Job(Supplier<HaaSClient> haasClientSupplier) {
this.haasClientSupplier = haasClientSupplier;
}
public boolean needsDownload() {
return needsDownload != null && needsDownload;
}
synchronized public long getJobId() {
if (jobId == null) {
......@@ -103,7 +99,7 @@ public class Job {
}
public void download(Progress notifier) {
download(x -> true, notifier, false);
download(x -> true, notifier);
}
public Path storeDataInWorkdirectory(UploadingFile uploadingFile) throws IOException {
......@@ -114,19 +110,8 @@ public class Job {
return result;
}
synchronized public void download(Predicate<String> predicate, Progress notifier, boolean allowAgain) {
if (!allowAgain && !needsDownload()) {
throw new IllegalStateException("Job: " + getJobId() + " doesn't need download");
}
synchronized public void download(Predicate<String> predicate, Progress notifier) {
haasClientSupplier.get().download(getJobId(), jobDir, predicate, new P_ProgressNotifierAdapter(notifier));
if(!allowAgain) {
needsDownload = false;
try {
saveJobinfo();
} catch (IOException e) {
log.error(e);
}
}
}
public JobState getState() {
......@@ -157,14 +142,12 @@ public class Job {
return Files.newInputStream(jobDir.resolve(name));
}
public void setProperty(String name, String value) throws IOException {
Properties prop = loadPropertiesIfExists();
prop.setProperty(name, value);
storeProperties(prop);
public void setProperty(String name, String value) {
propertyHolder.setValue(name, value);
}
public String getProperty(String name) throws IOException {
return loadPropertiesIfExists().getProperty(name);
public String getProperty(String name) {
return propertyHolder.getValue(name);
}
public void updateInfo() {
......@@ -174,49 +157,7 @@ public class Job {
public Path getDirectory() {
return jobDir;
}
synchronized private void updateNeedsDownload() throws IOException {
if (needsDownload == null
&& EnumSet.of(JobState.Failed, JobState.Finished, JobState.Canceled).contains(getState())) {
needsDownload = true;
}
saveJobinfo();
}
private synchronized void saveJobinfo() throws IOException {
Properties prop = loadPropertiesIfExists();
if (needsDownload != null) {
prop.setProperty(JOB_HAS_DATA_TO_DOWNLOAD_PROPERTY, needsDownload.toString());
}
prop.setProperty(JOB_NAME, name);
storeProperties(prop);
}
private void storeProperties(Properties prop) throws IOException {
try (OutputStream ow = Files.newOutputStream(jobDir.resolve(JOB_INFO_FILE),
StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE)) {
prop.store(ow, null);
}
}
private synchronized void loadJobInfo() throws IOException {
Properties prop = loadPropertiesIfExists();
if (prop.containsKey(JOB_HAS_DATA_TO_DOWNLOAD_PROPERTY)) {
needsDownload = Boolean.parseBoolean(prop.getProperty(JOB_HAS_DATA_TO_DOWNLOAD_PROPERTY));
}
name = prop.getProperty(JOB_NAME);
}
private Properties loadPropertiesIfExists() throws IOException {
Properties prop = new Properties();
if (Files.exists(jobDir.resolve(JOB_INFO_FILE))) {
try (InputStream is = Files.newInputStream(jobDir.resolve(JOB_INFO_FILE))) {
prop.load(is);
}
}
return prop;
}
private JobInfo getJobInfo() {
if (jobInfo == null) {
updateJobInfo();
......@@ -226,11 +167,6 @@ public class Job {
private void updateJobInfo() {
jobInfo = haasClientSupplier.get().obtainJobInfo(getJobId());
try {
updateNeedsDownload();
} catch (IOException e) {
log.error(e.getMessage(), e);
}
}
private static boolean isValidPath(Path path) {
......
haas-imagej-client/src/main/java/cz/it4i/fiji/haas/JobManager.java
+ 14
− 21
View file @ 8444af16
......@@ -63,20 +63,17 @@ public class JobManager {
return result;
}
public Iterable<JobInfo> getJobsNeedingDownload() {
return () -> jobs.stream().filter(j -> j.needsDownload()).map(j -> new JobInfo(j)).iterator();
}
public Collection<JobInfo> getJobs() throws IOException {
public Collection<JobInfo> getJobs() {
if (jobs == null) {
jobs = new LinkedList<>();
Files.list(this.workDirectory).filter(p -> Files.isDirectory(p) && Job.isJobPath(p)).forEach(p -> {
try {
jobs.add(new Job(p, this::getHaasClient));
} catch (IOException e) {
e.printStackTrace();
}
});
try {
Files.list(this.workDirectory).filter(p -> Files.isDirectory(p) && Job.isJobPath(p)).forEach(p -> {
jobs.add(new Job(p, this::getHaasClient));
});
} catch (IOException e) {
throw new RuntimeException(e);
}
}
return jobs.stream().map(j -> new JobInfo(j)).collect(Collectors.toList());
}
......@@ -146,10 +143,6 @@ public class JobManager {
return job.getState();
}
public boolean needsDownload() {
return job.needsDownload();
}
public String getCreationTime() {
return getStringFromTimeSafely(job.getCreationTime());
}
......@@ -163,11 +156,11 @@ public class JobManager {
}
public void downloadData(Progress notifier) {
downloadData(x -> true, notifier, false);
downloadData(x -> true, notifier);
}
public void downloadData(Predicate<String> predicate, Progress notifier, boolean allowAgain) {
job.download(predicate, notifier, allowAgain);
public void downloadData(Predicate<String> predicate, Progress notifier) {
job.download(predicate, notifier);
fireValueChangedEvent();
}
......@@ -202,12 +195,12 @@ public class JobManager {
return job.openLocalFile(name);
}
public void setProperty(String name, String value) throws IOException {
public void setProperty(String name, String value) {
job.setProperty(name, value);
}
public String getProperty(String name) throws IOException {
public String getProperty(String name) {
return job.getProperty(name);
}
......
haas-imagej-client/src/main/java/cz/it4i/fiji/haas/PropertyHolder.java 0 → 100644
+ 62
− 0
View file @ 8444af16
package cz.it4i.fiji.haas;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.Properties;
public class PropertyHolder {
private Path storage;
private Properties properties;
public PropertyHolder(Path storage) {
super();
this.storage = storage;
}
public String getValue(String key) {
Properties properties = getProperties();
return properties.getProperty(key);
}
public void setValue(String key, String value) {
Properties prop = getProperties();
prop.setProperty(key, value);
try {
storeProperties(prop);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
private Properties getProperties() {
if (properties == null) {
try {
properties = loadPropertiesIfExists();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
return properties;
}
private Properties loadPropertiesIfExists() throws IOException {
Properties prop = new Properties();
if (Files.exists(storage)) {
try (InputStream is = Files.newInputStream(storage)) {
prop.load(is);
}
}
return prop;
}
private void storeProperties(Properties prop) throws IOException {
try (OutputStream ow = Files.newOutputStream(storage, StandardOpenOption.TRUNCATE_EXISTING,
StandardOpenOption.CREATE)) {
prop.store(ow, null);
}
}
}
haas-snakemake-spim/src/main/java/cz/it4i/fiji/haas_snakemake_spim/commands/CheckStatusOfHaaS.java
+ 21
− 24
View file @ 8444af16
......@@ -2,7 +2,6 @@ package cz.it4i.fiji.haas_snakemake_spim.commands;
import java.awt.Frame;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Collection;
......@@ -51,29 +50,27 @@ public class CheckStatusOfHaaS implements Command {
@Override
public void run() {
try {
jobManager = new JobManager(getWorkingDirectoryPath(),TestingConstants.getSettings());
if (uiService.isHeadless()) {
downloadAll();
} else {
CheckStatusOfHaaSWindow window;
window = ModalDialogs.doModal(new CheckStatusOfHaaSWindow(getFrame(), context),WindowConstants.DISPOSE_ON_CLOSE);
ProgressDialog dialog = ModalDialogs.doModal(new ProgressDialog(window),WindowConstants.DO_NOTHING_ON_CLOSE);
dialog.setTitle("Downloading info about jobs");
Collection<JobInfo> jobs = jobManager.getJobs();
int count = 0;
for (JobInfo ji : jobs) {
String item;
dialog.addItem(item = "job id:" + ji.getId());
window.addJob(ji);
dialog.itemDone(item);
dialog.setCount(count, jobs.size());
count++;
}
dialog.done();
jobManager = new JobManager(getWorkingDirectoryPath(), TestingConstants.getSettings());
if (uiService.isHeadless()) {
downloadAll();
} else {
CheckStatusOfHaaSWindow window;
window = ModalDialogs.doModal(new CheckStatusOfHaaSWindow(getFrame(), context),
WindowConstants.DISPOSE_ON_CLOSE);
ProgressDialog dialog = ModalDialogs.doModal(new ProgressDialog(window),
WindowConstants.DO_NOTHING_ON_CLOSE);
dialog.setTitle("Downloading info about jobs");
Collection<JobInfo> jobs = jobManager.getJobs();
int count = 0;
for (JobInfo ji : jobs) {
String item;
dialog.addItem(item = "job id:" + ji.getId());
window.addJob(ji);
dialog.itemDone(item);
dialog.setCount(count, jobs.size());
count++;
}
} catch (IOException e) {
log.error(e);
dialog.done();
}
}
......@@ -92,7 +89,7 @@ public class CheckStatusOfHaaS implements Command {
}
private void downloadAll() {
for (JobInfo id : jobManager.getJobsNeedingDownload()) {
for (JobInfo id : jobManager.getJobs()) {
System.out.println("Job " + id.getId() + " needs download");
jobManager.downloadJob(id.getId(), new DummyProgress());
}
......
haas-snakemake-spim/src/main/java/cz/it4i/fiji/haas_snakemake_spim/ui/CheckStatusOfHaaSController.java
+ 2
− 2
View file @ 8444af16
......@@ -69,7 +69,7 @@ public class CheckStatusOfHaaSController implements FXFrame.Controller {
}
JobInfo job = jobs.getSelectionModel().getSelectedItem();
if (job != null && job.needsDownload()) {
if (job != null) {
download.setDisable(false);
} else {
download.setDisable(true);
......@@ -80,7 +80,7 @@ public class CheckStatusOfHaaSController implements FXFrame.Controller {
private void initTable() {
setCellValueFactory(0, j -> j.getId().toString());
setCellValueFactory(1, j -> j.getState().toString() + (j.needsDownload() ? " - needs download" : ""));
setCellValueFactory(1, j -> j.getState().toString());
setCellValueFactory(2, j -> j.getCreationTime().toString());
setCellValueFactory(3, j -> j.getStartTime().toString());
setCellValueFactory(4, j -> j.getEndTime().toString());
......
haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/commands/ManageSPIMBenchmark.java
+ 1
− 2
View file @ 8444af16
......@@ -85,8 +85,7 @@ public class ManageSPIMBenchmark implements Command {
// Launch ImageJ as usual.
final ImageJ ij = new ImageJ();
ij.launch(args);
//ij.command().run(ManageSPIMBenchmark.class, true);
ij.command().run(ManageSPIMBenchmark.class, true);
}
}
haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/core/BenchmarkJobManager.java
+ 22
− 10
View file @ 8444af16
......@@ -33,6 +33,8 @@ import net.imagej.updater.util.Progress;
public class BenchmarkJobManager {
private static final String JOB_HAS_DATA_TO_DOWNLOAD_PROPERTY = "job.needDownload";
private static Logger log = LoggerFactory
.getLogger(cz.it4i.fiji.haas_spim_benchmark.core.BenchmarkJobManager.class);
......@@ -51,6 +53,7 @@ public class BenchmarkJobManager {
String outputName = getOutputName(jobInfo.openLocalFile(Constants.CONFIG_YAML));
jobInfo.submit();
jobInfo.setProperty(Constants.SPIM_OUTPUT_FILENAME_PATTERN, outputName);
setDownloaded(false);
}
public JobState getState() {
......@@ -59,19 +62,18 @@ public class BenchmarkJobManager {
public void downloadData(Progress progress) throws IOException {
JobInfo ji = jobInfo;
if (ji.needsDownload()) {
if (ji.getState() == JobState.Finished) {
String filePattern = ji.getProperty(Constants.SPIM_OUTPUT_FILENAME_PATTERN);
ji.downloadData(downloadFinishedData(filePattern), progress, false);
} else if (ji.getState() == JobState.Failed) {
ji.downloadData(downloadFailedData(), progress, false);
}
if (ji.getState() == JobState.Finished) {
String filePattern = ji.getProperty(Constants.SPIM_OUTPUT_FILENAME_PATTERN);
ji.downloadData(downloadFinishedData(filePattern), progress);
} else if (ji.getState() == JobState.Failed) {
ji.downloadData(downloadFailedData(), progress);
}
setDownloaded(true);
}
public void downloadStatistics(Progress progress) throws IOException {
JobInfo ji = jobInfo;
ji.downloadData(BenchmarkJobManager.downloadStatistics(), progress, true);
ji.downloadData(BenchmarkJobManager.downloadStatistics(), progress);
Path resultFile = ji.getDirectory().resolve(Constants.BENCHMARK_RESULT_FILE);
if (resultFile != null)
BenchmarkJobManager.formatResultFile(resultFile);
......@@ -125,7 +127,7 @@ public class BenchmarkJobManager {
}
public boolean downloaded() {
return !jobInfo.needsDownload();
return getDownloaded();
}
public Job update() {
......@@ -136,6 +138,16 @@ public class BenchmarkJobManager {
public Path getDirectory() {
return jobInfo.getDirectory();
}
private void setDownloaded(boolean b) {
jobInfo.setProperty(JOB_HAS_DATA_TO_DOWNLOAD_PROPERTY, b + "");
}
private boolean getDownloaded() {
String downloadedStr = jobInfo.getProperty(JOB_HAS_DATA_TO_DOWNLOAD_PROPERTY);
return downloadedStr != null && Boolean.parseBoolean(downloadedStr);
}
}
private JobManager jobManager;
......@@ -151,10 +163,10 @@ public class BenchmarkJobManager {
}
public Collection<Job> getJobs() throws IOException {
return jobManager.getJobs().stream().map(this::convertJob).collect(Collectors.toList());
}
private HaaSClient.UploadingFile getUploadingFile() {
return new UploadingFileFromResource("", Constants.CONFIG_YAML);
}
......
haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/core/Constants.java
+ 1
− 1
View file @ 8444af16
......@@ -4,7 +4,7 @@ import java.util.HashMap;
import java.util.Map;
public interface Constants {
long HAAS_UPDATE_TIMEOUT = 1000;
long HAAS_UPDATE_TIMEOUT = 30000;
String HAAS_JOB_NAME = "HaaSSPIMBenchmark";
int HAAS_CLUSTER_NODE_TYPE = 6;
int HAAS_TEMPLATE_ID = 4;
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment