Skip to content
Snippets Groups Projects
Commit 36d156a0 authored by Jan Kožusznik's avatar Jan Kožusznik
Browse files

iss1007 

moving properties of job to property holder
changin update timeout
parent 474ff59a
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 143 additions and 144 deletions
haas-imagej-client/src/main/java/cz/it4i/fiji/haas/Job.java
+ 20
− 84
View file @ 36d156a0
......@@ -2,15 +2,11 @@ package cz.it4i.fiji.haas;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.Calendar;
import java.util.Collections;
import java.util.EnumSet;
import java.util.List;
import java.util.Properties;
import java.util.function.Predicate;
import java.util.function.Supplier;
import java.util.stream.Collectors;
......@@ -29,8 +25,6 @@ import net.imagej.updater.util.Progress;
public class Job {
private static final String JOB_HAS_DATA_TO_DOWNLOAD_PROPERTY = "job.needDownload";
private static final String JOB_NAME = "job.name";
public static boolean isJobPath(Path p) {
......@@ -47,27 +41,31 @@ public class Job {
private Supplier<HaaSClient> haasClientSupplier;
// private JobState state;
private Boolean needsDownload;
//private Boolean needsDownload;
private JobInfo jobInfo;
private Long jobId;
private String name;
private PropertyHolder propertyHolder;
public Job(String name, Path basePath, Supplier<HaaSClient> haasClientSupplier) throws IOException {
this(haasClientSupplier);
HaaSClient client = this.haasClientSupplier.get();
long id = client.createJob(name, Collections.emptyList());
jobDir = basePath.resolve("" + id);
this.name = name;
propertyHolder = new PropertyHolder(jobDir.resolve(JOB_INFO_FILE));
setName(name);
Files.createDirectory(jobDir);
updateNeedsDownload();
}
public Job(Path p, Supplier<HaaSClient> haasClientSupplier) throws IOException {
public void setName(String name) {
setProperty(JOB_NAME, name);
}
public Job(Path p, Supplier<HaaSClient> haasClientSupplier) {
this(haasClientSupplier);
jobDir = p;
loadJobInfo();
updateNeedsDownload();
propertyHolder = new PropertyHolder(jobDir.resolve(JOB_INFO_FILE));
}
public void uploadFiles(Iterable<UploadingFile> files, Progress notifier) {
......@@ -87,13 +85,11 @@ public class Job {
client.submitJob(jobId);
}
private Job(Supplier<HaaSClient> haasClientSupplier) throws IOException {
private Job(Supplier<HaaSClient> haasClientSupplier) {
this.haasClientSupplier = haasClientSupplier;
}
public boolean needsDownload() {
return needsDownload != null && needsDownload;
}
synchronized public long getJobId() {
if (jobId == null) {
......@@ -103,7 +99,7 @@ public class Job {
}
public void download(Progress notifier) {
download(x -> true, notifier, false);
download(x -> true, notifier);
}
public Path storeDataInWorkdirectory(UploadingFile uploadingFile) throws IOException {
......@@ -114,19 +110,8 @@ public class Job {
return result;
}
synchronized public void download(Predicate<String> predicate, Progress notifier, boolean allowAgain) {
if (!allowAgain && !needsDownload()) {
throw new IllegalStateException("Job: " + getJobId() + " doesn't need download");
}
synchronized public void download(Predicate<String> predicate, Progress notifier) {
haasClientSupplier.get().download(getJobId(), jobDir, predicate, new P_ProgressNotifierAdapter(notifier));
if(!allowAgain) {
needsDownload = false;
try {
saveJobinfo();
} catch (IOException e) {
log.error(e);
}
}
}
public JobState getState() {
......@@ -157,14 +142,12 @@ public class Job {
return Files.newInputStream(jobDir.resolve(name));
}
public void setProperty(String name, String value) throws IOException {
Properties prop = loadPropertiesIfExists();
prop.setProperty(name, value);
storeProperties(prop);
public void setProperty(String name, String value) {
propertyHolder.setValue(name, value);
}
public String getProperty(String name) throws IOException {
return loadPropertiesIfExists().getProperty(name);
public String getProperty(String name) {
return propertyHolder.getValue(name);
}
public void updateInfo() {
......@@ -174,49 +157,7 @@ public class Job {
public Path getDirectory() {
return jobDir;
}
synchronized private void updateNeedsDownload() throws IOException {
if (needsDownload == null
&& EnumSet.of(JobState.Failed, JobState.Finished, JobState.Canceled).contains(getState())) {
needsDownload = true;
}
saveJobinfo();
}
private synchronized void saveJobinfo() throws IOException {
Properties prop = loadPropertiesIfExists();
if (needsDownload != null) {
prop.setProperty(JOB_HAS_DATA_TO_DOWNLOAD_PROPERTY, needsDownload.toString());
}
prop.setProperty(JOB_NAME, name);
storeProperties(prop);
}
private void storeProperties(Properties prop) throws IOException {
try (OutputStream ow = Files.newOutputStream(jobDir.resolve(JOB_INFO_FILE),
StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE)) {
prop.store(ow, null);
}
}
private synchronized void loadJobInfo() throws IOException {
Properties prop = loadPropertiesIfExists();
if (prop.containsKey(JOB_HAS_DATA_TO_DOWNLOAD_PROPERTY)) {
needsDownload = Boolean.parseBoolean(prop.getProperty(JOB_HAS_DATA_TO_DOWNLOAD_PROPERTY));
}
name = prop.getProperty(JOB_NAME);
}
private Properties loadPropertiesIfExists() throws IOException {
Properties prop = new Properties();
if (Files.exists(jobDir.resolve(JOB_INFO_FILE))) {
try (InputStream is = Files.newInputStream(jobDir.resolve(JOB_INFO_FILE))) {
prop.load(is);
}
}
return prop;
}
private JobInfo getJobInfo() {
if (jobInfo == null) {
updateJobInfo();
......@@ -226,11 +167,6 @@ public class Job {
private void updateJobInfo() {
jobInfo = haasClientSupplier.get().obtainJobInfo(getJobId());
try {
updateNeedsDownload();
} catch (IOException e) {
log.error(e.getMessage(), e);
}
}
private static boolean isValidPath(Path path) {
......
haas-imagej-client/src/main/java/cz/it4i/fiji/haas/JobManager.java
+ 14
− 21
View file @ 36d156a0
......@@ -63,20 +63,17 @@ public class JobManager {
return result;
}
public Iterable<JobInfo> getJobsNeedingDownload() {
return () -> jobs.stream().filter(j -> j.needsDownload()).map(j -> new JobInfo(j)).iterator();
}
public Collection<JobInfo> getJobs() throws IOException {
public Collection<JobInfo> getJobs() {
if (jobs == null) {
jobs = new LinkedList<>();
Files.list(this.workDirectory).filter(p -> Files.isDirectory(p) && Job.isJobPath(p)).forEach(p -> {
try {
jobs.add(new Job(p, this::getHaasClient));
} catch (IOException e) {
e.printStackTrace();
}
});
try {
Files.list(this.workDirectory).filter(p -> Files.isDirectory(p) && Job.isJobPath(p)).forEach(p -> {
jobs.add(new Job(p, this::getHaasClient));
});
} catch (IOException e) {
throw new RuntimeException(e);
}
}
return jobs.stream().map(j -> new JobInfo(j)).collect(Collectors.toList());
}
......@@ -146,10 +143,6 @@ public class JobManager {
return job.getState();
}
public boolean needsDownload() {
return job.needsDownload();
}
public String getCreationTime() {
return getStringFromTimeSafely(job.getCreationTime());
}
......@@ -163,11 +156,11 @@ public class JobManager {
}
public void downloadData(Progress notifier) {
downloadData(x -> true, notifier, false);
downloadData(x -> true, notifier);
}
public void downloadData(Predicate<String> predicate, Progress notifier, boolean allowAgain) {
job.download(predicate, notifier, allowAgain);
public void downloadData(Predicate<String> predicate, Progress notifier) {
job.download(predicate, notifier);
fireValueChangedEvent();
}
......@@ -202,12 +195,12 @@ public class JobManager {
return job.openLocalFile(name);
}
public void setProperty(String name, String value) throws IOException {
public void setProperty(String name, String value) {
job.setProperty(name, value);
}
public String getProperty(String name) throws IOException {
public String getProperty(String name) {
return job.getProperty(name);
}
......
haas-imagej-client/src/main/java/cz/it4i/fiji/haas/PropertyHolder.java 0 → 100644
+ 62
− 0
View file @ 36d156a0
package cz.it4i.fiji.haas;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.Properties;
public class PropertyHolder {
private Path storage;
private Properties properties;
public PropertyHolder(Path storage) {
super();
this.storage = storage;
}
public String getValue(String key) {
Properties properties = getProperties();
return properties.getProperty(key);
}
public void setValue(String key, String value) {
Properties prop = getProperties();
prop.setProperty(key, value);
try {
storeProperties(prop);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
private Properties getProperties() {
if (properties == null) {
try {
properties = loadPropertiesIfExists();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
return properties;
}
private Properties loadPropertiesIfExists() throws IOException {
Properties prop = new Properties();
if (Files.exists(storage)) {
try (InputStream is = Files.newInputStream(storage)) {
prop.load(is);
}
}
return prop;
}
private void storeProperties(Properties prop) throws IOException {
try (OutputStream ow = Files.newOutputStream(storage, StandardOpenOption.TRUNCATE_EXISTING,
StandardOpenOption.CREATE)) {
prop.store(ow, null);
}
}
}
haas-snakemake-spim/src/main/java/cz/it4i/fiji/haas_snakemake_spim/commands/CheckStatusOfHaaS.java
+ 21
− 24
View file @ 36d156a0
......@@ -2,7 +2,6 @@ package cz.it4i.fiji.haas_snakemake_spim.commands;
import java.awt.Frame;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Collection;
......@@ -51,29 +50,27 @@ public class CheckStatusOfHaaS implements Command {
@Override
public void run() {
try {
jobManager = new JobManager(getWorkingDirectoryPath(),TestingConstants.getSettings());
if (uiService.isHeadless()) {
downloadAll();
} else {
CheckStatusOfHaaSWindow window;
window = ModalDialogs.doModal(new CheckStatusOfHaaSWindow(getFrame(), context),WindowConstants.DISPOSE_ON_CLOSE);
ProgressDialog dialog = ModalDialogs.doModal(new ProgressDialog(window),WindowConstants.DO_NOTHING_ON_CLOSE);
dialog.setTitle("Downloading info about jobs");
Collection<JobInfo> jobs = jobManager.getJobs();
int count = 0;
for (JobInfo ji : jobs) {
String item;
dialog.addItem(item = "job id:" + ji.getId());
window.addJob(ji);
dialog.itemDone(item);
dialog.setCount(count, jobs.size());
count++;
}
dialog.done();
jobManager = new JobManager(getWorkingDirectoryPath(), TestingConstants.getSettings());
if (uiService.isHeadless()) {
downloadAll();
} else {
CheckStatusOfHaaSWindow window;
window = ModalDialogs.doModal(new CheckStatusOfHaaSWindow(getFrame(), context),
WindowConstants.DISPOSE_ON_CLOSE);
ProgressDialog dialog = ModalDialogs.doModal(new ProgressDialog(window),
WindowConstants.DO_NOTHING_ON_CLOSE);
dialog.setTitle("Downloading info about jobs");
Collection<JobInfo> jobs = jobManager.getJobs();
int count = 0;
for (JobInfo ji : jobs) {
String item;
dialog.addItem(item = "job id:" + ji.getId());
window.addJob(ji);
dialog.itemDone(item);
dialog.setCount(count, jobs.size());
count++;
}
} catch (IOException e) {
log.error(e);
dialog.done();
}
}
......@@ -92,7 +89,7 @@ public class CheckStatusOfHaaS implements Command {
}
private void downloadAll() {
for (JobInfo id : jobManager.getJobsNeedingDownload()) {
for (JobInfo id : jobManager.getJobs()) {
System.out.println("Job " + id.getId() + " needs download");
jobManager.downloadJob(id.getId(), new DummyProgress());
}
......
haas-snakemake-spim/src/main/java/cz/it4i/fiji/haas_snakemake_spim/ui/CheckStatusOfHaaSController.java
+ 2
− 2
View file @ 36d156a0
......@@ -69,7 +69,7 @@ public class CheckStatusOfHaaSController implements FXFrame.Controller {
}
JobInfo job = jobs.getSelectionModel().getSelectedItem();
if (job != null && job.needsDownload()) {
if (job != null) {
download.setDisable(false);
} else {
download.setDisable(true);
......@@ -80,7 +80,7 @@ public class CheckStatusOfHaaSController implements FXFrame.Controller {
private void initTable() {
setCellValueFactory(0, j -> j.getId().toString());
setCellValueFactory(1, j -> j.getState().toString() + (j.needsDownload() ? " - needs download" : ""));
setCellValueFactory(1, j -> j.getState().toString());
setCellValueFactory(2, j -> j.getCreationTime().toString());
setCellValueFactory(3, j -> j.getStartTime().toString());
setCellValueFactory(4, j -> j.getEndTime().toString());
......
haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/commands/ManageSPIMBenchmark.java
+ 1
− 2
View file @ 36d156a0
......@@ -85,8 +85,7 @@ public class ManageSPIMBenchmark implements Command {
// Launch ImageJ as usual.
final ImageJ ij = new ImageJ();
ij.launch(args);
//ij.command().run(ManageSPIMBenchmark.class, true);
ij.command().run(ManageSPIMBenchmark.class, true);
}
}
haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/core/BenchmarkJobManager.java
+ 22
− 10
View file @ 36d156a0
......@@ -33,6 +33,8 @@ import net.imagej.updater.util.Progress;
public class BenchmarkJobManager {
private static final String JOB_HAS_DATA_TO_DOWNLOAD_PROPERTY = "job.needDownload";
private static Logger log = LoggerFactory
.getLogger(cz.it4i.fiji.haas_spim_benchmark.core.BenchmarkJobManager.class);
......@@ -51,6 +53,7 @@ public class BenchmarkJobManager {
String outputName = getOutputName(jobInfo.openLocalFile(Constants.CONFIG_YAML));
jobInfo.submit();
jobInfo.setProperty(Constants.SPIM_OUTPUT_FILENAME_PATTERN, outputName);
setDownloaded(false);
}
public JobState getState() {
......@@ -59,19 +62,18 @@ public class BenchmarkJobManager {
public void downloadData(Progress progress) throws IOException {
JobInfo ji = jobInfo;
if (ji.needsDownload()) {
if (ji.getState() == JobState.Finished) {
String filePattern = ji.getProperty(Constants.SPIM_OUTPUT_FILENAME_PATTERN);
ji.downloadData(downloadFinishedData(filePattern), progress, false);
} else if (ji.getState() == JobState.Failed) {
ji.downloadData(downloadFailedData(), progress, false);
}
if (ji.getState() == JobState.Finished) {
String filePattern = ji.getProperty(Constants.SPIM_OUTPUT_FILENAME_PATTERN);
ji.downloadData(downloadFinishedData(filePattern), progress);
} else if (ji.getState() == JobState.Failed) {
ji.downloadData(downloadFailedData(), progress);
}
setDownloaded(true);
}
public void downloadStatistics(Progress progress) throws IOException {
JobInfo ji = jobInfo;
ji.downloadData(BenchmarkJobManager.downloadStatistics(), progress, true);
ji.downloadData(BenchmarkJobManager.downloadStatistics(), progress);
Path resultFile = ji.getDirectory().resolve(Constants.BENCHMARK_RESULT_FILE);
if (resultFile != null)
BenchmarkJobManager.formatResultFile(resultFile);
......@@ -125,7 +127,7 @@ public class BenchmarkJobManager {
}
public boolean downloaded() {
return !jobInfo.needsDownload();
return getDownloaded();
}
public Job update() {
......@@ -136,6 +138,16 @@ public class BenchmarkJobManager {
public Path getDirectory() {
return jobInfo.getDirectory();
}
private void setDownloaded(boolean b) {
jobInfo.setProperty(JOB_HAS_DATA_TO_DOWNLOAD_PROPERTY, b + "");
}
private boolean getDownloaded() {
String downloadedStr = jobInfo.getProperty(JOB_HAS_DATA_TO_DOWNLOAD_PROPERTY);
return downloadedStr != null && Boolean.parseBoolean(downloadedStr);
}
}
private JobManager jobManager;
......@@ -151,10 +163,10 @@ public class BenchmarkJobManager {
}
public Collection<Job> getJobs() throws IOException {
return jobManager.getJobs().stream().map(this::convertJob).collect(Collectors.toList());
}
private HaaSClient.UploadingFile getUploadingFile() {
return new UploadingFileFromResource("", Constants.CONFIG_YAML);
}
......
haas-spim-benchmark/src/main/java/cz/it4i/fiji/haas_spim_benchmark/core/Constants.java
+ 1
− 1
View file @ 36d156a0
......@@ -4,7 +4,7 @@ import java.util.HashMap;
import java.util.Map;
public interface Constants {
long HAAS_UPDATE_TIMEOUT = 1000;
long HAAS_UPDATE_TIMEOUT = 30000;
String HAAS_JOB_NAME = "HaaSSPIMBenchmark";
int HAAS_CLUSTER_NODE_TYPE = 6;
int HAAS_TEMPLATE_ID = 4;
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment