Merge pull request #8 from schmiedc/master

Added sysconfcpus option. it's very flexible now. ready to go ...

spim_registration/timelapse/Snakefile

@@ -26,7 +26,7 @@ xml_merge_in = produce_xml_merge_job_files(datasets)
 rule done:
     input: [ ds + "_output_hdf5" for ds in datasets ]
 
-localrules: define_xml_czi, define_xml_tif, hdf5_xml, xml_merge, timelapse, 
+localrules: define_xml_czi, define_xml_tif, xml_merge, timelapse, 
 	    duplicate_transformations, external_transform, define_output,
 	    hdf5_xml_output
 	    
@@ -109,7 +109,7 @@ rule hdf5_xml:
     log: "logs/b1_hdf5_xml.log"
     run:
         part_string = produce_string(
-        	"""{fiji-prefix} {fiji-app} \
+        	"""{fiji-prefix} {sysconfcpus} {num_cores_hdf5_xml} {fiji-app} \
                 -Dimage_file_directory={jdir} \
                 -Dfirst_xml_filename={first_xml_filename} \
                 -Dhdf5_xml_filename={hdf5_xml_filename} \
@@ -139,7 +139,7 @@ rule resave_hdf5:
     log: "logs/b2_resave_hdf5-{file_id}.log"
     run:
         part_string = produce_string(
-        	"""{fiji-prefix} {fiji-app} \
+        	"""{fiji-prefix} {sysconfcpus} {num_cores_resave_hdf5} {fiji-app} \
         	-Dimage_file_directory={jdir} \
                 -Dfirst_xml_filename={first_xml_filename} \
                 -Dhdf5_xml_filename={input_xml_base} \
@@ -168,7 +168,7 @@ rule registration:
     log: "logs/c_{xml_base}-{file_id}-registration.log"
     run:
         cmd_string = produce_string(
-           """{fiji-prefix} {fiji-app} \
+           """{fiji-prefix} {sysconfcpus} {num_cores_reg} {fiji-app} \
         -Dparallel_timepoints={file_id_w} \
         -Dimage_file_directory={jdir} \
         -Dxml_filename={input_xml} \
@@ -300,7 +300,7 @@ rule fusion:
     log: "logs/e1_{xml_base}-{file_id,\d+}-00-fusion.log"
     run:
         cmd_string = produce_string(
-        	"""{fiji-prefix} {fiji-app} \
+        	"""{fiji-prefix} {sysconfcpus} {num_cores_fusion} {fiji-app} \
         -Dimage_file_directory={jdir} \
     	-Dparallel_timepoints={file_id_w} \
     	-Dmerged_xml={merged_xml_file} \
@@ -371,7 +371,7 @@ rule deconvolution:
     log: "logs/e2_{xml_base}-{file_id,\d+}-00-deconvolution.log"
     run:
         cmd_string = produce_string(
-        	"""{fiji-prefix} {fiji-app} \
+        	"""{fiji-prefix} {sysconfcpus} {num_cores_deco} {fiji-app} \
         -Dimage_file_directory={jdir} \
         -Ddeco_output_file_directory={jdir} \
         -Dmerged_xml={merged_xml_file} \
@@ -451,7 +451,7 @@ rule hdf5_xml_output:
     log: "logs/f2_output_hdf5_xml.log"
     run:
         part_string = produce_string(
-        	"""{fiji-prefix} {fiji-app} \
+        	"""{fiji-prefix} {sysconfcpus} {num_cores_hdf5_xml_output} {fiji-app} \
                 -Dimage_file_directory={jdir} \
                 -Dfirst_xml_filename={output_xml} \
                 -Dhdf5_xml_filename={output_hdf5_xml} \
@@ -482,7 +482,7 @@ rule resave_hdf5_output:
     log: "logs/f3_resave_output-{file_id}.log"
     run:
         part_string = produce_string(
-        	"""{fiji-prefix} {fiji-app} \
+        	"""{fiji-prefix} {sysconfcpus} {num_cores_resave_hdf5_output} {fiji-app} \
                 -Dimage_file_directory={jdir} \
                 -Dfirst_xml_filename={output_xml} \
                 -Dhdf5_xml_filename={output_hdf5_xml} \

spim_registration/timelapse/cluster.json

@@ -7,12 +7,12 @@
 
     "resave_hdf5" :	
     {
- 	"lsf_extra" : "-n 7 -R \"span[hosts=1] rusage[mem=50000]\""
+ 	"lsf_extra" : "-n 2 -R \"span[hosts=1] rusage[mem=50000]\""
     },
 
     "registration" :
     {
-        "lsf_extra" : "-R \"span[hosts=1] rusage[mem=100000]\""
+        "lsf_extra" : "-n 2 -R \"span[hosts=1] rusage[mem=100000]\""
     },
     
     "timelapse" : 
@@ -27,17 +27,17 @@
 
     "fusion" : 
     {
-        "lsf_extra" : "-n 12 -R \"span[hosts=1] rusage[mem=100000]\""
+        "lsf_extra" : "-n 6 -R \"span[hosts=1] rusage[mem=100000]\""
     },
 
     "deconvolution" :
     {
-        "lsf_extra" : "-n 7 -R \"span[hosts=1] rusage[mem=50000]\"",
+        "lsf_extra" : "-n 2 -R \"span[hosts=1] rusage[mem=50000]\"",
 	"lsf_q" : "gpu"
     },
     
     "resave_hdf5_output" :
     {
-	"lsf_extra" : "-n 7 -R \"span[hosts=1] rusage[mem=50000]\""
+	"lsf_extra" : "-n 2 -R \"span[hosts=1] rusage[mem=50000]\""
     }
 }

spim_registration/timelapse/fusion.bsh

@@ -99,7 +99,7 @@ IJ.run("Fuse/Deconvolve Dataset",
 	"process_views_in_paralell=All " +
 	"interpolation=[" + interpolation + "] " + 
 	"blend " +
-	"content-based " +
+	//"content-based " +
 	"output_file_directory=" + image_file_directory + " " +
 	"lossless " +
 	"imglib2_data_container=[" + imglib2_data_container + "]");

spim_registration/timelapse/single_test.yaml

@@ -13,6 +13,7 @@ common: {
   # Directory that contains the current working Fiji
   fiji-app: "/sw/users/schmied/packages/2015-06-30_Fiji.app.cuda/ImageJ-linux64",
   fiji-prefix: "/sw/users/schmied/packages/xvfb-run -a",       # calls xvfb for Fiji headless mode
+  sysconfcpus: "sysconfcpus -n",
   # ============================================================================
   #   Processing switches
   # Description: Use switches to decide which processing steps you need:
@@ -208,6 +209,8 @@ resave_hdf5: {
   resave_angle: '"All angles"',
   resave_channel: '"All channels"',
   resave_illumination: '"All illuminations"',
+  num_cores_hdf5_xml: "2",
+  num_cores_resave_hdf5: "2",
   bsh_file: "export.bsh"
   }
 
@@ -237,6 +240,7 @@ registration: {
   model_to_regularize_with: "Rigid",
   lambda: '0.10',
   imglib_container: '"ArrayImg (faster)"',
+  num_cores_reg: "2",
   bsh_file: "registration.bsh"  # .bsh script for registration
   }
 
@@ -273,6 +277,7 @@ fusion: {
   imglib2_data_container: '"ArrayImg (faster)"',
   process_views_in_paralell: '"All"',
   xml_output: '"Save every XML with user-provided unique id"',
+  num_cores_fusion: "6",
   bsh_file: "fusion.bsh"
   }
 
@@ -312,6 +317,7 @@ deconvolution: {
   psf_size_y: '19',
   psf_size_z: '25',
   imglib2_container: '"ArrayImg"',
+  num_cores_deco: "7",
   bsh_file: "deconvolution.bsh"
   }
   
@@ -329,6 +335,8 @@ hdf5_output: {
   output_multiple_angles: '"NO (one angle)"',
   output_illumination_directions: '"NO (one illumination direction)"',
   output_imglib_container: '"ArrayImg (faster)"',
+  num_cores_hdf5_xml_output: "2",
+  num_cores_resave_hdf5_output: "2",
   bsh_file_define: "define_output.bsh", # .bsh script for defining the dataset
   bsh_file_hdf5: "export_output.bsh"    # .bsh script for resaving into hdf5
   }