Added tools for file format pre-processing

spim_registration/tools/#master_preprocessing#

+#!/bin/bash
+#===============================================================================
+#
+#	  FILE: master
+#
+#  DESCRIPTION: source file for pre-processing of file formats
+#
+#       AUTHOR: Christopher Schmied, schmied@mpi-cbg.de
+#    INSTITUTE: Max Planck Institute for Molecular Cell Biology and Genetics
+#         BUGS:
+#        NOTES:
+#      Version: 1.0
+#      CREATED: 2015-07-05
+#     REVISION: 2015-07-05
+#
+# Preprocessing
+#	1) rename .czi files
+#	2) resave .czi files into .tif or .zip
+#	3) resave ome.tiff files into .tif
+#	4) Splitting output for Channel is
+#		c=0,1 etc
+#		spim_TL{tt}_Angle{a}_Channel{c}.tif
+#===============================================================================
+image_file_directory="/projects/pilot_spim/Christopher/Test_pipeline_3.0/czi/"
+
+# --- jobs directory -----------------------------------------------------------
+job_directory="/projects/pilot_spim/Christopher/snakemake-workflows/spim_registration/tools/"
+#-------------------------------------------------------------------------------
+# Resaving, Renaming files and Splitting: General
+#
+# Important: For renaming and resaving .czi files the first .czi file has to
+# carry the index (0)
+#-------------------------------------------------------------------------------
+
+pad="3"		# for padded zeros
+angle_prep="1" # angles format: "1 2 3"
+
+#--- Renaming ------------------------------------------------------------------
+
+first_index="0"		# First index of czi files
+last_index="391"		# Last index of czi files
+first_timepoint="0"	# The first timepoint
+angles_renaming=(1)	# Angles format: (1 2 3)
+
+source_pattern=2014-10-23_H2A_gsb_G3\(\{index\}\).czi # Name of .czi files
+target_pattern=spim_TL\{timepoint\}_Angle\{angle\}.czi	# The output pattern of renaming
+
+#-------------------------------------------------------------------------------
+# Fiji settings
+#-------------------------------------------------------------------------------
+XVFB_RUN="/sw/bin/xvfb-run"				 # virtual frame buffer
+# working Fiji
+#Fiji="/sw/users/schmied/packages/2015-05-21_Fiji.app.cuda/ImageJ-linux64"			# woriking Fiji
+Fiji="/sw/users/schmied/packages/2015-06-08_Fiji.app.cuda/ImageJ-linux64"
+
+#Fiji for Dual Channel timelapse and Dual Channel Deconvolution
+FijiDualTimelapse="/sw/users/schmied/packages/2015-05-29_Fiji-2.3.9-SNAP.app.cuda/ImageJ-linux64"
+
+Fiji_resave="/sw/users/schmied/lifeline/Fiji.app.lifeline2/ImageJ-linux64" 	# Fiji that works for resaving
+Fiji_Deconvolution=${FijiDualTimelapse}		# Fiji that works for deconvolution
+
+#-------------------------------------------------------------------------------
+# Pre-processing
+#-------------------------------------------------------------------------------
+#--- Resaving .czi into .tif files----------------------------------------------
+jobs_resaving=${job_directory}"czi_resave"  	# directory .czi resaving
+resaving=${jobs_resaving}"/resaving.bsh" 	# script .czi resaving
+#--- Resaving ome.tiff into .tif files -----------------------------------------
+jobs_resaving_ometiff=${job_directory}"ometiff_resave" 	 # directory .ome.tiff resaving
+resaving_ometiff=${jobs_resaving_ometiff}"/resaving-ometiff.bsh" # script .ome.tiff resaving
+#--- Compress dataset;----------------------------------------------------------
+jobs_compress=${job_directory}"compress" 	# directory .czi to .zip resaving
+czi_compress=${jobs_compress}"/for_czi.bsh" 	# script .czi to .zip resaving
+#--- Split channels-------------------------------------------------------------
+jobs_split=${job_directory}"split_channels" 			# directory
+split=${jobs_split}"/split.bsh" 				# script

spim_registration/tools/#rename-zeiss-files.sh#

+#!/bin/bash
+# path of master file
+source ../master_preprocessing.sh
+
+# path of source and target files
+source_pattern=${image_file_directory}${source_pattern}
+target_pattern=${image_file_directory}${target_pattern}
+
+# ------------------------------------------------------------------------------
+
+i=${first_index}
+t=${first_timepoint}
+t=`printf "%0${pad}d" "${t}"`
+
+while [ $i -le $last_index ]; do
+
+        for a in "${angles_renaming[@]}"; do
+
+        	source=${source_pattern/\{index\}/${i}}
+                tmp=${target_pattern/\{timepoint\}/${t}}
+                target=${tmp/\{angle\}/${a}
+
+                echo ${source} ${target} # displays source file and target file with path
+
+                mv ${source} ${target} # renames source file into target pattern
+                #cp ${source} ${target} # alternatively copy source file and resave into target pattern
+                let i=i+1
+
+        done
+	t=$(( 10#${t} ))
+        let t=t+1
+        t=`printf "%0${pad}d" "${t}"`
+
+done

spim_registration/tools/compress/create-compress-jobs.sh

+#!/bin/bash
+
+# path of master file
+source ../../master_preprocessing.sh
+
+# creates directory for job files if not present
+mkdir -p $jobs_compress
+
+echo $jobs_compress
+echo $czi_compress
+
+# splits up resaving into 1 job per .czi file and writes the given parameters
+# into the job file
+for i in $parallel_timepoints
+
+do
+	for a in $angle_prep
+	do
+		job="$jobs_compress/compress-$i-$a.job"
+		echo $job
+		echo "$XVFB_RUN -a $Fiji_resave \
+			-Ddir=$image_file_directory \
+			-Dtimepoint=$i \
+			-Dangle=$a \
+			-Dpad=$pad \
+			-- --no-splash $czi_compress" >> "$job"
+		chmod a+x "$job"
+	done
+done
+

spim_registration/tools/compress/for_czi.bsh

+// Loads Fiji dependencies
+import ij.IJ;
+import ij.ImagePlus;
+import java.lang.Runtime;
+import java.io.File;
+import java.io.FilenameFilter;
+
+runtime = Runtime.getRuntime();
+
+// Loads parameters form job file
+System.out.println( "=======================================================" );
+System.out.println( "Load Parameters" ) ;
+
+dir = System.getProperty( "dir" );
+int timepoint = Integer.parseInt( System.getProperty( "timepoint" ) );
+angle = System.getProperty( "angle" );
+int pad = Integer.parseInt( System.getProperty( "pad" ) );
+
+// Prints Parameters into output file
+System.out.println( "directory = " + dir );
+System.out.println( "timepoint = " + timepoint );
+System.out.println( "angle = " + angle );
+System.out.println( "pad = " + pad );
+
+
+// Executes Fiji Plugin "Bioformats Importer" to open .czi file
+System.out.println( "=======================================================" );
+System.out.println( "Opening Image" ) ;
+IJ.run("Bio-Formats Importer", 
+	"open=" + dir + "spim_TL" + IJ.pad( timepoint, pad ) + "_Angle" + angle + ".czi" + " " +
+	"autoscale " +
+	"color_mode=Default " +
+	"specify_range " +
+	"view=[Standard ImageJ] " +
+	"stack_order=Default " +
+	"t_begin=1000 " +
+	"t_end=1000 " +
+	"t_step=1");
+
+// Resaves .czi files as .zip file
+System.out.println( "Save as compressed image" ) ;
+IJ.saveAs("ZIP ", dir + "spim_TL" + IJ.pad( timepoint, pad ) + "_Angle" + angle + ".zip");
+
+/* shutdown */
+runtime.exit(0);
+

spim_registration/tools/compress/resubmit-failed-jobs.sh

+names=`grep "exit code 1" out.* -l`
+for name in $names; do
+	job=`sed -n '4,4p' $name | sed -n "s/Job <\([^>]*\)>.*/\1/p"`
+	echo bsub -q short -n 12 -R rusage[mem=110000] -R span[hosts=1] -o "out.%J" -e "err.%J" ${job}
+       bsub -q short -n 12 -R rusage[mem=110000] -R span[hosts=1] -o "out.%J" -e "err.%J" ${job}
+done

spim_registration/tools/compress/submit-jobs

+#!/bin/bash
+
+for file in `ls ${1} | grep ".job$"`
+do
+	bsub -q short -n 4 -R span[hosts=1] -o "out.%J" -e "err.%J" ${1}/$file
+done

spim_registration/tools/czi_resave/#checkpoint.sh#

+#!/bin/bash
+
+source ../../master_preprocessing.sh
+
+timepoint=`seq 0 391`
+dir=/projects/pilot_spim/Christopher/2014-10-23_H2A_gsb_G3/
+num_angles=1
+pad=3
+
+job_dir=/projects/pilot_spim/Christopher/pipeline_3.0/jobs_alpha_3.1/czi_resave/
+
+
+for i in $timepoint
+
+	do
+		i=`printf "%0${pad}d" "$i"`
+		num=$(ls $dir/spim_TL"$i"_Angle*.tif |wc -l)
+
+			if [ $num  -ne $num_angles ]
+
+	then
+		echo "TL"$i": TP or angles missing"
+		//bsub -q short -n 4 -R span[hosts=1] -o "out.%J" -e "err.%J" ${1}/*${i}*
+
+	else
+		echo "TL"$i": Correct"
+
+fi
+
+done

spim_registration/tools/czi_resave/checkpoint.sh

+#!/bin/bash
+
+#source ../../master_3.3.sh
+
+timepoint=`seq 0 391`
+dir=/projects/pilot_spim/Christopher/2014-10-23_H2A_gsb_G3/
+num_angles=1
+pad=3
+
+job_dir=/projects/pilot_spim/Christopher/pipeline_3.0/jobs_alpha_3.1/czi_resave/
+
+
+for i in $timepoint
+
+	do
+		i=`printf "%0${pad}d" "$i"`
+		num=$(ls $dir/spim_TL"$i"_Angle*.tif |wc -l)
+
+			if [ $num  -ne $num_angles ]
+
+	then
+		echo "TL"$i": TP or angles missing"
+		//bsub -q short -n 4 -R span[hosts=1] -o "out.%J" -e "err.%J" ${1}/*${i}*
+
+	else
+		echo "TL"$i": Correct"
+
+fi
+
+done

spim_registration/tools/czi_resave/create-resaving-jobs.sh

+#!/bin/bash
+
+# path of master file
+source ../../master_preprocessing.sh
+
+# creates directory for job files if not present
+mkdir -p $jobs_resaving
+
+# splits up resaving into 1 job per .czi file and writes the given parameters
+# into the job file
+for i in $parallel_timepoints
+
+do
+	for a in $angle_prep
+	do
+		job="$jobs_resaving/resave-$i-$a.job"
+		echo $job
+		echo "$XVFB_RUN -a $Fiji_resave \
+			-Ddir=$image_file_directory \
+			-Dtimepoint=$i \
+			-Dangle=$a \
+			-Dpad=$pad \
+			-- --no-splash $resaving" >> "$job"
+		chmod a+x "$job"
+	done
+done

spim_registration/tools/czi_resave/resaving.bsh

+// Loads Fiji dependencies
+import ij.IJ;
+import ij.ImagePlus;
+import java.lang.Runtime;
+import java.io.File;
+import java.io.FilenameFilter;
+
+runtime = Runtime.getRuntime();
+
+// Loads parameters form job file
+System.out.println( "=======================================================" );
+System.out.println( "Load Parameters" ) ;
+
+dir = System.getProperty( "dir" );
+int timepoint = Integer.parseInt( System.getProperty( "timepoint" ) );
+angle = System.getProperty( "angle" );
+int pad = Integer.parseInt( System.getProperty( "pad" ) );
+
+// Prints Parameters into output file
+System.out.println( "directory = " + dir );
+System.out.println( "timepoint = " + timepoint );
+System.out.println( "angle = " + angle );
+System.out.println( "pad = " + pad );
+
+// Executes Fiji Plugin "Bioformats Importer" to open .czi file
+System.out.println( "=======================================================" );
+System.out.println( "Opening Image" ) ;
+IJ.run("Bio-Formats Importer", 
+	"open=" + dir + "spim_TL" + IJ.pad( timepoint, pad ) + "_Angle" + angle + ".czi" + " " +
+	"autoscale " +
+	"color_mode=Default " +
+	"specify_range " +
+	"view=[Standard ImageJ] " +
+	"stack_order=Default " +
+	"t_begin=1000 " +
+	"t_end=1000 " +
+	"t_step=1");
+
+// Resaves .czi files as .tif file
+System.out.println( "Save as  .tif" ) ;
+IJ.saveAs("Tiff ", dir + "spim_TL" + IJ.pad( timepoint, pad ) + "_Angle" + angle + ".tif");
+
+/* shutdown */
+runtime.exit(0);
+

spim_registration/tools/czi_resave/resubmit-failed-jobs.sh

+names=`grep "exit code 1" out.* -l`
+for name in $names; do
+	job=`sed -n '4,4p' $name | sed -n "s/Job <\([^>]*\)>.*/\1/p"`
+	echo bsub -q short -n 12 -R rusage[mem=110000] -R span[hosts=1] -o "out.%J" -e "err.%J" ${job}
+       bsub -q short -n 12 -R rusage[mem=110000] -R span[hosts=1] -o "out.%J" -e "err.%J" ${job}
+done

spim_registration/tools/czi_resave/submit-jobs

+#!/bin/bash
+
+for file in `ls ${1} | grep ".job$"`
+do
+	bsub -q short -n 4 -R span[hosts=1] -o "out.%J" -e "err.%J" ${1}/$file
+done

spim_registration/tools/master_preprocessing.sh

+#!/bin/bash
+#===============================================================================
+#
+#	  FILE: master_preprocessing.sh
+#
+#  DESCRIPTION: source file for pre-processing of file formats
+#
+#       AUTHOR: Christopher Schmied, schmied@mpi-cbg.de
+#    INSTITUTE: Max Planck Institute for Molecular Cell Biology and Genetics
+#         BUGS:
+#        NOTES:
+#      Version: 1.0
+#      CREATED: 2015-07-05
+#     REVISION: 2015-07-05
+#
+# Preprocessing
+#	1) rename .czi files
+#	2) resave .czi files into .tif or .zip
+#	3) resave ome.tiff files into .tif
+#	4) Splitting output for Channel is
+#		c=0,1 etc
+#		spim_TL{tt}_Angle{a}_Channel{c}.tif
+#===============================================================================
+image_file_directory="/projects/pilot_spim/Christopher/Test_pipeline_3.0/czi/"
+
+# --- jobs directory -----------------------------------------------------------
+job_directory="/projects/pilot_spim/Christopher/snakemake-workflows/spim_registration/tools/"
+#-------------------------------------------------------------------------------
+# Resaving, Renaming files and Splitting: General
+#
+# Important: For renaming and resaving .czi files the first .czi file has to
+# carry the index (0)
+#-------------------------------------------------------------------------------
+
+pad="3"		# for padded zeros
+angle_prep="1" # angles format: "1 2 3"
+
+#--- Renaming ------------------------------------------------------------------
+
+first_index="0"		# First index of czi files
+last_index="391"		# Last index of czi files
+first_timepoint="0"	# Starts with 0
+angles_renaming=(1 2 3 4 5)	# Angles format: (1 2 3)
+
+source_pattern=2014-10-23_H2A_gsb_G3\(\{index\}\).czi # Name of .czi files
+target_pattern=spim_TL\{timepoint\}_Angle\{angle\}.czi	# The output pattern of renaming
+
+#-------------------------------------------------------------------------------
+# Fiji settings
+#-------------------------------------------------------------------------------
+XVFB_RUN="/sw/bin/xvfb-run" # virtual frame buffer
+
+Fiji_resave="/sw/users/schmied/lifeline/Fiji.app.lifeline2/ImageJ-linux64" # Fiji that works for resaving
+#-------------------------------------------------------------------------------
+# Pre-processing
+#-------------------------------------------------------------------------------
+#--- Resaving .czi into .tif files----------------------------------------------
+jobs_resaving=${job_directory}"czi_resave"  	# directory .czi resaving
+resaving=${jobs_resaving}"/resaving.bsh" 	# script .czi resaving
+#--- Resaving ome.tiff into .tif files -----------------------------------------
+jobs_resaving_ometiff=${job_directory}"ometiff_resave" 	 # directory .ome.tiff resaving
+resaving_ometiff=${jobs_resaving_ometiff}"/resaving-ometiff.bsh" # script .ome.tiff resaving
+#--- Compress dataset;----------------------------------------------------------
+jobs_compress=${job_directory}"compress" 	# directory .czi to .zip resaving
+czi_compress=${jobs_compress}"/for_czi.bsh" 	# script .czi to .zip resaving
+#--- Split channels-------------------------------------------------------------
+jobs_split=${job_directory}"split_channels" 			# directory
+split=${jobs_split}"/split.bsh" 				# script

spim_registration/tools/rename-zeiss-files.sh

+#!/bin/bash
+# path of master file
+source ../master_3.3.sh
+
+# path of source and target files
+source_pattern=${image_file_directory}${source_pattern}
+target_pattern=${image_file_directory}${target_pattern}
+
+# ------------------------------------------------------------------------------
+
+i=${first_index}
+t=${first_timepoint}
+t=`printf "%0${pad}d" "${t}"`
+
+while [ $i -le $last_index ]; do
+
+        for a in "${angles_renaming[@]}"; do
+
+        	source=${source_pattern/\{index\}/${i}}
+                tmp=${target_pattern/\{timepoint\}/${t}}
+                target=${tmp/\{angle\}/${a}
+
+                echo ${source} ${target} # displays source file and target file with path
+
+                mv ${source} ${target} # renames source file into target pattern
+                #cp ${source} ${target} # alternatively copy source file and resave into target pattern
+                let i=i+1
+
+        done
+	t=$(( 10#${t} ))
+        let t=t+1
+        t=`printf "%0${pad}d" "${t}"`
+
+done

spim_registration/tools/split_channels/create-split-jobs.sh

+#!/bin/bash
+
+source ../../master_preprocessing.sh
+
+mkdir -p ${jobs_split}
+
+for i in $parallel_timepoints
+
+	do
+		for a in $angle_prep
+
+			do
+				job="$jobs_split/split-$i-$a.job"
+				echo $job
+				echo "#!/bin/bash" > "$job"
+				echo "$XVFB_RUN -a $Fiji \
+				-Dimage_file_directory=$image_file_directory \
+				-Dparallel_timepoints=$i \
+				-Dangle_prep=$a \
+				-Dpad=$pad \
+				-Dtarget_split=$image_file_directory \
+				-- --no-splash \
+				$split" >> "$job"
+chmod a+x "$job"
+        done
+done
+

spim_registration/tools/split_channels/split.bsh

+import ij.IJ;
+import ij.ImagePlus;
+import ij.ImageStack;
+import java.lang.Runtime;
+import java.io.File;
+import java.io.FilenameFilter;
+
+
+runtime = Runtime.getRuntime();
+
+image_file_directory = System.getProperty( "image_file_directory" ); 
+int parallel_timepoints = Integer.parseInt( System.getProperty( "parallel_timepoints" ) );
+angle_prep = System.getProperty( "angle_prep" );
+target_split = System.getProperty( "target_split" );
+int pad = Integer.parseInt( System.getProperty( "pad" ) );
+
+System.out.println( "directory = " + image_file_directory );
+System.out.println( "timpoint = " + parallel_timepoints );
+System.out.println( "angles = " + angle_prep );
+System.out.println( "target_split = " + target_split );
+System.out.println( "pad = " + pad );
+
+//open image
+
+imp = new ImagePlus( image_file_directory + "spim_TL" + IJ.pad( parallel_timepoints , pad ) + "_Angle" + angle_prep + ".tif" );
+System.out.println( imp.getTitle() );
+
+/* split channels */
+stack = imp.getStack();
+for ( c = 0; c < imp.getNChannels(); ++c )
+{
+	channelStack = new ImageStack( imp.getWidth(), imp.getHeight() );
+	for ( z = 0; z < imp.getNSlices(); ++z )
+		channelStack.addSlice(
+			"",
+			stack.getProcessor(
+				imp.getStackIndex( c + 1, z + 1, 1 ) ) );
+	impc = new ImagePlus( imp.getTitle() + " #" + ( c + 1 ), channelStack );
+	IJ.save( impc, target_split + imp.getTitle().replaceFirst( ".tif$", "_Channel" + ( c ) + ".tif" ) );
+}
+
+/* shutdown */
+runtime.exit(0);
+

spim_registration/tools/split_channels/submit-jobs

+#!/bin/bash
+
+for file in `ls ${1} | grep ".job$"`
+do
+	bsub -q short -n 3  -R span[hosts=1] -o "out.%J" -e "err.%J" ${1}/$file
+done