diff --git a/spim_registration/tools/#master_preprocessing# b/spim_registration/tools/#master_preprocessing# new file mode 100755 index 0000000000000000000000000000000000000000..4d09285d69ea5908a9d762cd3f949ca1951f76bf --- /dev/null +++ b/spim_registration/tools/#master_preprocessing# @@ -0,0 +1,76 @@ +#!/bin/bash +#=============================================================================== +# +# FILE: master +# +# DESCRIPTION: source file for pre-processing of file formats +# +# AUTHOR: Christopher Schmied, schmied@mpi-cbg.de +# INSTITUTE: Max Planck Institute for Molecular Cell Biology and Genetics +# BUGS: +# NOTES: +# Version: 1.0 +# CREATED: 2015-07-05 +# REVISION: 2015-07-05 +# +# Preprocessing +# 1) rename .czi files +# 2) resave .czi files into .tif or .zip +# 3) resave ome.tiff files into .tif +# 4) Splitting output for Channel is +# c=0,1 etc +# spim_TL{tt}_Angle{a}_Channel{c}.tif +#=============================================================================== +image_file_directory="/projects/pilot_spim/Christopher/Test_pipeline_3.0/czi/" + +# --- jobs directory ----------------------------------------------------------- +job_directory="/projects/pilot_spim/Christopher/snakemake-workflows/spim_registration/tools/" +#------------------------------------------------------------------------------- +# Resaving, Renaming files and Splitting: General +# +# Important: For renaming and resaving .czi files the first .czi file has to +# carry the index (0) +#------------------------------------------------------------------------------- + +pad="3" # for padded zeros +angle_prep="1" # angles format: "1 2 3" + +#--- Renaming ------------------------------------------------------------------ + +first_index="0" # First index of czi files +last_index="391" # Last index of czi files +first_timepoint="0" # The first timepoint +angles_renaming=(1) # Angles format: (1 2 3) + +source_pattern=2014-10-23_H2A_gsb_G3\(\{index\}\).czi # Name of .czi files +target_pattern=spim_TL\{timepoint\}_Angle\{angle\}.czi # The output pattern of renaming + +#------------------------------------------------------------------------------- +# Fiji settings +#------------------------------------------------------------------------------- +XVFB_RUN="/sw/bin/xvfb-run" # virtual frame buffer +# working Fiji +#Fiji="/sw/users/schmied/packages/2015-05-21_Fiji.app.cuda/ImageJ-linux64" # woriking Fiji +Fiji="/sw/users/schmied/packages/2015-06-08_Fiji.app.cuda/ImageJ-linux64" + +#Fiji for Dual Channel timelapse and Dual Channel Deconvolution +FijiDualTimelapse="/sw/users/schmied/packages/2015-05-29_Fiji-2.3.9-SNAP.app.cuda/ImageJ-linux64" + +Fiji_resave="/sw/users/schmied/lifeline/Fiji.app.lifeline2/ImageJ-linux64" # Fiji that works for resaving +Fiji_Deconvolution=${FijiDualTimelapse} # Fiji that works for deconvolution + +#------------------------------------------------------------------------------- +# Pre-processing +#------------------------------------------------------------------------------- +#--- Resaving .czi into .tif files---------------------------------------------- +jobs_resaving=${job_directory}"czi_resave" # directory .czi resaving +resaving=${jobs_resaving}"/resaving.bsh" # script .czi resaving +#--- Resaving ome.tiff into .tif files ----------------------------------------- +jobs_resaving_ometiff=${job_directory}"ometiff_resave" # directory .ome.tiff resaving +resaving_ometiff=${jobs_resaving_ometiff}"/resaving-ometiff.bsh" # script .ome.tiff resaving +#--- Compress dataset;---------------------------------------------------------- +jobs_compress=${job_directory}"compress" # directory .czi to .zip resaving +czi_compress=${jobs_compress}"/for_czi.bsh" # script .czi to .zip resaving +#--- Split channels------------------------------------------------------------- +jobs_split=${job_directory}"split_channels" # directory +split=${jobs_split}"/split.bsh" # script diff --git a/spim_registration/tools/#rename-zeiss-files.sh# b/spim_registration/tools/#rename-zeiss-files.sh# new file mode 100755 index 0000000000000000000000000000000000000000..fff09797e01d8b14729ec78358d6d345a90eb168 --- /dev/null +++ b/spim_registration/tools/#rename-zeiss-files.sh# @@ -0,0 +1,34 @@ +#!/bin/bash +# path of master file +source ../master_preprocessing.sh + +# path of source and target files +source_pattern=${image_file_directory}${source_pattern} +target_pattern=${image_file_directory}${target_pattern} + +# ------------------------------------------------------------------------------ + +i=${first_index} +t=${first_timepoint} +t=`printf "%0${pad}d" "${t}"` + +while [ $i -le $last_index ]; do + + for a in "${angles_renaming[@]}"; do + + source=${source_pattern/\{index\}/${i}} + tmp=${target_pattern/\{timepoint\}/${t}} + target=${tmp/\{angle\}/${a} + + echo ${source} ${target} # displays source file and target file with path + + mv ${source} ${target} # renames source file into target pattern + #cp ${source} ${target} # alternatively copy source file and resave into target pattern + let i=i+1 + + done + t=$(( 10#${t} )) + let t=t+1 + t=`printf "%0${pad}d" "${t}"` + +done diff --git a/spim_registration/tools/compress/create-compress-jobs.sh b/spim_registration/tools/compress/create-compress-jobs.sh new file mode 100755 index 0000000000000000000000000000000000000000..2f5009a713f19d503da43e276bcced3cc8197648 --- /dev/null +++ b/spim_registration/tools/compress/create-compress-jobs.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# path of master file +source ../../master_preprocessing.sh + +# creates directory for job files if not present +mkdir -p $jobs_compress + +echo $jobs_compress +echo $czi_compress + +# splits up resaving into 1 job per .czi file and writes the given parameters +# into the job file +for i in $parallel_timepoints + +do + for a in $angle_prep + do + job="$jobs_compress/compress-$i-$a.job" + echo $job + echo "$XVFB_RUN -a $Fiji_resave \ + -Ddir=$image_file_directory \ + -Dtimepoint=$i \ + -Dangle=$a \ + -Dpad=$pad \ + -- --no-splash $czi_compress" >> "$job" + chmod a+x "$job" + done +done + diff --git a/spim_registration/tools/compress/for_czi.bsh b/spim_registration/tools/compress/for_czi.bsh new file mode 100755 index 0000000000000000000000000000000000000000..e860cf53e8038a8a241c844fb8935b6a4a219162 --- /dev/null +++ b/spim_registration/tools/compress/for_czi.bsh @@ -0,0 +1,46 @@ +// Loads Fiji dependencies +import ij.IJ; +import ij.ImagePlus; +import java.lang.Runtime; +import java.io.File; +import java.io.FilenameFilter; + +runtime = Runtime.getRuntime(); + +// Loads parameters form job file +System.out.println( "=======================================================" ); +System.out.println( "Load Parameters" ) ; + +dir = System.getProperty( "dir" ); +int timepoint = Integer.parseInt( System.getProperty( "timepoint" ) ); +angle = System.getProperty( "angle" ); +int pad = Integer.parseInt( System.getProperty( "pad" ) ); + +// Prints Parameters into output file +System.out.println( "directory = " + dir ); +System.out.println( "timepoint = " + timepoint ); +System.out.println( "angle = " + angle ); +System.out.println( "pad = " + pad ); + + +// Executes Fiji Plugin "Bioformats Importer" to open .czi file +System.out.println( "=======================================================" ); +System.out.println( "Opening Image" ) ; +IJ.run("Bio-Formats Importer", + "open=" + dir + "spim_TL" + IJ.pad( timepoint, pad ) + "_Angle" + angle + ".czi" + " " + + "autoscale " + + "color_mode=Default " + + "specify_range " + + "view=[Standard ImageJ] " + + "stack_order=Default " + + "t_begin=1000 " + + "t_end=1000 " + + "t_step=1"); + +// Resaves .czi files as .zip file +System.out.println( "Save as compressed image" ) ; +IJ.saveAs("ZIP ", dir + "spim_TL" + IJ.pad( timepoint, pad ) + "_Angle" + angle + ".zip"); + +/* shutdown */ +runtime.exit(0); + diff --git a/spim_registration/tools/compress/resubmit-failed-jobs.sh b/spim_registration/tools/compress/resubmit-failed-jobs.sh new file mode 100755 index 0000000000000000000000000000000000000000..7f2b0eff00cefb3c37d98211532199914fb10b01 --- /dev/null +++ b/spim_registration/tools/compress/resubmit-failed-jobs.sh @@ -0,0 +1,6 @@ +names=`grep "exit code 1" out.* -l` +for name in $names; do + job=`sed -n '4,4p' $name | sed -n "s/Job <\([^>]*\)>.*/\1/p"` + echo bsub -q short -n 12 -R rusage[mem=110000] -R span[hosts=1] -o "out.%J" -e "err.%J" ${job} + bsub -q short -n 12 -R rusage[mem=110000] -R span[hosts=1] -o "out.%J" -e "err.%J" ${job} +done diff --git a/spim_registration/tools/compress/submit-jobs b/spim_registration/tools/compress/submit-jobs new file mode 100755 index 0000000000000000000000000000000000000000..945496d595d8700e1d3976be2deba06c0947e800 --- /dev/null +++ b/spim_registration/tools/compress/submit-jobs @@ -0,0 +1,6 @@ +#!/bin/bash + +for file in `ls ${1} | grep ".job$"` +do + bsub -q short -n 4 -R span[hosts=1] -o "out.%J" -e "err.%J" ${1}/$file +done diff --git a/spim_registration/tools/czi_resave/#checkpoint.sh# b/spim_registration/tools/czi_resave/#checkpoint.sh# new file mode 100755 index 0000000000000000000000000000000000000000..7442db5909ada1ddf48c8ce377c2b2bfe4ce69bb --- /dev/null +++ b/spim_registration/tools/czi_resave/#checkpoint.sh# @@ -0,0 +1,30 @@ +#!/bin/bash + +source ../../master_preprocessing.sh + +timepoint=`seq 0 391` +dir=/projects/pilot_spim/Christopher/2014-10-23_H2A_gsb_G3/ +num_angles=1 +pad=3 + +job_dir=/projects/pilot_spim/Christopher/pipeline_3.0/jobs_alpha_3.1/czi_resave/ + + +for i in $timepoint + + do + i=`printf "%0${pad}d" "$i"` + num=$(ls $dir/spim_TL"$i"_Angle*.tif |wc -l) + + if [ $num -ne $num_angles ] + + then + echo "TL"$i": TP or angles missing" + //bsub -q short -n 4 -R span[hosts=1] -o "out.%J" -e "err.%J" ${1}/*${i}* + + else + echo "TL"$i": Correct" + +fi + +done diff --git a/spim_registration/tools/czi_resave/checkpoint.sh b/spim_registration/tools/czi_resave/checkpoint.sh new file mode 100755 index 0000000000000000000000000000000000000000..f24f5f65767c6859c3ddae328e6d985bf5af6532 --- /dev/null +++ b/spim_registration/tools/czi_resave/checkpoint.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +#source ../../master_3.3.sh + +timepoint=`seq 0 391` +dir=/projects/pilot_spim/Christopher/2014-10-23_H2A_gsb_G3/ +num_angles=1 +pad=3 + +job_dir=/projects/pilot_spim/Christopher/pipeline_3.0/jobs_alpha_3.1/czi_resave/ + + +for i in $timepoint + + do + i=`printf "%0${pad}d" "$i"` + num=$(ls $dir/spim_TL"$i"_Angle*.tif |wc -l) + + if [ $num -ne $num_angles ] + + then + echo "TL"$i": TP or angles missing" + //bsub -q short -n 4 -R span[hosts=1] -o "out.%J" -e "err.%J" ${1}/*${i}* + + else + echo "TL"$i": Correct" + +fi + +done diff --git a/spim_registration/tools/czi_resave/create-resaving-jobs.sh b/spim_registration/tools/czi_resave/create-resaving-jobs.sh new file mode 100755 index 0000000000000000000000000000000000000000..5c21db6049b1a22e1102db1a4bc36b05f1ae8dcf --- /dev/null +++ b/spim_registration/tools/czi_resave/create-resaving-jobs.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# path of master file +source ../../master_preprocessing.sh + +# creates directory for job files if not present +mkdir -p $jobs_resaving + +# splits up resaving into 1 job per .czi file and writes the given parameters +# into the job file +for i in $parallel_timepoints + +do + for a in $angle_prep + do + job="$jobs_resaving/resave-$i-$a.job" + echo $job + echo "$XVFB_RUN -a $Fiji_resave \ + -Ddir=$image_file_directory \ + -Dtimepoint=$i \ + -Dangle=$a \ + -Dpad=$pad \ + -- --no-splash $resaving" >> "$job" + chmod a+x "$job" + done +done diff --git a/spim_registration/tools/czi_resave/resaving.bsh b/spim_registration/tools/czi_resave/resaving.bsh new file mode 100755 index 0000000000000000000000000000000000000000..eb3c14e4ee60fd67c2b22c77ee3e62a259e4220a --- /dev/null +++ b/spim_registration/tools/czi_resave/resaving.bsh @@ -0,0 +1,45 @@ +// Loads Fiji dependencies +import ij.IJ; +import ij.ImagePlus; +import java.lang.Runtime; +import java.io.File; +import java.io.FilenameFilter; + +runtime = Runtime.getRuntime(); + +// Loads parameters form job file +System.out.println( "=======================================================" ); +System.out.println( "Load Parameters" ) ; + +dir = System.getProperty( "dir" ); +int timepoint = Integer.parseInt( System.getProperty( "timepoint" ) ); +angle = System.getProperty( "angle" ); +int pad = Integer.parseInt( System.getProperty( "pad" ) ); + +// Prints Parameters into output file +System.out.println( "directory = " + dir ); +System.out.println( "timepoint = " + timepoint ); +System.out.println( "angle = " + angle ); +System.out.println( "pad = " + pad ); + +// Executes Fiji Plugin "Bioformats Importer" to open .czi file +System.out.println( "=======================================================" ); +System.out.println( "Opening Image" ) ; +IJ.run("Bio-Formats Importer", + "open=" + dir + "spim_TL" + IJ.pad( timepoint, pad ) + "_Angle" + angle + ".czi" + " " + + "autoscale " + + "color_mode=Default " + + "specify_range " + + "view=[Standard ImageJ] " + + "stack_order=Default " + + "t_begin=1000 " + + "t_end=1000 " + + "t_step=1"); + +// Resaves .czi files as .tif file +System.out.println( "Save as .tif" ) ; +IJ.saveAs("Tiff ", dir + "spim_TL" + IJ.pad( timepoint, pad ) + "_Angle" + angle + ".tif"); + +/* shutdown */ +runtime.exit(0); + diff --git a/spim_registration/tools/czi_resave/resubmit-failed-jobs.sh b/spim_registration/tools/czi_resave/resubmit-failed-jobs.sh new file mode 100755 index 0000000000000000000000000000000000000000..7f2b0eff00cefb3c37d98211532199914fb10b01 --- /dev/null +++ b/spim_registration/tools/czi_resave/resubmit-failed-jobs.sh @@ -0,0 +1,6 @@ +names=`grep "exit code 1" out.* -l` +for name in $names; do + job=`sed -n '4,4p' $name | sed -n "s/Job <\([^>]*\)>.*/\1/p"` + echo bsub -q short -n 12 -R rusage[mem=110000] -R span[hosts=1] -o "out.%J" -e "err.%J" ${job} + bsub -q short -n 12 -R rusage[mem=110000] -R span[hosts=1] -o "out.%J" -e "err.%J" ${job} +done diff --git a/spim_registration/tools/czi_resave/submit-jobs b/spim_registration/tools/czi_resave/submit-jobs new file mode 100755 index 0000000000000000000000000000000000000000..945496d595d8700e1d3976be2deba06c0947e800 --- /dev/null +++ b/spim_registration/tools/czi_resave/submit-jobs @@ -0,0 +1,6 @@ +#!/bin/bash + +for file in `ls ${1} | grep ".job$"` +do + bsub -q short -n 4 -R span[hosts=1] -o "out.%J" -e "err.%J" ${1}/$file +done diff --git a/spim_registration/tools/master_preprocessing.sh b/spim_registration/tools/master_preprocessing.sh new file mode 100755 index 0000000000000000000000000000000000000000..be31db3ece9a4612f28bd591104ae739f7293d60 --- /dev/null +++ b/spim_registration/tools/master_preprocessing.sh @@ -0,0 +1,68 @@ +#!/bin/bash +#=============================================================================== +# +# FILE: master_preprocessing.sh +# +# DESCRIPTION: source file for pre-processing of file formats +# +# AUTHOR: Christopher Schmied, schmied@mpi-cbg.de +# INSTITUTE: Max Planck Institute for Molecular Cell Biology and Genetics +# BUGS: +# NOTES: +# Version: 1.0 +# CREATED: 2015-07-05 +# REVISION: 2015-07-05 +# +# Preprocessing +# 1) rename .czi files +# 2) resave .czi files into .tif or .zip +# 3) resave ome.tiff files into .tif +# 4) Splitting output for Channel is +# c=0,1 etc +# spim_TL{tt}_Angle{a}_Channel{c}.tif +#=============================================================================== +image_file_directory="/projects/pilot_spim/Christopher/Test_pipeline_3.0/czi/" + +# --- jobs directory ----------------------------------------------------------- +job_directory="/projects/pilot_spim/Christopher/snakemake-workflows/spim_registration/tools/" +#------------------------------------------------------------------------------- +# Resaving, Renaming files and Splitting: General +# +# Important: For renaming and resaving .czi files the first .czi file has to +# carry the index (0) +#------------------------------------------------------------------------------- + +pad="3" # for padded zeros +angle_prep="1" # angles format: "1 2 3" + +#--- Renaming ------------------------------------------------------------------ + +first_index="0" # First index of czi files +last_index="391" # Last index of czi files +first_timepoint="0" # Starts with 0 +angles_renaming=(1 2 3 4 5) # Angles format: (1 2 3) + +source_pattern=2014-10-23_H2A_gsb_G3\(\{index\}\).czi # Name of .czi files +target_pattern=spim_TL\{timepoint\}_Angle\{angle\}.czi # The output pattern of renaming + +#------------------------------------------------------------------------------- +# Fiji settings +#------------------------------------------------------------------------------- +XVFB_RUN="/sw/bin/xvfb-run" # virtual frame buffer + +Fiji_resave="/sw/users/schmied/lifeline/Fiji.app.lifeline2/ImageJ-linux64" # Fiji that works for resaving +#------------------------------------------------------------------------------- +# Pre-processing +#------------------------------------------------------------------------------- +#--- Resaving .czi into .tif files---------------------------------------------- +jobs_resaving=${job_directory}"czi_resave" # directory .czi resaving +resaving=${jobs_resaving}"/resaving.bsh" # script .czi resaving +#--- Resaving ome.tiff into .tif files ----------------------------------------- +jobs_resaving_ometiff=${job_directory}"ometiff_resave" # directory .ome.tiff resaving +resaving_ometiff=${jobs_resaving_ometiff}"/resaving-ometiff.bsh" # script .ome.tiff resaving +#--- Compress dataset;---------------------------------------------------------- +jobs_compress=${job_directory}"compress" # directory .czi to .zip resaving +czi_compress=${jobs_compress}"/for_czi.bsh" # script .czi to .zip resaving +#--- Split channels------------------------------------------------------------- +jobs_split=${job_directory}"split_channels" # directory +split=${jobs_split}"/split.bsh" # script diff --git a/spim_registration/tools/rename-zeiss-files.sh b/spim_registration/tools/rename-zeiss-files.sh new file mode 100755 index 0000000000000000000000000000000000000000..18adc3ac7a9bdc19697436ed687650a609a3a06b --- /dev/null +++ b/spim_registration/tools/rename-zeiss-files.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# path of master file +source ../master_3.3.sh + +# path of source and target files +source_pattern=${image_file_directory}${source_pattern} +target_pattern=${image_file_directory}${target_pattern} + +# ------------------------------------------------------------------------------ + +i=${first_index} +t=${first_timepoint} +t=`printf "%0${pad}d" "${t}"` + +while [ $i -le $last_index ]; do + + for a in "${angles_renaming[@]}"; do + + source=${source_pattern/\{index\}/${i}} + tmp=${target_pattern/\{timepoint\}/${t}} + target=${tmp/\{angle\}/${a} + + echo ${source} ${target} # displays source file and target file with path + + mv ${source} ${target} # renames source file into target pattern + #cp ${source} ${target} # alternatively copy source file and resave into target pattern + let i=i+1 + + done + t=$(( 10#${t} )) + let t=t+1 + t=`printf "%0${pad}d" "${t}"` + +done diff --git a/spim_registration/tools/split_channels/create-split-jobs.sh b/spim_registration/tools/split_channels/create-split-jobs.sh new file mode 100755 index 0000000000000000000000000000000000000000..2a1b873a3587ff33dc6c964a6c333d0674b871c8 --- /dev/null +++ b/spim_registration/tools/split_channels/create-split-jobs.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +source ../../master_preprocessing.sh + +mkdir -p ${jobs_split} + +for i in $parallel_timepoints + + do + for a in $angle_prep + + do + job="$jobs_split/split-$i-$a.job" + echo $job + echo "#!/bin/bash" > "$job" + echo "$XVFB_RUN -a $Fiji \ + -Dimage_file_directory=$image_file_directory \ + -Dparallel_timepoints=$i \ + -Dangle_prep=$a \ + -Dpad=$pad \ + -Dtarget_split=$image_file_directory \ + -- --no-splash \ + $split" >> "$job" +chmod a+x "$job" + done +done + diff --git a/spim_registration/tools/split_channels/split.bsh b/spim_registration/tools/split_channels/split.bsh new file mode 100755 index 0000000000000000000000000000000000000000..e562f2c0a2cd1e7a21f26ec124dda0fd18e80365 --- /dev/null +++ b/spim_registration/tools/split_channels/split.bsh @@ -0,0 +1,44 @@ +import ij.IJ; +import ij.ImagePlus; +import ij.ImageStack; +import java.lang.Runtime; +import java.io.File; +import java.io.FilenameFilter; + + +runtime = Runtime.getRuntime(); + +image_file_directory = System.getProperty( "image_file_directory" ); +int parallel_timepoints = Integer.parseInt( System.getProperty( "parallel_timepoints" ) ); +angle_prep = System.getProperty( "angle_prep" ); +target_split = System.getProperty( "target_split" ); +int pad = Integer.parseInt( System.getProperty( "pad" ) ); + +System.out.println( "directory = " + image_file_directory ); +System.out.println( "timpoint = " + parallel_timepoints ); +System.out.println( "angles = " + angle_prep ); +System.out.println( "target_split = " + target_split ); +System.out.println( "pad = " + pad ); + +//open image + +imp = new ImagePlus( image_file_directory + "spim_TL" + IJ.pad( parallel_timepoints , pad ) + "_Angle" + angle_prep + ".tif" ); +System.out.println( imp.getTitle() ); + +/* split channels */ +stack = imp.getStack(); +for ( c = 0; c < imp.getNChannels(); ++c ) +{ + channelStack = new ImageStack( imp.getWidth(), imp.getHeight() ); + for ( z = 0; z < imp.getNSlices(); ++z ) + channelStack.addSlice( + "", + stack.getProcessor( + imp.getStackIndex( c + 1, z + 1, 1 ) ) ); + impc = new ImagePlus( imp.getTitle() + " #" + ( c + 1 ), channelStack ); + IJ.save( impc, target_split + imp.getTitle().replaceFirst( ".tif$", "_Channel" + ( c ) + ".tif" ) ); +} + +/* shutdown */ +runtime.exit(0); + diff --git a/spim_registration/tools/split_channels/submit-jobs b/spim_registration/tools/split_channels/submit-jobs new file mode 100755 index 0000000000000000000000000000000000000000..d631b4efa2d22f1174da389ee8da22662df47c55 --- /dev/null +++ b/spim_registration/tools/split_channels/submit-jobs @@ -0,0 +1,6 @@ +#!/bin/bash + +for file in `ls ${1} | grep ".job$"` +do + bsub -q short -n 3 -R span[hosts=1] -o "out.%J" -e "err.%J" ${1}/$file +done