diff --git a/src/main/java/bdv/ij/ExportImagePlusPlugIn.java b/src/main/java/bdv/ij/ExportImagePlusPlugIn.java index 6a09940bef4659902a1d7029fd45bdd2b9117ed4..20069a51189a32e05bcc690b5697b2bbb0cc0549 100644 --- a/src/main/java/bdv/ij/ExportImagePlusPlugIn.java +++ b/src/main/java/bdv/ij/ExportImagePlusPlugIn.java @@ -19,7 +19,6 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.Map; -import mpicbg.spim.data.generic.sequence.BasicImgLoader; import mpicbg.spim.data.generic.sequence.BasicViewSetup; import mpicbg.spim.data.registration.ViewRegistration; import mpicbg.spim.data.registration.ViewRegistrations; @@ -28,13 +27,14 @@ import mpicbg.spim.data.sequence.FinalVoxelDimensions; import mpicbg.spim.data.sequence.TimePoint; import mpicbg.spim.data.sequence.TimePoints; import net.imglib2.FinalDimensions; +import net.imglib2.RandomAccessibleInterval; import net.imglib2.realtransform.AffineTransform3D; -import net.imglib2.type.numeric.integer.UnsignedShortType; import bdv.export.ExportMipmapInfo; import bdv.export.ProgressWriter; import bdv.export.ProposeMipmaps; import bdv.export.SubTaskProgressWriter; import bdv.export.WriteSequenceToHdf5; +import bdv.export.WriteSequenceToHdf5.LoopbackHeuristic; import bdv.ij.export.imgloader.ImagePlusImgLoader; import bdv.ij.export.imgloader.ImagePlusImgLoader.MinMaxOption; import bdv.ij.util.PluginHelper; @@ -116,7 +116,7 @@ public class ExportImagePlusPlugIn implements PlugIn progressWriter.out().println( "starting export..." ); // create ImgLoader wrapping the image - final BasicImgLoader< UnsignedShortType > imgLoader; + final ImagePlusImgLoader< ? > imgLoader; switch ( imp.getType() ) { case ImagePlus.GRAY8: @@ -157,6 +157,62 @@ public class ExportImagePlusPlugIn implements PlugIn for ( final BasicViewSetup setup : seq.getViewSetupsOrdered() ) perSetupExportMipmapInfo.put( setup.getId(), mipmapInfo ); + // TODO: explain & test + final boolean isVirtual = imp.getStack().isVirtual(); + final long planeSizeInBytes = imp.getWidth() * imp.getHeight() * imp.getBytesPerPixel(); + final long ijMaxMemory = IJ.maxMemory(); + final LoopbackHeuristic loopbackHeuristic = new LoopbackHeuristic() + { + @Override + public boolean decide( final RandomAccessibleInterval< ? > originalImg, final int[] factorsToOriginalImg, final int previousLevel, final int[] factorsToPreviousLevel, final int[] chunkSize ) + { + if ( previousLevel < 0 ) + return false; + + if ( isVirtual ) + { + final long requiredCacheSize = planeSizeInBytes * factorsToOriginalImg[ 2 ] * chunkSize[ 2 ]; + System.out.println( "requiredCacheSize = " + planeSizeInBytes + " * " + factorsToOriginalImg[ 2 ] + " * " + chunkSize[ 2 ] + " = " + requiredCacheSize ); + System.out.println( "ijMaxMemory = " + ijMaxMemory ); + if ( requiredCacheSize > ijMaxMemory / 3 ) + { + System.out.println( "--> use loopback" ); + return true; + } + } + else + { + if ( WriteSequenceToHdf5.numElements( factorsToOriginalImg ) / WriteSequenceToHdf5.numElements( factorsToPreviousLevel ) >= 8 ) + return true; + } + + return false; + } + + @Override + public void afterEachPlane() + { + if ( isVirtual ) + { + final long free = Runtime.getRuntime().freeMemory(); + final long total = Runtime.getRuntime().totalMemory(); + final long max = Runtime.getRuntime().maxMemory(); + final long actuallyFree = max - total + free; + + System.out.println( "freeMemory = " + ( free / 1024 / 1024 ) + "MB" ); + System.out.println( "freeMemory (corrected) = " + ( actuallyFree / 1024 / 1024 ) + "MB" ); + System.out.println( "totalMemory = " + ( total / 1024 / 1024 ) + "MB" ); + System.out.println( "maxMemory = " + ( max / 1024 / 1024 ) + "MB" ); + + if ( actuallyFree < max / 2 ) + { + System.out.println( "clearCache" ); + imgLoader.clearCache(); + } + } + } + }; + final ArrayList< Partition > partitions; if ( params.split ) { @@ -168,14 +224,14 @@ public class ExportImagePlusPlugIn implements PlugIn { final Partition partition = partitions.get( i ); final ProgressWriter p = new SubTaskProgressWriter( progressWriter, 0, 0.95 * i / partitions.size() ); - WriteSequenceToHdf5.writeHdf5PartitionFile( seq, perSetupExportMipmapInfo, params.deflate, partition, p ); + WriteSequenceToHdf5.writeHdf5PartitionFile( seq, perSetupExportMipmapInfo, params.deflate, partition, loopbackHeuristic, p ); } WriteSequenceToHdf5.writeHdf5PartitionLinkFile( seq, perSetupExportMipmapInfo, partitions, params.hdf5File ); } else { partitions = null; - WriteSequenceToHdf5.writeHdf5File( seq, perSetupExportMipmapInfo, params.deflate, params.hdf5File, new SubTaskProgressWriter( progressWriter, 0, 0.95 ) ); + WriteSequenceToHdf5.writeHdf5File( seq, perSetupExportMipmapInfo, params.deflate, params.hdf5File, loopbackHeuristic, new SubTaskProgressWriter( progressWriter, 0, 0.95 ) ); } // write xml sequence description diff --git a/src/main/java/bdv/ij/ExportSpimFusionPlugIn.java b/src/main/java/bdv/ij/ExportSpimFusionPlugIn.java index 6f86470732fd1f4b9fda6e2a778a3130888d47ce..911d42f31eb11b64699adf54036cf67510ad6c6c 100644 --- a/src/main/java/bdv/ij/ExportSpimFusionPlugIn.java +++ b/src/main/java/bdv/ij/ExportSpimFusionPlugIn.java @@ -294,7 +294,7 @@ public class ExportSpimFusionPlugIn implements PlugIn for ( final Partition partition : newPartitions ) { final SubTaskProgressWriter subtaskProgress = new SubTaskProgressWriter( progress, complete, complete + completionStep ); - WriteSequenceToHdf5.writeHdf5PartitionFile( fusionSeq, perSetupExportMipmapInfo, params.deflate, partition, subtaskProgress ); + WriteSequenceToHdf5.writeHdf5PartitionFile( fusionSeq, perSetupExportMipmapInfo, params.deflate, partition, null, subtaskProgress ); complete += completionStep; } @@ -346,13 +346,13 @@ public class ExportSpimFusionPlugIn implements PlugIn { final Partition partition = partitions.get( i ); final ProgressWriter p = new SubTaskProgressWriter( progress, 0, 0.95 * i / partitions.size() ); - WriteSequenceToHdf5.writeHdf5PartitionFile( desc, perSetupExportMipmapInfo, params.deflate, partition, p ); + WriteSequenceToHdf5.writeHdf5PartitionFile( desc, perSetupExportMipmapInfo, params.deflate, partition, null, p ); } WriteSequenceToHdf5.writeHdf5PartitionLinkFile( desc, perSetupExportMipmapInfo, partitions, params.hdf5File ); } else { - WriteSequenceToHdf5.writeHdf5File( desc, perSetupExportMipmapInfo, params.deflate, params.hdf5File, new SubTaskProgressWriter( progress, 0, 0.95 ) ); + WriteSequenceToHdf5.writeHdf5File( desc, perSetupExportMipmapInfo, params.deflate, params.hdf5File, null, new SubTaskProgressWriter( progress, 0, 0.95 ) ); } // write xml file diff --git a/src/main/java/bdv/ij/ExportSpimSequencePlugIn.java b/src/main/java/bdv/ij/ExportSpimSequencePlugIn.java index 7b1e2e514c86ddc68ab337bd9c91ad630e3ce673..29b79429e0e8df7b5e35accc03a3f4a119b705e5 100644 --- a/src/main/java/bdv/ij/ExportSpimSequencePlugIn.java +++ b/src/main/java/bdv/ij/ExportSpimSequencePlugIn.java @@ -86,14 +86,14 @@ public class ExportSpimSequencePlugIn implements PlugIn { final Partition partition = partitions.get( i ); final ProgressWriter p = new SubTaskProgressWriter( progress, 0, 0.95 * i / partitions.size() ); - WriteSequenceToHdf5.writeHdf5PartitionFile( desc, perSetupExportMipmapInfo, params.deflate, partition, p ); + WriteSequenceToHdf5.writeHdf5PartitionFile( desc, perSetupExportMipmapInfo, params.deflate, partition, null, p ); } WriteSequenceToHdf5.writeHdf5PartitionLinkFile( desc, perSetupExportMipmapInfo, partitions, params.hdf5File ); } else { partitions = null; - WriteSequenceToHdf5.writeHdf5File( desc, perSetupExportMipmapInfo, params.deflate, params.hdf5File, new SubTaskProgressWriter( progress, 0, 0.95 ) ); + WriteSequenceToHdf5.writeHdf5File( desc, perSetupExportMipmapInfo, params.deflate, params.hdf5File, null, new SubTaskProgressWriter( progress, 0, 0.95 ) ); } final Hdf5ImageLoader loader = new Hdf5ImageLoader( params.hdf5File, partitions, null, false ); diff --git a/src/main/java/bdv/ij/export/Scripting.java b/src/main/java/bdv/ij/export/Scripting.java index 38ac2431bda86ae493f2c3325424dafe2a3aaadc..2f98561ef0690c89e5e6283916fe86635837b9b5 100644 --- a/src/main/java/bdv/ij/export/Scripting.java +++ b/src/main/java/bdv/ij/export/Scripting.java @@ -203,7 +203,7 @@ public class Scripting public void writePartition( final int index ) { if ( index >= 0 && index < partitions.size() ) - WriteSequenceToHdf5.writeHdf5PartitionFile( spimData.getSequenceDescription(), perSetupMipmapInfo, deflate, partitions.get( index ), null ); + WriteSequenceToHdf5.writeHdf5PartitionFile( spimData.getSequenceDescription(), perSetupMipmapInfo, deflate, partitions.get( index ), null, null ); } public void writeXmlAndLinks() throws SpimDataException diff --git a/src/main/java/bdv/ij/export/imgloader/ImagePlusImgLoader.java b/src/main/java/bdv/ij/export/imgloader/ImagePlusImgLoader.java index 06916cedbf2e91dee2ae3a3d7feebdf664aef9f9..fbefc4ad89bcf786835780ca1350b1ac408e9a93 100644 --- a/src/main/java/bdv/ij/export/imgloader/ImagePlusImgLoader.java +++ b/src/main/java/bdv/ij/export/imgloader/ImagePlusImgLoader.java @@ -72,7 +72,7 @@ public class ImagePlusImgLoader< T extends RealType< T > & NativeType< T > > imp protected final BasicImgLoader< T > loader; - VolatileGlobalCellCache< ? > loadercache; + protected VolatileGlobalCellCache< ? > loadercache; protected double impMin; @@ -129,6 +129,16 @@ public class ImagePlusImgLoader< T extends RealType< T > & NativeType< T > > imp } } + public void clearCache() + { + if ( loadercache != null ) + { + loadercache.clearCache(); + System.runFinalization(); + System.gc(); + } + } + @Override public RandomAccessibleInterval< UnsignedShortType > getImage( final ViewId view ) { diff --git a/src/main/java/bdv/ij/export/tiles/CellVoyagerDataExporter.java b/src/main/java/bdv/ij/export/tiles/CellVoyagerDataExporter.java index 0625243aec2939f7e91178aefc69ddcbd0f6c880..b4b035cd78be3ac43cabb8a94fa740f8a54949dc 100644 --- a/src/main/java/bdv/ij/export/tiles/CellVoyagerDataExporter.java +++ b/src/main/java/bdv/ij/export/tiles/CellVoyagerDataExporter.java @@ -315,7 +315,7 @@ public class CellVoyagerDataExporter * Write to HDF5 */ - WriteSequenceToHdf5.writeHdf5File( sequenceDescriptionHDF5, resolutions, chunks, true, hdf5File, progressWriter ); + WriteSequenceToHdf5.writeHdf5File( sequenceDescriptionHDF5, resolutions, chunks, true, hdf5File, null, progressWriter ); /* * write XML sequence description