export to HDF5: use null LoopbackHeuristic, except for ExportImagePlusPlugin:

 * if saving more than 8x on pixel reads use loopback over original image
 * for virtual stacks also consider the cache size that would be required to original planes
 * explicit cache clearing for virtual stacks

src/main/java/bdv/ij/ExportImagePlusPlugIn.java

@@ -19,7 +19,6 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Map;
 
-import mpicbg.spim.data.generic.sequence.BasicImgLoader;
 import mpicbg.spim.data.generic.sequence.BasicViewSetup;
 import mpicbg.spim.data.registration.ViewRegistration;
 import mpicbg.spim.data.registration.ViewRegistrations;
@@ -28,13 +27,14 @@ import mpicbg.spim.data.sequence.FinalVoxelDimensions;
 import mpicbg.spim.data.sequence.TimePoint;
 import mpicbg.spim.data.sequence.TimePoints;
 import net.imglib2.FinalDimensions;
+import net.imglib2.RandomAccessibleInterval;
 import net.imglib2.realtransform.AffineTransform3D;
-import net.imglib2.type.numeric.integer.UnsignedShortType;
 import bdv.export.ExportMipmapInfo;
 import bdv.export.ProgressWriter;
 import bdv.export.ProposeMipmaps;
 import bdv.export.SubTaskProgressWriter;
 import bdv.export.WriteSequenceToHdf5;
+import bdv.export.WriteSequenceToHdf5.LoopbackHeuristic;
 import bdv.ij.export.imgloader.ImagePlusImgLoader;
 import bdv.ij.export.imgloader.ImagePlusImgLoader.MinMaxOption;
 import bdv.ij.util.PluginHelper;
@@ -116,7 +116,7 @@ public class ExportImagePlusPlugIn implements PlugIn
 		progressWriter.out().println( "starting export..." );
 
 		// create ImgLoader wrapping the image
-		final BasicImgLoader< UnsignedShortType > imgLoader;
+		final ImagePlusImgLoader< ? > imgLoader;
 		switch ( imp.getType() )
 		{
 		case ImagePlus.GRAY8:
@@ -157,6 +157,62 @@ public class ExportImagePlusPlugIn implements PlugIn
 		for ( final BasicViewSetup setup : seq.getViewSetupsOrdered() )
 			perSetupExportMipmapInfo.put( setup.getId(), mipmapInfo );
 
+		// TODO: explain & test
+		final boolean isVirtual = imp.getStack().isVirtual();
+		final long planeSizeInBytes = imp.getWidth() * imp.getHeight() * imp.getBytesPerPixel();
+		final long ijMaxMemory = IJ.maxMemory();
+		final LoopbackHeuristic loopbackHeuristic = new LoopbackHeuristic()
+		{
+			@Override
+			public boolean decide( final RandomAccessibleInterval< ? > originalImg, final int[] factorsToOriginalImg, final int previousLevel, final int[] factorsToPreviousLevel, final int[] chunkSize )
+			{
+				if ( previousLevel < 0 )
+					return false;
+
+				if ( isVirtual )
+				{
+					final long requiredCacheSize = planeSizeInBytes * factorsToOriginalImg[ 2 ] * chunkSize[ 2 ];
+					System.out.println( "requiredCacheSize = " + planeSizeInBytes + " * " + factorsToOriginalImg[ 2 ] + " * " + chunkSize[ 2 ] + " = " + requiredCacheSize );
+					System.out.println( "ijMaxMemory = " + ijMaxMemory );
+					if ( requiredCacheSize > ijMaxMemory / 3 )
+					{
+						System.out.println( "--> use loopback" );
+						return true;
+					}
+				}
+				else
+				{
+					if ( WriteSequenceToHdf5.numElements( factorsToOriginalImg ) / WriteSequenceToHdf5.numElements( factorsToPreviousLevel ) >= 8 )
+						return true;
+				}
+
+				return false;
+			}
+
+			@Override
+			public void afterEachPlane()
+			{
+				if ( isVirtual )
+				{
+					final long free = Runtime.getRuntime().freeMemory();
+					final long total = Runtime.getRuntime().totalMemory();
+					final long max = Runtime.getRuntime().maxMemory();
+					final long actuallyFree = max - total + free;
+
+					System.out.println( "freeMemory = " + ( free / 1024 / 1024 ) + "MB" );
+					System.out.println( "freeMemory (corrected) = " + ( actuallyFree / 1024 / 1024 ) + "MB" );
+					System.out.println( "totalMemory = " + ( total / 1024 / 1024 ) + "MB" );
+					System.out.println( "maxMemory = " + ( max / 1024 / 1024 ) + "MB" );
+
+					if ( actuallyFree < max / 2 )
+					{
+						System.out.println( "clearCache" );
+						imgLoader.clearCache();
+					}
+				}
+			}
+		};
+
 		final ArrayList< Partition > partitions;
 		if ( params.split )
 		{
@@ -168,14 +224,14 @@ public class ExportImagePlusPlugIn implements PlugIn
 			{
 				final Partition partition = partitions.get( i );
 				final ProgressWriter p = new SubTaskProgressWriter( progressWriter, 0, 0.95 * i / partitions.size() );
-				WriteSequenceToHdf5.writeHdf5PartitionFile( seq, perSetupExportMipmapInfo, params.deflate, partition, p );
+				WriteSequenceToHdf5.writeHdf5PartitionFile( seq, perSetupExportMipmapInfo, params.deflate, partition, loopbackHeuristic, p );
 			}
 			WriteSequenceToHdf5.writeHdf5PartitionLinkFile( seq, perSetupExportMipmapInfo, partitions, params.hdf5File );
 		}
 		else
 		{
 			partitions = null;
-			WriteSequenceToHdf5.writeHdf5File( seq, perSetupExportMipmapInfo, params.deflate, params.hdf5File, new SubTaskProgressWriter( progressWriter, 0, 0.95 ) );
+			WriteSequenceToHdf5.writeHdf5File( seq, perSetupExportMipmapInfo, params.deflate, params.hdf5File, loopbackHeuristic, new SubTaskProgressWriter( progressWriter, 0, 0.95 ) );
 		}
 
 		// write xml sequence description

src/main/java/bdv/ij/ExportSpimFusionPlugIn.java

@@ -294,7 +294,7 @@ public class ExportSpimFusionPlugIn implements PlugIn
 		for ( final Partition partition : newPartitions )
 		{
 			final SubTaskProgressWriter subtaskProgress = new SubTaskProgressWriter( progress, complete, complete + completionStep );
-			WriteSequenceToHdf5.writeHdf5PartitionFile( fusionSeq, perSetupExportMipmapInfo, params.deflate, partition, subtaskProgress );
+			WriteSequenceToHdf5.writeHdf5PartitionFile( fusionSeq, perSetupExportMipmapInfo, params.deflate, partition, null, subtaskProgress );
 			complete += completionStep;
 		}
 
@@ -346,13 +346,13 @@ public class ExportSpimFusionPlugIn implements PlugIn
 			{
 				final Partition partition = partitions.get( i );
 				final ProgressWriter p = new SubTaskProgressWriter( progress, 0, 0.95 * i / partitions.size() );
-				WriteSequenceToHdf5.writeHdf5PartitionFile( desc, perSetupExportMipmapInfo, params.deflate, partition, p );
+				WriteSequenceToHdf5.writeHdf5PartitionFile( desc, perSetupExportMipmapInfo, params.deflate, partition, null, p );
 			}
 			WriteSequenceToHdf5.writeHdf5PartitionLinkFile( desc, perSetupExportMipmapInfo, partitions, params.hdf5File );
 		}
 		else
 		{
-			WriteSequenceToHdf5.writeHdf5File( desc, perSetupExportMipmapInfo, params.deflate, params.hdf5File, new SubTaskProgressWriter( progress, 0, 0.95 ) );
+			WriteSequenceToHdf5.writeHdf5File( desc, perSetupExportMipmapInfo, params.deflate, params.hdf5File, null, new SubTaskProgressWriter( progress, 0, 0.95 ) );
 		}
 
 		// write xml file

src/main/java/bdv/ij/ExportSpimSequencePlugIn.java

@@ -86,14 +86,14 @@ public class ExportSpimSequencePlugIn implements PlugIn
 			{
 				final Partition partition = partitions.get( i );
 				final ProgressWriter p = new SubTaskProgressWriter( progress, 0, 0.95 * i / partitions.size() );
-				WriteSequenceToHdf5.writeHdf5PartitionFile( desc, perSetupExportMipmapInfo, params.deflate, partition, p );
+				WriteSequenceToHdf5.writeHdf5PartitionFile( desc, perSetupExportMipmapInfo, params.deflate, partition, null, p );
 			}
 			WriteSequenceToHdf5.writeHdf5PartitionLinkFile( desc, perSetupExportMipmapInfo, partitions, params.hdf5File );
 		}
 		else
 		{
 			partitions = null;
-			WriteSequenceToHdf5.writeHdf5File( desc, perSetupExportMipmapInfo, params.deflate, params.hdf5File, new SubTaskProgressWriter( progress, 0, 0.95 ) );
+			WriteSequenceToHdf5.writeHdf5File( desc, perSetupExportMipmapInfo, params.deflate, params.hdf5File, null, new SubTaskProgressWriter( progress, 0, 0.95 ) );
 		}
 
 		final Hdf5ImageLoader loader = new Hdf5ImageLoader( params.hdf5File, partitions, null, false );

src/main/java/bdv/ij/export/Scripting.java

@@ -203,7 +203,7 @@ public class Scripting
 		public void writePartition( final int index )
 		{
 			if ( index >= 0 && index < partitions.size() )
-				WriteSequenceToHdf5.writeHdf5PartitionFile( spimData.getSequenceDescription(), perSetupMipmapInfo, deflate, partitions.get( index ), null );
+				WriteSequenceToHdf5.writeHdf5PartitionFile( spimData.getSequenceDescription(), perSetupMipmapInfo, deflate, partitions.get( index ), null, null );
 		}
 
 		public void writeXmlAndLinks() throws SpimDataException

src/main/java/bdv/ij/export/imgloader/ImagePlusImgLoader.java

@@ -72,7 +72,7 @@ public class ImagePlusImgLoader< T extends RealType< T > & NativeType< T > > imp
 
 	protected final BasicImgLoader< T > loader;
 
-	VolatileGlobalCellCache< ? > loadercache;
+	protected VolatileGlobalCellCache< ? > loadercache;
 
 	protected double impMin;
 
@@ -129,6 +129,16 @@ public class ImagePlusImgLoader< T extends RealType< T > & NativeType< T > > imp
 		}
 	}
 
+	public void clearCache()
+	{
+		if ( loadercache != null )
+		{
+			loadercache.clearCache();
+			System.runFinalization();
+			System.gc();
+		}
+	}
+
 	@Override
 	public RandomAccessibleInterval< UnsignedShortType > getImage( final ViewId view )
 	{

src/main/java/bdv/ij/export/tiles/CellVoyagerDataExporter.java

@@ -315,7 +315,7 @@ public class CellVoyagerDataExporter
 		 * Write to HDF5
 		 */
 
-		WriteSequenceToHdf5.writeHdf5File( sequenceDescriptionHDF5, resolutions, chunks, true, hdf5File, progressWriter );
+		WriteSequenceToHdf5.writeHdf5File( sequenceDescriptionHDF5, resolutions, chunks, true, hdf5File, null, progressWriter );
 
 		/*
 		 * write XML sequence description