Skip to content
Snippets Groups Projects
Commit 776975ee authored by Tobias Pietzsch's avatar Tobias Pietzsch
Browse files

load full FloatType image from HDF5 without copying from ushort

parent a4d6ee7b
No related branches found
No related tags found
No related merge requests found
package bdv.img.hdf5; package bdv.img.hdf5;
import static bdv.img.hdf5.Util.reorder; import static bdv.img.hdf5.Util.reorder;
import ch.systemsx.cisd.base.mdarray.MDFloatArray;
import ch.systemsx.cisd.base.mdarray.MDShortArray; import ch.systemsx.cisd.base.mdarray.MDShortArray;
import ch.systemsx.cisd.hdf5.HDF5DataSetInformation; import ch.systemsx.cisd.hdf5.HDF5DataSetInformation;
import ch.systemsx.cisd.hdf5.IHDF5Reader; import ch.systemsx.cisd.hdf5.IHDF5Reader;
...@@ -42,7 +43,6 @@ public class HDF5Access implements IHDF5Access ...@@ -42,7 +43,6 @@ public class HDF5Access implements IHDF5Access
throw new InterruptedException(); throw new InterruptedException();
Util.reorder( dimensions, reorderedDimensions ); Util.reorder( dimensions, reorderedDimensions );
Util.reorder( min, reorderedMin ); Util.reorder( min, reorderedMin );
hdf5Reader.int16().readMDArray( Util.getCellsPath( timepoint, setup, level ) );
final MDShortArray array = hdf5Reader.int16().readMDArrayBlockWithOffset( Util.getCellsPath( timepoint, setup, level ), reorderedDimensions, reorderedMin ); final MDShortArray array = hdf5Reader.int16().readMDArrayBlockWithOffset( Util.getCellsPath( timepoint, setup, level ), reorderedDimensions, reorderedMin );
return array.getAsFlatArray(); return array.getAsFlatArray();
} }
...@@ -53,4 +53,22 @@ public class HDF5Access implements IHDF5Access ...@@ -53,4 +53,22 @@ public class HDF5Access implements IHDF5Access
System.arraycopy( readShortMDArrayBlockWithOffset( timepoint, setup, level, dimensions, min ), 0, dataBlock, 0, dataBlock.length ); System.arraycopy( readShortMDArrayBlockWithOffset( timepoint, setup, level, dimensions, min ), 0, dataBlock, 0, dataBlock.length );
return dataBlock; return dataBlock;
} }
@Override
public float[] readShortMDArrayBlockWithOffsetAsFloat( final int timepoint, final int setup, final int level, final int[] dimensions, final long[] min ) throws InterruptedException
{
if ( Thread.interrupted() )
throw new InterruptedException();
Util.reorder( dimensions, reorderedDimensions );
Util.reorder( min, reorderedMin );
final MDFloatArray array = hdf5Reader.float32().readMDArrayBlockWithOffset( Util.getCellsPath( timepoint, setup, level ), reorderedDimensions, reorderedMin );
return array.getAsFlatArray();
}
@Override
public float[] readShortMDArrayBlockWithOffsetAsFloat( final int timepoint, final int setup, final int level, final int[] dimensions, final long[] min, final float[] dataBlock ) throws InterruptedException
{
System.arraycopy( readShortMDArrayBlockWithOffsetAsFloat( timepoint, setup, level, dimensions, min ), 0, dataBlock, 0, dataBlock.length );
return dataBlock;
}
} }
...@@ -12,6 +12,7 @@ import static ch.systemsx.cisd.hdf5.hdf5lib.H5S.H5Sselect_hyperslab; ...@@ -12,6 +12,7 @@ import static ch.systemsx.cisd.hdf5.hdf5lib.H5S.H5Sselect_hyperslab;
import static ch.systemsx.cisd.hdf5.hdf5lib.HDF5Constants.H5P_DEFAULT; import static ch.systemsx.cisd.hdf5.hdf5lib.HDF5Constants.H5P_DEFAULT;
import static ch.systemsx.cisd.hdf5.hdf5lib.HDF5Constants.H5S_MAX_RANK; import static ch.systemsx.cisd.hdf5.hdf5lib.HDF5Constants.H5S_MAX_RANK;
import static ch.systemsx.cisd.hdf5.hdf5lib.HDF5Constants.H5S_SELECT_SET; import static ch.systemsx.cisd.hdf5.hdf5lib.HDF5Constants.H5S_SELECT_SET;
import static ch.systemsx.cisd.hdf5.hdf5lib.HDF5Constants.H5T_NATIVE_FLOAT;
import static ch.systemsx.cisd.hdf5.hdf5lib.HDF5Constants.H5T_NATIVE_INT16; import static ch.systemsx.cisd.hdf5.hdf5lib.HDF5Constants.H5T_NATIVE_INT16;
import java.lang.reflect.Field; import java.lang.reflect.Field;
...@@ -171,6 +172,32 @@ public class HDF5AccessHack implements IHDF5Access ...@@ -171,6 +172,32 @@ public class HDF5AccessHack implements IHDF5Access
return dataBlock; return dataBlock;
} }
@Override
public float[] readShortMDArrayBlockWithOffsetAsFloat( final int timepoint, final int setup, final int level, final int[] dimensions, final long[] min ) throws InterruptedException
{
final float[] dataBlock = new float[ dimensions[ 0 ] * dimensions[ 1 ] * dimensions[ 2 ] ];
readShortMDArrayBlockWithOffsetAsFloat( timepoint, setup, level, dimensions, min, dataBlock );
return dataBlock;
}
@Override
public float[] readShortMDArrayBlockWithOffsetAsFloat( final int timepoint, final int setup, final int level, final int[] dimensions, final long[] min, final float[] dataBlock ) throws InterruptedException
{
if ( Thread.interrupted() )
throw new InterruptedException();
Util.reorder( dimensions, reorderedDimensions );
Util.reorder( min, reorderedMin );
final OpenDataSet dataset = openDataSetCache.getDataSet( new ViewLevelId( timepoint, setup, level ) );
final int memorySpaceId = H5Screate_simple( reorderedDimensions.length, reorderedDimensions, null );
H5Sselect_hyperslab( dataset.fileSpaceId, H5S_SELECT_SET, reorderedMin, null, reorderedDimensions, null );
H5Dread( dataset.dataSetId, H5T_NATIVE_FLOAT, memorySpaceId, dataset.fileSpaceId, numericConversionXferPropertyListID, dataBlock );
H5Sclose( memorySpaceId );
return dataBlock;
}
@Override @Override
protected void finalize() throws Throwable protected void finalize() throws Throwable
{ {
......
...@@ -22,12 +22,13 @@ import net.imglib2.Cursor; ...@@ -22,12 +22,13 @@ import net.imglib2.Cursor;
import net.imglib2.Dimensions; import net.imglib2.Dimensions;
import net.imglib2.FinalDimensions; import net.imglib2.FinalDimensions;
import net.imglib2.FinalInterval; import net.imglib2.FinalInterval;
import net.imglib2.IterableInterval;
import net.imglib2.RandomAccess; import net.imglib2.RandomAccess;
import net.imglib2.RandomAccessibleInterval; import net.imglib2.RandomAccessibleInterval;
import net.imglib2.algorithm.stats.Normalize;
import net.imglib2.img.Img; import net.imglib2.img.Img;
import net.imglib2.img.NativeImg; import net.imglib2.img.NativeImg;
import net.imglib2.img.array.ArrayImgs; import net.imglib2.img.array.ArrayImgs;
import net.imglib2.img.basictypeaccess.array.FloatArray;
import net.imglib2.img.basictypeaccess.array.ShortArray; import net.imglib2.img.basictypeaccess.array.ShortArray;
import net.imglib2.img.basictypeaccess.volatiles.array.VolatileShortArray; import net.imglib2.img.basictypeaccess.volatiles.array.VolatileShortArray;
import net.imglib2.img.cell.CellImg; import net.imglib2.img.cell.CellImg;
...@@ -375,10 +376,10 @@ public class Hdf5ImageLoader extends AbstractViewerImgLoader< UnsignedShortType, ...@@ -375,10 +376,10 @@ public class Hdf5ImageLoader extends AbstractViewerImgLoader< UnsignedShortType,
public class MonolithicImageLoader implements ImgLoader< UnsignedShortType > public class MonolithicImageLoader implements ImgLoader< UnsignedShortType >
{ {
@Override @Override
public RandomAccessibleInterval< UnsignedShortType > getImage( final ViewId view ) public RandomAccessibleInterval< UnsignedShortType > getImage( final ViewId view )
{ {
Img< UnsignedShortType > img = null;
final int timepoint = view.getTimePointId(); final int timepoint = view.getTimePointId();
final int setup = view.getViewSetupId(); final int setup = view.getViewSetupId();
final int level = 0; final int level = 0;
...@@ -400,31 +401,18 @@ public class Hdf5ImageLoader extends AbstractViewerImgLoader< UnsignedShortType, ...@@ -400,31 +401,18 @@ public class Hdf5ImageLoader extends AbstractViewerImgLoader< UnsignedShortType,
} }
catch ( final InterruptedException e ) catch ( final InterruptedException e )
{} {}
return ArrayImgs.unsignedShorts( data, dimsLong ); img = ArrayImgs.unsignedShorts( data, dimsLong );
} }
else else
{ {
// TODO final int[] cellDimensions = computeCellDimensions(
final int[] cellDimensions = new int[ n ]; dimsLong,
long s = Integer.MAX_VALUE; perSetupMipmapInfo.get( setup ).getSubdivisions()[ level ] );
for ( int d = 0; d < n; ++d )
{
final long ns = s / dimsLong[ d ];
if ( ns > 0 )
cellDimensions[ d ] = ( int ) dimsLong[ d ];
else
{
cellDimensions[ d ] = ( int ) ( s % dimsLong[ d ] );
for ( ++d; d < n; ++d )
cellDimensions[ d ] = 1;
}
s = ns;
}
final CellImgFactory< UnsignedShortType > factory = new CellImgFactory< UnsignedShortType >( cellDimensions ); final CellImgFactory< UnsignedShortType > factory = new CellImgFactory< UnsignedShortType >( cellDimensions );
@SuppressWarnings( "unchecked" ) @SuppressWarnings( "unchecked" )
final CellImg< UnsignedShortType, ShortArray, DefaultCell< ShortArray > > img = final CellImg< UnsignedShortType, ShortArray, DefaultCell< ShortArray > > cellImg =
( CellImg< UnsignedShortType, ShortArray, DefaultCell< ShortArray > > ) factory.create( dimsLong, new UnsignedShortType() ); ( CellImg< UnsignedShortType, ShortArray, DefaultCell< ShortArray > > ) factory.create( dimsLong, new UnsignedShortType() );
final Cursor< DefaultCell< ShortArray > > cursor = img.getCells().cursor(); final Cursor< DefaultCell< ShortArray > > cursor = cellImg.getCells().cursor();
while ( cursor.hasNext() ) while ( cursor.hasNext() )
{ {
final DefaultCell< ShortArray > cell = cursor.next(); final DefaultCell< ShortArray > cell = cursor.next();
...@@ -438,75 +426,99 @@ public class Hdf5ImageLoader extends AbstractViewerImgLoader< UnsignedShortType, ...@@ -438,75 +426,99 @@ public class Hdf5ImageLoader extends AbstractViewerImgLoader< UnsignedShortType,
catch ( final InterruptedException e ) catch ( final InterruptedException e )
{} {}
} }
return img; img = cellImg;
} }
return img;
} }
@Override @Override
public RandomAccessibleInterval< FloatType > getFloatImage( final ViewId view, final boolean normalize ) public RandomAccessibleInterval< FloatType > getFloatImage( final ViewId view, final boolean normalize )
{ {
final RandomAccessibleInterval< UnsignedShortType > ushortImg = getImage( view ); Img< FloatType > img = null;
final int timepoint = view.getTimePointId();
// copy unsigned short img to float img final int setup = view.getViewSetupId();
final FloatType f = new FloatType(); final int level = 0;
final Img< FloatType > floatImg = net.imglib2.util.Util.getArrayOrCellImgFactory( ushortImg, f ).create( ushortImg, f ); final Dimensions dims = getImageSize( view );
final int n = dims.numDimensions();
// set up executor service final long[] dimsLong = new long[ n ];
final int numProcessors = Runtime.getRuntime().availableProcessors(); dims.dimensions( dimsLong );
final ExecutorService taskExecutor = Executors.newFixedThreadPool( numProcessors ); final int[] dimsInt = new int[ n ];
final ArrayList< Callable< Void > > tasks = new ArrayList< Callable< Void > >(); final long[] min = new long[ n ];
if ( Intervals.numElements( dims ) <= Integer.MAX_VALUE )
// set up all tasks
final int numPortions = numProcessors * 2;
final long threadChunkSize = floatImg.size() / numPortions;
final long threadChunkMod = floatImg.size() % numPortions;
for ( int portionID = 0; portionID < numPortions; ++portionID )
{
// move to the starting position of the current thread
final long startPosition = portionID * threadChunkSize;
// the last thread may has to run longer if the number of pixels cannot be divided by the number of threads
final long loopSize = ( portionID == numPortions - 1 ) ? threadChunkSize + threadChunkMod : threadChunkSize;
tasks.add( new Callable< Void >()
{
@Override
public Void call() throws Exception
{ {
final Cursor< UnsignedShortType > in = Views.iterable( ushortImg ).localizingCursor(); // use ArrayImg
final RandomAccess< FloatType > out = floatImg.randomAccess(); for ( int d = 0; d < dimsInt.length; ++d )
dimsInt[ d ] = ( int ) dimsLong[ d ];
in.jumpFwd( startPosition ); float[] data = null;
try
for ( long j = 0; j < loopSize; ++j )
{ {
final UnsignedShortType vin = in.next(); data = hdf5Access.readShortMDArrayBlockWithOffsetAsFloat( timepoint, setup, level, dimsInt, min );
out.setPosition( in );
out.get().set( vin.getRealFloat() );
} }
catch ( final InterruptedException e )
return null; {}
} img = ArrayImgs.floats( data, dimsLong );
});
} }
else
{
final int[] cellDimensions = computeCellDimensions(
dimsLong,
perSetupMipmapInfo.get( setup ).getSubdivisions()[ level ] );
final CellImgFactory< FloatType > factory = new CellImgFactory< FloatType >( cellDimensions );
@SuppressWarnings( "unchecked" )
final CellImg< FloatType, FloatArray, DefaultCell< FloatArray > > cellImg =
( CellImg< FloatType, FloatArray, DefaultCell< FloatArray > > ) factory.create( dimsLong, new FloatType() );
final Cursor< DefaultCell< FloatArray > > cursor = cellImg.getCells().cursor();
while ( cursor.hasNext() )
{
final DefaultCell< FloatArray > cell = cursor.next();
final float[] dataBlock = cell.getData().getCurrentStorageArray();
cell.dimensions( dimsInt );
cell.min( min );
try try
{ {
// invokeAll() returns when all tasks are complete hdf5Access.readShortMDArrayBlockWithOffsetAsFloat( timepoint, setup, level, dimsInt, min, dataBlock );
taskExecutor.invokeAll( tasks );
taskExecutor.shutdown();
} }
catch ( final InterruptedException e ) catch ( final InterruptedException e )
{ {}
return null; }
img = cellImg;
} }
if ( normalize ) if ( normalize )
// normalize the image to 0...1 // normalize the image to 0...1
Normalize.normalize( floatImg, new FloatType( 0 ), new FloatType( 1 ) ); normalize( img );
return floatImg; return img;
}
private int[] computeCellDimensions( final long[] dimsLong, final int[] chunkSize )
{
final int n = dimsLong.length;
final long[] dimsInChunks = new long[ n ];
int elementsPerChunk = 1;
for ( int d = 0; d < n; ++d )
{
dimsInChunks[ d ] = ( dimsLong[ d ] + chunkSize[ d ] - 1 ) / chunkSize[ d ];
elementsPerChunk *= chunkSize[ d ];
}
final int[] cellDimensions = new int[ n ];
long s = Integer.MAX_VALUE / elementsPerChunk;
for ( int d = 0; d < n; ++d )
{
final long ns = s / dimsInChunks[ d ];
if ( ns > 0 )
cellDimensions[ d ] = chunkSize[ d ] * ( int ) ( dimsInChunks[ d ] );
else
{
cellDimensions[ d ] = chunkSize[ d ] * ( int ) ( s % dimsInChunks[ d ] );
for ( ++d; d < n; ++d )
cellDimensions[ d ] = chunkSize[ d ];
}
s = ns;
}
return cellDimensions;
} }
@Override @Override
...@@ -593,11 +605,32 @@ public class Hdf5ImageLoader extends AbstractViewerImgLoader< UnsignedShortType, ...@@ -593,11 +605,32 @@ public class Hdf5ImageLoader extends AbstractViewerImgLoader< UnsignedShortType,
if ( normalize ) if ( normalize )
// normalize the image to 0...1 // normalize the image to 0...1
Normalize.normalize( floatImg, new FloatType( 0 ), new FloatType( 1 ) ); normalize( floatImg );
return floatImg; return floatImg;
} }
/**
* normalize img to 0...1
*/
protected static void normalize( final IterableInterval< FloatType > img )
{
float currentMax = img.firstElement().get();
float currentMin = currentMax;
for ( final FloatType t : img )
{
final float f = t.get();
if ( f > currentMax )
currentMax = f;
else if ( f < currentMin )
currentMin = f;
}
final float scale = ( float ) ( 1.0 / ( currentMax - currentMin ) );
for ( final FloatType t : img )
t.set( ( t.get() - currentMin ) * scale );
}
@Override @Override
public Dimensions getImageSize( final ViewId view ) public Dimensions getImageSize( final ViewId view )
{ {
......
...@@ -7,4 +7,8 @@ public interface IHDF5Access ...@@ -7,4 +7,8 @@ public interface IHDF5Access
public short[] readShortMDArrayBlockWithOffset( final int timepoint, final int setup, final int level, final int[] dimensions, final long[] min ) throws InterruptedException; public short[] readShortMDArrayBlockWithOffset( final int timepoint, final int setup, final int level, final int[] dimensions, final long[] min ) throws InterruptedException;
public short[] readShortMDArrayBlockWithOffset( final int timepoint, final int setup, final int level, final int[] dimensions, final long[] min, final short[] dataBlock ) throws InterruptedException; public short[] readShortMDArrayBlockWithOffset( final int timepoint, final int setup, final int level, final int[] dimensions, final long[] min, final short[] dataBlock ) throws InterruptedException;
public float[] readShortMDArrayBlockWithOffsetAsFloat( final int timepoint, final int setup, final int level, final int[] dimensions, final long[] min ) throws InterruptedException;
public float[] readShortMDArrayBlockWithOffsetAsFloat( final int timepoint, final int setup, final int level, final int[] dimensions, final long[] min, final float[] dataBlock ) throws InterruptedException;
} }
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment