diff --git a/src/main/java/bdv/img/openconnectome/OpenConnectomeDataset.java b/src/main/java/bdv/img/openconnectome/OpenConnectomeDataset.java new file mode 100644 index 0000000000000000000000000000000000000000..40969a82366b3a312081fa3637c2e64ca673b12e --- /dev/null +++ b/src/main/java/bdv/img/openconnectome/OpenConnectomeDataset.java @@ -0,0 +1,18 @@ +package bdv.img.openconnectome; + +import java.io.Serializable; +import java.util.HashMap; + +public class OpenConnectomeDataset implements Serializable +{ + private static final long serialVersionUID = -3203047697934754547L; + + public HashMap< String, int[] > cube_dimension; + public HashMap< String, long[] > imagesize; + public HashMap< String, long[] > isotropic_slicerange; + public HashMap< String, Long > neariso_scaledown; + public long[] resolutions; + public long[] slicerange; + public HashMap< String, Double > zscale; + public HashMap< String, long[] > zscaled_slicerange; +} \ No newline at end of file diff --git a/src/main/java/bdv/img/openconnectome/OpenConnectomeImageLoader.java b/src/main/java/bdv/img/openconnectome/OpenConnectomeImageLoader.java new file mode 100644 index 0000000000000000000000000000000000000000..83f8f8b02f879f7848e68ea6c7a754ffbbf36817 --- /dev/null +++ b/src/main/java/bdv/img/openconnectome/OpenConnectomeImageLoader.java @@ -0,0 +1,276 @@ +package bdv.img.openconnectome; + +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.URL; + +import mpicbg.spim.data.View; +import net.imglib2.RandomAccessibleInterval; +import net.imglib2.img.NativeImg; +import net.imglib2.img.basictypeaccess.volatiles.array.VolatileByteArray; +import net.imglib2.img.cell.CellImg; +import net.imglib2.realtransform.AffineTransform3D; +import net.imglib2.type.NativeType; +import net.imglib2.type.numeric.ARGBType; +import net.imglib2.type.numeric.integer.UnsignedByteType; +import net.imglib2.type.numeric.real.FloatType; +import net.imglib2.type.volatiles.VolatileARGBType; +import net.imglib2.type.volatiles.VolatileUnsignedByteType; + +import org.jdom2.Element; + +import bdv.ViewerImgLoader; +import bdv.img.cache.Cache; +import bdv.img.cache.VolatileCell; +import bdv.img.cache.VolatileGlobalCellCache; +import bdv.img.cache.VolatileGlobalCellCache.LoadingStrategy; +import bdv.img.cache.VolatileImgCells; +import bdv.img.cache.VolatileImgCells.CellCache; + +import com.google.gson.Gson; +import com.google.gson.JsonIOException; +import com.google.gson.JsonSyntaxException; + +public class OpenConnectomeImageLoader implements ViewerImgLoader< UnsignedByteType, VolatileUnsignedByteType > +{ + /** + * URL of the + */ + private String baseUrl; + + private String token; + + private String mode; + + private int numScales; + + private double[][] mipmapResolutions; + + private long[][] imageDimensions; + + private int[][] blockDimensions; + + private AffineTransform3D[] mipmapTransforms; + + protected VolatileGlobalCellCache< VolatileByteArray > cache; + + /** + * Fetch the list of public tokens from an OpenConnectome volume cutout + * service, e.g. + * {@linkplain http://openconnecto.me/emca/public_tokens/}. + * + * @param baseUrl e.g. "http://openconnecto.me/emca" + * @return a list of {@link String Strings} + * @throws JsonSyntaxException + * @throws JsonIOException + * @throws IOException + */ + final static public String[] fetchTokenList( final String baseUrl ) + throws JsonSyntaxException, JsonIOException, IOException + { + final Gson gson = new Gson(); + final URL url = new URL( baseUrl + "/public_tokens/" ); + final String[] tokens = gson.fromJson( new InputStreamReader( url.openStream() ), String[].class ); + return tokens; + } + + /** + * Fetch information for a token from an OpenConnectome volume cutout + * service, e.g. + * {@linkplain http://openconnecto.me/emca/<token>/info/}. + * + * @param baseUrl e.g. "http://openconnecto.me/emca" + * @param token + * @return an {@link OpenConnectomeTokenInfo} instance that carries the token information + * @throws JsonSyntaxException + * @throws JsonIOException + * @throws IOException + */ + final static public OpenConnectomeTokenInfo fetchTokenInfo( final String baseUrl, final String token ) + throws JsonSyntaxException, JsonIOException, IOException + { + final Gson gson = new Gson(); + final URL url = new URL( baseUrl + "/" + token + "/info/" ); + return gson.fromJson( new InputStreamReader( url.openStream() ), OpenConnectomeTokenInfo.class ); + } + + /** + * Try to fetch the list of public tokens from an OpenConnectome volume cutout + * service, e.g. + * {@linkplain http://openconnecto.me/emca/public_tokens/}. + * + * @param baseUrl e.g. "http://openconnecto.me/emca" + * @param maxNumTrials the maximum number of trials + * + * @return a list of {@link String Strings} or <code>null</code> if + * <code>maxNumTrials</code> were executed without success + */ + final static public String[] tryFetchTokenList( final String baseUrl, final int maxNumTrials ) + { + String[] tokens = null; + for ( int i = 0; i < maxNumTrials && tokens == null; ++i ) + { + try + { + tokens = fetchTokenList( baseUrl ); + break; + } + catch ( final Exception e ) {} + try + { + Thread.sleep( 100 ); + } + catch ( final InterruptedException e ) {} + } + return tokens; + } + + + /** + * Try to fetch information for a token from an OpenConnectome volume cutout + * service, e.g. + * {@linkplain http://openconnecto.me/emca/<token>/info/}. + * + * @param baseUrl e.g. "http://openconnecto.me/emca" + * @param token + * @param maxNumTrials + * @return an {@link OpenConnectomeTokenInfo} instance that carries the + * token information or <code>null</code> if <code>maxNumTrials</code> + * were executed without success + */ + final static public OpenConnectomeTokenInfo tryFetchTokenInfo( final String baseUrl, final String token, final int maxNumTrials ) + { + OpenConnectomeTokenInfo info = null; + for ( int i = 0; i < maxNumTrials && info == null; ++i ) + { + try + { + info = fetchTokenInfo( baseUrl, token ); + break; + } + catch ( final Exception e ) {} + try + { + Thread.sleep( 100 ); + } + catch ( final InterruptedException e ) {} + } + return info; + } + + @Override + public void init( final Element elem, final File basePath ) + { + baseUrl = elem.getChildText( "baseUrl" ); + token = elem.getChildText( "token" ); + mode = elem.getChildText( "mode" ); + + final OpenConnectomeTokenInfo info = tryFetchTokenInfo( baseUrl, token, 20 ); + + numScales = info.dataset.cube_dimension.size(); + + mipmapResolutions = info.getLevelScales( mode ); + imageDimensions = info.getLevelDimensions( mode ); + blockDimensions = info.getLevelCellDimensions(); + mipmapTransforms = info.getLevelTransforms( mode ); + + final int[] maxLevels = new int[]{ numScales - 1 }; + + cache = new VolatileGlobalCellCache< VolatileByteArray >( + new OpenConnectomeVolatileArrayLoader( baseUrl, token, mode, info.getMinZ() ), 1, 1, numScales, maxLevels, 10 ); + } + + @Override + public Element toXml( final File basePath ) + { + throw new UnsupportedOperationException( "not implemented" ); + } + + @Override + public RandomAccessibleInterval< FloatType > getFloatImage( final View view ) + { + throw new UnsupportedOperationException( "not implemented" ); + } + + @Override + public RandomAccessibleInterval< UnsignedByteType > getImage( final View view ) + { + return getImage( view, 0 ); + } + + @Override + public RandomAccessibleInterval< UnsignedByteType > getImage( final View view, final int level ) + { + final CellImg< UnsignedByteType, VolatileByteArray, VolatileCell< VolatileByteArray > > img = prepareCachedImage( view, level, LoadingStrategy.BLOCKING ); + final UnsignedByteType linkedType = new UnsignedByteType( img ); + img.setLinkedType( linkedType ); + return img; + } + + @Override + public RandomAccessibleInterval< VolatileUnsignedByteType > getVolatileImage( final View view, final int level ) + { + final CellImg< VolatileUnsignedByteType, VolatileByteArray, VolatileCell< VolatileByteArray > > img = prepareCachedImage( view, level, LoadingStrategy.VOLATILE ); + final VolatileUnsignedByteType linkedType = new VolatileUnsignedByteType( img ); + img.setLinkedType( linkedType ); + return img; + } + + @Override + public double[][] getMipmapResolutions( final int setup ) + { + return mipmapResolutions; + } + + @Override + public int numMipmapLevels( final int setup ) + { + return numScales; + } + + /** + * (Almost) create a {@link CellImg} backed by the cache. + * The created image needs a {@link NativeImg#setLinkedType(net.imglib2.type.Type) linked type} before it can be used. + * The type should be either {@link ARGBType} and {@link VolatileARGBType}. + */ + protected < T extends NativeType< T > > CellImg< T, VolatileByteArray, VolatileCell< VolatileByteArray > > prepareCachedImage( final View view, final int level, final LoadingStrategy loadingStrategy ) + { + final long[] dimensions = imageDimensions[ level ]; + final int[] cellDimensions = blockDimensions[ level ]; + + final CellCache< VolatileByteArray > c = cache.new VolatileCellCache( view.getTimepointIndex(), view.getSetupIndex(), level, loadingStrategy ); + final VolatileImgCells< VolatileByteArray > cells = new VolatileImgCells< VolatileByteArray >( c, 1, dimensions, cellDimensions ); + final CellImg< T, VolatileByteArray, VolatileCell< VolatileByteArray > > img = new CellImg< T, VolatileByteArray, VolatileCell< VolatileByteArray > >( null, cells ); + return img; + } + + @Override + public Cache getCache() + { + return cache; + } + + private final UnsignedByteType type = new UnsignedByteType(); + + private final VolatileUnsignedByteType volatileType = new VolatileUnsignedByteType(); + + @Override + public UnsignedByteType getImageType() + { + return type; + } + + @Override + public VolatileUnsignedByteType getVolatileImageType() + { + return volatileType; + } + + @Override + public AffineTransform3D[] getMipmapTransforms( int setup ) + { + return mipmapTransforms; + } + +} diff --git a/src/main/java/bdv/img/openconnectome/OpenConnectomeProject.java b/src/main/java/bdv/img/openconnectome/OpenConnectomeProject.java new file mode 100644 index 0000000000000000000000000000000000000000..2202178b3fe1807745edeb555e26e30679327b88 --- /dev/null +++ b/src/main/java/bdv/img/openconnectome/OpenConnectomeProject.java @@ -0,0 +1,17 @@ +package bdv.img.openconnectome; + +import java.io.Serializable; + +public class OpenConnectomeProject implements Serializable +{ + private static final long serialVersionUID = 3296461682832630651L; + + public String dataset; + public String dataurl; + public String dbname; + public boolean exceptions; + public String host; + public int projecttype; + public boolean readonly; + public int resolution; +} \ No newline at end of file diff --git a/src/main/java/bdv/img/openconnectome/OpenConnectomeTokenInfo.java b/src/main/java/bdv/img/openconnectome/OpenConnectomeTokenInfo.java new file mode 100644 index 0000000000000000000000000000000000000000..7ae6a345a75eb2129bab6a341f0e72b96535ee67 --- /dev/null +++ b/src/main/java/bdv/img/openconnectome/OpenConnectomeTokenInfo.java @@ -0,0 +1,112 @@ +package bdv.img.openconnectome; + +import java.io.Serializable; + +import net.imglib2.realtransform.AffineTransform3D; + +public class OpenConnectomeTokenInfo implements Serializable +{ + private static final long serialVersionUID = -560051267067033900L; + + public OpenConnectomeDataset dataset; + public OpenConnectomeProject project; + + public long[][] getLevelDimensions( final String mode ) + { + final long[][] levelDimensions = new long[ dataset.imagesize.size() ][ 3 ]; + + if ( mode.equals( "neariso" ) ) + { + for ( int i = 0; i < dataset.imagesize.size(); ++i ) + { + final long[] xy = dataset.imagesize.get( new Integer( i ).toString() ); + final long[] slicerange = dataset.isotropic_slicerange.get( new Integer( i ).toString() ); + levelDimensions[ i ][ 0 ] = xy[ 0 ]; + levelDimensions[ i ][ 1 ] = xy[ 1 ]; + levelDimensions[ i ][ 2 ] = slicerange[ 1 ] - slicerange[ 0 ]; + } + } + else + { + for ( int i = 0; i < dataset.imagesize.size(); ++i ) + { + final long[] xy = dataset.imagesize.get( new Integer( i ).toString() ); + levelDimensions[ i ][ 0 ] = xy[ 0 ]; + levelDimensions[ i ][ 1 ] = xy[ 1 ]; + levelDimensions[ i ][ 2 ] = dataset.slicerange[ 1 ] - dataset.slicerange[ 0 ]; + } + } + + return levelDimensions; + } + + public int[][] getLevelCellDimensions() + { + final int[][] levelCellDimensions = new int[ dataset.cube_dimension.size() ][]; + + for ( int i = 0; i < dataset.cube_dimension.size(); ++i ) + levelCellDimensions[ i ] = dataset.cube_dimension.get( new Integer( i ).toString() ).clone(); + return levelCellDimensions; + } + + public double[][] getLevelScales( final String mode ) + { + final double[][] levelScales = new double[ dataset.zscale.size() ][ 3 ]; + long s = 1; + if ( mode.equals( "neariso" ) ) + { + final double zScale0 = dataset.zscale.get( "0" ); + for ( int i = 0; i < dataset.neariso_scaledown.size(); ++i, s <<= 1 ) + { + levelScales[ i ][ 0 ] = s; + levelScales[ i ][ 1 ] = s; + levelScales[ i ][ 2 ] = zScale0 * dataset.neariso_scaledown.get( new Integer( i ).toString() ); + } + } + else + { + for ( int i = 0; i < dataset.zscale.size(); ++i, s <<= 1 ) + { + levelScales[ i ][ 0 ] = s; + levelScales[ i ][ 1 ] = s; + levelScales[ i ][ 2 ] = dataset.zscale.get( new Integer( i ).toString() ) * s; + } + } + + return levelScales; + } + + public long getMinZ() + { + return dataset.slicerange[ 0 ]; + } + + public AffineTransform3D[] getLevelTransforms( final String mode ) + { + final int n = dataset.cube_dimension.size(); + final AffineTransform3D[] levelTransforms = new AffineTransform3D[ n ]; + final boolean neariso = mode.equals( "neariso" ); + final double zScale0 = dataset.zscale.get( "0" ); + + long s = 1; + for ( int i = 0; i < n; ++i, s <<= 1 ) + { + final AffineTransform3D levelTransform = new AffineTransform3D(); + levelTransform.set( s, 0, 0 ); + levelTransform.set( s, 1, 1 ); + + levelTransform.set( -0.5 * ( s - 1 ), 0, 3 ); + levelTransform.set( -0.5 * ( s - 1 ), 1, 3 ); + + if ( neariso ) + { + final double zScale = zScale0 * dataset.neariso_scaledown.get( new Integer( i ).toString() ); + levelTransform.set( zScale, 2, 2 ); + levelTransform.set( 0.5 * ( zScale - 1 ), 2, 3 ); + } + levelTransforms[ i ] = levelTransform; + } + + return levelTransforms; + } +} \ No newline at end of file diff --git a/src/main/java/bdv/img/openconnectome/OpenConnectomeVolatileArrayLoader.java b/src/main/java/bdv/img/openconnectome/OpenConnectomeVolatileArrayLoader.java new file mode 100644 index 0000000000000000000000000000000000000000..b3c92ad04dfe9603a7cd399e773dea8939e6b612 --- /dev/null +++ b/src/main/java/bdv/img/openconnectome/OpenConnectomeVolatileArrayLoader.java @@ -0,0 +1,144 @@ +package bdv.img.openconnectome; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.util.zip.DataFormatException; +import java.util.zip.Inflater; + +import net.imglib2.img.basictypeaccess.volatiles.array.VolatileByteArray; +import bdv.img.cache.CacheArrayLoader; + +public class OpenConnectomeVolatileArrayLoader implements CacheArrayLoader< VolatileByteArray > +{ + private VolatileByteArray theEmptyArray; + + final private String tokenUrl; + + final private String mode; + + final private long zMin; + + /** + * <p>Create a {@link CacheArrayLoader} for a source provided by the + * <a href="http://hssl.cs.jhu.edu/wiki/doku.php?id=randal:hssl:research:brain:data_set_description">Open + * Connectome Volume Cutout Service</a>.</p> + * + * <p>It is created with a base URL, e.g. + * <a href="http://openconnecto.me/emca/kasthuri11">http://openconnecto.me/emca/kasthuri11</a> + * the cell dimensions, and an offset in <em>z</em>. This offset constitutes the + * 0-coordinate in <em>z</em> and should point to the first slice of the + * dataset.</p> + * + * @param baseUrl e.g. + * <a href="http://openconnecto.me/emca">http://openconnecto.me/emca</a> + * @param token e.g. "kasthuri11" + * @param mode z-scaling mode, either of [null, "", "neariso"] + * @param zMin first z-index + */ + public OpenConnectomeVolatileArrayLoader( + final String baseUrl, + final String token, + final String mode, + final long zMin ) + { + theEmptyArray = new VolatileByteArray( 1, false ); + this.tokenUrl = baseUrl + "/" + token + "/zip/"; + this.mode = "/" + mode + ( mode == null || mode.equals( "" ) ? "" : "/" ); + this.zMin = zMin; + } + + @Override + public int getBytesPerElement() + { + return 1; + } + + @Override + public VolatileByteArray loadArray( + final int timepoint, + final int setup, + final int level, + final int[] dimensions, + final long[] min ) throws InterruptedException + { + try + { + return tryLoadArray( timepoint, setup, level, dimensions, min ); + } + catch ( final OutOfMemoryError e ) + { + System.gc(); + return tryLoadArray( timepoint, setup, level, dimensions, min ); + } + } + + public VolatileByteArray tryLoadArray( + final int timepoint, + final int setup, + final int level, + final int[] dimensions, + final long[] min ) throws InterruptedException + { + final byte[] data = new byte[ dimensions[ 0 ] * dimensions[ 1 ] * dimensions[ 2 ] ]; + + final StringBuffer url = new StringBuffer( tokenUrl ); + + final long z = min[ 2 ] + zMin; + + url.append( level ); + url.append( "/" ); + url.append( min[ 0 ] ); + url.append( "," ); + url.append( min[ 0 ] + dimensions[ 0 ] ); + url.append( "/" ); + url.append( min[ 1 ] ); + url.append( "," ); + url.append( min[ 1 ] + + dimensions[ 1 ] ); + url.append( "/" ); + url.append( z ); + url.append( "," ); + url.append( z + dimensions[ 2 ] ); + url.append( mode ); + + try + { + final URL file = new URL( url.toString() ); + final InputStream in = file.openStream(); + final ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + final byte[] chunk = new byte[ 4096 ]; + int l; + for ( l = in.read( chunk ); l > 0; l = in.read( chunk ) ) + byteStream.write( chunk, 0, l ); + + final byte[] zippedData = byteStream.toByteArray(); + final Inflater inflater = new Inflater(); + inflater.setInput( zippedData ); + inflater.inflate( data ); + inflater.end(); + byteStream.close(); + } + catch ( final IOException e ) + { + System.out.println( "failed loading x=" + min[ 0 ] + " y=" + min[ 1 ] + " z=" + min[ 2 ] + " url(" + url.toString() + ")" ); + } + catch( final DataFormatException e ) + { + System.out.println( "failed unpacking x=" + min[ 0 ] + " y=" + min[ 1 ] + " z=" + min[ 2 ] + " url(" + url.toString() + ")" ); + } + + return new VolatileByteArray( data, true ); + } + + @Override + public VolatileByteArray emptyArray( final int[] dimensions ) + { + int numEntities = 1; + for ( int i = 0; i < dimensions.length; ++i ) + numEntities *= dimensions[ i ]; + if ( theEmptyArray.getCurrentStorageArray().length < numEntities ) + theEmptyArray = new VolatileByteArray( numEntities, false ); + return theEmptyArray; + } +}