From 5981d62ff543eb0ed45b318e1364228b7edbbede Mon Sep 17 00:00:00 2001
From: Vojtech Moravec <vojtech.moravec.st@vsb.cz>
Date: Tue, 19 Nov 2019 13:45:56 +0100
Subject: [PATCH] Fixed imports.
---
pom.xml | 8 +++-
src/main/java/bdv/server/BigDataServer.java | 40 ++++++++++++++++---
src/main/java/bdv/server/CellHandler.java | 29 ++++++++++----
.../bdv/server/JsonDatasetListHandler.java | 2 +-
src/main/java/bdv/server/ManagerHandler.java | 4 +-
5 files changed, 67 insertions(+), 16 deletions(-)
diff --git a/pom.xml b/pom.xml
index a61c695..2004d0c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -117,5 +117,11 @@
<artifactId>commons-cli</artifactId>
<version>${commons-cli.version}</version>
</dependency>
- </dependencies>
+ <dependency>
+ <groupId>org.scijava</groupId>
+ <artifactId>DataCompressor</artifactId>
+ <version>1.0-SNAPSHOT</version>
+ <scope>compile</scope>
+ </dependency>
+ </dependencies>
</project>
diff --git a/src/main/java/bdv/server/BigDataServer.java b/src/main/java/bdv/server/BigDataServer.java
index 0ea54df..9e49eed 100644
--- a/src/main/java/bdv/server/BigDataServer.java
+++ b/src/main/java/bdv/server/BigDataServer.java
@@ -1,5 +1,7 @@
package bdv.server;
+import compression.quantization.scalar.LloydMaxU16ScalarQuantization;
+import gnu.trove.impl.sync.TSynchronizedShortByteMap;
import mpicbg.spim.data.SpimDataException;
import org.apache.commons.cli.*;
@@ -49,12 +51,14 @@ import java.util.Map.Entry;
* {@link Constants#ENABLE_EXPERIMENTAL_FEATURES} set to {@code true}.
*
* @author Tobias Pietzsch <tobias.pietzsch@gmail.com>
- * @author HongKee Moon <moon@mpi-cbg.de>
+ * @author HongKee Moon <moon@mpi-cbg.quantization.de>
*/
public class BigDataServer
{
private static final org.eclipse.jetty.util.log.Logger LOG = Log.getLogger( BigDataServer.class );
+ private static LloydMaxU16ScalarQuantization quantizer;
+
static Parameters getDefaultParameters()
{
final int port = 8080;
@@ -69,7 +73,7 @@ public class BigDataServer
}
final String thumbnailDirectory = null;
final boolean enableManagerContext = false;
- return new Parameters( port, hostname, new HashMap< String, String >(), thumbnailDirectory, enableManagerContext );
+ return new Parameters( port, hostname, new HashMap< String, String >(), thumbnailDirectory, enableManagerContext, CellHandler.DumpFile);
}
public static void main( final String[] args ) throws Exception
@@ -80,6 +84,8 @@ public class BigDataServer
if ( params == null )
return;
+ CellHandler.DumpFile = params.getDumpFile();
+
final String thumbnailsDirectoryName = getThumbnailDirectoryPath( params );
// Threadpool for multiple connections
@@ -93,6 +99,10 @@ public class BigDataServer
server.setConnectors( new Connector[] { connector } );
final String baseURL = "http://" + server.getURI().getHost() + ":" + params.getPort();
+
+ quantizer = new LloydMaxU16ScalarQuantization("D:\\tmp\\server-dump\\initial_load.bin",8);
+ quantizer.train(true);
+
// Handler initialization
final HandlerCollection handlers = new HandlerCollection();
@@ -114,6 +124,8 @@ public class BigDataServer
handler = statHandler;
}
+
+
LOG.info( "Set handler: " + handler );
server.setHandler( handler );
LOG.info( "Server Base URL: " + baseURL );
@@ -138,15 +150,18 @@ public class BigDataServer
private final String thumbnailDirectory;
+ private final String dumpFile;
+
private final boolean enableManagerContext;
- Parameters( final int port, final String hostname, final Map< String, String > datasetNameToXml, final String thumbnailDirectory, final boolean enableManagerContext )
+ Parameters( final int port, final String hostname, final Map< String, String > datasetNameToXml, final String thumbnailDirectory, final boolean enableManagerContext, final String dumpFile)
{
this.port = port;
this.hostname = hostname;
this.datasetNameToXml = datasetNameToXml;
this.thumbnailDirectory = thumbnailDirectory;
this.enableManagerContext = enableManagerContext;
+ this.dumpFile = dumpFile;
}
public int getPort()
@@ -178,6 +193,10 @@ public class BigDataServer
{
return enableManagerContext;
}
+
+ public String getDumpFile() {
+ return dumpFile;
+ }
}
@SuppressWarnings( "static-access" )
@@ -217,6 +236,13 @@ public class BigDataServer
.withArgName( "DIRECTORY" )
.create( "t" ) );
+
+ options.addOption( OptionBuilder
+ .withDescription( "File in which to store data dump" )
+ .hasArg()
+ .withArgName( "DUMP" )
+ .create( "dump" ) );
+
if ( Constants.ENABLE_EXPERIMENTAL_FEATURES )
{
options.addOption( OptionBuilder
@@ -233,12 +259,16 @@ public class BigDataServer
final String portString = cmd.getOptionValue( "p", Integer.toString( defaultParameters.getPort() ) );
final int port = Integer.parseInt( portString );
+ final String dumpFile = cmd.getOptionValue("dump", defaultParameters.getDumpFile());
+
// Getting server name option
final String serverName = cmd.getOptionValue( "s", defaultParameters.getHostname() );
// Getting thumbnail directory option
final String thumbnailDirectory = cmd.getOptionValue( "t", defaultParameters.getThumbnailDirectory() );
+
+
final HashMap< String, String > datasets = new HashMap< String, String >( defaultParameters.getDatasets() );
boolean enableManagerContext = false;
@@ -294,7 +324,7 @@ public class BigDataServer
if ( datasets.isEmpty() )
throw new IllegalArgumentException( "Dataset list is empty." );
- return new Parameters( port, serverName, datasets, thumbnailDirectory, enableManagerContext );
+ return new Parameters( port, serverName, datasets, thumbnailDirectory, enableManagerContext, dumpFile );
}
catch ( final ParseException | IllegalArgumentException e )
{
@@ -360,7 +390,7 @@ public class BigDataServer
final String name = entry.getKey();
final String xmlpath = entry.getValue();
final String context = "/" + name;
- final CellHandler ctx = new CellHandler( baseURL + context + "/", xmlpath, name, thumbnailsDirectoryName );
+ final CellHandler ctx = new CellHandler( baseURL + context + "/", xmlpath, name, thumbnailsDirectoryName, quantizer );
ctx.setContextPath( context );
handlers.addHandler( ctx );
}
diff --git a/src/main/java/bdv/server/CellHandler.java b/src/main/java/bdv/server/CellHandler.java
index b4ee268..ec80358 100644
--- a/src/main/java/bdv/server/CellHandler.java
+++ b/src/main/java/bdv/server/CellHandler.java
@@ -1,11 +1,7 @@
package bdv.server;
import java.awt.image.BufferedImage;
-import java.io.File;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.PrintWriter;
-import java.io.StringWriter;
+import java.io.*;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -14,6 +10,7 @@ import javax.imageio.ImageIO;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
+import compression.quantization.scalar.LloydMaxU16ScalarQuantization;
import org.eclipse.jetty.server.Request;
import org.eclipse.jetty.server.handler.ContextHandler;
import org.eclipse.jetty.util.log.Log;
@@ -51,6 +48,8 @@ public class CellHandler extends ContextHandler
private static final org.eclipse.jetty.util.log.Logger LOG = Log.getLogger( CellHandler.class );
+ public static String DumpFile = "";
+ private int counter = 0;
private final VolatileGlobalCellCache cache;
private final Hdf5VolatileShortArrayLoader loader;
@@ -93,12 +92,15 @@ public class CellHandler extends ContextHandler
*/
private final String thumbnailFilename;
- public CellHandler( final String baseUrl, final String xmlFilename, final String datasetName, final String thumbnailsDirectory ) throws SpimDataException, IOException
+ private LloydMaxU16ScalarQuantization quantizer;
+
+ public CellHandler(final String baseUrl, final String xmlFilename, final String datasetName, final String thumbnailsDirectory, final LloydMaxU16ScalarQuantization quantizer) throws SpimDataException, IOException
{
final XmlIoSpimDataMinimal io = new XmlIoSpimDataMinimal();
final SpimDataMinimal spimData = io.load( xmlFilename );
final SequenceDescriptionMinimal seq = spimData.getSequenceDescription();
final Hdf5ImageLoader imgLoader = ( Hdf5ImageLoader ) seq.getImgLoader();
+ this.quantizer = quantizer;
cache = imgLoader.getCacheControl();
loader = imgLoader.getShortArrayLoader();
@@ -162,8 +164,12 @@ public class CellHandler extends ContextHandler
cell = cache.getLoadingVolatileCache().get( key, cacheHints, new VolatileCellLoader<>( loader, timepoint, setup, level, cellDims, cellMin ) );
}
+
@SuppressWarnings( "unchecked" )
- final short[] data = ( ( VolatileCell< VolatileShortArray > ) cell ).getData().getCurrentStorageArray();
+ short[] data = ((VolatileCell<VolatileShortArray>) cell).getData().getCurrentStorageArray();
+ if (quantizer != null) {
+ data = quantizer.quantize(data);
+ }
/*
* NOTE(Moravec): This is possible place, where to compress data. Image data are inside data array, but we access only part of the image.
@@ -181,6 +187,15 @@ public class CellHandler extends ContextHandler
buf[ j++ ] = ( byte ) ( s & 0xff );
}
+ if (!DumpFile.equals("")) {
+ //String requestLog = String.format("%s\\request_%d_%d.data", DumpFile, buf.length, counter++);
+ FileOutputStream dumpStream = new FileOutputStream(DumpFile, true);
+ dumpStream.write(buf);
+ dumpStream.flush();
+ dumpStream.close();
+ }
+
+
transferedDataSize += buf.length;
LOG.info(String.format("Total transfered data: [%d KB] [%d MB]", (transferedDataSize/1000), ((transferedDataSize/1000)/1000)));
diff --git a/src/main/java/bdv/server/JsonDatasetListHandler.java b/src/main/java/bdv/server/JsonDatasetListHandler.java
index 8e8ec1a..d976b5a 100644
--- a/src/main/java/bdv/server/JsonDatasetListHandler.java
+++ b/src/main/java/bdv/server/JsonDatasetListHandler.java
@@ -16,7 +16,7 @@ import java.net.URISyntaxException;
/**
* Provides a list of available datasets on this {@link BigDataServer}
*
- * @author HongKee Moon <moon@mpi-cbg.de>
+ * @author HongKee Moon <moon@mpi-cbg.quantization.de>
*/
public class JsonDatasetListHandler extends ContextHandler
{
diff --git a/src/main/java/bdv/server/ManagerHandler.java b/src/main/java/bdv/server/ManagerHandler.java
index d806e4d..c458969 100644
--- a/src/main/java/bdv/server/ManagerHandler.java
+++ b/src/main/java/bdv/server/ManagerHandler.java
@@ -24,7 +24,7 @@ import java.net.URISyntaxException;
import java.text.DecimalFormat;
/**
- * @author HongKee Moon <moon@mpi-cbg.de>
+ * @author HongKee Moon <moon@mpi-cbg.quantization.de>
* @author Tobias Pietzsch <tobias.pietzsch@gmail.com>
*/
public class ManagerHandler extends ContextHandler
@@ -175,7 +175,7 @@ public class ManagerHandler extends ContextHandler
CellHandler ctx = null;
try
{
- ctx = new CellHandler( baseURL + context + "/", fileLocation, datasetName, thumbnailsDirectoryName );
+ ctx = new CellHandler( baseURL + context + "/", fileLocation, datasetName, thumbnailsDirectoryName, null );
}
catch ( final SpimDataException e )
{
--
GitLab