diff --git a/src/main/java/bdv/server/BigDataServer.java b/src/main/java/bdv/server/BigDataServer.java
index 9e49eedc64f4ce9934add2bb61cea083e4323fbc..76ee54afb15490275834bb7609e7cd3119dbcade 100644
--- a/src/main/java/bdv/server/BigDataServer.java
+++ b/src/main/java/bdv/server/BigDataServer.java
@@ -1,7 +1,6 @@
package bdv.server;
import compression.quantization.scalar.LloydMaxU16ScalarQuantization;
-import gnu.trove.impl.sync.TSynchronizedShortByteMap;
import mpicbg.spim.data.SpimDataException;
import org.apache.commons.cli.*;
@@ -46,355 +45,378 @@ import java.util.Map.Entry;
* by default.)
* -m enable statistics and manager context. EXPERIMENTAL!
* </pre>
- *
+ * <p>
* To enable the {@code -m} option, build with
* {@link Constants#ENABLE_EXPERIMENTAL_FEATURES} set to {@code true}.
*
* @author Tobias Pietzsch <tobias.pietzsch@gmail.com>
* @author HongKee Moon <moon@mpi-cbg.quantization.de>
*/
-public class BigDataServer
-{
- private static final org.eclipse.jetty.util.log.Logger LOG = Log.getLogger( BigDataServer.class );
-
- private static LloydMaxU16ScalarQuantization quantizer;
-
- static Parameters getDefaultParameters()
- {
- final int port = 8080;
- String hostname;
- try
- {
- hostname = InetAddress.getLocalHost().getHostName();
- }
- catch ( final UnknownHostException e )
- {
- hostname = "localhost";
- }
- final String thumbnailDirectory = null;
- final boolean enableManagerContext = false;
- return new Parameters( port, hostname, new HashMap< String, String >(), thumbnailDirectory, enableManagerContext, CellHandler.DumpFile);
- }
-
- public static void main( final String[] args ) throws Exception
- {
- System.setProperty( "org.eclipse.jetty.util.log.class", "org.eclipse.jetty.util.log.StdErrLog" );
-
- final Parameters params = processOptions( args, getDefaultParameters() );
- if ( params == null )
- return;
-
- CellHandler.DumpFile = params.getDumpFile();
-
- final String thumbnailsDirectoryName = getThumbnailDirectoryPath( params );
-
- // Threadpool for multiple connections
- final Server server = new Server( new QueuedThreadPool( 200, 8 ) );
-
- // ServerConnector configuration
- final ServerConnector connector = new ServerConnector( server );
- connector.setHost( params.getHostname() );
- connector.setPort( params.getPort() );
- LOG.info( "Set connectors: " + connector );
- server.setConnectors( new Connector[] { connector } );
- final String baseURL = "http://" + server.getURI().getHost() + ":" + params.getPort();
-
-
- quantizer = new LloydMaxU16ScalarQuantization("D:\\tmp\\server-dump\\initial_load.bin",8);
- quantizer.train(true);
-
- // Handler initialization
- final HandlerCollection handlers = new HandlerCollection();
-
- final ContextHandlerCollection datasetHandlers = createHandlers( baseURL, params.getDatasets(), thumbnailsDirectoryName );
- handlers.addHandler( datasetHandlers );
- handlers.addHandler( new JsonDatasetListHandler( server, datasetHandlers ) );
-
- Handler handler = handlers;
- if ( params.enableManagerContext() )
- {
- // Add Statistics bean to the connector
- final ConnectorStatistics connectorStats = new ConnectorStatistics();
- connector.addBean( connectorStats );
-
- // create StatisticsHandler wrapper and ManagerHandler
- final StatisticsHandler statHandler = new StatisticsHandler();
- handlers.addHandler( new ManagerHandler( baseURL, server, connectorStats, statHandler, datasetHandlers, thumbnailsDirectoryName ) );
- statHandler.setHandler( handlers );
- handler = statHandler;
- }
-
-
-
- LOG.info( "Set handler: " + handler );
- server.setHandler( handler );
- LOG.info( "Server Base URL: " + baseURL );
- LOG.info( "BigDataServer starting" );
- server.start();
- server.join();
- }
-
- /**
- * Server parameters: hostname, port, datasets.
- */
- private static class Parameters
- {
- private final int port;
-
- private final String hostname;
-
- /**
- * maps from dataset name to dataset xml path.
- */
- private final Map< String, String > datasetNameToXml;
-
- private final String thumbnailDirectory;
-
- private final String dumpFile;
-
- private final boolean enableManagerContext;
-
- Parameters( final int port, final String hostname, final Map< String, String > datasetNameToXml, final String thumbnailDirectory, final boolean enableManagerContext, final String dumpFile)
- {
- this.port = port;
- this.hostname = hostname;
- this.datasetNameToXml = datasetNameToXml;
- this.thumbnailDirectory = thumbnailDirectory;
- this.enableManagerContext = enableManagerContext;
- this.dumpFile = dumpFile;
- }
-
- public int getPort()
- {
- return port;
- }
-
- public String getHostname()
- {
- return hostname;
- }
-
- public String getThumbnailDirectory()
- {
- return thumbnailDirectory;
- }
-
- /**
- * Get datasets.
- *
- * @return datasets as a map from dataset name to dataset xml path.
- */
- public Map< String, String > getDatasets()
- {
- return datasetNameToXml;
- }
-
- public boolean enableManagerContext()
- {
- return enableManagerContext;
- }
-
- public String getDumpFile() {
- return dumpFile;
- }
- }
-
- @SuppressWarnings( "static-access" )
- static private Parameters processOptions( final String[] args, final Parameters defaultParameters ) throws IOException
- {
- // create Options object
- final Options options = new Options();
-
- final String cmdLineSyntax = "BigDataServer [OPTIONS] [NAME XML] ...\n";
-
- final String description =
- "Serves one or more XML/HDF5 datasets for remote access over HTTP.\n" +
- "Provide (NAME XML) pairs on the command line or in a dataset file, where NAME is the name under which the dataset should be made accessible and XML is the path to the XML file of the dataset.";
-
- options.addOption( OptionBuilder
- .withDescription( "Hostname of the server.\n(default: " + defaultParameters.getHostname() + ")" )
- .hasArg()
- .withArgName( "HOSTNAME" )
- .create( "s" ) );
-
- options.addOption( OptionBuilder
- .withDescription( "Listening port.\n(default: " + defaultParameters.getPort() + ")" )
- .hasArg()
- .withArgName( "PORT" )
- .create( "p" ) );
-
- // -d or multiple {name name.xml} pairs
- options.addOption( OptionBuilder
- .withDescription( "Dataset file: A plain text file specifying one dataset per line. Each line is formatted as \"NAME <TAB> XML\"." )
- .hasArg()
- .withArgName( "FILE" )
- .create( "d" ) );
-
- options.addOption( OptionBuilder
- .withDescription( "Directory to store thumbnails. (new temporary directory by default.)" )
- .hasArg()
- .withArgName( "DIRECTORY" )
- .create( "t" ) );
-
-
- options.addOption( OptionBuilder
- .withDescription( "File in which to store data dump" )
- .hasArg()
- .withArgName( "DUMP" )
- .create( "dump" ) );
-
- if ( Constants.ENABLE_EXPERIMENTAL_FEATURES )
- {
- options.addOption( OptionBuilder
- .withDescription( "enable statistics and manager context. EXPERIMENTAL!" )
- .create( "m" ) );
- }
-
- try
- {
- final CommandLineParser parser = new BasicParser();
- final CommandLine cmd = parser.parse( options, args );
-
- // Getting port number option
- final String portString = cmd.getOptionValue( "p", Integer.toString( defaultParameters.getPort() ) );
- final int port = Integer.parseInt( portString );
-
- final String dumpFile = cmd.getOptionValue("dump", defaultParameters.getDumpFile());
-
- // Getting server name option
- final String serverName = cmd.getOptionValue( "s", defaultParameters.getHostname() );
-
- // Getting thumbnail directory option
- final String thumbnailDirectory = cmd.getOptionValue( "t", defaultParameters.getThumbnailDirectory() );
-
-
-
- final HashMap< String, String > datasets = new HashMap< String, String >( defaultParameters.getDatasets() );
-
- boolean enableManagerContext = false;
- if ( Constants.ENABLE_EXPERIMENTAL_FEATURES )
- {
- if ( cmd.hasOption( "m" ) )
- enableManagerContext = true;
- }
-
- if ( cmd.hasOption( "d" ) )
- {
- // process the file given with "-d"
- final String datasetFile = cmd.getOptionValue( "d" );
-
- // check the file presence
- final Path path = Paths.get( datasetFile );
-
- if ( Files.notExists( path ) )
- throw new IllegalArgumentException( "Dataset list file does not exist." );
-
- // Process dataset list file
- final List< String > lines = Files.readAllLines( path, StandardCharsets.UTF_8 );
-
- for ( final String str : lines )
- {
- final String[] tokens = str.split( "\\s*\\t\\s*" );
- if ( tokens.length == 2 && StringUtils.isNotEmpty( tokens[ 0 ].trim() ) && StringUtils.isNotEmpty( tokens[ 1 ].trim() ) )
- {
- final String name = tokens[ 0 ].trim();
- final String xmlpath = tokens[ 1 ].trim();
- tryAddDataset( datasets, name, xmlpath );
- }
- else
- {
- LOG.warn( "Invalid dataset file line (will be skipped): {" + str + "}" );
- }
- }
- }
-
- // process additional {name, name.xml} pairs given on the
- // command-line
- final String[] leftoverArgs = cmd.getArgs();
- if ( leftoverArgs.length % 2 != 0 )
- throw new IllegalArgumentException( "Dataset list has an error while processing." );
-
- for ( int i = 0; i < leftoverArgs.length; i += 2 )
- {
- final String name = leftoverArgs[ i ];
- final String xmlpath = leftoverArgs[ i + 1 ];
- tryAddDataset( datasets, name, xmlpath );
- }
-
- if ( datasets.isEmpty() )
- throw new IllegalArgumentException( "Dataset list is empty." );
-
- return new Parameters( port, serverName, datasets, thumbnailDirectory, enableManagerContext, dumpFile );
- }
- catch ( final ParseException | IllegalArgumentException e )
- {
- LOG.warn( e.getMessage() );
- System.out.println();
- final HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp( cmdLineSyntax, description, options, null );
- }
- return null;
- }
-
- private static void tryAddDataset( final HashMap< String, String > datasetNameToXML, final String name, final String xmlpath ) throws IllegalArgumentException
- {
- for ( final String reserved : Constants.RESERVED_CONTEXT_NAMES )
- if ( name.equals( reserved ) )
- throw new IllegalArgumentException( "Cannot use dataset name: \"" + name + "\" (reserved for internal use)." );
- if ( datasetNameToXML.containsKey( name ) )
- throw new IllegalArgumentException( "Duplicate dataset name: \"" + name + "\"" );
- if ( Files.notExists( Paths.get( xmlpath ) ) )
- throw new IllegalArgumentException( "Dataset file does not exist: \"" + xmlpath + "\"" );
- datasetNameToXML.put( name, xmlpath );
- LOG.info( "Dataset added: {" + name + ", " + xmlpath + "}" );
- }
-
- private static String getThumbnailDirectoryPath( final Parameters params ) throws IOException
- {
- final String thumbnailDirectoryName = params.getThumbnailDirectory();
- if ( thumbnailDirectoryName != null )
- {
- Path thumbnails = Paths.get( thumbnailDirectoryName );
- if ( ! Files.exists( thumbnails ) )
- {
- try
- {
- thumbnails = Files.createDirectories( thumbnails );
- return thumbnails.toFile().getAbsolutePath();
- }
- catch ( final IOException e )
- {
- LOG.warn( e.getMessage() );
- LOG.warn( "Could not create thumbnails directory \"" + thumbnailDirectoryName + "\".\n Trying to create temporary directory." );
- }
- }
- else
- {
- if ( ! Files.isDirectory( thumbnails ) )
- LOG.warn( "Thumbnails directory \"" + thumbnailDirectoryName + "\" is not a directory.\n Trying to create temporary directory." );
- else
- return thumbnails.toFile().getAbsolutePath();
- }
- }
- final Path thumbnails = Files.createTempDirectory( "thumbnails" );
- thumbnails.toFile().deleteOnExit();
- return thumbnails.toFile().getAbsolutePath();
- }
-
- private static ContextHandlerCollection createHandlers( final String baseURL, final Map< String, String > dataSet, final String thumbnailsDirectoryName ) throws SpimDataException, IOException
- {
- final ContextHandlerCollection handlers = new ContextHandlerCollection();
-
- for ( final Entry< String, String > entry : dataSet.entrySet() )
- {
- final String name = entry.getKey();
- final String xmlpath = entry.getValue();
- final String context = "/" + name;
- final CellHandler ctx = new CellHandler( baseURL + context + "/", xmlpath, name, thumbnailsDirectoryName, quantizer );
- ctx.setContextPath( context );
- handlers.addHandler( ctx );
- }
-
- return handlers;
- }
+public class BigDataServer {
+ private static final org.eclipse.jetty.util.log.Logger LOG = Log.getLogger(BigDataServer.class);
+
+ private static LloydMaxU16ScalarQuantization quantizer;
+
+ static Parameters getDefaultParameters() {
+ final int port = 8080;
+ String hostname;
+ try {
+ hostname = InetAddress.getLocalHost().getHostName();
+ } catch (final UnknownHostException e) {
+ hostname = "localhost";
+ }
+ final String thumbnailDirectory = null;
+ final boolean enableManagerContext = false;
+ return new Parameters(port, hostname, new HashMap<String, String>(), thumbnailDirectory, enableManagerContext,
+ new CustomCompressionParameters("", "", 8, false, false));
+ }
+
+ public static void main(final String[] args) throws Exception {
+ System.setProperty("org.eclipse.jetty.util.log.class", "org.eclipse.jetty.util.log.StdErrLog");
+
+ final Parameters params = processOptions(args, getDefaultParameters());
+ if (params == null)
+ return;
+
+ final String thumbnailsDirectoryName = getThumbnailDirectoryPath(params);
+
+ // Threadpool for multiple connections
+ final Server server = new Server(new QueuedThreadPool(200, 8));
+
+ // ServerConnector configuration
+ final ServerConnector connector = new ServerConnector(server);
+ connector.setHost(params.getHostname());
+ connector.setPort(params.getPort());
+ LOG.info("Set connectors: " + connector);
+ server.setConnectors(new Connector[]{connector});
+ final String baseURL = "http://" + server.getURI().getHost() + ":" + params.getPort();
+
+
+ final CustomCompressionParameters compParams = params.getCompressionParams();
+ if (compParams.shouldCompressData() || compParams.renderDifference()) {
+ //TODO(Moravec): Replace LloydMaxU16ScalarQuantization with some ICompressor.
+
+ quantizer = new LloydMaxU16ScalarQuantization(compParams.getTrainFile(), compParams.getBitTarget());
+ quantizer.train(true);
+ }
+
+
+ // Handler initialization
+ final HandlerCollection handlers = new HandlerCollection();
+
+ final ContextHandlerCollection datasetHandlers = createHandlers(baseURL, params, thumbnailsDirectoryName);
+ handlers.addHandler(datasetHandlers);
+ handlers.addHandler(new JsonDatasetListHandler(server, datasetHandlers));
+
+ Handler handler = handlers;
+ if (params.enableManagerContext()) {
+ // Add Statistics bean to the connector
+ final ConnectorStatistics connectorStats = new ConnectorStatistics();
+ connector.addBean(connectorStats);
+
+ // create StatisticsHandler wrapper and ManagerHandler
+ final StatisticsHandler statHandler = new StatisticsHandler();
+ handlers.addHandler(new ManagerHandler(baseURL, server, connectorStats, statHandler, datasetHandlers, thumbnailsDirectoryName));
+ statHandler.setHandler(handlers);
+ handler = statHandler;
+ }
+
+
+ LOG.info("Set handler: " + handler);
+ server.setHandler(handler);
+ LOG.info("Server Base URL: " + baseURL);
+ LOG.info("BigDataServer starting");
+ server.start();
+ server.join();
+ }
+
+ /**
+ * Server parameters: hostname, port, datasets.
+ */
+ private static class Parameters {
+ private final int port;
+
+ private final String hostname;
+
+ /**
+ * maps from dataset name to dataset xml path.
+ */
+ private final Map<String, String> datasetNameToXml;
+
+ private final String thumbnailDirectory;
+
+
+ private final CustomCompressionParameters compressionParam;
+
+ private final boolean enableManagerContext;
+
+ Parameters(final int port, final String hostname, final Map<String, String> datasetNameToXml,
+ final String thumbnailDirectory, final boolean enableManagerContext,
+ final CustomCompressionParameters customCompressionParameters) {
+ this.port = port;
+ this.hostname = hostname;
+ this.datasetNameToXml = datasetNameToXml;
+ this.thumbnailDirectory = thumbnailDirectory;
+ this.enableManagerContext = enableManagerContext;
+ this.compressionParam = customCompressionParameters;
+ }
+
+ public int getPort() {
+ return port;
+ }
+
+ public String getHostname() {
+ return hostname;
+ }
+
+ public String getThumbnailDirectory() {
+ return thumbnailDirectory;
+ }
+
+ /**
+ * Get datasets.
+ *
+ * @return datasets as a map from dataset name to dataset xml path.
+ */
+ public Map<String, String> getDatasets() {
+ return datasetNameToXml;
+ }
+
+ public boolean enableManagerContext() {
+ return enableManagerContext;
+ }
+
+ public CustomCompressionParameters getCompressionParams() {
+ return compressionParam;
+ }
+
+ }
+
+
+ @SuppressWarnings("static-access")
+ static private Parameters processOptions(final String[] args, final Parameters defaultParameters) throws IOException {
+ final String BIT_TARGET = "bits";
+ final String ENABLE_COMPRESSION = "compression";
+ final String ENABLE_COMPRESSION_DIFF = "diff";
+ final String DUMP_FILE = "dump";
+ final String TRAIN_FILE = "train";
+ // create Options object
+ final Options options = new Options();
+
+ final String cmdLineSyntax = "BigDataServer [OPTIONS] [NAME XML] ...\n";
+
+ final String description =
+ "Serves one or more XML/HDF5 datasets for remote access over HTTP.\n" +
+ "Provide (NAME XML) pairs on the command line or in a dataset file, where NAME is the name under which the dataset should be made accessible and XML is the path to the XML file of the dataset.";
+
+ options.addOption(OptionBuilder
+ .withDescription("Hostname of the server.\n(default: " + defaultParameters.getHostname() + ")")
+ .hasArg()
+ .withArgName("HOSTNAME")
+ .create("s"));
+
+ options.addOption(OptionBuilder
+ .withDescription("Listening port.\n(default: " + defaultParameters.getPort() + ")")
+ .hasArg()
+ .withArgName("PORT")
+ .create("p"));
+
+ // -d or multiple {name name.xml} pairs
+ options.addOption(OptionBuilder
+ .withDescription("Dataset file: A plain text file specifying one dataset per line. Each line is formatted as \"NAME <TAB> XML\".")
+ .hasArg()
+ .withArgName("FILE")
+ .create("d"));
+
+ options.addOption(OptionBuilder
+ .withDescription("Directory to store thumbnails. (new temporary directory by default.)")
+ .hasArg()
+ .withArgName("DIRECTORY")
+ .create("t"));
+
+
+ options.addOption(OptionBuilder
+ .withDescription("File in which to store request data dump")
+ .hasArg()
+ .withArgName("DUMP")
+ .create(DUMP_FILE));
+
+ options.addOption(OptionBuilder
+ .withDescription("Enable request compression")
+ .create(ENABLE_COMPRESSION));
+
+ options.addOption(OptionBuilder
+ .withDescription("Compression train file")
+ .hasArg()
+ .withArgName("TRAINFILE")
+ .create(TRAIN_FILE));
+
+ options.addOption(OptionBuilder
+ .withDescription("Compression bit target")
+ .hasArg()
+ .withArgName("BITS")
+ .create(BIT_TARGET));
+
+ options.addOption(OptionBuilder
+ .withDescription("Send compression difference")
+ .create(ENABLE_COMPRESSION_DIFF));
+
+ if (Constants.ENABLE_EXPERIMENTAL_FEATURES) {
+ options.addOption(OptionBuilder
+ .withDescription("enable statistics and manager context. EXPERIMENTAL!")
+ .create("m"));
+ }
+
+ try {
+ final CommandLineParser parser = new BasicParser();
+ final CommandLine cmd = parser.parse(options, args);
+
+ // Getting port number option
+ final String portString = cmd.getOptionValue("p", Integer.toString(defaultParameters.getPort()));
+ final int port = Integer.parseInt(portString);
+
+
+ // Getting server name option
+ final String serverName = cmd.getOptionValue("s", defaultParameters.getHostname());
+
+ // Getting thumbnail directory option
+ final String thumbnailDirectory = cmd.getOptionValue("t", defaultParameters.getThumbnailDirectory());
+
+ final HashMap<String, String> datasets = new HashMap<String, String>(defaultParameters.getDatasets());
+
+ // Custom compression parameters
+ //cmd.hasOption()
+
+ final String dumpFile = cmd.getOptionValue(DUMP_FILE, "");
+ final boolean enableCompression = cmd.hasOption(ENABLE_COMPRESSION);
+ final boolean enableCompressionDiff = cmd.hasOption(ENABLE_COMPRESSION_DIFF);
+ final String trainFile = cmd.getOptionValue(TRAIN_FILE, "");
+ final int bitTarget = Integer.parseInt(cmd.getOptionValue(BIT_TARGET, "8"));
+
+ if ((enableCompression || enableCompressionDiff) && (trainFile.isEmpty())) {
+ throw new MissingArgumentException(String.format("!!! %s must be specified when %s or %s is specified !!!",
+ TRAIN_FILE, ENABLE_COMPRESSION, ENABLE_COMPRESSION_DIFF));
+ }
+
+ final CustomCompressionParameters customCompParams = new CustomCompressionParameters(dumpFile, trainFile, bitTarget,
+ enableCompression, enableCompressionDiff);
+
+ LOG.info("Compression is " + (enableCompression ? "Matched" : "Not matched"));
+ LOG.info("Compression-Diff is " + (enableCompressionDiff ? "Matched" : "Not matched"));
+
+ boolean enableManagerContext = false;
+ if (Constants.ENABLE_EXPERIMENTAL_FEATURES) {
+ if (cmd.hasOption("m"))
+ enableManagerContext = true;
+ }
+
+
+ if (cmd.hasOption("d")) {
+ // process the file given with "-d"
+ final String datasetFile = cmd.getOptionValue("d");
+
+ // check the file presence
+ final Path path = Paths.get(datasetFile);
+
+ if (Files.notExists(path))
+ throw new IllegalArgumentException("Dataset list file does not exist.");
+
+ // Process dataset list file
+ final List<String> lines = Files.readAllLines(path, StandardCharsets.UTF_8);
+
+ for (final String str : lines) {
+ final String[] tokens = str.split("\\s*\\t\\s*");
+ if (tokens.length == 2 && StringUtils.isNotEmpty(tokens[0].trim()) && StringUtils.isNotEmpty(tokens[1].trim())) {
+ final String name = tokens[0].trim();
+ final String xmlpath = tokens[1].trim();
+ tryAddDataset(datasets, name, xmlpath);
+ } else {
+ LOG.warn("Invalid dataset file line (will be skipped): {" + str + "}");
+ }
+ }
+ }
+
+ // process additional {name, name.xml} pairs given on the
+ // command-line
+ final String[] leftoverArgs = cmd.getArgs();
+ if (leftoverArgs.length % 2 != 0)
+ throw new IllegalArgumentException("Dataset list has an error while processing.");
+
+ for (int i = 0; i < leftoverArgs.length; i += 2) {
+ final String name = leftoverArgs[i];
+ final String xmlpath = leftoverArgs[i + 1];
+ tryAddDataset(datasets, name, xmlpath);
+ }
+
+ if (datasets.isEmpty())
+ throw new IllegalArgumentException("Dataset list is empty.");
+
+ return new Parameters(port, serverName, datasets, thumbnailDirectory, enableManagerContext,
+ customCompParams);
+ } catch (final ParseException | IllegalArgumentException e) {
+ LOG.warn(e.getMessage());
+ System.out.println();
+ final HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp(cmdLineSyntax, description, options, null);
+ }
+ return null;
+ }
+
+ private static void tryAddDataset(final HashMap<String, String> datasetNameToXML, final String name, final String xmlpath) throws IllegalArgumentException {
+ for (final String reserved : Constants.RESERVED_CONTEXT_NAMES)
+ if (name.equals(reserved))
+ throw new IllegalArgumentException("Cannot use dataset name: \"" + name + "\" (reserved for internal use).");
+ if (datasetNameToXML.containsKey(name))
+ throw new IllegalArgumentException("Duplicate dataset name: \"" + name + "\"");
+ if (Files.notExists(Paths.get(xmlpath)))
+ throw new IllegalArgumentException("Dataset file does not exist: \"" + xmlpath + "\"");
+ datasetNameToXML.put(name, xmlpath);
+ LOG.info("Dataset added: {" + name + ", " + xmlpath + "}");
+ }
+
+ private static String getThumbnailDirectoryPath(final Parameters params) throws IOException {
+ final String thumbnailDirectoryName = params.getThumbnailDirectory();
+ if (thumbnailDirectoryName != null) {
+ Path thumbnails = Paths.get(thumbnailDirectoryName);
+ if (!Files.exists(thumbnails)) {
+ try {
+ thumbnails = Files.createDirectories(thumbnails);
+ return thumbnails.toFile().getAbsolutePath();
+ } catch (final IOException e) {
+ LOG.warn(e.getMessage());
+ LOG.warn("Could not create thumbnails directory \"" + thumbnailDirectoryName + "\".\n Trying to create temporary directory.");
+ }
+ } else {
+ if (!Files.isDirectory(thumbnails))
+ LOG.warn("Thumbnails directory \"" + thumbnailDirectoryName + "\" is not a directory.\n Trying to create temporary directory.");
+ else
+ return thumbnails.toFile().getAbsolutePath();
+ }
+ }
+ final Path thumbnails = Files.createTempDirectory("thumbnails");
+ thumbnails.toFile().deleteOnExit();
+ return thumbnails.toFile().getAbsolutePath();
+ }
+
+ private static ContextHandlerCollection createHandlers(final String baseURL,
+ final Parameters params,
+ final String thumbnailsDirectoryName) throws SpimDataException, IOException {
+
+ final ContextHandlerCollection handlers = new ContextHandlerCollection();
+
+ final Map<String, String> dataSet = params.getDatasets();
+ for (final Entry<String, String> entry : dataSet.entrySet()) {
+ final String name = entry.getKey();
+ final String xmlpath = entry.getValue();
+ final String context = "/" + name;
+ final CellHandler ctx = new CellHandler(baseURL + context + "/", xmlpath, name,
+ thumbnailsDirectoryName,
+ params.getCompressionParams(), quantizer);
+
+ ctx.setContextPath(context);
+ handlers.addHandler(ctx);
+ }
+
+ return handlers;
+ }
}
diff --git a/src/main/java/bdv/server/CellHandler.java b/src/main/java/bdv/server/CellHandler.java
index ec80358a54527a18212baba7fef4d0af95b63776..b294558259bc012d9ed5a71cec203cae3448a711 100644
--- a/src/main/java/bdv/server/CellHandler.java
+++ b/src/main/java/bdv/server/CellHandler.java
@@ -11,6 +11,7 @@ import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import compression.quantization.scalar.LloydMaxU16ScalarQuantization;
+import compression.utilities.Utils;
import org.eclipse.jetty.server.Request;
import org.eclipse.jetty.server.handler.ContextHandler;
import org.eclipse.jetty.util.log.Log;
@@ -42,309 +43,288 @@ import mpicbg.spim.data.SpimDataException;
import net.imglib2.img.basictypeaccess.volatiles.array.VolatileShortArray;
import net.imglib2.realtransform.AffineTransform3D;
-public class CellHandler extends ContextHandler
-{
- private long transferedDataSize = 0;
-
- private static final org.eclipse.jetty.util.log.Logger LOG = Log.getLogger( CellHandler.class );
-
- public static String DumpFile = "";
- private int counter = 0;
- private final VolatileGlobalCellCache cache;
-
- private final Hdf5VolatileShortArrayLoader loader;
-
- private final CacheHints cacheHints;
-
- /**
- * Full path of the dataset xml file this {@link CellHandler} is serving.
- */
- private final String xmlFilename;
-
- /**
- * Full path of the dataset xml file this {@link CellHandler} is serving,
- * without the ".xml" suffix.
- */
- private final String baseFilename;
-
- private final String dataSetURL;
-
- /**
- * Cached dataset XML to be send to and opened by {@link BigDataViewer}
- * clients.
- */
- private final String datasetXmlString;
-
- /**
- * Cached JSON representation of the {@link RemoteImageLoaderMetaData} to be
- * send to clients.
- */
- private final String metadataJson;
-
- /**
- * Cached dataset.settings XML to be send to clients. May be null if no
- * settings file exists for the dataset.
- */
- private final String settingsXmlString;
-
- /**
- * Full path to thumbnail png.
- */
- private final String thumbnailFilename;
-
- private LloydMaxU16ScalarQuantization quantizer;
-
- public CellHandler(final String baseUrl, final String xmlFilename, final String datasetName, final String thumbnailsDirectory, final LloydMaxU16ScalarQuantization quantizer) throws SpimDataException, IOException
- {
- final XmlIoSpimDataMinimal io = new XmlIoSpimDataMinimal();
- final SpimDataMinimal spimData = io.load( xmlFilename );
- final SequenceDescriptionMinimal seq = spimData.getSequenceDescription();
- final Hdf5ImageLoader imgLoader = ( Hdf5ImageLoader ) seq.getImgLoader();
- this.quantizer = quantizer;
-
- cache = imgLoader.getCacheControl();
- loader = imgLoader.getShortArrayLoader();
- cacheHints = new CacheHints( LoadingStrategy.BLOCKING, 0, false );
-
- // dataSetURL property is used for providing the XML file by replace
- // SequenceDescription>ImageLoader>baseUrl
- this.xmlFilename = xmlFilename;
- baseFilename = xmlFilename.endsWith( ".xml" ) ? xmlFilename.substring( 0, xmlFilename.length() - ".xml".length() ) : xmlFilename;
- dataSetURL = baseUrl;
-
- datasetXmlString = buildRemoteDatasetXML( io, spimData, baseUrl );
- metadataJson = buildMetadataJsonString( imgLoader, seq );
- settingsXmlString = buildSettingsXML( baseFilename );
- thumbnailFilename = createThumbnail( spimData, baseFilename, datasetName, thumbnailsDirectory );
- }
-
- @Override
- public void doHandle( final String target, final Request baseRequest, final HttpServletRequest request, final HttpServletResponse response ) throws IOException
- {
- if ( target.equals( "/settings" ) )
- {
- if ( settingsXmlString != null )
- respondWithString( baseRequest, response, "application/xml", settingsXmlString );
- return;
- }
-
- if ( target.equals( "/png" ) )
- {
- provideThumbnail( baseRequest, response );
- return;
- }
-
- final String cellString = request.getParameter( "p" );
-
- if ( cellString == null )
- {
- respondWithString( baseRequest, response, "application/xml", datasetXmlString );
- return;
- }
-
- final String[] parts = cellString.split( "/" );
- if ( parts[ 0 ].equals( "cell" ) )
- {
- final int index = Integer.parseInt( parts[ 1 ] );
- final int timepoint = Integer.parseInt( parts[ 2 ] );
- final int setup = Integer.parseInt( parts[ 3 ] );
- final int level = Integer.parseInt( parts[ 4 ] );
- final Key key = new VolatileGlobalCellCache.Key( timepoint, setup, level, index );
- VolatileCell< ? > cell = cache.getLoadingVolatileCache().getIfPresent( key, cacheHints );
- if ( cell == null )
- {
- final int[] cellDims = new int[] {
- Integer.parseInt( parts[ 5 ] ),
- Integer.parseInt( parts[ 6 ] ),
- Integer.parseInt( parts[ 7 ] ) };
- final long[] cellMin = new long[] {
- Long.parseLong( parts[ 8 ] ),
- Long.parseLong( parts[ 9 ] ),
- Long.parseLong( parts[ 10 ] ) };
- cell = cache.getLoadingVolatileCache().get( key, cacheHints, new VolatileCellLoader<>( loader, timepoint, setup, level, cellDims, cellMin ) );
- }
-
-
- @SuppressWarnings( "unchecked" )
- short[] data = ((VolatileCell<VolatileShortArray>) cell).getData().getCurrentStorageArray();
- if (quantizer != null) {
- data = quantizer.quantize(data);
- }
-
- /*
- * NOTE(Moravec): This is possible place, where to compress data. Image data are inside data array, but we access only part of the image.
- * if (compressionEnabled)
- * {
- * data = compress(data);
- * }
- * */
-
- final byte[] buf = new byte[ 2 * data.length ];
- for ( int i = 0, j = 0; i < data.length; i++ )
- {
- final short s = data[ i ];
- buf[ j++ ] = ( byte ) ( ( s >> 8 ) & 0xff );
- buf[ j++ ] = ( byte ) ( s & 0xff );
- }
-
- if (!DumpFile.equals("")) {
- //String requestLog = String.format("%s\\request_%d_%d.data", DumpFile, buf.length, counter++);
- FileOutputStream dumpStream = new FileOutputStream(DumpFile, true);
- dumpStream.write(buf);
- dumpStream.flush();
- dumpStream.close();
- }
-
-
- transferedDataSize += buf.length;
- LOG.info(String.format("Total transfered data: [%d KB] [%d MB]", (transferedDataSize/1000), ((transferedDataSize/1000)/1000)));
-
- response.setContentType( "application/octet-stream" );
- response.setContentLength( buf.length );
- response.setStatus( HttpServletResponse.SC_OK );
- baseRequest.setHandled( true );
- final OutputStream os = response.getOutputStream();
- os.write( buf );
- os.close();
- }
- else if ( parts[ 0 ].equals( "init" ) )
- {
- respondWithString( baseRequest, response, "application/json", metadataJson );
- }
- }
-
- private void provideThumbnail( final Request baseRequest, final HttpServletResponse response ) throws IOException
- {
- final Path path = Paths.get( thumbnailFilename );
- if ( Files.exists( path ) )
- {
- final byte[] imageData = Files.readAllBytes(path);
- if ( imageData != null )
- {
- response.setContentType( "image/png" );
- response.setContentLength( imageData.length );
- response.setStatus( HttpServletResponse.SC_OK );
- baseRequest.setHandled( true );
-
- final OutputStream os = response.getOutputStream();
- os.write( imageData );
- os.close();
- }
- }
- }
-
- public String getXmlFile()
- {
- return xmlFilename;
- }
-
- public String getDataSetURL()
- {
- return dataSetURL;
- }
-
- public String getThumbnailUrl()
- {
- return dataSetURL + "png";
- }
-
- public String getDescription()
- {
- throw new UnsupportedOperationException();
- }
-
- /**
- * Create a JSON representation of the {@link RemoteImageLoaderMetaData}
- * (image sizes and resolutions) provided by the given
- * {@link Hdf5ImageLoader}.
- */
- private static String buildMetadataJsonString( final Hdf5ImageLoader imgLoader, final SequenceDescriptionMinimal seq )
- {
- final RemoteImageLoaderMetaData metadata = new RemoteImageLoaderMetaData( imgLoader, seq );
- final GsonBuilder gsonBuilder = new GsonBuilder();
- gsonBuilder.registerTypeAdapter( AffineTransform3D.class, new AffineTransform3DJsonSerializer() );
- gsonBuilder.enableComplexMapKeySerialization();
- return gsonBuilder.create().toJson( metadata );
- }
-
- /**
- * Create a modified dataset XML by replacing the ImageLoader with an
- * {@link RemoteImageLoader} pointing to the data we are serving.
- */
- private static String buildRemoteDatasetXML( final XmlIoSpimDataMinimal io, final SpimDataMinimal spimData, final String baseUrl ) throws IOException, SpimDataException
- {
- final SpimDataMinimal s = new SpimDataMinimal( spimData, new RemoteImageLoader( baseUrl, false ) );
- final Document doc = new Document( io.toXml( s, s.getBasePath() ) );
- final XMLOutputter xout = new XMLOutputter( Format.getPrettyFormat() );
- final StringWriter sw = new StringWriter();
- xout.output( doc, sw );
- return sw.toString();
- }
-
- /**
- * Read {@code baseFilename.settings.xml} into a string if it exists.
- *
- * @return contents of {@code baseFilename.settings.xml} or {@code null} if
- * that file couldn't be read.
- */
- private static String buildSettingsXML( final String baseFilename )
- {
- final String settings = baseFilename + ".settings.xml";
- if ( new File( settings ).exists() )
- {
- try
- {
- final SAXBuilder sax = new SAXBuilder();
- final Document doc = sax.build( settings );
- final XMLOutputter xout = new XMLOutputter( Format.getPrettyFormat() );
- final StringWriter sw = new StringWriter();
- xout.output( doc, sw );
- return sw.toString();
- }
- catch ( JDOMException | IOException e )
- {
- LOG.warn( "Could not read settings file \"" + settings + "\"" );
- LOG.warn( e.getMessage() );
- }
- }
- return null;
- }
-
- /**
- * Create PNG thumbnail file named "{@code <baseFilename>.png}".
- */
- private static String createThumbnail( final SpimDataMinimal spimData, final String baseFilename, final String datasetName, final String thumbnailsDirectory )
- {
- final String thumbnailFileName = thumbnailsDirectory + "/" + datasetName + ".png";
- final File thumbnailFile = new File( thumbnailFileName );
- if ( !thumbnailFile.isFile() ) // do not recreate thumbnail if it already exists
- {
- final BufferedImage bi = ThumbnailGenerator.makeThumbnail( spimData, baseFilename, Constants.THUMBNAIL_WIDTH, Constants.THUMBNAIL_HEIGHT );
- try
- {
- ImageIO.write( bi, "png", thumbnailFile );
- }
- catch ( final IOException e )
- {
- LOG.warn( "Could not create thumbnail png for dataset \"" + baseFilename + "\"" );
- LOG.warn( e.getMessage() );
- }
- }
- return thumbnailFileName;
- }
-
- /**
- * Handle request by sending a UTF-8 string.
- */
- private static void respondWithString( final Request baseRequest, final HttpServletResponse response, final String contentType, final String string ) throws IOException
- {
- response.setContentType( contentType );
- response.setCharacterEncoding( "UTF-8" );
- response.setStatus( HttpServletResponse.SC_OK );
- baseRequest.setHandled( true );
-
- final PrintWriter ow = response.getWriter();
- ow.write( string );
- ow.close();
- }
+public class CellHandler extends ContextHandler {
+ private long transferedDataSize = 0;
+
+ private static final org.eclipse.jetty.util.log.Logger LOG = Log.getLogger(CellHandler.class);
+
+ private int counter = 0;
+ private final VolatileGlobalCellCache cache;
+
+ private final Hdf5VolatileShortArrayLoader loader;
+
+ private final CacheHints cacheHints;
+
+ /**
+ * Full path of the dataset xml file this {@link CellHandler} is serving.
+ */
+ private final String xmlFilename;
+
+ /**
+ * Full path of the dataset xml file this {@link CellHandler} is serving,
+ * without the ".xml" suffix.
+ */
+ private final String baseFilename;
+
+ private final String dataSetURL;
+
+ /**
+ * Cached dataset XML to be send to and opened by {@link BigDataViewer}
+ * clients.
+ */
+ private final String datasetXmlString;
+
+ /**
+ * Cached JSON representation of the {@link RemoteImageLoaderMetaData} to be
+ * send to clients.
+ */
+ private final String metadataJson;
+
+ /**
+ * Cached dataset.settings XML to be send to clients. May be null if no
+ * settings file exists for the dataset.
+ */
+ private final String settingsXmlString;
+
+ /**
+ * Full path to thumbnail png.
+ */
+ private final String thumbnailFilename;
+ final CustomCompressionParameters compressionParams;
+ private LloydMaxU16ScalarQuantization quantizer;
+
+ public CellHandler(final String baseUrl, final String xmlFilename, final String datasetName, final String thumbnailsDirectory,
+ final CustomCompressionParameters compressionParams,
+ final LloydMaxU16ScalarQuantization quantizer) throws SpimDataException, IOException {
+
+ final XmlIoSpimDataMinimal io = new XmlIoSpimDataMinimal();
+ final SpimDataMinimal spimData = io.load(xmlFilename);
+ final SequenceDescriptionMinimal seq = spimData.getSequenceDescription();
+ final Hdf5ImageLoader imgLoader = (Hdf5ImageLoader) seq.getImgLoader();
+ this.quantizer = quantizer;
+ this.compressionParams = compressionParams;
+
+ cache = imgLoader.getCacheControl();
+ loader = imgLoader.getShortArrayLoader();
+ cacheHints = new CacheHints(LoadingStrategy.BLOCKING, 0, false);
+
+ // dataSetURL property is used for providing the XML file by replace
+ // SequenceDescription>ImageLoader>baseUrl
+ this.xmlFilename = xmlFilename;
+ baseFilename = xmlFilename.endsWith(".xml") ? xmlFilename.substring(0, xmlFilename.length() - ".xml".length()) : xmlFilename;
+ dataSetURL = baseUrl;
+
+ datasetXmlString = buildRemoteDatasetXML(io, spimData, baseUrl);
+ metadataJson = buildMetadataJsonString(imgLoader, seq);
+ settingsXmlString = buildSettingsXML(baseFilename);
+ thumbnailFilename = createThumbnail(spimData, baseFilename, datasetName, thumbnailsDirectory);
+ }
+
+ @Override
+ public void doHandle(final String target, final Request baseRequest, final HttpServletRequest request, final HttpServletResponse response) throws IOException {
+ if (target.equals("/settings")) {
+ if (settingsXmlString != null)
+ respondWithString(baseRequest, response, "application/xml", settingsXmlString);
+ return;
+ }
+
+ if (target.equals("/png")) {
+ provideThumbnail(baseRequest, response);
+ return;
+ }
+
+ final String cellString = request.getParameter("p");
+
+ if (cellString == null) {
+ respondWithString(baseRequest, response, "application/xml", datasetXmlString);
+ return;
+ }
+
+ final String[] parts = cellString.split("/");
+ if (parts[0].equals("cell")) {
+ final int index = Integer.parseInt(parts[1]);
+ final int timepoint = Integer.parseInt(parts[2]);
+ final int setup = Integer.parseInt(parts[3]);
+ final int level = Integer.parseInt(parts[4]);
+ final Key key = new VolatileGlobalCellCache.Key(timepoint, setup, level, index);
+ VolatileCell<?> cell = cache.getLoadingVolatileCache().getIfPresent(key, cacheHints);
+ if (cell == null) {
+ final int[] cellDims = new int[]{
+ Integer.parseInt(parts[5]),
+ Integer.parseInt(parts[6]),
+ Integer.parseInt(parts[7])};
+ final long[] cellMin = new long[]{
+ Long.parseLong(parts[8]),
+ Long.parseLong(parts[9]),
+ Long.parseLong(parts[10])};
+ cell = cache.getLoadingVolatileCache().get(key, cacheHints, new VolatileCellLoader<>(loader, timepoint, setup, level, cellDims, cellMin));
+ }
+
+
+ @SuppressWarnings("unchecked")
+ short[] data = ((VolatileCell<VolatileShortArray>) cell).getData().getCurrentStorageArray();
+ if (compressionParams.shouldCompressData()) {
+ assert (quantizer != null) : "Compressor wasn't created";
+ data = quantizer.quantize(data);
+ } else if (compressionParams.renderDifference()) {
+ assert (quantizer != null) : "Compressor wasn't created";
+ short[] compressedData = quantizer.quantize(data);
+
+ for (int i = 0; i < data.length; i++) {
+ // Original - Compressed
+ data[i] = Utils.u16BitsToShort(data[i]-compressedData[i]);
+ // Compressed - Original
+ //data[i] = Utils.u16BitsToShort(compressedData[i]-data[i]);
+ }
+
+ //LOG.warn("Not yet implemented.");
+ }
+
+ final byte[] buf = new byte[2 * data.length];
+ for (int i = 0, j = 0; i < data.length; i++) {
+ final short s = data[i];
+ buf[j++] = (byte) ((s >> 8) & 0xff);
+ buf[j++] = (byte) (s & 0xff);
+ }
+
+ if (compressionParams.shouldDumpRequestData()) {
+ FileOutputStream dumpStream = new FileOutputStream(compressionParams.getDumpFile(), true);
+ dumpStream.write(buf);
+ dumpStream.flush();
+ dumpStream.close();
+ }
+
+ transferedDataSize += buf.length;
+
+ LOG.info(String.format("I:%d;T:%d;S:%d;L:%d Total transfered data: [%d KB] [%d MB]",
+ index, timepoint, setup, level,
+ (transferedDataSize / 1000), ((transferedDataSize / 1000) / 1000)));
+
+ response.setContentType("application/octet-stream");
+ response.setContentLength(buf.length);
+ response.setStatus(HttpServletResponse.SC_OK);
+ baseRequest.setHandled(true);
+ final OutputStream os = response.getOutputStream();
+ os.write(buf);
+ os.close();
+ } else if (parts[0].equals("init")) {
+ respondWithString(baseRequest, response, "application/json", metadataJson);
+ }
+ }
+
+ private void provideThumbnail(final Request baseRequest, final HttpServletResponse response) throws IOException {
+ final Path path = Paths.get(thumbnailFilename);
+ if (Files.exists(path)) {
+ final byte[] imageData = Files.readAllBytes(path);
+ if (imageData != null) {
+ response.setContentType("image/png");
+ response.setContentLength(imageData.length);
+ response.setStatus(HttpServletResponse.SC_OK);
+ baseRequest.setHandled(true);
+
+ final OutputStream os = response.getOutputStream();
+ os.write(imageData);
+ os.close();
+ }
+ }
+ }
+
+ public String getXmlFile() {
+ return xmlFilename;
+ }
+
+ public String getDataSetURL() {
+ return dataSetURL;
+ }
+
+ public String getThumbnailUrl() {
+ return dataSetURL + "png";
+ }
+
+ public String getDescription() {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Create a JSON representation of the {@link RemoteImageLoaderMetaData}
+ * (image sizes and resolutions) provided by the given
+ * {@link Hdf5ImageLoader}.
+ */
+ private static String buildMetadataJsonString(final Hdf5ImageLoader imgLoader, final SequenceDescriptionMinimal seq) {
+ final RemoteImageLoaderMetaData metadata = new RemoteImageLoaderMetaData(imgLoader, seq);
+ final GsonBuilder gsonBuilder = new GsonBuilder();
+ gsonBuilder.registerTypeAdapter(AffineTransform3D.class, new AffineTransform3DJsonSerializer());
+ gsonBuilder.enableComplexMapKeySerialization();
+ return gsonBuilder.create().toJson(metadata);
+ }
+
+ /**
+ * Create a modified dataset XML by replacing the ImageLoader with an
+ * {@link RemoteImageLoader} pointing to the data we are serving.
+ */
+ private static String buildRemoteDatasetXML(final XmlIoSpimDataMinimal io, final SpimDataMinimal spimData, final String baseUrl) throws IOException, SpimDataException {
+ final SpimDataMinimal s = new SpimDataMinimal(spimData, new RemoteImageLoader(baseUrl, false));
+ final Document doc = new Document(io.toXml(s, s.getBasePath()));
+ final XMLOutputter xout = new XMLOutputter(Format.getPrettyFormat());
+ final StringWriter sw = new StringWriter();
+ xout.output(doc, sw);
+ return sw.toString();
+ }
+
+ /**
+ * Read {@code baseFilename.settings.xml} into a string if it exists.
+ *
+ * @return contents of {@code baseFilename.settings.xml} or {@code null} if
+ * that file couldn't be read.
+ */
+ private static String buildSettingsXML(final String baseFilename) {
+ final String settings = baseFilename + ".settings.xml";
+ if (new File(settings).exists()) {
+ try {
+ final SAXBuilder sax = new SAXBuilder();
+ final Document doc = sax.build(settings);
+ final XMLOutputter xout = new XMLOutputter(Format.getPrettyFormat());
+ final StringWriter sw = new StringWriter();
+ xout.output(doc, sw);
+ return sw.toString();
+ } catch (JDOMException | IOException e) {
+ LOG.warn("Could not read settings file \"" + settings + "\"");
+ LOG.warn(e.getMessage());
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Create PNG thumbnail file named "{@code <baseFilename>.png}".
+ */
+ private static String createThumbnail(final SpimDataMinimal spimData, final String baseFilename, final String datasetName, final String thumbnailsDirectory) {
+ final String thumbnailFileName = thumbnailsDirectory + "/" + datasetName + ".png";
+ final File thumbnailFile = new File(thumbnailFileName);
+ if (!thumbnailFile.isFile()) // do not recreate thumbnail if it already exists
+ {
+ final BufferedImage bi = ThumbnailGenerator.makeThumbnail(spimData, baseFilename, Constants.THUMBNAIL_WIDTH, Constants.THUMBNAIL_HEIGHT);
+ try {
+ ImageIO.write(bi, "png", thumbnailFile);
+ } catch (final IOException e) {
+ LOG.warn("Could not create thumbnail png for dataset \"" + baseFilename + "\"");
+ LOG.warn(e.getMessage());
+ }
+ }
+ return thumbnailFileName;
+ }
+
+ /**
+ * Handle request by sending a UTF-8 string.
+ */
+ private static void respondWithString(final Request baseRequest, final HttpServletResponse response, final String contentType, final String string) throws IOException {
+ response.setContentType(contentType);
+ response.setCharacterEncoding("UTF-8");
+ response.setStatus(HttpServletResponse.SC_OK);
+ baseRequest.setHandled(true);
+
+ final PrintWriter ow = response.getWriter();
+ ow.write(string);
+ ow.close();
+ }
}
diff --git a/src/main/java/bdv/server/CustomCompressionParameters.java b/src/main/java/bdv/server/CustomCompressionParameters.java
new file mode 100644
index 0000000000000000000000000000000000000000..6b2fc917818465eea5e36a92a9217d82a07e0555
--- /dev/null
+++ b/src/main/java/bdv/server/CustomCompressionParameters.java
@@ -0,0 +1,49 @@
+package bdv.server;
+
+public class CustomCompressionParameters {
+ private final boolean dumpRequestData;
+ private final String dumpFile;
+ private final String trainFile;
+ private final int bitTarget;
+
+ public String getTrainFile() {
+ return trainFile;
+ }
+
+ public int getBitTarget() {
+ return bitTarget;
+ }
+
+ private final boolean enableRequestCompression;
+ private final boolean renderDifference;
+
+ public CustomCompressionParameters(final String dumpFile,
+ final String trainFile,
+ final int bitTarget,
+ final boolean enableRequestCompression,
+ final boolean renderDifference) {
+ this.dumpFile = dumpFile;
+ this.trainFile = trainFile;
+ this.bitTarget=bitTarget;
+ this.dumpRequestData = !this.dumpFile.isEmpty();
+ this.enableRequestCompression = enableRequestCompression;
+ this.renderDifference = renderDifference;
+ }
+
+ public boolean shouldDumpRequestData() {
+ return dumpRequestData;
+ }
+
+ public String getDumpFile() {
+ return dumpFile;
+ }
+
+ public boolean shouldCompressData() {
+ return enableRequestCompression;
+ }
+
+ public boolean renderDifference() {
+ return renderDifference;
+ }
+
+}
diff --git a/src/main/java/bdv/server/ManagerHandler.java b/src/main/java/bdv/server/ManagerHandler.java
index c45896942f74d058e060bcb10dd2813f95b0e442..b05e7c5f2d63041053b079e97776287d9b654858 100644
--- a/src/main/java/bdv/server/ManagerHandler.java
+++ b/src/main/java/bdv/server/ManagerHandler.java
@@ -27,212 +27,186 @@ import java.text.DecimalFormat;
* @author HongKee Moon <moon@mpi-cbg.quantization.de>
* @author Tobias Pietzsch <tobias.pietzsch@gmail.com>
*/
-public class ManagerHandler extends ContextHandler
-{
- private static final org.eclipse.jetty.util.log.Logger LOG = Log.getLogger( ManagerHandler.class );
-
- private final String baseURL;
-
- private final Server server;
-
- private final ContextHandlerCollection handlers;
-
- private final StatisticsHandler statHandler;
-
- private final ConnectorStatistics connectorStats;
-
- private String contexts = null;
-
- private int noDataSets = 0;
-
- private long sizeDataSets = 0;
-
- private final String thumbnailsDirectoryName;
-
- public ManagerHandler(
- final String baseURL,
- final Server server,
- final ConnectorStatistics connectorStats,
- final StatisticsHandler statHandler,
- final ContextHandlerCollection handlers,
- final String thumbnailsDirectoryName )
- throws IOException, URISyntaxException
- {
- this.baseURL = baseURL;
- this.server = server;
- this.handlers = handlers;
- this.statHandler = statHandler;
- this.connectorStats = connectorStats;
- this.thumbnailsDirectoryName = thumbnailsDirectoryName;
- setContextPath( "/" + Constants.MANAGER_CONTEXT_NAME );
- }
-
- @Override
- public void doHandle( final String target, final Request baseRequest, final HttpServletRequest request, final HttpServletResponse response ) throws IOException, ServletException
- {
- final String op = request.getParameter( "op" );
-
- if ( op == null )
- {
- list( baseRequest, response );
- }
- else if ( op.equals( "deploy" ) )
- {
- final String ds = request.getParameter( "ds" );
- final String file = request.getParameter( "file" );
- deploy( ds, file, baseRequest, response );
- }
- else if ( op.equals( "undeploy" ) )
- {
- final String ds = request.getParameter( "ds" );
- undeploy( ds, baseRequest, response );
- }
- }
-
- public String getByteSizeString( final long size )
- {
- if ( size <= 0 )
- return "0";
- final String[] units = new String[] { "B", "kB", "MB", "GB", "TB" };
- final int digitGroups = ( int ) ( Math.log10( size ) / Math.log10( 1024 ) );
- return new DecimalFormat( "#,##0.#" ).format( size / Math.pow( 1024, digitGroups ) ) + " " + units[ digitGroups ];
- }
-
- private void list( final Request baseRequest, final HttpServletResponse response ) throws IOException
- {
- response.setContentType( "text/html" );
- response.setStatus( HttpServletResponse.SC_OK );
- baseRequest.setHandled( true );
-
- final PrintWriter ow = response.getWriter();
-
- ow.write( getHtml() );
- ow.close();
- }
-
- private String getHtml()
- {
- final StringTemplateGroup templates = new StringTemplateGroup( "manager" );
- final StringTemplate t = templates.getInstanceOf( "templates/manager" );
-
- t.setAttribute( "bytesSent", getByteSizeString( statHandler.getResponsesBytesTotal() ) );
- t.setAttribute( "msgPerSec", connectorStats.getMessagesOutPerSecond() );
- t.setAttribute( "openConnections", connectorStats.getConnectionsOpen() );
- t.setAttribute( "maxOpenConnections", connectorStats.getConnectionsOpenMax() );
-
- getContexts();
-
- t.setAttribute( "contexts", contexts );
-
- t.setAttribute( "noDataSets", noDataSets );
- t.setAttribute( "sizeDataSets", getByteSizeString( sizeDataSets ) );
-
- t.setAttribute( "statHtml", statHandler.toStatsHTML() );
-
- return t.toString();
- }
-
- private void getContexts()
- {
- if ( contexts == null )
- {
- noDataSets = 0;
- sizeDataSets = 0;
-
- final StringBuilder sb = new StringBuilder();
- for ( final Handler handler : server.getChildHandlersByClass( CellHandler.class ) )
- {
- sb.append( "<tr>\n<th>" );
- final CellHandler contextHandler = ( CellHandler ) handler;
- sb.append( contextHandler.getContextPath() + "</th>\n<td>" );
- sb.append( contextHandler.getXmlFile() + "</td>\n</tr>\n" );
- noDataSets++;
- sizeDataSets += new File( contextHandler.getXmlFile().replace( ".xml", ".h5" ) ).length();
- }
- contexts = sb.toString();
- }
- }
-
- private void deploy( final String datasetName, final String fileLocation, final Request baseRequest, final HttpServletResponse response ) throws IOException
- {
- LOG.info( "Add new context: " + datasetName );
- final String context = "/" + datasetName;
-
- boolean alreadyExists = false;
- for ( final Handler handler : server.getChildHandlersByClass( CellHandler.class ) )
- {
- final CellHandler contextHandler = ( CellHandler ) handler;
- if ( context.equals( contextHandler.getContextPath() ) )
- {
- LOG.info( "Context " + datasetName + " already exists.");
- alreadyExists = true;
- break;
- }
- }
-
- if ( ! alreadyExists )
- {
- CellHandler ctx = null;
- try
- {
- ctx = new CellHandler( baseURL + context + "/", fileLocation, datasetName, thumbnailsDirectoryName, null );
- }
- catch ( final SpimDataException e )
- {
- LOG.warn( "Failed to create a CellHandler", e );
- e.printStackTrace();
- }
- ctx.setContextPath( context );
- handlers.addHandler( ctx );
- }
-
- response.setContentType( "text/html" );
- response.setStatus( HttpServletResponse.SC_OK );
- baseRequest.setHandled( true );
-
- final PrintWriter ow = response.getWriter();
- if ( alreadyExists )
- ow.write( datasetName + " already exists. Not registered." );
- else
- ow.write( datasetName + " registered." );
- ow.close();
- }
-
- private void undeploy( final String datasetName, final Request baseRequest, final HttpServletResponse response ) throws IOException
- {
- LOG.info( "Remove the context: " + datasetName );
- boolean ret = false;
-
- final String context = "/" + datasetName;
- for ( final Handler handler : server.getChildHandlersByClass( CellHandler.class ) )
- {
- final CellHandler contextHandler = ( CellHandler ) handler;
- if ( context.equals( contextHandler.getContextPath() ) )
- {
- try
- {
- contextHandler.stop();
- }
- catch ( final Exception e )
- {
- e.printStackTrace();
- }
- contextHandler.destroy();
- handlers.removeHandler( contextHandler );
- ret = true;
- break;
- }
- }
-
- if ( ret )
- {
- response.setContentType( "text/html" );
- response.setStatus( HttpServletResponse.SC_OK );
- baseRequest.setHandled( true );
-
- final PrintWriter ow = response.getWriter();
- ow.write( datasetName + " removed." );
- ow.close();
- }
- }
+public class ManagerHandler extends ContextHandler {
+ private static final org.eclipse.jetty.util.log.Logger LOG = Log.getLogger(ManagerHandler.class);
+
+ private final String baseURL;
+
+ private final Server server;
+
+ private final ContextHandlerCollection handlers;
+
+ private final StatisticsHandler statHandler;
+
+ private final ConnectorStatistics connectorStats;
+
+ private String contexts = null;
+
+ private int noDataSets = 0;
+
+ private long sizeDataSets = 0;
+
+ private final String thumbnailsDirectoryName;
+
+ public ManagerHandler(
+ final String baseURL,
+ final Server server,
+ final ConnectorStatistics connectorStats,
+ final StatisticsHandler statHandler,
+ final ContextHandlerCollection handlers,
+ final String thumbnailsDirectoryName)
+ throws IOException, URISyntaxException {
+ this.baseURL = baseURL;
+ this.server = server;
+ this.handlers = handlers;
+ this.statHandler = statHandler;
+ this.connectorStats = connectorStats;
+ this.thumbnailsDirectoryName = thumbnailsDirectoryName;
+ setContextPath("/" + Constants.MANAGER_CONTEXT_NAME);
+ }
+
+ @Override
+ public void doHandle(final String target, final Request baseRequest, final HttpServletRequest request, final HttpServletResponse response) throws IOException, ServletException {
+ final String op = request.getParameter("op");
+
+ if (op == null) {
+ list(baseRequest, response);
+ } else if (op.equals("deploy")) {
+ final String ds = request.getParameter("ds");
+ final String file = request.getParameter("file");
+ deploy(ds, file, baseRequest, response);
+ } else if (op.equals("undeploy")) {
+ final String ds = request.getParameter("ds");
+ undeploy(ds, baseRequest, response);
+ }
+ }
+
+ public String getByteSizeString(final long size) {
+ if (size <= 0)
+ return "0";
+ final String[] units = new String[]{"B", "kB", "MB", "GB", "TB"};
+ final int digitGroups = (int) (Math.log10(size) / Math.log10(1024));
+ return new DecimalFormat("#,##0.#").format(size / Math.pow(1024, digitGroups)) + " " + units[digitGroups];
+ }
+
+ private void list(final Request baseRequest, final HttpServletResponse response) throws IOException {
+ response.setContentType("text/html");
+ response.setStatus(HttpServletResponse.SC_OK);
+ baseRequest.setHandled(true);
+
+ final PrintWriter ow = response.getWriter();
+
+ ow.write(getHtml());
+ ow.close();
+ }
+
+ private String getHtml() {
+ final StringTemplateGroup templates = new StringTemplateGroup("manager");
+ final StringTemplate t = templates.getInstanceOf("templates/manager");
+
+ t.setAttribute("bytesSent", getByteSizeString(statHandler.getResponsesBytesTotal()));
+ t.setAttribute("msgPerSec", connectorStats.getMessagesOutPerSecond());
+ t.setAttribute("openConnections", connectorStats.getConnectionsOpen());
+ t.setAttribute("maxOpenConnections", connectorStats.getConnectionsOpenMax());
+
+ getContexts();
+
+ t.setAttribute("contexts", contexts);
+
+ t.setAttribute("noDataSets", noDataSets);
+ t.setAttribute("sizeDataSets", getByteSizeString(sizeDataSets));
+
+ t.setAttribute("statHtml", statHandler.toStatsHTML());
+
+ return t.toString();
+ }
+
+ private void getContexts() {
+ if (contexts == null) {
+ noDataSets = 0;
+ sizeDataSets = 0;
+
+ final StringBuilder sb = new StringBuilder();
+ for (final Handler handler : server.getChildHandlersByClass(CellHandler.class)) {
+ sb.append("<tr>\n<th>");
+ final CellHandler contextHandler = (CellHandler) handler;
+ sb.append(contextHandler.getContextPath() + "</th>\n<td>");
+ sb.append(contextHandler.getXmlFile() + "</td>\n</tr>\n");
+ noDataSets++;
+ sizeDataSets += new File(contextHandler.getXmlFile().replace(".xml", ".h5")).length();
+ }
+ contexts = sb.toString();
+ }
+ }
+
+ private void deploy(final String datasetName, final String fileLocation, final Request baseRequest, final HttpServletResponse response) throws IOException {
+ LOG.info("Add new context: " + datasetName);
+ final String context = "/" + datasetName;
+
+ boolean alreadyExists = false;
+ for (final Handler handler : server.getChildHandlersByClass(CellHandler.class)) {
+ final CellHandler contextHandler = (CellHandler) handler;
+ if (context.equals(contextHandler.getContextPath())) {
+ LOG.info("Context " + datasetName + " already exists.");
+ alreadyExists = true;
+ break;
+ }
+ }
+
+ if (!alreadyExists) {
+ CellHandler ctx = null;
+ try {
+ LOG.warn("We are creating CellHandler without compression params!");
+ ctx = new CellHandler(baseURL + context + "/", fileLocation, datasetName, thumbnailsDirectoryName,
+ null, null);
+ } catch (final SpimDataException e) {
+ LOG.warn("Failed to create a CellHandler", e);
+ e.printStackTrace();
+ }
+ ctx.setContextPath(context);
+ handlers.addHandler(ctx);
+ }
+
+ response.setContentType("text/html");
+ response.setStatus(HttpServletResponse.SC_OK);
+ baseRequest.setHandled(true);
+
+ final PrintWriter ow = response.getWriter();
+ if (alreadyExists)
+ ow.write(datasetName + " already exists. Not registered.");
+ else
+ ow.write(datasetName + " registered.");
+ ow.close();
+ }
+
+ private void undeploy(final String datasetName, final Request baseRequest, final HttpServletResponse response) throws IOException {
+ LOG.info("Remove the context: " + datasetName);
+ boolean ret = false;
+
+ final String context = "/" + datasetName;
+ for (final Handler handler : server.getChildHandlersByClass(CellHandler.class)) {
+ final CellHandler contextHandler = (CellHandler) handler;
+ if (context.equals(contextHandler.getContextPath())) {
+ try {
+ contextHandler.stop();
+ } catch (final Exception e) {
+ e.printStackTrace();
+ }
+ contextHandler.destroy();
+ handlers.removeHandler(contextHandler);
+ ret = true;
+ break;
+ }
+ }
+
+ if (ret) {
+ response.setContentType("text/html");
+ response.setStatus(HttpServletResponse.SC_OK);
+ baseRequest.setHandled(true);
+
+ final PrintWriter ow = response.getWriter();
+ ow.write(datasetName + " removed.");
+ ow.close();
+ }
+ }
}
diff --git a/src/main/resources/templates/manager.st b/src/main/resources/templates/manager.st
deleted file mode 100644
index aa0abf2548d8766bb196770545b7d0a4ebad4b0f..0000000000000000000000000000000000000000
--- a/src/main/resources/templates/manager.st
+++ /dev/null
@@ -1,74 +0,0 @@
-<!DOCTYPE html>
-<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
-<head>
- <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
- <meta http-equiv="refresh" content="5"/>
-
- <title>BigDataServer</title>
-
- <!-- Latest compiled and minified CSS -->
- <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/css/bootstrap.min.css">
-
- <!-- Optional theme -->
- <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/css/bootstrap-theme.min.css">
-
- <!-- Latest compiled and minified JavaScript -->
- <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/js/bootstrap.min.js"></script>
-</head>
-
-<body class="tundra">
-
-This page is refreshed in every 5 secs.<br/>
-
-<div class="contentelement">
-
- <h1>$title$</h1>
-
- <hr>
-
- <table cellspacing="2">
- <tr>
- <th>Bytes sent total:</th>
- <td>$bytesSent$</td>
- </tr>
- <tr>
- <th>Messages per second:</th>
- <td>$msgPerSec$</td>
- </tr>
- <tr>
- <th>Open connections:</th>
- <td>$openConnections$</td>
- </tr>
- <tr>
- <th>Max open connections:</th>
- <td>$maxOpenConnections$</td>
- </tr>
- <tr>
- <th>Number of datasets:</th>
- <td>$noDataSets$</td>
- </tr>
- <tr>
- <th>Total size of datasets:</th>
- <td>$sizeDataSets$</td>
- </tr>
- </table>
-
- <hr>
-
- <h1> Datasets: </h1>
- <table cellspacing="2">
- $contexts$
- </table>
-
- <hr>
- <table cellspacing="2">
- <tr>
- <td>
- $statHtml$
- </td>
- </tr>
- </table>
-
-</div>
-</body>
-</html>
\ No newline at end of file