From 606c6f3d46a0aeeb501eb63781276212d797acae Mon Sep 17 00:00:00 2001 From: Vojtech Moravec <vojtech.moravec.st@vsb.cz> Date: Fri, 22 Nov 2019 10:45:09 +0100 Subject: [PATCH] Custom parameters from CLI option. Save our CLI options to custom parameter class. Also reformatted some class with automatic formatting. --- src/main/java/bdv/server/BigDataServer.java | 714 +++++++++--------- src/main/java/bdv/server/CellHandler.java | 590 +++++++-------- .../server/CustomCompressionParameters.java | 49 ++ src/main/java/bdv/server/ManagerHandler.java | 390 +++++----- src/main/resources/templates/manager.st | 74 -- 5 files changed, 884 insertions(+), 933 deletions(-) create mode 100644 src/main/java/bdv/server/CustomCompressionParameters.java delete mode 100644 src/main/resources/templates/manager.st diff --git a/src/main/java/bdv/server/BigDataServer.java b/src/main/java/bdv/server/BigDataServer.java index 9e49eed..76ee54a 100644 --- a/src/main/java/bdv/server/BigDataServer.java +++ b/src/main/java/bdv/server/BigDataServer.java @@ -1,7 +1,6 @@ package bdv.server; import compression.quantization.scalar.LloydMaxU16ScalarQuantization; -import gnu.trove.impl.sync.TSynchronizedShortByteMap; import mpicbg.spim.data.SpimDataException; import org.apache.commons.cli.*; @@ -46,355 +45,378 @@ import java.util.Map.Entry; * by default.) * -m enable statistics and manager context. EXPERIMENTAL! * </pre> - * + * <p> * To enable the {@code -m} option, build with * {@link Constants#ENABLE_EXPERIMENTAL_FEATURES} set to {@code true}. * * @author Tobias Pietzsch <tobias.pietzsch@gmail.com> * @author HongKee Moon <moon@mpi-cbg.quantization.de> */ -public class BigDataServer -{ - private static final org.eclipse.jetty.util.log.Logger LOG = Log.getLogger( BigDataServer.class ); - - private static LloydMaxU16ScalarQuantization quantizer; - - static Parameters getDefaultParameters() - { - final int port = 8080; - String hostname; - try - { - hostname = InetAddress.getLocalHost().getHostName(); - } - catch ( final UnknownHostException e ) - { - hostname = "localhost"; - } - final String thumbnailDirectory = null; - final boolean enableManagerContext = false; - return new Parameters( port, hostname, new HashMap< String, String >(), thumbnailDirectory, enableManagerContext, CellHandler.DumpFile); - } - - public static void main( final String[] args ) throws Exception - { - System.setProperty( "org.eclipse.jetty.util.log.class", "org.eclipse.jetty.util.log.StdErrLog" ); - - final Parameters params = processOptions( args, getDefaultParameters() ); - if ( params == null ) - return; - - CellHandler.DumpFile = params.getDumpFile(); - - final String thumbnailsDirectoryName = getThumbnailDirectoryPath( params ); - - // Threadpool for multiple connections - final Server server = new Server( new QueuedThreadPool( 200, 8 ) ); - - // ServerConnector configuration - final ServerConnector connector = new ServerConnector( server ); - connector.setHost( params.getHostname() ); - connector.setPort( params.getPort() ); - LOG.info( "Set connectors: " + connector ); - server.setConnectors( new Connector[] { connector } ); - final String baseURL = "http://" + server.getURI().getHost() + ":" + params.getPort(); - - - quantizer = new LloydMaxU16ScalarQuantization("D:\\tmp\\server-dump\\initial_load.bin",8); - quantizer.train(true); - - // Handler initialization - final HandlerCollection handlers = new HandlerCollection(); - - final ContextHandlerCollection datasetHandlers = createHandlers( baseURL, params.getDatasets(), thumbnailsDirectoryName ); - handlers.addHandler( datasetHandlers ); - handlers.addHandler( new JsonDatasetListHandler( server, datasetHandlers ) ); - - Handler handler = handlers; - if ( params.enableManagerContext() ) - { - // Add Statistics bean to the connector - final ConnectorStatistics connectorStats = new ConnectorStatistics(); - connector.addBean( connectorStats ); - - // create StatisticsHandler wrapper and ManagerHandler - final StatisticsHandler statHandler = new StatisticsHandler(); - handlers.addHandler( new ManagerHandler( baseURL, server, connectorStats, statHandler, datasetHandlers, thumbnailsDirectoryName ) ); - statHandler.setHandler( handlers ); - handler = statHandler; - } - - - - LOG.info( "Set handler: " + handler ); - server.setHandler( handler ); - LOG.info( "Server Base URL: " + baseURL ); - LOG.info( "BigDataServer starting" ); - server.start(); - server.join(); - } - - /** - * Server parameters: hostname, port, datasets. - */ - private static class Parameters - { - private final int port; - - private final String hostname; - - /** - * maps from dataset name to dataset xml path. - */ - private final Map< String, String > datasetNameToXml; - - private final String thumbnailDirectory; - - private final String dumpFile; - - private final boolean enableManagerContext; - - Parameters( final int port, final String hostname, final Map< String, String > datasetNameToXml, final String thumbnailDirectory, final boolean enableManagerContext, final String dumpFile) - { - this.port = port; - this.hostname = hostname; - this.datasetNameToXml = datasetNameToXml; - this.thumbnailDirectory = thumbnailDirectory; - this.enableManagerContext = enableManagerContext; - this.dumpFile = dumpFile; - } - - public int getPort() - { - return port; - } - - public String getHostname() - { - return hostname; - } - - public String getThumbnailDirectory() - { - return thumbnailDirectory; - } - - /** - * Get datasets. - * - * @return datasets as a map from dataset name to dataset xml path. - */ - public Map< String, String > getDatasets() - { - return datasetNameToXml; - } - - public boolean enableManagerContext() - { - return enableManagerContext; - } - - public String getDumpFile() { - return dumpFile; - } - } - - @SuppressWarnings( "static-access" ) - static private Parameters processOptions( final String[] args, final Parameters defaultParameters ) throws IOException - { - // create Options object - final Options options = new Options(); - - final String cmdLineSyntax = "BigDataServer [OPTIONS] [NAME XML] ...\n"; - - final String description = - "Serves one or more XML/HDF5 datasets for remote access over HTTP.\n" + - "Provide (NAME XML) pairs on the command line or in a dataset file, where NAME is the name under which the dataset should be made accessible and XML is the path to the XML file of the dataset."; - - options.addOption( OptionBuilder - .withDescription( "Hostname of the server.\n(default: " + defaultParameters.getHostname() + ")" ) - .hasArg() - .withArgName( "HOSTNAME" ) - .create( "s" ) ); - - options.addOption( OptionBuilder - .withDescription( "Listening port.\n(default: " + defaultParameters.getPort() + ")" ) - .hasArg() - .withArgName( "PORT" ) - .create( "p" ) ); - - // -d or multiple {name name.xml} pairs - options.addOption( OptionBuilder - .withDescription( "Dataset file: A plain text file specifying one dataset per line. Each line is formatted as \"NAME <TAB> XML\"." ) - .hasArg() - .withArgName( "FILE" ) - .create( "d" ) ); - - options.addOption( OptionBuilder - .withDescription( "Directory to store thumbnails. (new temporary directory by default.)" ) - .hasArg() - .withArgName( "DIRECTORY" ) - .create( "t" ) ); - - - options.addOption( OptionBuilder - .withDescription( "File in which to store data dump" ) - .hasArg() - .withArgName( "DUMP" ) - .create( "dump" ) ); - - if ( Constants.ENABLE_EXPERIMENTAL_FEATURES ) - { - options.addOption( OptionBuilder - .withDescription( "enable statistics and manager context. EXPERIMENTAL!" ) - .create( "m" ) ); - } - - try - { - final CommandLineParser parser = new BasicParser(); - final CommandLine cmd = parser.parse( options, args ); - - // Getting port number option - final String portString = cmd.getOptionValue( "p", Integer.toString( defaultParameters.getPort() ) ); - final int port = Integer.parseInt( portString ); - - final String dumpFile = cmd.getOptionValue("dump", defaultParameters.getDumpFile()); - - // Getting server name option - final String serverName = cmd.getOptionValue( "s", defaultParameters.getHostname() ); - - // Getting thumbnail directory option - final String thumbnailDirectory = cmd.getOptionValue( "t", defaultParameters.getThumbnailDirectory() ); - - - - final HashMap< String, String > datasets = new HashMap< String, String >( defaultParameters.getDatasets() ); - - boolean enableManagerContext = false; - if ( Constants.ENABLE_EXPERIMENTAL_FEATURES ) - { - if ( cmd.hasOption( "m" ) ) - enableManagerContext = true; - } - - if ( cmd.hasOption( "d" ) ) - { - // process the file given with "-d" - final String datasetFile = cmd.getOptionValue( "d" ); - - // check the file presence - final Path path = Paths.get( datasetFile ); - - if ( Files.notExists( path ) ) - throw new IllegalArgumentException( "Dataset list file does not exist." ); - - // Process dataset list file - final List< String > lines = Files.readAllLines( path, StandardCharsets.UTF_8 ); - - for ( final String str : lines ) - { - final String[] tokens = str.split( "\\s*\\t\\s*" ); - if ( tokens.length == 2 && StringUtils.isNotEmpty( tokens[ 0 ].trim() ) && StringUtils.isNotEmpty( tokens[ 1 ].trim() ) ) - { - final String name = tokens[ 0 ].trim(); - final String xmlpath = tokens[ 1 ].trim(); - tryAddDataset( datasets, name, xmlpath ); - } - else - { - LOG.warn( "Invalid dataset file line (will be skipped): {" + str + "}" ); - } - } - } - - // process additional {name, name.xml} pairs given on the - // command-line - final String[] leftoverArgs = cmd.getArgs(); - if ( leftoverArgs.length % 2 != 0 ) - throw new IllegalArgumentException( "Dataset list has an error while processing." ); - - for ( int i = 0; i < leftoverArgs.length; i += 2 ) - { - final String name = leftoverArgs[ i ]; - final String xmlpath = leftoverArgs[ i + 1 ]; - tryAddDataset( datasets, name, xmlpath ); - } - - if ( datasets.isEmpty() ) - throw new IllegalArgumentException( "Dataset list is empty." ); - - return new Parameters( port, serverName, datasets, thumbnailDirectory, enableManagerContext, dumpFile ); - } - catch ( final ParseException | IllegalArgumentException e ) - { - LOG.warn( e.getMessage() ); - System.out.println(); - final HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp( cmdLineSyntax, description, options, null ); - } - return null; - } - - private static void tryAddDataset( final HashMap< String, String > datasetNameToXML, final String name, final String xmlpath ) throws IllegalArgumentException - { - for ( final String reserved : Constants.RESERVED_CONTEXT_NAMES ) - if ( name.equals( reserved ) ) - throw new IllegalArgumentException( "Cannot use dataset name: \"" + name + "\" (reserved for internal use)." ); - if ( datasetNameToXML.containsKey( name ) ) - throw new IllegalArgumentException( "Duplicate dataset name: \"" + name + "\"" ); - if ( Files.notExists( Paths.get( xmlpath ) ) ) - throw new IllegalArgumentException( "Dataset file does not exist: \"" + xmlpath + "\"" ); - datasetNameToXML.put( name, xmlpath ); - LOG.info( "Dataset added: {" + name + ", " + xmlpath + "}" ); - } - - private static String getThumbnailDirectoryPath( final Parameters params ) throws IOException - { - final String thumbnailDirectoryName = params.getThumbnailDirectory(); - if ( thumbnailDirectoryName != null ) - { - Path thumbnails = Paths.get( thumbnailDirectoryName ); - if ( ! Files.exists( thumbnails ) ) - { - try - { - thumbnails = Files.createDirectories( thumbnails ); - return thumbnails.toFile().getAbsolutePath(); - } - catch ( final IOException e ) - { - LOG.warn( e.getMessage() ); - LOG.warn( "Could not create thumbnails directory \"" + thumbnailDirectoryName + "\".\n Trying to create temporary directory." ); - } - } - else - { - if ( ! Files.isDirectory( thumbnails ) ) - LOG.warn( "Thumbnails directory \"" + thumbnailDirectoryName + "\" is not a directory.\n Trying to create temporary directory." ); - else - return thumbnails.toFile().getAbsolutePath(); - } - } - final Path thumbnails = Files.createTempDirectory( "thumbnails" ); - thumbnails.toFile().deleteOnExit(); - return thumbnails.toFile().getAbsolutePath(); - } - - private static ContextHandlerCollection createHandlers( final String baseURL, final Map< String, String > dataSet, final String thumbnailsDirectoryName ) throws SpimDataException, IOException - { - final ContextHandlerCollection handlers = new ContextHandlerCollection(); - - for ( final Entry< String, String > entry : dataSet.entrySet() ) - { - final String name = entry.getKey(); - final String xmlpath = entry.getValue(); - final String context = "/" + name; - final CellHandler ctx = new CellHandler( baseURL + context + "/", xmlpath, name, thumbnailsDirectoryName, quantizer ); - ctx.setContextPath( context ); - handlers.addHandler( ctx ); - } - - return handlers; - } +public class BigDataServer { + private static final org.eclipse.jetty.util.log.Logger LOG = Log.getLogger(BigDataServer.class); + + private static LloydMaxU16ScalarQuantization quantizer; + + static Parameters getDefaultParameters() { + final int port = 8080; + String hostname; + try { + hostname = InetAddress.getLocalHost().getHostName(); + } catch (final UnknownHostException e) { + hostname = "localhost"; + } + final String thumbnailDirectory = null; + final boolean enableManagerContext = false; + return new Parameters(port, hostname, new HashMap<String, String>(), thumbnailDirectory, enableManagerContext, + new CustomCompressionParameters("", "", 8, false, false)); + } + + public static void main(final String[] args) throws Exception { + System.setProperty("org.eclipse.jetty.util.log.class", "org.eclipse.jetty.util.log.StdErrLog"); + + final Parameters params = processOptions(args, getDefaultParameters()); + if (params == null) + return; + + final String thumbnailsDirectoryName = getThumbnailDirectoryPath(params); + + // Threadpool for multiple connections + final Server server = new Server(new QueuedThreadPool(200, 8)); + + // ServerConnector configuration + final ServerConnector connector = new ServerConnector(server); + connector.setHost(params.getHostname()); + connector.setPort(params.getPort()); + LOG.info("Set connectors: " + connector); + server.setConnectors(new Connector[]{connector}); + final String baseURL = "http://" + server.getURI().getHost() + ":" + params.getPort(); + + + final CustomCompressionParameters compParams = params.getCompressionParams(); + if (compParams.shouldCompressData() || compParams.renderDifference()) { + //TODO(Moravec): Replace LloydMaxU16ScalarQuantization with some ICompressor. + + quantizer = new LloydMaxU16ScalarQuantization(compParams.getTrainFile(), compParams.getBitTarget()); + quantizer.train(true); + } + + + // Handler initialization + final HandlerCollection handlers = new HandlerCollection(); + + final ContextHandlerCollection datasetHandlers = createHandlers(baseURL, params, thumbnailsDirectoryName); + handlers.addHandler(datasetHandlers); + handlers.addHandler(new JsonDatasetListHandler(server, datasetHandlers)); + + Handler handler = handlers; + if (params.enableManagerContext()) { + // Add Statistics bean to the connector + final ConnectorStatistics connectorStats = new ConnectorStatistics(); + connector.addBean(connectorStats); + + // create StatisticsHandler wrapper and ManagerHandler + final StatisticsHandler statHandler = new StatisticsHandler(); + handlers.addHandler(new ManagerHandler(baseURL, server, connectorStats, statHandler, datasetHandlers, thumbnailsDirectoryName)); + statHandler.setHandler(handlers); + handler = statHandler; + } + + + LOG.info("Set handler: " + handler); + server.setHandler(handler); + LOG.info("Server Base URL: " + baseURL); + LOG.info("BigDataServer starting"); + server.start(); + server.join(); + } + + /** + * Server parameters: hostname, port, datasets. + */ + private static class Parameters { + private final int port; + + private final String hostname; + + /** + * maps from dataset name to dataset xml path. + */ + private final Map<String, String> datasetNameToXml; + + private final String thumbnailDirectory; + + + private final CustomCompressionParameters compressionParam; + + private final boolean enableManagerContext; + + Parameters(final int port, final String hostname, final Map<String, String> datasetNameToXml, + final String thumbnailDirectory, final boolean enableManagerContext, + final CustomCompressionParameters customCompressionParameters) { + this.port = port; + this.hostname = hostname; + this.datasetNameToXml = datasetNameToXml; + this.thumbnailDirectory = thumbnailDirectory; + this.enableManagerContext = enableManagerContext; + this.compressionParam = customCompressionParameters; + } + + public int getPort() { + return port; + } + + public String getHostname() { + return hostname; + } + + public String getThumbnailDirectory() { + return thumbnailDirectory; + } + + /** + * Get datasets. + * + * @return datasets as a map from dataset name to dataset xml path. + */ + public Map<String, String> getDatasets() { + return datasetNameToXml; + } + + public boolean enableManagerContext() { + return enableManagerContext; + } + + public CustomCompressionParameters getCompressionParams() { + return compressionParam; + } + + } + + + @SuppressWarnings("static-access") + static private Parameters processOptions(final String[] args, final Parameters defaultParameters) throws IOException { + final String BIT_TARGET = "bits"; + final String ENABLE_COMPRESSION = "compression"; + final String ENABLE_COMPRESSION_DIFF = "diff"; + final String DUMP_FILE = "dump"; + final String TRAIN_FILE = "train"; + // create Options object + final Options options = new Options(); + + final String cmdLineSyntax = "BigDataServer [OPTIONS] [NAME XML] ...\n"; + + final String description = + "Serves one or more XML/HDF5 datasets for remote access over HTTP.\n" + + "Provide (NAME XML) pairs on the command line or in a dataset file, where NAME is the name under which the dataset should be made accessible and XML is the path to the XML file of the dataset."; + + options.addOption(OptionBuilder + .withDescription("Hostname of the server.\n(default: " + defaultParameters.getHostname() + ")") + .hasArg() + .withArgName("HOSTNAME") + .create("s")); + + options.addOption(OptionBuilder + .withDescription("Listening port.\n(default: " + defaultParameters.getPort() + ")") + .hasArg() + .withArgName("PORT") + .create("p")); + + // -d or multiple {name name.xml} pairs + options.addOption(OptionBuilder + .withDescription("Dataset file: A plain text file specifying one dataset per line. Each line is formatted as \"NAME <TAB> XML\".") + .hasArg() + .withArgName("FILE") + .create("d")); + + options.addOption(OptionBuilder + .withDescription("Directory to store thumbnails. (new temporary directory by default.)") + .hasArg() + .withArgName("DIRECTORY") + .create("t")); + + + options.addOption(OptionBuilder + .withDescription("File in which to store request data dump") + .hasArg() + .withArgName("DUMP") + .create(DUMP_FILE)); + + options.addOption(OptionBuilder + .withDescription("Enable request compression") + .create(ENABLE_COMPRESSION)); + + options.addOption(OptionBuilder + .withDescription("Compression train file") + .hasArg() + .withArgName("TRAINFILE") + .create(TRAIN_FILE)); + + options.addOption(OptionBuilder + .withDescription("Compression bit target") + .hasArg() + .withArgName("BITS") + .create(BIT_TARGET)); + + options.addOption(OptionBuilder + .withDescription("Send compression difference") + .create(ENABLE_COMPRESSION_DIFF)); + + if (Constants.ENABLE_EXPERIMENTAL_FEATURES) { + options.addOption(OptionBuilder + .withDescription("enable statistics and manager context. EXPERIMENTAL!") + .create("m")); + } + + try { + final CommandLineParser parser = new BasicParser(); + final CommandLine cmd = parser.parse(options, args); + + // Getting port number option + final String portString = cmd.getOptionValue("p", Integer.toString(defaultParameters.getPort())); + final int port = Integer.parseInt(portString); + + + // Getting server name option + final String serverName = cmd.getOptionValue("s", defaultParameters.getHostname()); + + // Getting thumbnail directory option + final String thumbnailDirectory = cmd.getOptionValue("t", defaultParameters.getThumbnailDirectory()); + + final HashMap<String, String> datasets = new HashMap<String, String>(defaultParameters.getDatasets()); + + // Custom compression parameters + //cmd.hasOption() + + final String dumpFile = cmd.getOptionValue(DUMP_FILE, ""); + final boolean enableCompression = cmd.hasOption(ENABLE_COMPRESSION); + final boolean enableCompressionDiff = cmd.hasOption(ENABLE_COMPRESSION_DIFF); + final String trainFile = cmd.getOptionValue(TRAIN_FILE, ""); + final int bitTarget = Integer.parseInt(cmd.getOptionValue(BIT_TARGET, "8")); + + if ((enableCompression || enableCompressionDiff) && (trainFile.isEmpty())) { + throw new MissingArgumentException(String.format("!!! %s must be specified when %s or %s is specified !!!", + TRAIN_FILE, ENABLE_COMPRESSION, ENABLE_COMPRESSION_DIFF)); + } + + final CustomCompressionParameters customCompParams = new CustomCompressionParameters(dumpFile, trainFile, bitTarget, + enableCompression, enableCompressionDiff); + + LOG.info("Compression is " + (enableCompression ? "Matched" : "Not matched")); + LOG.info("Compression-Diff is " + (enableCompressionDiff ? "Matched" : "Not matched")); + + boolean enableManagerContext = false; + if (Constants.ENABLE_EXPERIMENTAL_FEATURES) { + if (cmd.hasOption("m")) + enableManagerContext = true; + } + + + if (cmd.hasOption("d")) { + // process the file given with "-d" + final String datasetFile = cmd.getOptionValue("d"); + + // check the file presence + final Path path = Paths.get(datasetFile); + + if (Files.notExists(path)) + throw new IllegalArgumentException("Dataset list file does not exist."); + + // Process dataset list file + final List<String> lines = Files.readAllLines(path, StandardCharsets.UTF_8); + + for (final String str : lines) { + final String[] tokens = str.split("\\s*\\t\\s*"); + if (tokens.length == 2 && StringUtils.isNotEmpty(tokens[0].trim()) && StringUtils.isNotEmpty(tokens[1].trim())) { + final String name = tokens[0].trim(); + final String xmlpath = tokens[1].trim(); + tryAddDataset(datasets, name, xmlpath); + } else { + LOG.warn("Invalid dataset file line (will be skipped): {" + str + "}"); + } + } + } + + // process additional {name, name.xml} pairs given on the + // command-line + final String[] leftoverArgs = cmd.getArgs(); + if (leftoverArgs.length % 2 != 0) + throw new IllegalArgumentException("Dataset list has an error while processing."); + + for (int i = 0; i < leftoverArgs.length; i += 2) { + final String name = leftoverArgs[i]; + final String xmlpath = leftoverArgs[i + 1]; + tryAddDataset(datasets, name, xmlpath); + } + + if (datasets.isEmpty()) + throw new IllegalArgumentException("Dataset list is empty."); + + return new Parameters(port, serverName, datasets, thumbnailDirectory, enableManagerContext, + customCompParams); + } catch (final ParseException | IllegalArgumentException e) { + LOG.warn(e.getMessage()); + System.out.println(); + final HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp(cmdLineSyntax, description, options, null); + } + return null; + } + + private static void tryAddDataset(final HashMap<String, String> datasetNameToXML, final String name, final String xmlpath) throws IllegalArgumentException { + for (final String reserved : Constants.RESERVED_CONTEXT_NAMES) + if (name.equals(reserved)) + throw new IllegalArgumentException("Cannot use dataset name: \"" + name + "\" (reserved for internal use)."); + if (datasetNameToXML.containsKey(name)) + throw new IllegalArgumentException("Duplicate dataset name: \"" + name + "\""); + if (Files.notExists(Paths.get(xmlpath))) + throw new IllegalArgumentException("Dataset file does not exist: \"" + xmlpath + "\""); + datasetNameToXML.put(name, xmlpath); + LOG.info("Dataset added: {" + name + ", " + xmlpath + "}"); + } + + private static String getThumbnailDirectoryPath(final Parameters params) throws IOException { + final String thumbnailDirectoryName = params.getThumbnailDirectory(); + if (thumbnailDirectoryName != null) { + Path thumbnails = Paths.get(thumbnailDirectoryName); + if (!Files.exists(thumbnails)) { + try { + thumbnails = Files.createDirectories(thumbnails); + return thumbnails.toFile().getAbsolutePath(); + } catch (final IOException e) { + LOG.warn(e.getMessage()); + LOG.warn("Could not create thumbnails directory \"" + thumbnailDirectoryName + "\".\n Trying to create temporary directory."); + } + } else { + if (!Files.isDirectory(thumbnails)) + LOG.warn("Thumbnails directory \"" + thumbnailDirectoryName + "\" is not a directory.\n Trying to create temporary directory."); + else + return thumbnails.toFile().getAbsolutePath(); + } + } + final Path thumbnails = Files.createTempDirectory("thumbnails"); + thumbnails.toFile().deleteOnExit(); + return thumbnails.toFile().getAbsolutePath(); + } + + private static ContextHandlerCollection createHandlers(final String baseURL, + final Parameters params, + final String thumbnailsDirectoryName) throws SpimDataException, IOException { + + final ContextHandlerCollection handlers = new ContextHandlerCollection(); + + final Map<String, String> dataSet = params.getDatasets(); + for (final Entry<String, String> entry : dataSet.entrySet()) { + final String name = entry.getKey(); + final String xmlpath = entry.getValue(); + final String context = "/" + name; + final CellHandler ctx = new CellHandler(baseURL + context + "/", xmlpath, name, + thumbnailsDirectoryName, + params.getCompressionParams(), quantizer); + + ctx.setContextPath(context); + handlers.addHandler(ctx); + } + + return handlers; + } } diff --git a/src/main/java/bdv/server/CellHandler.java b/src/main/java/bdv/server/CellHandler.java index ec80358..b294558 100644 --- a/src/main/java/bdv/server/CellHandler.java +++ b/src/main/java/bdv/server/CellHandler.java @@ -11,6 +11,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import compression.quantization.scalar.LloydMaxU16ScalarQuantization; +import compression.utilities.Utils; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.handler.ContextHandler; import org.eclipse.jetty.util.log.Log; @@ -42,309 +43,288 @@ import mpicbg.spim.data.SpimDataException; import net.imglib2.img.basictypeaccess.volatiles.array.VolatileShortArray; import net.imglib2.realtransform.AffineTransform3D; -public class CellHandler extends ContextHandler -{ - private long transferedDataSize = 0; - - private static final org.eclipse.jetty.util.log.Logger LOG = Log.getLogger( CellHandler.class ); - - public static String DumpFile = ""; - private int counter = 0; - private final VolatileGlobalCellCache cache; - - private final Hdf5VolatileShortArrayLoader loader; - - private final CacheHints cacheHints; - - /** - * Full path of the dataset xml file this {@link CellHandler} is serving. - */ - private final String xmlFilename; - - /** - * Full path of the dataset xml file this {@link CellHandler} is serving, - * without the ".xml" suffix. - */ - private final String baseFilename; - - private final String dataSetURL; - - /** - * Cached dataset XML to be send to and opened by {@link BigDataViewer} - * clients. - */ - private final String datasetXmlString; - - /** - * Cached JSON representation of the {@link RemoteImageLoaderMetaData} to be - * send to clients. - */ - private final String metadataJson; - - /** - * Cached dataset.settings XML to be send to clients. May be null if no - * settings file exists for the dataset. - */ - private final String settingsXmlString; - - /** - * Full path to thumbnail png. - */ - private final String thumbnailFilename; - - private LloydMaxU16ScalarQuantization quantizer; - - public CellHandler(final String baseUrl, final String xmlFilename, final String datasetName, final String thumbnailsDirectory, final LloydMaxU16ScalarQuantization quantizer) throws SpimDataException, IOException - { - final XmlIoSpimDataMinimal io = new XmlIoSpimDataMinimal(); - final SpimDataMinimal spimData = io.load( xmlFilename ); - final SequenceDescriptionMinimal seq = spimData.getSequenceDescription(); - final Hdf5ImageLoader imgLoader = ( Hdf5ImageLoader ) seq.getImgLoader(); - this.quantizer = quantizer; - - cache = imgLoader.getCacheControl(); - loader = imgLoader.getShortArrayLoader(); - cacheHints = new CacheHints( LoadingStrategy.BLOCKING, 0, false ); - - // dataSetURL property is used for providing the XML file by replace - // SequenceDescription>ImageLoader>baseUrl - this.xmlFilename = xmlFilename; - baseFilename = xmlFilename.endsWith( ".xml" ) ? xmlFilename.substring( 0, xmlFilename.length() - ".xml".length() ) : xmlFilename; - dataSetURL = baseUrl; - - datasetXmlString = buildRemoteDatasetXML( io, spimData, baseUrl ); - metadataJson = buildMetadataJsonString( imgLoader, seq ); - settingsXmlString = buildSettingsXML( baseFilename ); - thumbnailFilename = createThumbnail( spimData, baseFilename, datasetName, thumbnailsDirectory ); - } - - @Override - public void doHandle( final String target, final Request baseRequest, final HttpServletRequest request, final HttpServletResponse response ) throws IOException - { - if ( target.equals( "/settings" ) ) - { - if ( settingsXmlString != null ) - respondWithString( baseRequest, response, "application/xml", settingsXmlString ); - return; - } - - if ( target.equals( "/png" ) ) - { - provideThumbnail( baseRequest, response ); - return; - } - - final String cellString = request.getParameter( "p" ); - - if ( cellString == null ) - { - respondWithString( baseRequest, response, "application/xml", datasetXmlString ); - return; - } - - final String[] parts = cellString.split( "/" ); - if ( parts[ 0 ].equals( "cell" ) ) - { - final int index = Integer.parseInt( parts[ 1 ] ); - final int timepoint = Integer.parseInt( parts[ 2 ] ); - final int setup = Integer.parseInt( parts[ 3 ] ); - final int level = Integer.parseInt( parts[ 4 ] ); - final Key key = new VolatileGlobalCellCache.Key( timepoint, setup, level, index ); - VolatileCell< ? > cell = cache.getLoadingVolatileCache().getIfPresent( key, cacheHints ); - if ( cell == null ) - { - final int[] cellDims = new int[] { - Integer.parseInt( parts[ 5 ] ), - Integer.parseInt( parts[ 6 ] ), - Integer.parseInt( parts[ 7 ] ) }; - final long[] cellMin = new long[] { - Long.parseLong( parts[ 8 ] ), - Long.parseLong( parts[ 9 ] ), - Long.parseLong( parts[ 10 ] ) }; - cell = cache.getLoadingVolatileCache().get( key, cacheHints, new VolatileCellLoader<>( loader, timepoint, setup, level, cellDims, cellMin ) ); - } - - - @SuppressWarnings( "unchecked" ) - short[] data = ((VolatileCell<VolatileShortArray>) cell).getData().getCurrentStorageArray(); - if (quantizer != null) { - data = quantizer.quantize(data); - } - - /* - * NOTE(Moravec): This is possible place, where to compress data. Image data are inside data array, but we access only part of the image. - * if (compressionEnabled) - * { - * data = compress(data); - * } - * */ - - final byte[] buf = new byte[ 2 * data.length ]; - for ( int i = 0, j = 0; i < data.length; i++ ) - { - final short s = data[ i ]; - buf[ j++ ] = ( byte ) ( ( s >> 8 ) & 0xff ); - buf[ j++ ] = ( byte ) ( s & 0xff ); - } - - if (!DumpFile.equals("")) { - //String requestLog = String.format("%s\\request_%d_%d.data", DumpFile, buf.length, counter++); - FileOutputStream dumpStream = new FileOutputStream(DumpFile, true); - dumpStream.write(buf); - dumpStream.flush(); - dumpStream.close(); - } - - - transferedDataSize += buf.length; - LOG.info(String.format("Total transfered data: [%d KB] [%d MB]", (transferedDataSize/1000), ((transferedDataSize/1000)/1000))); - - response.setContentType( "application/octet-stream" ); - response.setContentLength( buf.length ); - response.setStatus( HttpServletResponse.SC_OK ); - baseRequest.setHandled( true ); - final OutputStream os = response.getOutputStream(); - os.write( buf ); - os.close(); - } - else if ( parts[ 0 ].equals( "init" ) ) - { - respondWithString( baseRequest, response, "application/json", metadataJson ); - } - } - - private void provideThumbnail( final Request baseRequest, final HttpServletResponse response ) throws IOException - { - final Path path = Paths.get( thumbnailFilename ); - if ( Files.exists( path ) ) - { - final byte[] imageData = Files.readAllBytes(path); - if ( imageData != null ) - { - response.setContentType( "image/png" ); - response.setContentLength( imageData.length ); - response.setStatus( HttpServletResponse.SC_OK ); - baseRequest.setHandled( true ); - - final OutputStream os = response.getOutputStream(); - os.write( imageData ); - os.close(); - } - } - } - - public String getXmlFile() - { - return xmlFilename; - } - - public String getDataSetURL() - { - return dataSetURL; - } - - public String getThumbnailUrl() - { - return dataSetURL + "png"; - } - - public String getDescription() - { - throw new UnsupportedOperationException(); - } - - /** - * Create a JSON representation of the {@link RemoteImageLoaderMetaData} - * (image sizes and resolutions) provided by the given - * {@link Hdf5ImageLoader}. - */ - private static String buildMetadataJsonString( final Hdf5ImageLoader imgLoader, final SequenceDescriptionMinimal seq ) - { - final RemoteImageLoaderMetaData metadata = new RemoteImageLoaderMetaData( imgLoader, seq ); - final GsonBuilder gsonBuilder = new GsonBuilder(); - gsonBuilder.registerTypeAdapter( AffineTransform3D.class, new AffineTransform3DJsonSerializer() ); - gsonBuilder.enableComplexMapKeySerialization(); - return gsonBuilder.create().toJson( metadata ); - } - - /** - * Create a modified dataset XML by replacing the ImageLoader with an - * {@link RemoteImageLoader} pointing to the data we are serving. - */ - private static String buildRemoteDatasetXML( final XmlIoSpimDataMinimal io, final SpimDataMinimal spimData, final String baseUrl ) throws IOException, SpimDataException - { - final SpimDataMinimal s = new SpimDataMinimal( spimData, new RemoteImageLoader( baseUrl, false ) ); - final Document doc = new Document( io.toXml( s, s.getBasePath() ) ); - final XMLOutputter xout = new XMLOutputter( Format.getPrettyFormat() ); - final StringWriter sw = new StringWriter(); - xout.output( doc, sw ); - return sw.toString(); - } - - /** - * Read {@code baseFilename.settings.xml} into a string if it exists. - * - * @return contents of {@code baseFilename.settings.xml} or {@code null} if - * that file couldn't be read. - */ - private static String buildSettingsXML( final String baseFilename ) - { - final String settings = baseFilename + ".settings.xml"; - if ( new File( settings ).exists() ) - { - try - { - final SAXBuilder sax = new SAXBuilder(); - final Document doc = sax.build( settings ); - final XMLOutputter xout = new XMLOutputter( Format.getPrettyFormat() ); - final StringWriter sw = new StringWriter(); - xout.output( doc, sw ); - return sw.toString(); - } - catch ( JDOMException | IOException e ) - { - LOG.warn( "Could not read settings file \"" + settings + "\"" ); - LOG.warn( e.getMessage() ); - } - } - return null; - } - - /** - * Create PNG thumbnail file named "{@code <baseFilename>.png}". - */ - private static String createThumbnail( final SpimDataMinimal spimData, final String baseFilename, final String datasetName, final String thumbnailsDirectory ) - { - final String thumbnailFileName = thumbnailsDirectory + "/" + datasetName + ".png"; - final File thumbnailFile = new File( thumbnailFileName ); - if ( !thumbnailFile.isFile() ) // do not recreate thumbnail if it already exists - { - final BufferedImage bi = ThumbnailGenerator.makeThumbnail( spimData, baseFilename, Constants.THUMBNAIL_WIDTH, Constants.THUMBNAIL_HEIGHT ); - try - { - ImageIO.write( bi, "png", thumbnailFile ); - } - catch ( final IOException e ) - { - LOG.warn( "Could not create thumbnail png for dataset \"" + baseFilename + "\"" ); - LOG.warn( e.getMessage() ); - } - } - return thumbnailFileName; - } - - /** - * Handle request by sending a UTF-8 string. - */ - private static void respondWithString( final Request baseRequest, final HttpServletResponse response, final String contentType, final String string ) throws IOException - { - response.setContentType( contentType ); - response.setCharacterEncoding( "UTF-8" ); - response.setStatus( HttpServletResponse.SC_OK ); - baseRequest.setHandled( true ); - - final PrintWriter ow = response.getWriter(); - ow.write( string ); - ow.close(); - } +public class CellHandler extends ContextHandler { + private long transferedDataSize = 0; + + private static final org.eclipse.jetty.util.log.Logger LOG = Log.getLogger(CellHandler.class); + + private int counter = 0; + private final VolatileGlobalCellCache cache; + + private final Hdf5VolatileShortArrayLoader loader; + + private final CacheHints cacheHints; + + /** + * Full path of the dataset xml file this {@link CellHandler} is serving. + */ + private final String xmlFilename; + + /** + * Full path of the dataset xml file this {@link CellHandler} is serving, + * without the ".xml" suffix. + */ + private final String baseFilename; + + private final String dataSetURL; + + /** + * Cached dataset XML to be send to and opened by {@link BigDataViewer} + * clients. + */ + private final String datasetXmlString; + + /** + * Cached JSON representation of the {@link RemoteImageLoaderMetaData} to be + * send to clients. + */ + private final String metadataJson; + + /** + * Cached dataset.settings XML to be send to clients. May be null if no + * settings file exists for the dataset. + */ + private final String settingsXmlString; + + /** + * Full path to thumbnail png. + */ + private final String thumbnailFilename; + final CustomCompressionParameters compressionParams; + private LloydMaxU16ScalarQuantization quantizer; + + public CellHandler(final String baseUrl, final String xmlFilename, final String datasetName, final String thumbnailsDirectory, + final CustomCompressionParameters compressionParams, + final LloydMaxU16ScalarQuantization quantizer) throws SpimDataException, IOException { + + final XmlIoSpimDataMinimal io = new XmlIoSpimDataMinimal(); + final SpimDataMinimal spimData = io.load(xmlFilename); + final SequenceDescriptionMinimal seq = spimData.getSequenceDescription(); + final Hdf5ImageLoader imgLoader = (Hdf5ImageLoader) seq.getImgLoader(); + this.quantizer = quantizer; + this.compressionParams = compressionParams; + + cache = imgLoader.getCacheControl(); + loader = imgLoader.getShortArrayLoader(); + cacheHints = new CacheHints(LoadingStrategy.BLOCKING, 0, false); + + // dataSetURL property is used for providing the XML file by replace + // SequenceDescription>ImageLoader>baseUrl + this.xmlFilename = xmlFilename; + baseFilename = xmlFilename.endsWith(".xml") ? xmlFilename.substring(0, xmlFilename.length() - ".xml".length()) : xmlFilename; + dataSetURL = baseUrl; + + datasetXmlString = buildRemoteDatasetXML(io, spimData, baseUrl); + metadataJson = buildMetadataJsonString(imgLoader, seq); + settingsXmlString = buildSettingsXML(baseFilename); + thumbnailFilename = createThumbnail(spimData, baseFilename, datasetName, thumbnailsDirectory); + } + + @Override + public void doHandle(final String target, final Request baseRequest, final HttpServletRequest request, final HttpServletResponse response) throws IOException { + if (target.equals("/settings")) { + if (settingsXmlString != null) + respondWithString(baseRequest, response, "application/xml", settingsXmlString); + return; + } + + if (target.equals("/png")) { + provideThumbnail(baseRequest, response); + return; + } + + final String cellString = request.getParameter("p"); + + if (cellString == null) { + respondWithString(baseRequest, response, "application/xml", datasetXmlString); + return; + } + + final String[] parts = cellString.split("/"); + if (parts[0].equals("cell")) { + final int index = Integer.parseInt(parts[1]); + final int timepoint = Integer.parseInt(parts[2]); + final int setup = Integer.parseInt(parts[3]); + final int level = Integer.parseInt(parts[4]); + final Key key = new VolatileGlobalCellCache.Key(timepoint, setup, level, index); + VolatileCell<?> cell = cache.getLoadingVolatileCache().getIfPresent(key, cacheHints); + if (cell == null) { + final int[] cellDims = new int[]{ + Integer.parseInt(parts[5]), + Integer.parseInt(parts[6]), + Integer.parseInt(parts[7])}; + final long[] cellMin = new long[]{ + Long.parseLong(parts[8]), + Long.parseLong(parts[9]), + Long.parseLong(parts[10])}; + cell = cache.getLoadingVolatileCache().get(key, cacheHints, new VolatileCellLoader<>(loader, timepoint, setup, level, cellDims, cellMin)); + } + + + @SuppressWarnings("unchecked") + short[] data = ((VolatileCell<VolatileShortArray>) cell).getData().getCurrentStorageArray(); + if (compressionParams.shouldCompressData()) { + assert (quantizer != null) : "Compressor wasn't created"; + data = quantizer.quantize(data); + } else if (compressionParams.renderDifference()) { + assert (quantizer != null) : "Compressor wasn't created"; + short[] compressedData = quantizer.quantize(data); + + for (int i = 0; i < data.length; i++) { + // Original - Compressed + data[i] = Utils.u16BitsToShort(data[i]-compressedData[i]); + // Compressed - Original + //data[i] = Utils.u16BitsToShort(compressedData[i]-data[i]); + } + + //LOG.warn("Not yet implemented."); + } + + final byte[] buf = new byte[2 * data.length]; + for (int i = 0, j = 0; i < data.length; i++) { + final short s = data[i]; + buf[j++] = (byte) ((s >> 8) & 0xff); + buf[j++] = (byte) (s & 0xff); + } + + if (compressionParams.shouldDumpRequestData()) { + FileOutputStream dumpStream = new FileOutputStream(compressionParams.getDumpFile(), true); + dumpStream.write(buf); + dumpStream.flush(); + dumpStream.close(); + } + + transferedDataSize += buf.length; + + LOG.info(String.format("I:%d;T:%d;S:%d;L:%d Total transfered data: [%d KB] [%d MB]", + index, timepoint, setup, level, + (transferedDataSize / 1000), ((transferedDataSize / 1000) / 1000))); + + response.setContentType("application/octet-stream"); + response.setContentLength(buf.length); + response.setStatus(HttpServletResponse.SC_OK); + baseRequest.setHandled(true); + final OutputStream os = response.getOutputStream(); + os.write(buf); + os.close(); + } else if (parts[0].equals("init")) { + respondWithString(baseRequest, response, "application/json", metadataJson); + } + } + + private void provideThumbnail(final Request baseRequest, final HttpServletResponse response) throws IOException { + final Path path = Paths.get(thumbnailFilename); + if (Files.exists(path)) { + final byte[] imageData = Files.readAllBytes(path); + if (imageData != null) { + response.setContentType("image/png"); + response.setContentLength(imageData.length); + response.setStatus(HttpServletResponse.SC_OK); + baseRequest.setHandled(true); + + final OutputStream os = response.getOutputStream(); + os.write(imageData); + os.close(); + } + } + } + + public String getXmlFile() { + return xmlFilename; + } + + public String getDataSetURL() { + return dataSetURL; + } + + public String getThumbnailUrl() { + return dataSetURL + "png"; + } + + public String getDescription() { + throw new UnsupportedOperationException(); + } + + /** + * Create a JSON representation of the {@link RemoteImageLoaderMetaData} + * (image sizes and resolutions) provided by the given + * {@link Hdf5ImageLoader}. + */ + private static String buildMetadataJsonString(final Hdf5ImageLoader imgLoader, final SequenceDescriptionMinimal seq) { + final RemoteImageLoaderMetaData metadata = new RemoteImageLoaderMetaData(imgLoader, seq); + final GsonBuilder gsonBuilder = new GsonBuilder(); + gsonBuilder.registerTypeAdapter(AffineTransform3D.class, new AffineTransform3DJsonSerializer()); + gsonBuilder.enableComplexMapKeySerialization(); + return gsonBuilder.create().toJson(metadata); + } + + /** + * Create a modified dataset XML by replacing the ImageLoader with an + * {@link RemoteImageLoader} pointing to the data we are serving. + */ + private static String buildRemoteDatasetXML(final XmlIoSpimDataMinimal io, final SpimDataMinimal spimData, final String baseUrl) throws IOException, SpimDataException { + final SpimDataMinimal s = new SpimDataMinimal(spimData, new RemoteImageLoader(baseUrl, false)); + final Document doc = new Document(io.toXml(s, s.getBasePath())); + final XMLOutputter xout = new XMLOutputter(Format.getPrettyFormat()); + final StringWriter sw = new StringWriter(); + xout.output(doc, sw); + return sw.toString(); + } + + /** + * Read {@code baseFilename.settings.xml} into a string if it exists. + * + * @return contents of {@code baseFilename.settings.xml} or {@code null} if + * that file couldn't be read. + */ + private static String buildSettingsXML(final String baseFilename) { + final String settings = baseFilename + ".settings.xml"; + if (new File(settings).exists()) { + try { + final SAXBuilder sax = new SAXBuilder(); + final Document doc = sax.build(settings); + final XMLOutputter xout = new XMLOutputter(Format.getPrettyFormat()); + final StringWriter sw = new StringWriter(); + xout.output(doc, sw); + return sw.toString(); + } catch (JDOMException | IOException e) { + LOG.warn("Could not read settings file \"" + settings + "\""); + LOG.warn(e.getMessage()); + } + } + return null; + } + + /** + * Create PNG thumbnail file named "{@code <baseFilename>.png}". + */ + private static String createThumbnail(final SpimDataMinimal spimData, final String baseFilename, final String datasetName, final String thumbnailsDirectory) { + final String thumbnailFileName = thumbnailsDirectory + "/" + datasetName + ".png"; + final File thumbnailFile = new File(thumbnailFileName); + if (!thumbnailFile.isFile()) // do not recreate thumbnail if it already exists + { + final BufferedImage bi = ThumbnailGenerator.makeThumbnail(spimData, baseFilename, Constants.THUMBNAIL_WIDTH, Constants.THUMBNAIL_HEIGHT); + try { + ImageIO.write(bi, "png", thumbnailFile); + } catch (final IOException e) { + LOG.warn("Could not create thumbnail png for dataset \"" + baseFilename + "\""); + LOG.warn(e.getMessage()); + } + } + return thumbnailFileName; + } + + /** + * Handle request by sending a UTF-8 string. + */ + private static void respondWithString(final Request baseRequest, final HttpServletResponse response, final String contentType, final String string) throws IOException { + response.setContentType(contentType); + response.setCharacterEncoding("UTF-8"); + response.setStatus(HttpServletResponse.SC_OK); + baseRequest.setHandled(true); + + final PrintWriter ow = response.getWriter(); + ow.write(string); + ow.close(); + } } diff --git a/src/main/java/bdv/server/CustomCompressionParameters.java b/src/main/java/bdv/server/CustomCompressionParameters.java new file mode 100644 index 0000000..6b2fc91 --- /dev/null +++ b/src/main/java/bdv/server/CustomCompressionParameters.java @@ -0,0 +1,49 @@ +package bdv.server; + +public class CustomCompressionParameters { + private final boolean dumpRequestData; + private final String dumpFile; + private final String trainFile; + private final int bitTarget; + + public String getTrainFile() { + return trainFile; + } + + public int getBitTarget() { + return bitTarget; + } + + private final boolean enableRequestCompression; + private final boolean renderDifference; + + public CustomCompressionParameters(final String dumpFile, + final String trainFile, + final int bitTarget, + final boolean enableRequestCompression, + final boolean renderDifference) { + this.dumpFile = dumpFile; + this.trainFile = trainFile; + this.bitTarget=bitTarget; + this.dumpRequestData = !this.dumpFile.isEmpty(); + this.enableRequestCompression = enableRequestCompression; + this.renderDifference = renderDifference; + } + + public boolean shouldDumpRequestData() { + return dumpRequestData; + } + + public String getDumpFile() { + return dumpFile; + } + + public boolean shouldCompressData() { + return enableRequestCompression; + } + + public boolean renderDifference() { + return renderDifference; + } + +} diff --git a/src/main/java/bdv/server/ManagerHandler.java b/src/main/java/bdv/server/ManagerHandler.java index c458969..b05e7c5 100644 --- a/src/main/java/bdv/server/ManagerHandler.java +++ b/src/main/java/bdv/server/ManagerHandler.java @@ -27,212 +27,186 @@ import java.text.DecimalFormat; * @author HongKee Moon <moon@mpi-cbg.quantization.de> * @author Tobias Pietzsch <tobias.pietzsch@gmail.com> */ -public class ManagerHandler extends ContextHandler -{ - private static final org.eclipse.jetty.util.log.Logger LOG = Log.getLogger( ManagerHandler.class ); - - private final String baseURL; - - private final Server server; - - private final ContextHandlerCollection handlers; - - private final StatisticsHandler statHandler; - - private final ConnectorStatistics connectorStats; - - private String contexts = null; - - private int noDataSets = 0; - - private long sizeDataSets = 0; - - private final String thumbnailsDirectoryName; - - public ManagerHandler( - final String baseURL, - final Server server, - final ConnectorStatistics connectorStats, - final StatisticsHandler statHandler, - final ContextHandlerCollection handlers, - final String thumbnailsDirectoryName ) - throws IOException, URISyntaxException - { - this.baseURL = baseURL; - this.server = server; - this.handlers = handlers; - this.statHandler = statHandler; - this.connectorStats = connectorStats; - this.thumbnailsDirectoryName = thumbnailsDirectoryName; - setContextPath( "/" + Constants.MANAGER_CONTEXT_NAME ); - } - - @Override - public void doHandle( final String target, final Request baseRequest, final HttpServletRequest request, final HttpServletResponse response ) throws IOException, ServletException - { - final String op = request.getParameter( "op" ); - - if ( op == null ) - { - list( baseRequest, response ); - } - else if ( op.equals( "deploy" ) ) - { - final String ds = request.getParameter( "ds" ); - final String file = request.getParameter( "file" ); - deploy( ds, file, baseRequest, response ); - } - else if ( op.equals( "undeploy" ) ) - { - final String ds = request.getParameter( "ds" ); - undeploy( ds, baseRequest, response ); - } - } - - public String getByteSizeString( final long size ) - { - if ( size <= 0 ) - return "0"; - final String[] units = new String[] { "B", "kB", "MB", "GB", "TB" }; - final int digitGroups = ( int ) ( Math.log10( size ) / Math.log10( 1024 ) ); - return new DecimalFormat( "#,##0.#" ).format( size / Math.pow( 1024, digitGroups ) ) + " " + units[ digitGroups ]; - } - - private void list( final Request baseRequest, final HttpServletResponse response ) throws IOException - { - response.setContentType( "text/html" ); - response.setStatus( HttpServletResponse.SC_OK ); - baseRequest.setHandled( true ); - - final PrintWriter ow = response.getWriter(); - - ow.write( getHtml() ); - ow.close(); - } - - private String getHtml() - { - final StringTemplateGroup templates = new StringTemplateGroup( "manager" ); - final StringTemplate t = templates.getInstanceOf( "templates/manager" ); - - t.setAttribute( "bytesSent", getByteSizeString( statHandler.getResponsesBytesTotal() ) ); - t.setAttribute( "msgPerSec", connectorStats.getMessagesOutPerSecond() ); - t.setAttribute( "openConnections", connectorStats.getConnectionsOpen() ); - t.setAttribute( "maxOpenConnections", connectorStats.getConnectionsOpenMax() ); - - getContexts(); - - t.setAttribute( "contexts", contexts ); - - t.setAttribute( "noDataSets", noDataSets ); - t.setAttribute( "sizeDataSets", getByteSizeString( sizeDataSets ) ); - - t.setAttribute( "statHtml", statHandler.toStatsHTML() ); - - return t.toString(); - } - - private void getContexts() - { - if ( contexts == null ) - { - noDataSets = 0; - sizeDataSets = 0; - - final StringBuilder sb = new StringBuilder(); - for ( final Handler handler : server.getChildHandlersByClass( CellHandler.class ) ) - { - sb.append( "<tr>\n<th>" ); - final CellHandler contextHandler = ( CellHandler ) handler; - sb.append( contextHandler.getContextPath() + "</th>\n<td>" ); - sb.append( contextHandler.getXmlFile() + "</td>\n</tr>\n" ); - noDataSets++; - sizeDataSets += new File( contextHandler.getXmlFile().replace( ".xml", ".h5" ) ).length(); - } - contexts = sb.toString(); - } - } - - private void deploy( final String datasetName, final String fileLocation, final Request baseRequest, final HttpServletResponse response ) throws IOException - { - LOG.info( "Add new context: " + datasetName ); - final String context = "/" + datasetName; - - boolean alreadyExists = false; - for ( final Handler handler : server.getChildHandlersByClass( CellHandler.class ) ) - { - final CellHandler contextHandler = ( CellHandler ) handler; - if ( context.equals( contextHandler.getContextPath() ) ) - { - LOG.info( "Context " + datasetName + " already exists."); - alreadyExists = true; - break; - } - } - - if ( ! alreadyExists ) - { - CellHandler ctx = null; - try - { - ctx = new CellHandler( baseURL + context + "/", fileLocation, datasetName, thumbnailsDirectoryName, null ); - } - catch ( final SpimDataException e ) - { - LOG.warn( "Failed to create a CellHandler", e ); - e.printStackTrace(); - } - ctx.setContextPath( context ); - handlers.addHandler( ctx ); - } - - response.setContentType( "text/html" ); - response.setStatus( HttpServletResponse.SC_OK ); - baseRequest.setHandled( true ); - - final PrintWriter ow = response.getWriter(); - if ( alreadyExists ) - ow.write( datasetName + " already exists. Not registered." ); - else - ow.write( datasetName + " registered." ); - ow.close(); - } - - private void undeploy( final String datasetName, final Request baseRequest, final HttpServletResponse response ) throws IOException - { - LOG.info( "Remove the context: " + datasetName ); - boolean ret = false; - - final String context = "/" + datasetName; - for ( final Handler handler : server.getChildHandlersByClass( CellHandler.class ) ) - { - final CellHandler contextHandler = ( CellHandler ) handler; - if ( context.equals( contextHandler.getContextPath() ) ) - { - try - { - contextHandler.stop(); - } - catch ( final Exception e ) - { - e.printStackTrace(); - } - contextHandler.destroy(); - handlers.removeHandler( contextHandler ); - ret = true; - break; - } - } - - if ( ret ) - { - response.setContentType( "text/html" ); - response.setStatus( HttpServletResponse.SC_OK ); - baseRequest.setHandled( true ); - - final PrintWriter ow = response.getWriter(); - ow.write( datasetName + " removed." ); - ow.close(); - } - } +public class ManagerHandler extends ContextHandler { + private static final org.eclipse.jetty.util.log.Logger LOG = Log.getLogger(ManagerHandler.class); + + private final String baseURL; + + private final Server server; + + private final ContextHandlerCollection handlers; + + private final StatisticsHandler statHandler; + + private final ConnectorStatistics connectorStats; + + private String contexts = null; + + private int noDataSets = 0; + + private long sizeDataSets = 0; + + private final String thumbnailsDirectoryName; + + public ManagerHandler( + final String baseURL, + final Server server, + final ConnectorStatistics connectorStats, + final StatisticsHandler statHandler, + final ContextHandlerCollection handlers, + final String thumbnailsDirectoryName) + throws IOException, URISyntaxException { + this.baseURL = baseURL; + this.server = server; + this.handlers = handlers; + this.statHandler = statHandler; + this.connectorStats = connectorStats; + this.thumbnailsDirectoryName = thumbnailsDirectoryName; + setContextPath("/" + Constants.MANAGER_CONTEXT_NAME); + } + + @Override + public void doHandle(final String target, final Request baseRequest, final HttpServletRequest request, final HttpServletResponse response) throws IOException, ServletException { + final String op = request.getParameter("op"); + + if (op == null) { + list(baseRequest, response); + } else if (op.equals("deploy")) { + final String ds = request.getParameter("ds"); + final String file = request.getParameter("file"); + deploy(ds, file, baseRequest, response); + } else if (op.equals("undeploy")) { + final String ds = request.getParameter("ds"); + undeploy(ds, baseRequest, response); + } + } + + public String getByteSizeString(final long size) { + if (size <= 0) + return "0"; + final String[] units = new String[]{"B", "kB", "MB", "GB", "TB"}; + final int digitGroups = (int) (Math.log10(size) / Math.log10(1024)); + return new DecimalFormat("#,##0.#").format(size / Math.pow(1024, digitGroups)) + " " + units[digitGroups]; + } + + private void list(final Request baseRequest, final HttpServletResponse response) throws IOException { + response.setContentType("text/html"); + response.setStatus(HttpServletResponse.SC_OK); + baseRequest.setHandled(true); + + final PrintWriter ow = response.getWriter(); + + ow.write(getHtml()); + ow.close(); + } + + private String getHtml() { + final StringTemplateGroup templates = new StringTemplateGroup("manager"); + final StringTemplate t = templates.getInstanceOf("templates/manager"); + + t.setAttribute("bytesSent", getByteSizeString(statHandler.getResponsesBytesTotal())); + t.setAttribute("msgPerSec", connectorStats.getMessagesOutPerSecond()); + t.setAttribute("openConnections", connectorStats.getConnectionsOpen()); + t.setAttribute("maxOpenConnections", connectorStats.getConnectionsOpenMax()); + + getContexts(); + + t.setAttribute("contexts", contexts); + + t.setAttribute("noDataSets", noDataSets); + t.setAttribute("sizeDataSets", getByteSizeString(sizeDataSets)); + + t.setAttribute("statHtml", statHandler.toStatsHTML()); + + return t.toString(); + } + + private void getContexts() { + if (contexts == null) { + noDataSets = 0; + sizeDataSets = 0; + + final StringBuilder sb = new StringBuilder(); + for (final Handler handler : server.getChildHandlersByClass(CellHandler.class)) { + sb.append("<tr>\n<th>"); + final CellHandler contextHandler = (CellHandler) handler; + sb.append(contextHandler.getContextPath() + "</th>\n<td>"); + sb.append(contextHandler.getXmlFile() + "</td>\n</tr>\n"); + noDataSets++; + sizeDataSets += new File(contextHandler.getXmlFile().replace(".xml", ".h5")).length(); + } + contexts = sb.toString(); + } + } + + private void deploy(final String datasetName, final String fileLocation, final Request baseRequest, final HttpServletResponse response) throws IOException { + LOG.info("Add new context: " + datasetName); + final String context = "/" + datasetName; + + boolean alreadyExists = false; + for (final Handler handler : server.getChildHandlersByClass(CellHandler.class)) { + final CellHandler contextHandler = (CellHandler) handler; + if (context.equals(contextHandler.getContextPath())) { + LOG.info("Context " + datasetName + " already exists."); + alreadyExists = true; + break; + } + } + + if (!alreadyExists) { + CellHandler ctx = null; + try { + LOG.warn("We are creating CellHandler without compression params!"); + ctx = new CellHandler(baseURL + context + "/", fileLocation, datasetName, thumbnailsDirectoryName, + null, null); + } catch (final SpimDataException e) { + LOG.warn("Failed to create a CellHandler", e); + e.printStackTrace(); + } + ctx.setContextPath(context); + handlers.addHandler(ctx); + } + + response.setContentType("text/html"); + response.setStatus(HttpServletResponse.SC_OK); + baseRequest.setHandled(true); + + final PrintWriter ow = response.getWriter(); + if (alreadyExists) + ow.write(datasetName + " already exists. Not registered."); + else + ow.write(datasetName + " registered."); + ow.close(); + } + + private void undeploy(final String datasetName, final Request baseRequest, final HttpServletResponse response) throws IOException { + LOG.info("Remove the context: " + datasetName); + boolean ret = false; + + final String context = "/" + datasetName; + for (final Handler handler : server.getChildHandlersByClass(CellHandler.class)) { + final CellHandler contextHandler = (CellHandler) handler; + if (context.equals(contextHandler.getContextPath())) { + try { + contextHandler.stop(); + } catch (final Exception e) { + e.printStackTrace(); + } + contextHandler.destroy(); + handlers.removeHandler(contextHandler); + ret = true; + break; + } + } + + if (ret) { + response.setContentType("text/html"); + response.setStatus(HttpServletResponse.SC_OK); + baseRequest.setHandled(true); + + final PrintWriter ow = response.getWriter(); + ow.write(datasetName + " removed."); + ow.close(); + } + } } diff --git a/src/main/resources/templates/manager.st b/src/main/resources/templates/manager.st deleted file mode 100644 index aa0abf2..0000000 --- a/src/main/resources/templates/manager.st +++ /dev/null @@ -1,74 +0,0 @@ -<!DOCTYPE html> -<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> -<head> - <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/> - <meta http-equiv="refresh" content="5"/> - - <title>BigDataServer</title> - - <!-- Latest compiled and minified CSS --> - <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/css/bootstrap.min.css"> - - <!-- Optional theme --> - <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/css/bootstrap-theme.min.css"> - - <!-- Latest compiled and minified JavaScript --> - <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/js/bootstrap.min.js"></script> -</head> - -<body class="tundra"> - -This page is refreshed in every 5 secs.<br/> - -<div class="contentelement"> - - <h1>$title$</h1> - - <hr> - - <table cellspacing="2"> - <tr> - <th>Bytes sent total:</th> - <td>$bytesSent$</td> - </tr> - <tr> - <th>Messages per second:</th> - <td>$msgPerSec$</td> - </tr> - <tr> - <th>Open connections:</th> - <td>$openConnections$</td> - </tr> - <tr> - <th>Max open connections:</th> - <td>$maxOpenConnections$</td> - </tr> - <tr> - <th>Number of datasets:</th> - <td>$noDataSets$</td> - </tr> - <tr> - <th>Total size of datasets:</th> - <td>$sizeDataSets$</td> - </tr> - </table> - - <hr> - - <h1> Datasets: </h1> - <table cellspacing="2"> - $contexts$ - </table> - - <hr> - <table cellspacing="2"> - <tr> - <td> - $statHtml$ - </td> - </tr> - </table> - -</div> -</body> -</html> \ No newline at end of file -- GitLab