From dc7978ff9be5b57089a61d439590817a7e21db61 Mon Sep 17 00:00:00 2001 From: Vojtech Moravec <vojtech.moravec.st@vsb.cz> Date: Sun, 22 Mar 2020 09:42:58 +0100 Subject: [PATCH] Deprecated reference plane index in favor of middle plane. --- README.md | 8 +-- .../benchmark/BenchmarkBase.java | 11 ++-- .../ScalarQuantizationBenchmark.java | 13 ++--- .../VectorQuantizationBenchmark.java | 13 ++--- .../java/azgracompress/cli/CliConstants.java | 12 ++--- .../azgracompress/cli/ParsedCliOptions.java | 34 +++--------- .../functions/MeasurePlaneErrorFunction.java | 3 -- .../CompressorDecompressorBase.java | 8 +++ .../compression/ImageCompressor.java | 4 +- .../compression/SQImageCompressor.java | 52 +++++++++---------- .../compression/SQImageDecompressor.java | 2 +- .../compression/VQImageCompressor.java | 21 ++++---- .../compression/VQImageDecompressor.java | 2 +- 13 files changed, 82 insertions(+), 101 deletions(-) diff --git a/README.md b/README.md index fa8b0d8..bf49431 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ usage: azgracompress.DataCompressor [options] input -h,--help Print help -i,--inspect Inspect the compressed file -o,--output <arg> Custom output file - -rp,--reference-plane <arg> Reference plane index + -mp,--middle-plane Use middle plane for codebook training. -sq,--scalar-quantization Use scalar quantization. -tcb,--train-codebook Train codebook and save learned codebook to cache file. @@ -39,9 +39,9 @@ usage: azgracompress.DataCompressor [options] input - QT is required - Set the bits per pixel using `-b` or `--bits` and integer value from 1 to 8. Codebook size is equal to (*2^bits*). - Normally the codebook is created for each image plane separately, if one wants to use general codebook for all planes, these are the options: - - Set the reference plane index using `-rp` or `--reference-plane`. Reference plane is used to create codebook for all planes. + - Set the middle plane index using `-mp` or `--middle-plane`. Middle plane is used to create codebook for all planes. - Set the cache folder by `cbc` or `--codebook-cache`, quantizer will look for cached codebook of given file and codebook size. -- For input file info seee Input File section +- For input file info see Input File section ### Decompress - Use with `-d` or `--decompress` @@ -50,7 +50,7 @@ usage: azgracompress.DataCompressor [options] input ### Inspect - Use with `-i` or `--inspect` -- Inspect the compressed file. Read compressed file header are write the informations from that header. +- Inspect the compressed file. Read compressed file header are write the information from that header. ### Train codebook - Use with `-tcb` or `--train-codebook` diff --git a/src/main/java/azgracompress/benchmark/BenchmarkBase.java b/src/main/java/azgracompress/benchmark/BenchmarkBase.java index 3a2e437..18aad38 100644 --- a/src/main/java/azgracompress/benchmark/BenchmarkBase.java +++ b/src/main/java/azgracompress/benchmark/BenchmarkBase.java @@ -23,8 +23,7 @@ abstract class BenchmarkBase { protected final int[] planes; protected final V3i rawImageDims; - protected final boolean hasReferencePlane; - protected final int referencePlaneIndex; + protected final boolean useMiddlePlane; protected final int codebookSize; protected final boolean hasCacheFolder; protected final String cacheFolder; @@ -41,8 +40,7 @@ abstract class BenchmarkBase { this.planes = planes; this.rawImageDims = rawImageDims; - hasReferencePlane = false; - referencePlaneIndex = -1; + useMiddlePlane = false; codebookSize = 256; hasCacheFolder = false; @@ -56,8 +54,7 @@ abstract class BenchmarkBase { this.inputFile = options.getInputFile(); this.outputDirectory = options.getOutputFile(); this.rawImageDims = options.getImageDimension(); - this.hasReferencePlane = options.hasReferencePlaneIndex(); - this.referencePlaneIndex = options.getReferencePlaneIndex(); + this.useMiddlePlane = options.shouldUseMiddlePlane(); this.codebookSize = (int) Math.pow(2, options.getBitsPerPixel()); if (options.hasPlaneIndexSet()) { @@ -81,7 +78,7 @@ abstract class BenchmarkBase { hasCacheFolder = options.hasCodebookCacheFolder(); cacheFolder = options.getCodebookCacheFolder(); - hasGeneralQuantizer = hasReferencePlane || hasCacheFolder; + hasGeneralQuantizer = useMiddlePlane || hasCacheFolder; workerCount = options.getWorkerCount(); } diff --git a/src/main/java/azgracompress/benchmark/ScalarQuantizationBenchmark.java b/src/main/java/azgracompress/benchmark/ScalarQuantizationBenchmark.java index 4cf0060..051ee8e 100644 --- a/src/main/java/azgracompress/benchmark/ScalarQuantizationBenchmark.java +++ b/src/main/java/azgracompress/benchmark/ScalarQuantizationBenchmark.java @@ -49,19 +49,20 @@ public class ScalarQuantizationBenchmark extends BenchmarkBase { return; } System.out.println("Created quantizer from cache"); - } else if (hasReferencePlane) { - final int[] refPlaneData = loadPlaneData(referencePlaneIndex); - if (refPlaneData.length == 0) { - System.err.println("Failed to load reference plane data."); + } else if (useMiddlePlane) { + final int middlePlaneIndex = rawImageDims.getZ() / 2; + final int[] middlePlaneData = loadPlaneData(middlePlaneIndex); + if (middlePlaneData.length == 0) { + System.err.println("Failed to load middle plane data."); return; } if (useDiffEvolution) { assert (false) : "DE is depracated"; quantizer = null; } else { - quantizer = trainLloydMaxQuantizer(refPlaneData, codebookSize); + quantizer = trainLloydMaxQuantizer(middlePlaneData, codebookSize); } - System.out.println("Created reference quantizer."); + System.out.println("Created quantizer from middle plane."); } for (final int planeIndex : planes) { diff --git a/src/main/java/azgracompress/benchmark/VectorQuantizationBenchmark.java b/src/main/java/azgracompress/benchmark/VectorQuantizationBenchmark.java index 7c01f3c..5224d38 100644 --- a/src/main/java/azgracompress/benchmark/VectorQuantizationBenchmark.java +++ b/src/main/java/azgracompress/benchmark/VectorQuantizationBenchmark.java @@ -76,19 +76,20 @@ public class VectorQuantizationBenchmark extends BenchmarkBase { return; } System.out.println("Created quantizer from cache"); - } else if (hasReferencePlane) { - final ImageU16 plane = loadPlane(referencePlaneIndex); + } else if (useMiddlePlane) { + final int middlePlaneIndex = rawImageDims.getZ() / 2; + final ImageU16 middlePlane = loadPlane(middlePlaneIndex); - if (plane == null) { - System.err.println("Failed to load reference plane data."); + if (middlePlane == null) { + System.err.println("Failed to load middle plane data."); return; } - final int[][] refPlaneData = getPlaneVectors(plane, qVector); + final int[][] refPlaneData = getPlaneVectors(middlePlane, qVector); LBGVectorQuantizer vqInitializer = new LBGVectorQuantizer(refPlaneData, codebookSize, workerCount); final LBGResult vqResult = vqInitializer.findOptimalCodebook(); quantizer = new VectorQuantizer(vqResult.getCodebook()); - System.out.println("Created reference quantizer."); + System.out.println("Created quantizer from middle plane."); } for (final int planeIndex : planes) { diff --git a/src/main/java/azgracompress/cli/CliConstants.java b/src/main/java/azgracompress/cli/CliConstants.java index 4044e30..aa5dd83 100644 --- a/src/main/java/azgracompress/cli/CliConstants.java +++ b/src/main/java/azgracompress/cli/CliConstants.java @@ -51,8 +51,8 @@ public class CliConstants { public static final String VECTOR_QUANTIZATION_SHORT = "vq"; public static final String VECTOR_QUANTIZATION_LONG = "vector-quantization"; - public static final String REFERENCE_PLANE_SHORT = "rp"; - public static final String REFERENCE_PLANE_LONG = "reference-plane"; + public static final String USE_MIDDLE_PLANE_SHORT = "md"; + public static final String USE_MIDDLE_PLANE_LONG = "middle-plane"; @NotNull public static Options getOptions() { @@ -105,10 +105,10 @@ public class CliConstants { options.addOptionGroup(compressionMethodGroup); options.addOption(CliConstants.BITS_SHORT, CliConstants.BITS_LONG, true, "Bit count per pixel [Default 8]"); - options.addOption(CliConstants.REFERENCE_PLANE_SHORT, - CliConstants.REFERENCE_PLANE_LONG, - true, - "Reference plane index"); + options.addOption(CliConstants.USE_MIDDLE_PLANE_SHORT, + CliConstants.USE_MIDDLE_PLANE_LONG, + false, + "Use middle plane for codebook creation"); options.addOption(new Option(CliConstants.VERBOSE_SHORT, CliConstants.VERBOSE_LONG, diff --git a/src/main/java/azgracompress/cli/ParsedCliOptions.java b/src/main/java/azgracompress/cli/ParsedCliOptions.java index eb4add1..b43ce7f 100644 --- a/src/main/java/azgracompress/cli/ParsedCliOptions.java +++ b/src/main/java/azgracompress/cli/ParsedCliOptions.java @@ -27,8 +27,7 @@ public class ParsedCliOptions { private boolean planeIndexSet = false; private int planeIndex; - private boolean refPlaneIndexSet = false; - private int referencePlaneIndex = -1; + private boolean useMiddlePlane = false; private boolean verbose; @@ -93,7 +92,7 @@ public class ParsedCliOptions { parseBitsPerPixel(cmd, errorBuilder); - parseReferencePlaneIndex(cmd, errorBuilder); + useMiddlePlane = cmd.hasOption(CliConstants.USE_MIDDLE_PLANE_LONG); final String[] fileInfo = cmd.getArgs(); parseInputFilePart(errorBuilder, fileInfo); @@ -219,23 +218,6 @@ public class ParsedCliOptions { } - private void parseReferencePlaneIndex(CommandLine cmd, StringBuilder errorBuilder) { - if (cmd.hasOption(CliConstants.REFERENCE_PLANE_LONG)) { - final String rpString = cmd.getOptionValue(CliConstants.REFERENCE_PLANE_LONG); - final ParseResult<Integer> parseResult = tryParseInt(rpString); - if (parseResult.isSuccess()) { - referencePlaneIndex = parseResult.getValue(); - refPlaneIndexSet = true; - } else { - errorOccurred = true; - errorBuilder.append("Failed to parse reference plane index").append('\n'); - errorBuilder.append(parseResult.getErrorMessage()).append('\n'); - } - } else { - refPlaneIndexSet = false; - } - } - private void parseBitsPerPixel(CommandLine cmd, StringBuilder errorBuilder) { if (cmd.hasOption(CliConstants.BITS_LONG)) { final String bitsString = cmd.getOptionValue(CliConstants.BITS_LONG); @@ -375,16 +357,12 @@ public class ParsedCliOptions { return planeIndex; } - public int getReferencePlaneIndex() { - return referencePlaneIndex; - } - public boolean isVerbose() { return verbose; } - public boolean hasReferencePlaneIndex() { - return refPlaneIndexSet; + public boolean shouldUseMiddlePlane() { + return useMiddlePlane; } public boolean hasPlaneIndexSet() { @@ -482,8 +460,8 @@ public class ParsedCliOptions { if (planeIndexSet) { sb.append("PlaneIndex: ").append(planeIndex).append('\n'); } - if (refPlaneIndexSet) { - sb.append("ReferencePlaneIndex: ").append(referencePlaneIndex).append('\n'); + if (useMiddlePlane) { + sb.append("Use middle plane for codebook training\n"); } if (planeRangeSet) { sb.append("FromPlaneIndex: ").append(fromPlaneIndex).append('\n'); diff --git a/src/main/java/azgracompress/cli/functions/MeasurePlaneErrorFunction.java b/src/main/java/azgracompress/cli/functions/MeasurePlaneErrorFunction.java index b48c91f..cdc8097 100644 --- a/src/main/java/azgracompress/cli/functions/MeasurePlaneErrorFunction.java +++ b/src/main/java/azgracompress/cli/functions/MeasurePlaneErrorFunction.java @@ -122,9 +122,6 @@ public class MeasurePlaneErrorFunction extends CustomFunctionBase { final int workerCount = 8; final V3i dims = new V3i(1041, 996, 946); final int planePixelCount = dims.getX() * dims.getY(); - // ImageU16 compressedPlane = null; - // ImageU16 originalPlane = null; - // ImageU16 differencePlane = null; PlaneError[] planeErrors = new PlaneError[dims.getZ()]; diff --git a/src/main/java/azgracompress/compression/CompressorDecompressorBase.java b/src/main/java/azgracompress/compression/CompressorDecompressorBase.java index f80ba04..4519b12 100644 --- a/src/main/java/azgracompress/compression/CompressorDecompressorBase.java +++ b/src/main/java/azgracompress/compression/CompressorDecompressorBase.java @@ -70,4 +70,12 @@ public abstract class CompressorDecompressorBase { System.err.println(message); } } + + /** + * Get index of the middle plane. + * @return Index of the middle plane. + */ + protected int getMiddlePlaneIndex() { + return (options.getImageDimension().getZ() / 2); + } } diff --git a/src/main/java/azgracompress/compression/ImageCompressor.java b/src/main/java/azgracompress/compression/ImageCompressor.java index 75ed35d..701725e 100644 --- a/src/main/java/azgracompress/compression/ImageCompressor.java +++ b/src/main/java/azgracompress/compression/ImageCompressor.java @@ -128,8 +128,8 @@ public class ImageCompressor extends CompressorDecompressorBase { header.setQuantizationType(options.getQuantizationType()); header.setBitsPerPixel((byte) options.getBitsPerPixel()); - // Codebook per plane is used only if reference plane isn't set nor is the cache folder. - final boolean oneCodebook = options.hasReferencePlaneIndex() || options.hasCodebookCacheFolder(); + // Codebook per plane is used only if middle plane isn't set nor is the cache folder. + final boolean oneCodebook = options.shouldUseMiddlePlane() || options.hasCodebookCacheFolder(); header.setCodebookPerPlane(!oneCodebook); header.setImageSizeX(options.getImageDimension().getX()); diff --git a/src/main/java/azgracompress/compression/SQImageCompressor.java b/src/main/java/azgracompress/compression/SQImageCompressor.java index 82d8eb8..20bb0a2 100644 --- a/src/main/java/azgracompress/compression/SQImageCompressor.java +++ b/src/main/java/azgracompress/compression/SQImageCompressor.java @@ -90,9 +90,7 @@ public class SQImageCompressor extends CompressorDecompressorBase implements IIm */ public long[] compress(DataOutputStream compressStream) throws ImageCompressionException { Stopwatch stopwatch = new Stopwatch(); - - final boolean hasGeneralQuantizer = options.hasCodebookCacheFolder() || options.hasReferencePlaneIndex(); - + final boolean hasGeneralQuantizer = options.hasCodebookCacheFolder() || options.shouldUseMiddlePlane(); ScalarQuantizer quantizer = null; Huffman huffman = null; final int[] huffmanSymbols = createHuffmanSymbols(); @@ -102,27 +100,27 @@ public class SQImageCompressor extends CompressorDecompressorBase implements IIm quantizer = loadQuantizerFromCache(); Log("Cached quantizer created."); writeCodebookToOutputStream(quantizer, compressStream); - } else if (options.hasReferencePlaneIndex()) { - // TODO(Moravec): Reference plane will be deprecated in favor of 'middle' plane. + } else if (options.shouldUseMiddlePlane()) { stopwatch.restart(); - ImageU16 referencePlane = null; + ImageU16 middlePlane = null; + final int middlePlaneIndex = getMiddlePlaneIndex(); try { - referencePlane = RawDataIO.loadImageU16(options.getInputFile(), - options.getImageDimension(), - options.getReferencePlaneIndex()); + middlePlane = RawDataIO.loadImageU16(options.getInputFile(), + options.getImageDimension(), + getMiddlePlaneIndex()); // TODO(Moravec): Create huffman. } catch (Exception ex) { - throw new ImageCompressionException("Unable to load reference plane data.", ex); + throw new ImageCompressionException("Unable to load plane data.", ex); } - Log(String.format("Training scalar quantizer from reference plane %d.", options.getReferencePlaneIndex())); - quantizer = trainScalarQuantizerFromData(referencePlane.getData()); + Log(String.format("Training scalar quantizer from middle plane %d.", middlePlaneIndex)); + quantizer = trainScalarQuantizerFromData(middlePlane.getData()); stopwatch.stop(); writeCodebookToOutputStream(quantizer, compressStream); - Log("Reference codebook created in: " + stopwatch.getElapsedTimeString()); + Log("Middle plane codebook created in: " + stopwatch.getElapsedTimeString()); } final int[] planeIndices = getPlaneIndicesForCompression(); @@ -157,19 +155,19 @@ public class SQImageCompressor extends CompressorDecompressorBase implements IIm Log("Compressing plane..."); final int[] indices = quantizer.quantizeIntoIndices(plane.getData(), 1); -// //////////////////////// -// for (int i = 0; i < indices.length; i++) { -// final boolean[] huffmanCode = huffman.getCode(indices[i]); -// HuffmanNode currentHuffmanNode = huffman.getRoot(); -// boolean bit; -// int index = 0; -// while (!currentHuffmanNode.isLeaf()) { -// bit = huffmanCode[index++]; -// currentHuffmanNode = currentHuffmanNode.traverse(bit); -// } -// assert (indices[i] == currentHuffmanNode.getSymbol()); -// } -// //////////////////////////////// + // //////////////////////// + // for (int i = 0; i < indices.length; i++) { + // final boolean[] huffmanCode = huffman.getCode(indices[i]); + // HuffmanNode currentHuffmanNode = huffman.getRoot(); + // boolean bit; + // int index = 0; + // while (!currentHuffmanNode.isLeaf()) { + // bit = huffmanCode[index++]; + // currentHuffmanNode = currentHuffmanNode.traverse(bit); + // } + // assert (indices[i] == currentHuffmanNode.getSymbol()); + // } + // //////////////////////////////// try (OutBitStream outBitStream = new OutBitStream(compressStream, options.getBitsPerPixel(), 2048)) { @@ -199,7 +197,7 @@ public class SQImageCompressor extends CompressorDecompressorBase implements IIm options.getImageDimension(), options.getPlaneIndex()).getData(); } catch (IOException e) { - throw new ImageCompressionException("Failed to load reference image data.", e); + throw new ImageCompressionException("Failed to load plane data.", e); } } else if (options.hasPlaneRangeSet()) { Log("Loading plane range data."); diff --git a/src/main/java/azgracompress/compression/SQImageDecompressor.java b/src/main/java/azgracompress/compression/SQImageDecompressor.java index 75eb8e2..5d71d0d 100644 --- a/src/main/java/azgracompress/compression/SQImageDecompressor.java +++ b/src/main/java/azgracompress/compression/SQImageDecompressor.java @@ -78,9 +78,9 @@ public class SQImageDecompressor extends CompressorDecompressorBase implements I Huffman huffman = null; if (!header.isCodebookPerPlane()) { // There is only one codebook. - Log("Loading reference codebook..."); huffman = null; // TODO(Moravec): Handle loading of Huffman. + Log("Loading codebook from cache..."); //quantizationValues = readScalarQuantizationValues(compressedStream, codebookSize); } diff --git a/src/main/java/azgracompress/compression/VQImageCompressor.java b/src/main/java/azgracompress/compression/VQImageCompressor.java index 2941305..8bf557a 100644 --- a/src/main/java/azgracompress/compression/VQImageCompressor.java +++ b/src/main/java/azgracompress/compression/VQImageCompressor.java @@ -88,7 +88,7 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm public long[] compress(DataOutputStream compressStream) throws ImageCompressionException { long[] planeDataSizes = new long[options.getImageDimension().getZ()]; Stopwatch stopwatch = new Stopwatch(); - final boolean hasGeneralQuantizer = options.hasCodebookCacheFolder() || options.hasReferencePlaneIndex(); + final boolean hasGeneralQuantizer = options.hasCodebookCacheFolder() || options.shouldUseMiddlePlane(); VectorQuantizer quantizer = null; if (options.hasCodebookCacheFolder()) { @@ -96,24 +96,25 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm quantizer = loadQuantizerFromCache(); Log("Cached quantizer created."); writeQuantizerToCompressStream(quantizer, compressStream); - } else if (options.hasReferencePlaneIndex()) { + } else if (options.shouldUseMiddlePlane()) { stopwatch.restart(); - ImageU16 referencePlane = null; + final int middlePlaneIndex = getMiddlePlaneIndex(); + ImageU16 middlePlane = null; try { - referencePlane = RawDataIO.loadImageU16(options.getInputFile(), + middlePlane = RawDataIO.loadImageU16(options.getInputFile(), options.getImageDimension(), - options.getReferencePlaneIndex()); + middlePlaneIndex); } catch (Exception ex) { - throw new ImageCompressionException("Unable to load reference plane data.", ex); + throw new ImageCompressionException("Unable to load plane data.", ex); } - Log(String.format("Training vector quantizer from reference plane %d.", options.getReferencePlaneIndex())); - final int[][] refPlaneVectors = referencePlane.toQuantizationVectors(options.getVectorDimension()); + Log(String.format("Training vector quantizer from middle plane %d.", middlePlaneIndex)); + final int[][] refPlaneVectors = middlePlane.toQuantizationVectors(options.getVectorDimension()); quantizer = trainVectorQuantizerFromPlaneVectors(refPlaneVectors); writeQuantizerToCompressStream(quantizer, compressStream); stopwatch.stop(); - Log("Reference codebook created in: " + stopwatch.getElapsedTimeString()); + Log("Middle plane codebook created in: " + stopwatch.getElapsedTimeString()); } final int[] planeIndices = getPlaneIndicesForCompression(); @@ -184,7 +185,7 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm try { trainData = loadPlaneQuantizationVectors(options.getPlaneIndex()); } catch (IOException e) { - throw new ImageCompressionException("Failed to load reference image data.", e); + throw new ImageCompressionException("Failed to load plane data.", e); } } else { Log(options.hasPlaneRangeSet() ? "VQ: Loading plane range data." : "VQ: Loading all planes data."); diff --git a/src/main/java/azgracompress/compression/VQImageDecompressor.java b/src/main/java/azgracompress/compression/VQImageDecompressor.java index 7e43243..5045688 100644 --- a/src/main/java/azgracompress/compression/VQImageDecompressor.java +++ b/src/main/java/azgracompress/compression/VQImageDecompressor.java @@ -102,7 +102,7 @@ public class VQImageDecompressor extends CompressorDecompressorBase implements I int[][] quantizationVectors = null; if (!header.isCodebookPerPlane()) { // There is only one codebook. - Log("Loading reference codebook..."); + Log("Loading codebook from cache..."); quantizationVectors = readCodebookVectors(compressedStream, codebookSize, vectorSize); } -- GitLab