Skip to content
Snippets Groups Projects
Commit 882bbd23 authored by Vojtech Moravec's avatar Vojtech Moravec
Browse files

Use the improved codebook API.

parent 02df1827
Branches
No related tags found
No related merge requests found
Showing
with 92 additions and 121 deletions
package cz.it4i.qcmp.cli.functions; package cz.it4i.qcmp.cli.functions;
import cz.it4i.qcmp.cache.QuantizationCacheManager;
import cz.it4i.qcmp.cache.VqQvcFile;
import cz.it4i.qcmp.cli.CompressionOptionsCLIParser; import cz.it4i.qcmp.cli.CompressionOptionsCLIParser;
import cz.it4i.qcmp.cli.CustomFunctionBase; import cz.it4i.qcmp.cli.CustomFunctionBase;
import cz.it4i.qcmp.huffman.HuffmanNode;
import cz.it4i.qcmp.huffman.HuffmanTreeBuilder;
import cz.it4i.qcmp.io.InBitStream;
import cz.it4i.qcmp.io.OutBitStream;
import cz.it4i.qcmp.quantization.vector.VQCodebook;
import cz.it4i.qcmp.utilities.Utils;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
@SuppressWarnings("ConstantConditions") @SuppressWarnings("ConstantConditions")
public class DebugFunction extends CustomFunctionBase { public class DebugFunction extends CustomFunctionBase {
...@@ -29,38 +17,37 @@ public class DebugFunction extends CustomFunctionBase { ...@@ -29,38 +17,37 @@ public class DebugFunction extends CustomFunctionBase {
@Override @Override
public boolean run() { public boolean run() {
final VQCodebook codebook = ((VqQvcFile) QuantizationCacheManager.readCacheFile("D:\\tmp\\codebook.qvc")).getCodebook(); // final VQCodebook codebook = ((VqQvcFile) QuantizationCacheManager.readCacheFile("D:\\tmp\\codebook.qvc")).getCodebook();
//
final int[] symbols = new int[codebook.getCodebookSize()]; // final int[] symbols = new int[codebook.getCodebookSize()];
for (int i = 0; i < codebook.getCodebookSize(); i++) { // for (int i = 0; i < codebook.getCodebookSize(); i++) {
symbols[i] = i; // symbols[i] = i;
} // }
//
final HuffmanTreeBuilder huffmanBuilder = new HuffmanTreeBuilder(symbols, codebook.getVectorFrequencies()); // final HuffmanTreeBuilder huffmanBuilder = new HuffmanTreeBuilder(symbols, codebook.getVectorFrequencies());
huffmanBuilder.buildHuffmanTree(); // huffmanBuilder.buildHuffmanTree();
//
final int bitsPerSymbol = (int) Utils.log2(codebook.getCodebookSize()); // final int bitsPerSymbol = (int) Utils.log2(codebook.getCodebookSize());
try (final OutBitStream bitStream = new OutBitStream(new FileOutputStream("D:\\tmp\\huffman_tree.data", false), // try (final OutBitStream bitStream = new OutBitStream(new FileOutputStream("D:\\tmp\\huffman_tree.data", false),
bitsPerSymbol, // bitsPerSymbol,
64)) { // 64)) {
huffmanBuilder.createEncoder().getRoot().writeToBinaryStream(bitStream); // huffmanBuilder.createEncoder().getRoot().writeToBinaryStream(bitStream);
} catch (final IOException e) { // } catch (final IOException e) {
e.printStackTrace(); // e.printStackTrace();
} // }
//
//
HuffmanNode readRoot = null; // HuffmanNode readRoot = null;
try (final InBitStream inBitStream = new InBitStream(new FileInputStream("D:\\tmp\\huffman_tree.data"), bitsPerSymbol, 256)) { // try (final InBitStream inBitStream = new InBitStream(new FileInputStream("D:\\tmp\\huffman_tree.data"), bitsPerSymbol,
readRoot = HuffmanNode.readFromStream(inBitStream); // 256)) {
} catch (final IOException ex) { // readRoot = HuffmanNode.readFromStream(inBitStream);
ex.printStackTrace(); // } catch (final IOException ex) {
} // ex.printStackTrace();
// }
readRoot. //
// final boolean equal = huffmanBuilder.createEncoder().getRoot().treeEqual(readRoot);
final boolean equal = huffmanBuilder.createEncoder().getRoot().treeEqual(readRoot); //
// System.out.println(readRoot != null);
System.out.println(readRoot != null);
return true; return true;
} }
......
...@@ -3,9 +3,7 @@ package cz.it4i.qcmp.compression; ...@@ -3,9 +3,7 @@ package cz.it4i.qcmp.compression;
import cz.it4i.qcmp.compression.exception.ImageCompressionException; import cz.it4i.qcmp.compression.exception.ImageCompressionException;
import cz.it4i.qcmp.compression.listeners.IProgressListener; import cz.it4i.qcmp.compression.listeners.IProgressListener;
import cz.it4i.qcmp.compression.listeners.IStatusListener; import cz.it4i.qcmp.compression.listeners.IStatusListener;
import cz.it4i.qcmp.huffman.HuffmanDecoder;
import cz.it4i.qcmp.huffman.HuffmanEncoder; import cz.it4i.qcmp.huffman.HuffmanEncoder;
import cz.it4i.qcmp.huffman.HuffmanTreeBuilder;
import cz.it4i.qcmp.io.InputData; import cz.it4i.qcmp.io.InputData;
import cz.it4i.qcmp.io.OutBitStream; import cz.it4i.qcmp.io.OutBitStream;
...@@ -101,19 +99,7 @@ public abstract class CompressorDecompressorBase { ...@@ -101,19 +99,7 @@ public abstract class CompressorDecompressorBase {
} }
return symbols; return symbols;
} }
protected HuffmanEncoder createHuffmanEncoder(final int[] symbols, final long[] frequencies) {
final HuffmanTreeBuilder huffman = new HuffmanTreeBuilder(symbols, frequencies);
huffman.buildHuffmanTree();
return huffman.createEncoder();
}
protected HuffmanDecoder createHuffmanDecoder(final int[] symbols, final long[] frequencies) {
final HuffmanTreeBuilder huffman = new HuffmanTreeBuilder(symbols, frequencies);
huffman.buildHuffmanTree();
return huffman.createDecoder();
}
protected int[] getPlaneIndicesForCompression(final InputData inputData) { protected int[] getPlaneIndicesForCompression(final InputData inputData) {
if (inputData.isPlaneIndexSet()) { if (inputData.isPlaneIndexSet()) {
return new int[]{inputData.getPlaneIndex()}; return new int[]{inputData.getPlaneIndex()};
......
...@@ -45,8 +45,7 @@ public class SQImageCompressor extends CompressorDecompressorBase implements IIm ...@@ -45,8 +45,7 @@ public class SQImageCompressor extends CompressorDecompressorBase implements IIm
public void preloadGlobalCodebook(final IQvcFile codebookCacheFile) { public void preloadGlobalCodebook(final IQvcFile codebookCacheFile) {
final SQCodebook cachedCodebook = ((SqQvcFile) codebookCacheFile).getCodebook(); final SQCodebook cachedCodebook = ((SqQvcFile) codebookCacheFile).getCodebook();
cachedQuantizer = new ScalarQuantizer(cachedCodebook); cachedQuantizer = new ScalarQuantizer(cachedCodebook);
cachedHuffmanEncoder = createHuffmanEncoder(createHuffmanSymbols(cachedCodebook.getCodebookSize()), cachedHuffmanEncoder = cachedCodebook.getHuffmanEncoder();
cachedCodebook.getSymbolFrequencies());
} }
/** /**
...@@ -89,8 +88,7 @@ public class SQImageCompressor extends CompressorDecompressorBase implements IIm ...@@ -89,8 +88,7 @@ public class SQImageCompressor extends CompressorDecompressorBase implements IIm
trainAndSaveCodebook(); trainAndSaveCodebook();
} }
final SQCodebook codebook = cacheManager.loadSQCodebook(options.getInputDataInfo().getCacheFileName(), final SQCodebook codebook = cacheManager.loadSQCodebook(options.getInputDataInfo().getCacheFileName(), getCodebookSize());
getCodebookSize());
if (codebook == null) { if (codebook == null) {
throw new ImageCompressionException("Failed to read quantization values from cache file."); throw new ImageCompressionException("Failed to read quantization values from cache file.");
} }
...@@ -123,7 +121,7 @@ public class SQImageCompressor extends CompressorDecompressorBase implements IIm ...@@ -123,7 +121,7 @@ public class SQImageCompressor extends CompressorDecompressorBase implements IIm
reportStatusToListeners("Loading codebook from cache file."); reportStatusToListeners("Loading codebook from cache file.");
quantizer = loadQuantizerFromCache(); quantizer = loadQuantizerFromCache();
huffmanEncoder = createHuffmanEncoder(huffmanSymbols, quantizer.getCodebook().getSymbolFrequencies()); huffmanEncoder = quantizer.getCodebook().getHuffmanEncoder();
reportStatusToListeners("Cached quantizer with huffman coder created."); reportStatusToListeners("Cached quantizer with huffman coder created.");
writeCodebookToOutputStream(quantizer, compressStream); writeCodebookToOutputStream(quantizer, compressStream);
...@@ -139,7 +137,7 @@ public class SQImageCompressor extends CompressorDecompressorBase implements IIm ...@@ -139,7 +137,7 @@ public class SQImageCompressor extends CompressorDecompressorBase implements IIm
reportStatusToListeners(String.format("Training scalar quantizer from middle plane %d.", middlePlaneIndex)); reportStatusToListeners(String.format("Training scalar quantizer from middle plane %d.", middlePlaneIndex));
quantizer = trainScalarQuantizerFromData(middlePlaneData); quantizer = trainScalarQuantizerFromData(middlePlaneData);
huffmanEncoder = createHuffmanEncoder(huffmanSymbols, quantizer.getCodebook().getSymbolFrequencies()); huffmanEncoder = quantizer.getCodebook().getHuffmanEncoder();
stopwatch.stop(); stopwatch.stop();
writeCodebookToOutputStream(quantizer, compressStream); writeCodebookToOutputStream(quantizer, compressStream);
...@@ -175,7 +173,7 @@ public class SQImageCompressor extends CompressorDecompressorBase implements IIm ...@@ -175,7 +173,7 @@ public class SQImageCompressor extends CompressorDecompressorBase implements IIm
quantizer = trainScalarQuantizerFromData(planeData); quantizer = trainScalarQuantizerFromData(planeData);
writeCodebookToOutputStream(quantizer, compressStream); writeCodebookToOutputStream(quantizer, compressStream);
huffmanEncoder = createHuffmanEncoder(huffmanSymbols, quantizer.getCodebook().getSymbolFrequencies()); huffmanEncoder = quantizer.getCodebook().getHuffmanEncoder();
} }
assert (quantizer != null) : "Scalar Quantizer wasn't initialized."; assert (quantizer != null) : "Scalar Quantizer wasn't initialized.";
...@@ -200,7 +198,7 @@ public class SQImageCompressor extends CompressorDecompressorBase implements IIm ...@@ -200,7 +198,7 @@ public class SQImageCompressor extends CompressorDecompressorBase implements IIm
} catch (final Exception e) { } catch (final Exception e) {
throw new ImageCompressionException("Unable to create SCIFIO reader. " + e.getMessage()); throw new ImageCompressionException("Unable to create SCIFIO reader. " + e.getMessage());
} }
int[] trainData = null; final int[] trainData;
if (options.getCodebookType() == CompressionOptions.CodebookType.MiddlePlane) { if (options.getCodebookType() == CompressionOptions.CodebookType.MiddlePlane) {
final int middlePlaneIndex = getMiddlePlaneIndex(); final int middlePlaneIndex = getMiddlePlaneIndex();
......
...@@ -5,6 +5,7 @@ import cz.it4i.qcmp.cache.SqQvcFile; ...@@ -5,6 +5,7 @@ import cz.it4i.qcmp.cache.SqQvcFile;
import cz.it4i.qcmp.compression.exception.ImageDecompressionException; import cz.it4i.qcmp.compression.exception.ImageDecompressionException;
import cz.it4i.qcmp.fileformat.QCMPFileHeader; import cz.it4i.qcmp.fileformat.QCMPFileHeader;
import cz.it4i.qcmp.huffman.HuffmanDecoder; import cz.it4i.qcmp.huffman.HuffmanDecoder;
import cz.it4i.qcmp.huffman.HuffmanTreeBuilder;
import cz.it4i.qcmp.io.InBitStream; import cz.it4i.qcmp.io.InBitStream;
import cz.it4i.qcmp.quantization.scalar.SQCodebook; import cz.it4i.qcmp.quantization.scalar.SQCodebook;
import cz.it4i.qcmp.utilities.Stopwatch; import cz.it4i.qcmp.utilities.Stopwatch;
...@@ -22,8 +23,7 @@ public class SQImageDecompressor extends CompressorDecompressorBase implements I ...@@ -22,8 +23,7 @@ public class SQImageDecompressor extends CompressorDecompressorBase implements I
super(options); super(options);
} }
private SQCodebook readScalarQuantizationValues(final DataInputStream compressedStream, private SQCodebook readSqCodebook(final DataInputStream compressedStream, final int codebookSize) throws ImageDecompressionException {
final int codebookSize) throws ImageDecompressionException {
final int[] quantizationValues = new int[codebookSize]; final int[] quantizationValues = new int[codebookSize];
final long[] symbolFrequencies = new long[codebookSize]; final long[] symbolFrequencies = new long[codebookSize];
try { try {
...@@ -36,7 +36,9 @@ public class SQImageDecompressor extends CompressorDecompressorBase implements I ...@@ -36,7 +36,9 @@ public class SQImageDecompressor extends CompressorDecompressorBase implements I
} catch (final IOException ioEx) { } catch (final IOException ioEx) {
throw new ImageDecompressionException("Unable to read quantization values from compressed stream.", ioEx); throw new ImageDecompressionException("Unable to read quantization values from compressed stream.", ioEx);
} }
return new SQCodebook(quantizationValues, symbolFrequencies); final HuffmanTreeBuilder builder = new HuffmanTreeBuilder(createHuffmanSymbols(codebookSize), symbolFrequencies);
builder.buildHuffmanTree();
return new SQCodebook(quantizationValues, builder.getRoot());
} }
@Override @Override
...@@ -45,7 +47,6 @@ public class SQImageDecompressor extends CompressorDecompressorBase implements I ...@@ -45,7 +47,6 @@ public class SQImageDecompressor extends CompressorDecompressorBase implements I
final QCMPFileHeader header) throws ImageDecompressionException { final QCMPFileHeader header) throws ImageDecompressionException {
final int codebookSize = (int) Math.pow(2, header.getBitsPerCodebookIndex()); final int codebookSize = (int) Math.pow(2, header.getBitsPerCodebookIndex());
final int[] huffmanSymbols = createHuffmanSymbols(codebookSize);
final int planeCountForDecompression = header.getImageSizeZ(); final int planeCountForDecompression = header.getImageSizeZ();
final int planePixelCount = header.getImageSizeX() * header.getImageSizeY(); final int planePixelCount = header.getImageSizeX() * header.getImageSizeY();
...@@ -55,8 +56,8 @@ public class SQImageDecompressor extends CompressorDecompressorBase implements I ...@@ -55,8 +56,8 @@ public class SQImageDecompressor extends CompressorDecompressorBase implements I
if (!header.isCodebookPerPlane()) { if (!header.isCodebookPerPlane()) {
// There is only one codebook. // There is only one codebook.
reportStatusToListeners("Loading single codebook and huffman coder."); reportStatusToListeners("Loading single codebook and huffman coder.");
codebook = readScalarQuantizationValues(compressedStream, codebookSize); codebook = readSqCodebook(compressedStream, codebookSize);
huffmanDecoder = createHuffmanDecoder(huffmanSymbols, codebook.getSymbolFrequencies()); huffmanDecoder = codebook.getHuffmanDecoder();
} }
final Stopwatch stopwatch = new Stopwatch(); final Stopwatch stopwatch = new Stopwatch();
...@@ -64,8 +65,8 @@ public class SQImageDecompressor extends CompressorDecompressorBase implements I ...@@ -64,8 +65,8 @@ public class SQImageDecompressor extends CompressorDecompressorBase implements I
stopwatch.restart(); stopwatch.restart();
if (header.isCodebookPerPlane()) { if (header.isCodebookPerPlane()) {
reportStatusToListeners("Loading plane codebook..."); reportStatusToListeners("Loading plane codebook...");
codebook = readScalarQuantizationValues(compressedStream, codebookSize); codebook = readSqCodebook(compressedStream, codebookSize);
huffmanDecoder = createHuffmanDecoder(huffmanSymbols, codebook.getSymbolFrequencies()); huffmanDecoder = codebook.getHuffmanDecoder();
} }
assert (codebook != null && huffmanDecoder != null); assert (codebook != null && huffmanDecoder != null);
...@@ -110,8 +111,7 @@ public class SQImageDecompressor extends CompressorDecompressorBase implements I ...@@ -110,8 +111,7 @@ public class SQImageDecompressor extends CompressorDecompressorBase implements I
final SqQvcFile codebookCache = (SqQvcFile) codebookCacheFile; final SqQvcFile codebookCache = (SqQvcFile) codebookCacheFile;
cachedCodebook = codebookCache.getCodebook(); cachedCodebook = codebookCache.getCodebook();
cachedHuffmanDecoder = createHuffmanDecoder(createHuffmanSymbols(cachedCodebook.getCodebookSize()), cachedHuffmanDecoder = cachedCodebook.getHuffmanDecoder();
cachedCodebook.getSymbolFrequencies());
} }
@Override @Override
...@@ -119,7 +119,6 @@ public class SQImageDecompressor extends CompressorDecompressorBase implements I ...@@ -119,7 +119,6 @@ public class SQImageDecompressor extends CompressorDecompressorBase implements I
final short[][] buffer, final short[][] buffer,
final QCMPFileHeader header) throws ImageDecompressionException { final QCMPFileHeader header) throws ImageDecompressionException {
final int codebookSize = (int) Math.pow(2, header.getBitsPerCodebookIndex()); final int codebookSize = (int) Math.pow(2, header.getBitsPerCodebookIndex());
final int[] huffmanSymbols = createHuffmanSymbols(codebookSize);
final int planeCountForDecompression = header.getImageSizeZ(); final int planeCountForDecompression = header.getImageSizeZ();
final int planePixelCount = header.getImageSizeX() * header.getImageSizeY(); final int planePixelCount = header.getImageSizeX() * header.getImageSizeY();
...@@ -128,15 +127,15 @@ public class SQImageDecompressor extends CompressorDecompressorBase implements I ...@@ -128,15 +127,15 @@ public class SQImageDecompressor extends CompressorDecompressorBase implements I
HuffmanDecoder huffmanDecoder = null; HuffmanDecoder huffmanDecoder = null;
if (!header.isCodebookPerPlane()) { if (!header.isCodebookPerPlane()) {
// There is only one codebook. // There is only one codebook.
codebook = readScalarQuantizationValues(compressedStream, codebookSize); codebook = readSqCodebook(compressedStream, codebookSize);
huffmanDecoder = createHuffmanDecoder(huffmanSymbols, codebook.getSymbolFrequencies()); huffmanDecoder = codebook.getHuffmanDecoder();
} }
for (int planeIndex = 0; planeIndex < planeCountForDecompression; planeIndex++) { for (int planeIndex = 0; planeIndex < planeCountForDecompression; planeIndex++) {
reportProgressToListeners(planeIndex, planeCountForDecompression, "Decompressing plane %d", planeIndex); reportProgressToListeners(planeIndex, planeCountForDecompression, "Decompressing plane %d", planeIndex);
if (header.isCodebookPerPlane()) { if (header.isCodebookPerPlane()) {
codebook = readScalarQuantizationValues(compressedStream, codebookSize); codebook = readSqCodebook(compressedStream, codebookSize);
huffmanDecoder = createHuffmanDecoder(huffmanSymbols, codebook.getSymbolFrequencies()); huffmanDecoder = codebook.getHuffmanDecoder();
} }
assert (codebook != null && huffmanDecoder != null); assert (codebook != null && huffmanDecoder != null);
......
...@@ -16,7 +16,6 @@ import cz.it4i.qcmp.quantization.vector.VQCodebook; ...@@ -16,7 +16,6 @@ import cz.it4i.qcmp.quantization.vector.VQCodebook;
import cz.it4i.qcmp.quantization.vector.VectorQuantizer; import cz.it4i.qcmp.quantization.vector.VectorQuantizer;
import cz.it4i.qcmp.utilities.Stopwatch; import cz.it4i.qcmp.utilities.Stopwatch;
import cz.it4i.qcmp.utilities.Utils; import cz.it4i.qcmp.utilities.Utils;
import org.jetbrains.annotations.NotNull;
import java.io.DataOutputStream; import java.io.DataOutputStream;
import java.io.IOException; import java.io.IOException;
...@@ -36,8 +35,7 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm ...@@ -36,8 +35,7 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm
public void preloadGlobalCodebook(final IQvcFile codebookCacheFile) { public void preloadGlobalCodebook(final IQvcFile codebookCacheFile) {
final VQCodebook cachedCodebook = ((VqQvcFile) codebookCacheFile).getCodebook(); final VQCodebook cachedCodebook = ((VqQvcFile) codebookCacheFile).getCodebook();
cachedQuantizer = new VectorQuantizer(cachedCodebook); cachedQuantizer = new VectorQuantizer(cachedCodebook);
cachedHuffmanEncoder = createHuffmanEncoder(createHuffmanSymbols(cachedCodebook.getCodebookSize()), cachedHuffmanEncoder = cachedCodebook.getHuffmanEncoder();
cachedCodebook.getVectorFrequencies());
} }
public boolean shouldUseKdTree() { public boolean shouldUseKdTree() {
...@@ -74,6 +72,7 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm ...@@ -74,6 +72,7 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm
*/ */
private void writeQuantizerToCompressStream(final VectorQuantizer quantizer, private void writeQuantizerToCompressStream(final VectorQuantizer quantizer,
final DataOutputStream compressStream) throws ImageCompressionException { final DataOutputStream compressStream) throws ImageCompressionException {
// TODO
final int[][] codebook = quantizer.getCodebookVectors(); final int[][] codebook = quantizer.getCodebookVectors();
try { try {
for (final int[] entry : codebook) { for (final int[] entry : codebook) {
...@@ -143,7 +142,6 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm ...@@ -143,7 +142,6 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm
return compress1D2DVectors(compressStream, true, inputData); return compress1D2DVectors(compressStream, true, inputData);
} }
@NotNull
private long[] compress1D2DVectors(final DataOutputStream compressStream, private long[] compress1D2DVectors(final DataOutputStream compressStream,
final boolean streamMode, final boolean streamMode,
final InputData inputData) throws ImageCompressionException { final InputData inputData) throws ImageCompressionException {
...@@ -166,7 +164,7 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm ...@@ -166,7 +164,7 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm
if (options.getCodebookType() == CompressionOptions.CodebookType.Global) { if (options.getCodebookType() == CompressionOptions.CodebookType.Global) {
reportStatusToListeners("Loading codebook from cache file."); reportStatusToListeners("Loading codebook from cache file.");
quantizer = loadQuantizerFromCache(); quantizer = loadQuantizerFromCache();
huffmanEncoder = createHuffmanEncoder(huffmanSymbols, quantizer.getFrequencies()); huffmanEncoder = quantizer.getCodebook().getHuffmanEncoder();
reportStatusToListeners("Cached quantizer with huffman coder created."); reportStatusToListeners("Cached quantizer with huffman coder created.");
writeQuantizerToCompressStream(quantizer, compressStream); writeQuantizerToCompressStream(quantizer, compressStream);
} else if (options.getCodebookType() == CompressionOptions.CodebookType.MiddlePlane) { } else if (options.getCodebookType() == CompressionOptions.CodebookType.MiddlePlane) {
...@@ -175,7 +173,7 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm ...@@ -175,7 +173,7 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm
final int[][] refPlaneVectors = planeLoader.loadVectorsFromPlaneRange(0, options, final int[][] refPlaneVectors = planeLoader.loadVectorsFromPlaneRange(0, options,
Utils.singlePlaneRange(getMiddlePlaneIndex())); Utils.singlePlaneRange(getMiddlePlaneIndex()));
quantizer = trainVectorQuantizerFromPlaneVectors(refPlaneVectors); quantizer = trainVectorQuantizerFromPlaneVectors(refPlaneVectors);
huffmanEncoder = createHuffmanEncoder(huffmanSymbols, quantizer.getFrequencies()); huffmanEncoder = quantizer.getCodebook().getHuffmanEncoder();
stopwatch.stop(); stopwatch.stop();
reportStatusToListeners("Middle plane codebook created in: " + stopwatch.getElapsedTimeString()); reportStatusToListeners("Middle plane codebook created in: " + stopwatch.getElapsedTimeString());
writeQuantizerToCompressStream(quantizer, compressStream); writeQuantizerToCompressStream(quantizer, compressStream);
...@@ -209,7 +207,7 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm ...@@ -209,7 +207,7 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm
if (!streamMode && !hasGeneralQuantizer) { if (!streamMode && !hasGeneralQuantizer) {
reportStatusToListeners(String.format("Training vector quantizer from plane %d.", planeIndex)); reportStatusToListeners(String.format("Training vector quantizer from plane %d.", planeIndex));
quantizer = trainVectorQuantizerFromPlaneVectors(planeVectors); quantizer = trainVectorQuantizerFromPlaneVectors(planeVectors);
huffmanEncoder = createHuffmanEncoder(huffmanSymbols, quantizer.getFrequencies()); huffmanEncoder = quantizer.getCodebook().getHuffmanEncoder();
writeQuantizerToCompressStream(quantizer, compressStream); writeQuantizerToCompressStream(quantizer, compressStream);
} }
...@@ -258,7 +256,6 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm ...@@ -258,7 +256,6 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm
final InputData inputData) throws ImageCompressionException { final InputData inputData) throws ImageCompressionException {
assert (options.getCodebookType() == CompressionOptions.CodebookType.Global); assert (options.getCodebookType() == CompressionOptions.CodebookType.Global);
final IPlaneLoader planeLoader; final IPlaneLoader planeLoader;
final int[] huffmanSymbols = createHuffmanSymbols(getCodebookSize());
try { try {
planeLoader = PlaneLoaderFactory.getPlaneLoaderForInputFile(inputData); planeLoader = PlaneLoaderFactory.getPlaneLoaderForInputFile(inputData);
planeLoader.setWorkerCount(options.getWorkerCount()); planeLoader.setWorkerCount(options.getWorkerCount());
...@@ -286,7 +283,7 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm ...@@ -286,7 +283,7 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm
final VectorQuantizer quantizer = (cachedQuantizer != null) ? cachedQuantizer : loadQuantizerFromCache(); final VectorQuantizer quantizer = (cachedQuantizer != null) ? cachedQuantizer : loadQuantizerFromCache();
final HuffmanEncoder huffmanEncoder = (cachedHuffmanEncoder != null) final HuffmanEncoder huffmanEncoder = (cachedHuffmanEncoder != null)
? cachedHuffmanEncoder ? cachedHuffmanEncoder
: createHuffmanEncoder(huffmanSymbols, quantizer.getFrequencies()); : quantizer.getCodebook().getHuffmanEncoder();
if (!streamMode) if (!streamMode)
writeQuantizerToCompressStream(quantizer, compressStream); writeQuantizerToCompressStream(quantizer, compressStream);
...@@ -387,8 +384,7 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm ...@@ -387,8 +384,7 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm
codebookTrainer.findOptimalCodebook(vqCodebook -> { codebookTrainer.findOptimalCodebook(vqCodebook -> {
try { try {
assert ((vqCodebook.getCodebookSize() == vqCodebook.getVectors().length) && assert (vqCodebook.getCodebookSize() == vqCodebook.getVectors().length)
(vqCodebook.getCodebookSize() == vqCodebook.getVectorFrequencies().length))
: "Codebook size, Vector count, Frequencies count mismatch"; : "Codebook size, Vector count, Frequencies count mismatch";
qcm.saveCodebook(options.getInputDataInfo().getCacheFileName(), vqCodebook); qcm.saveCodebook(options.getInputDataInfo().getCacheFileName(), vqCodebook);
} catch (final IOException e) { } catch (final IOException e) {
......
...@@ -7,6 +7,7 @@ import cz.it4i.qcmp.data.*; ...@@ -7,6 +7,7 @@ import cz.it4i.qcmp.data.*;
import cz.it4i.qcmp.fileformat.QCMPFileHeader; import cz.it4i.qcmp.fileformat.QCMPFileHeader;
import cz.it4i.qcmp.fileformat.QuantizationType; import cz.it4i.qcmp.fileformat.QuantizationType;
import cz.it4i.qcmp.huffman.HuffmanDecoder; import cz.it4i.qcmp.huffman.HuffmanDecoder;
import cz.it4i.qcmp.huffman.HuffmanTreeBuilder;
import cz.it4i.qcmp.io.InBitStream; import cz.it4i.qcmp.io.InBitStream;
import cz.it4i.qcmp.quantization.vector.VQCodebook; import cz.it4i.qcmp.quantization.vector.VQCodebook;
import cz.it4i.qcmp.utilities.Stopwatch; import cz.it4i.qcmp.utilities.Stopwatch;
...@@ -36,8 +37,8 @@ public class VQImageDecompressor extends CompressorDecompressorBase implements I ...@@ -36,8 +37,8 @@ public class VQImageDecompressor extends CompressorDecompressorBase implements I
} }
private long calculatePlaneVectorCount(final QCMPFileHeader header) { private long calculatePlaneVectorCount(final QCMPFileHeader header) {
final int vectorXCount = (int) Math.ceil((double) header.getImageSizeX() / (double) header.getVectorSizeX()); final long vectorXCount = (long) Math.ceil((double) header.getImageSizeX() / (double) header.getVectorSizeX());
final int vectorYCount = (int) Math.ceil((double) header.getImageSizeY() / (double) header.getVectorSizeY()); final long vectorYCount = (long) Math.ceil((double) header.getImageSizeY() / (double) header.getVectorSizeY());
// Number of vectors per plane. // Number of vectors per plane.
return (vectorXCount * vectorYCount); return (vectorXCount * vectorYCount);
} }
...@@ -62,8 +63,11 @@ public class VQImageDecompressor extends CompressorDecompressorBase implements I ...@@ -62,8 +63,11 @@ public class VQImageDecompressor extends CompressorDecompressorBase implements I
throw new ImageDecompressionException("Unable to read quantization values from compressed stream.", ioEx); throw new ImageDecompressionException("Unable to read quantization values from compressed stream.", ioEx);
} }
final HuffmanTreeBuilder builder = new HuffmanTreeBuilder(createHuffmanSymbols(codebookSize), frequencies);
builder.buildHuffmanTree();
// We don't care about vector dimensions in here. // We don't care about vector dimensions in here.
return new VQCodebook(new V3i(0), codebookVectors, frequencies); return new VQCodebook(new V3i(0), codebookVectors, builder.getRoot());
} }
@Override @Override
...@@ -72,8 +76,7 @@ public class VQImageDecompressor extends CompressorDecompressorBase implements I ...@@ -72,8 +76,7 @@ public class VQImageDecompressor extends CompressorDecompressorBase implements I
final VqQvcFile codebookCache = (VqQvcFile) codebookCacheFile; final VqQvcFile codebookCache = (VqQvcFile) codebookCacheFile;
cachedCodebook = codebookCache.getCodebook(); cachedCodebook = codebookCache.getCodebook();
cachedHuffmanDecoder = createHuffmanDecoder(createHuffmanSymbols(cachedCodebook.getCodebookSize()), cachedHuffmanDecoder = cachedCodebook.getHuffmanDecoder();
cachedCodebook.getVectorFrequencies());
} }
...@@ -118,21 +121,19 @@ public class VQImageDecompressor extends CompressorDecompressorBase implements I ...@@ -118,21 +121,19 @@ public class VQImageDecompressor extends CompressorDecompressorBase implements I
final int planeCountForDecompression = header.getImageSizeZ(); final int planeCountForDecompression = header.getImageSizeZ();
final long planeVectorCount = calculatePlaneVectorCount(header); final long planeVectorCount = calculatePlaneVectorCount(header);
final V2i qVector = new V2i(header.getVectorSizeX(), header.getVectorSizeY()); final V2i qVector = new V2i(header.getVectorSizeX(), header.getVectorSizeY());
final int[] huffmanSymbols = createHuffmanSymbols(codebookSize);
VQCodebook codebook = null; VQCodebook codebook = null;
HuffmanDecoder huffmanDecoder = null; HuffmanDecoder huffmanDecoder = null;
if (!header.isCodebookPerPlane()) { if (!header.isCodebookPerPlane()) {
// There is only one codebook. // There is only one codebook.
codebook = readCodebook(compressedStream, codebookSize, vectorSize); codebook = readCodebook(compressedStream, codebookSize, vectorSize);
huffmanDecoder = createHuffmanDecoder(huffmanSymbols, codebook.getVectorFrequencies()); huffmanDecoder = codebook.getHuffmanDecoder();
} }
for (int planeIndex = 0; planeIndex < planeCountForDecompression; planeIndex++) { for (int planeIndex = 0; planeIndex < planeCountForDecompression; planeIndex++) {
if (header.isCodebookPerPlane()) { if (header.isCodebookPerPlane()) {
codebook = readCodebook(compressedStream, codebookSize, vectorSize); codebook = readCodebook(compressedStream, codebookSize, vectorSize);
huffmanDecoder = createHuffmanDecoder(huffmanSymbols, codebook.getVectorFrequencies()); huffmanDecoder = codebook.getHuffmanDecoder();
} }
assert (codebook != null && huffmanDecoder != null); assert (codebook != null && huffmanDecoder != null);
...@@ -239,7 +240,7 @@ public class VQImageDecompressor extends CompressorDecompressorBase implements I ...@@ -239,7 +240,7 @@ public class VQImageDecompressor extends CompressorDecompressorBase implements I
final VQCodebook codebook = readCodebook(compressedStream, codebookSize, vectorSize); final VQCodebook codebook = readCodebook(compressedStream, codebookSize, vectorSize);
final HuffmanDecoder huffmanDecoder = createHuffmanDecoder(huffmanSymbols, codebook.getVectorFrequencies()); final HuffmanDecoder huffmanDecoder = codebook.getHuffmanDecoder();
final int voxelLayerCount = VQImageCompressor.calculateVoxelLayerCount(header.getImageSizeZ(), header.getVectorSizeZ()); final int voxelLayerCount = VQImageCompressor.calculateVoxelLayerCount(header.getImageSizeZ(), header.getVectorSizeZ());
final Stopwatch stopwatch = new Stopwatch(); final Stopwatch stopwatch = new Stopwatch();
......
...@@ -2,6 +2,7 @@ package cz.it4i.qcmp.quantization.scalar; ...@@ -2,6 +2,7 @@ package cz.it4i.qcmp.quantization.scalar;
import cz.it4i.qcmp.U16; import cz.it4i.qcmp.U16;
import cz.it4i.qcmp.compression.listeners.IStatusListener; import cz.it4i.qcmp.compression.listeners.IStatusListener;
import cz.it4i.qcmp.huffman.HuffmanTreeBuilder;
import cz.it4i.qcmp.quantization.QTrainIteration; import cz.it4i.qcmp.quantization.QTrainIteration;
import cz.it4i.qcmp.utilities.MinMaxResult; import cz.it4i.qcmp.utilities.MinMaxResult;
import cz.it4i.qcmp.utilities.Stopwatch; import cz.it4i.qcmp.utilities.Stopwatch;
...@@ -258,7 +259,9 @@ public class LloydMaxU16ScalarQuantization { ...@@ -258,7 +259,9 @@ public class LloydMaxU16ScalarQuantization {
} }
public SQCodebook getCodebook() { public SQCodebook getCodebook() {
return new SQCodebook(centroids, frequencies); final HuffmanTreeBuilder builder = new HuffmanTreeBuilder(codebookSize, frequencies);
builder.buildHuffmanTree();
return new SQCodebook(centroids, builder.getRoot());
} }
} }
package cz.it4i.qcmp.quantization.vector; package cz.it4i.qcmp.quantization.vector;
import cz.it4i.qcmp.data.V3i; import cz.it4i.qcmp.data.V3i;
import cz.it4i.qcmp.huffman.HuffmanTreeBuilder;
public class LBGResult { public class LBGResult {
...@@ -23,7 +24,9 @@ public class LBGResult { ...@@ -23,7 +24,9 @@ public class LBGResult {
} }
public VQCodebook getCodebook() { public VQCodebook getCodebook() {
return new VQCodebook(vectorDims, codebookVectors, frequencies); final HuffmanTreeBuilder builder = new HuffmanTreeBuilder(codebookVectors.length, frequencies);
builder.buildHuffmanTree();
return new VQCodebook(vectorDims, codebookVectors, builder.getRoot());
} }
public double getAverageMse() { public double getAverageMse() {
......
...@@ -3,6 +3,7 @@ package cz.it4i.qcmp.quantization.vector; ...@@ -3,6 +3,7 @@ package cz.it4i.qcmp.quantization.vector;
import cz.it4i.qcmp.U16; import cz.it4i.qcmp.U16;
import cz.it4i.qcmp.compression.listeners.IStatusListener; import cz.it4i.qcmp.compression.listeners.IStatusListener;
import cz.it4i.qcmp.data.V3i; import cz.it4i.qcmp.data.V3i;
import cz.it4i.qcmp.huffman.HuffmanTreeBuilder;
import cz.it4i.qcmp.utilities.Stopwatch; import cz.it4i.qcmp.utilities.Stopwatch;
import cz.it4i.qcmp.utilities.Utils; import cz.it4i.qcmp.utilities.Utils;
...@@ -207,9 +208,7 @@ public class LBGVectorQuantizer { ...@@ -207,9 +208,7 @@ public class LBGVectorQuantizer {
workers[wId] = new Thread(() -> { workers[wId] = new Thread(() -> {
final long[] workerFrequencies = new long[codebook.length]; final long[] workerFrequencies = new long[codebook.length];
final VectorQuantizer quantizer = new VectorQuantizer(new VQCodebook(vectorDimensions, final VectorQuantizer quantizer = new VectorQuantizer(new VQCodebook(vectorDimensions, codebook, null));
codebook,
frequencies));
double threadMse = 0.0; double threadMse = 0.0;
int[] vector; int[] vector;
...@@ -245,9 +244,7 @@ public class LBGVectorQuantizer { ...@@ -245,9 +244,7 @@ public class LBGVectorQuantizer {
} }
mse = _mse / (double) workerCount; mse = _mse / (double) workerCount;
} else { } else {
final VectorQuantizer quantizer = new VectorQuantizer(new VQCodebook(vectorDimensions, final VectorQuantizer quantizer = new VectorQuantizer(new VQCodebook(vectorDimensions, codebook, null));
codebook,
frequencies));
int qIndex; int qIndex;
int[] qVector; int[] qVector;
for (final TrainingVector trV : trainingVectors) { for (final TrainingVector trV : trainingVectors) {
...@@ -412,9 +409,10 @@ public class LBGVectorQuantizer { ...@@ -412,9 +409,10 @@ public class LBGVectorQuantizer {
final long[] codebookFrequencies = new long[codebook.length]; final long[] codebookFrequencies = new long[codebook.length];
System.arraycopy(frequencies, 0, codebookFrequencies, 0, codebook.length); System.arraycopy(frequencies, 0, codebookFrequencies, 0, codebook.length);
codebookFoundCallback.process(new VQCodebook(vectorDimensions,
learningCodebookToCodebook(codebook), final HuffmanTreeBuilder builder = new HuffmanTreeBuilder(codebook.length, frequencies);
codebookFrequencies)); builder.buildHuffmanTree();
codebookFoundCallback.process(new VQCodebook(vectorDimensions, learningCodebookToCodebook(codebook), builder.getRoot()));
} }
} }
return codebook; return codebook;
......
...@@ -32,6 +32,7 @@ public class VQCodebook extends Codebook { ...@@ -32,6 +32,7 @@ public class VQCodebook extends Codebook {
* @param huffmanRoot Root of the huffman tree. * @param huffmanRoot Root of the huffman tree.
*/ */
public VQCodebook(final V3i vectorDims, final int[][] vectors, final HuffmanNode huffmanRoot) { public VQCodebook(final V3i vectorDims, final int[][] vectors, final HuffmanNode huffmanRoot) {
super(huffmanRoot);
this.vectorDims = vectorDims; this.vectorDims = vectorDims;
this.vectors = vectors; this.vectors = vectors;
this.codebookSize = vectors.length; this.codebookSize = vectors.length;
......
...@@ -11,16 +11,15 @@ public class VectorQuantizer { ...@@ -11,16 +11,15 @@ public class VectorQuantizer {
private final VectorDistanceMetric metric = VectorDistanceMetric.Euclidean; private final VectorDistanceMetric metric = VectorDistanceMetric.Euclidean;
private final int[][] codebookVectors; private final int[][] codebookVectors;
private final int vectorSize; private final int vectorSize;
private final long[] frequencies; private final VQCodebook codebook;
// private final KDTree kdTree; // private final KDTree kdTree;
public VectorQuantizer(final VQCodebook codebook) { public VectorQuantizer(final VQCodebook codebook) {
this.codebook = codebook;
this.codebookVectors = codebook.getVectors(); this.codebookVectors = codebook.getVectors();
this.vectorSize = codebook.getVectors()[0].length; this.vectorSize = codebook.getVectors()[0].length;
this.frequencies = codebook.getVectorFrequencies(); // kdTree = new KDTreeBuilder(this.vectorSize, 8).buildTree(codebook.getVectors());
// kdTree = new KDTreeBuilder(this.vectorSize, 8).buildTree(codebook.getVectors());
} }
public int[] quantize(final int[] dataVector) { public int[] quantize(final int[] dataVector) {
...@@ -167,8 +166,8 @@ public class VectorQuantizer { ...@@ -167,8 +166,8 @@ public class VectorQuantizer {
return codebookVectors; return codebookVectors;
} }
public long[] getFrequencies() { public VQCodebook getCodebook() {
return frequencies; return codebook;
} }
} }
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment