Skip to content
Snippets Groups Projects
Commit 13724779 authored by Vojtech Moravec's avatar Vojtech Moravec
Browse files

Support stream compression/decompression of block vectors.

parent 4a227e80
No related branches found
No related tags found
No related merge requests found
...@@ -102,14 +102,12 @@ public abstract class CompressorDecompressorBase { ...@@ -102,14 +102,12 @@ public abstract class CompressorDecompressorBase {
return huffman; return huffman;
} }
protected int[] getPlaneIndicesForCompression() { protected int[] getPlaneIndicesForCompression(final InputData inputData) {
if (inputData.isPlaneIndexSet()) {
final InputData ifi = options.getInputDataInfo(); return new int[]{inputData.getPlaneIndex()};
if (ifi.isPlaneIndexSet()) { } else if (inputData.isPlaneRangeSet()) {
return new int[]{ifi.getPlaneIndex()}; final int from = inputData.getPlaneRange().getFrom();
} else if (ifi.isPlaneRangeSet()) { final int count = inputData.getPlaneRange().getTo() - from;
final int from = ifi.getPlaneRange().getFrom();
final int count = ifi.getPlaneRange().getTo() - from;
int[] indices = new int[count + 1]; int[] indices = new int[count + 1];
for (int i = 0; i <= count; i++) { for (int i = 0; i <= count; i++) {
...@@ -117,7 +115,7 @@ public abstract class CompressorDecompressorBase { ...@@ -117,7 +115,7 @@ public abstract class CompressorDecompressorBase {
} }
return indices; return indices;
} else { } else {
return generateAllPlaneIndices(ifi.getDimensions().getZ()); return generateAllPlaneIndices(inputData.getDimensions().getZ());
} }
} }
......
...@@ -146,7 +146,7 @@ public class SQImageCompressor extends CompressorDecompressorBase implements IIm ...@@ -146,7 +146,7 @@ public class SQImageCompressor extends CompressorDecompressorBase implements IIm
reportStatusToListeners("Middle plane codebook with huffman coder created in: " + stopwatch.getElapsedTimeString()); reportStatusToListeners("Middle plane codebook with huffman coder created in: " + stopwatch.getElapsedTimeString());
} }
final int[] planeIndices = getPlaneIndicesForCompression(); final int[] planeIndices = getPlaneIndicesForCompression(options.getInputDataInfo());
long[] planeDataSizes = new long[planeIndices.length]; long[] planeDataSizes = new long[planeIndices.length];
int planeCounter = 0; int planeCounter = 0;
for (final int planeIndex : planeIndices) { for (final int planeIndex : planeIndices) {
...@@ -201,7 +201,7 @@ public class SQImageCompressor extends CompressorDecompressorBase implements IIm ...@@ -201,7 +201,7 @@ public class SQImageCompressor extends CompressorDecompressorBase implements IIm
} }
} else if (inputDataInfo.isPlaneRangeSet()) { } else if (inputDataInfo.isPlaneRangeSet()) {
reportStatusToListeners("Loading plane range data."); reportStatusToListeners("Loading plane range data.");
final int[] planes = getPlaneIndicesForCompression(); final int[] planes = getPlaneIndicesForCompression(options.getInputDataInfo());
try { try {
trainData = planeLoader.loadPlanesU16Data(planes); trainData = planeLoader.loadPlanesU16Data(planes);
} catch (IOException e) { } catch (IOException e) {
......
...@@ -117,7 +117,7 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm ...@@ -117,7 +117,7 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm
return compressVoxels(compressStream, false, options.getInputDataInfo()); return compressVoxels(compressStream, false, options.getInputDataInfo());
} }
assert (options.getQuantizationVector().getZ() == 1); assert (options.getQuantizationVector().getZ() == 1);
return compress1D2DVectors(compressStream, false); return compress1D2DVectors(compressStream, false, options.getInputDataInfo());
} }
@Override @Override
...@@ -126,52 +126,54 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm ...@@ -126,52 +126,54 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm
return compressVoxels(compressStream, true, inputData); return compressVoxels(compressStream, true, inputData);
} }
assert (options.getQuantizationVector().getZ() == 1); assert (options.getQuantizationVector().getZ() == 1);
return compress1D2DVectors(compressStream, true); return compress1D2DVectors(compressStream, true, inputData);
} }
@NotNull @NotNull
private long[] compress1D2DVectors(final DataOutputStream compressStream, final boolean streamMode) throws ImageCompressionException { private long[] compress1D2DVectors(final DataOutputStream compressStream,
final boolean streamMode,
final InputData inputData) throws ImageCompressionException {
final InputData inputDataInfo = options.getInputDataInfo();
Stopwatch stopwatch = new Stopwatch(); Stopwatch stopwatch = new Stopwatch();
final boolean hasGeneralQuantizer = options.getCodebookType() != CompressionOptions.CodebookType.Individual; final boolean hasGeneralQuantizer = options.getCodebookType() != CompressionOptions.CodebookType.Individual;
final IPlaneLoader planeLoader; final IPlaneLoader planeLoader;
final int[] huffmanSymbols = createHuffmanSymbols(getCodebookSize()); final int[] huffmanSymbols = createHuffmanSymbols(getCodebookSize());
try { try {
planeLoader = PlaneLoaderFactory.getPlaneLoaderForInputFile(inputDataInfo); planeLoader = PlaneLoaderFactory.getPlaneLoaderForInputFile(inputData);
} catch (Exception e) { } catch (Exception e) {
throw new ImageCompressionException("Unable to create plane reader. " + e.getMessage()); throw new ImageCompressionException("Unable to create plane reader. " + e.getMessage());
} }
VectorQuantizer quantizer = null;
Huffman huffman = null; VectorQuantizer quantizer = cachedQuantizer;
Huffman huffman = cachedHuffman;
if (options.getCodebookType() == CompressionOptions.CodebookType.Global) { assert (!streamMode || ((quantizer != null) && (huffman != null)));
reportStatusToListeners("Loading codebook from cache file.");
quantizer = loadQuantizerFromCache(); if (!streamMode) {
huffman = createHuffmanCoder(huffmanSymbols, quantizer.getFrequencies()); if (options.getCodebookType() == CompressionOptions.CodebookType.Global) {
reportStatusToListeners("Cached quantizer with huffman coder created."); reportStatusToListeners("Loading codebook from cache file.");
if (!streamMode) quantizer = loadQuantizerFromCache();
writeQuantizerToCompressStream(quantizer, compressStream); huffman = createHuffmanCoder(huffmanSymbols, quantizer.getFrequencies());
} else if (options.getCodebookType() == CompressionOptions.CodebookType.MiddlePlane) { reportStatusToListeners("Cached quantizer with huffman coder created.");
stopwatch.restart(); } else if (options.getCodebookType() == CompressionOptions.CodebookType.MiddlePlane) {
reportStatusToListeners("Training vector quantizer from middle plane."); stopwatch.restart();
final int[][] refPlaneVectors = planeLoader.loadVectorsFromPlaneRange(options, Utils.singlePlaneRange(getMiddlePlaneIndex())); reportStatusToListeners("Training vector quantizer from middle plane.");
quantizer = trainVectorQuantizerFromPlaneVectors(refPlaneVectors); final int[][] refPlaneVectors = planeLoader.loadVectorsFromPlaneRange(options,
huffman = createHuffmanCoder(huffmanSymbols, quantizer.getFrequencies()); Utils.singlePlaneRange(getMiddlePlaneIndex()));
if (!streamMode) quantizer = trainVectorQuantizerFromPlaneVectors(refPlaneVectors);
writeQuantizerToCompressStream(quantizer, compressStream); huffman = createHuffmanCoder(huffmanSymbols, quantizer.getFrequencies());
stopwatch.stop(); stopwatch.stop();
reportStatusToListeners("Middle plane codebook created in: " + stopwatch.getElapsedTimeString()); reportStatusToListeners("Middle plane codebook created in: " + stopwatch.getElapsedTimeString());
}
writeQuantizerToCompressStream(quantizer, compressStream);
} }
final int[] planeIndices = getPlaneIndicesForCompression(); final int[] planeIndices = getPlaneIndicesForCompression(inputData);
if (streamMode) { if (streamMode) {
try { try {
final V3i imageDims = options.getInputDataInfo().getDimensions();
// Image dimensions // Image dimensions
compressStream.writeShort(imageDims.getX()); compressStream.writeShort(inputData.getDimensions().getX());
compressStream.writeShort(imageDims.getY()); compressStream.writeShort(inputData.getDimensions().getY());
compressStream.writeShort(imageDims.getZ()); compressStream.writeShort(inputData.getDimensions().getZ());
// Write voxel layer in stream mode. // Write voxel layer in stream mode.
compressStream.writeShort(planeIndices.length); compressStream.writeShort(planeIndices.length);
...@@ -189,18 +191,15 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm ...@@ -189,18 +191,15 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm
final int[][] planeVectors = planeLoader.loadVectorsFromPlaneRange(options, Utils.singlePlaneRange(planeIndex)); final int[][] planeVectors = planeLoader.loadVectorsFromPlaneRange(options, Utils.singlePlaneRange(planeIndex));
if (!hasGeneralQuantizer) { if (!streamMode && !hasGeneralQuantizer) {
reportStatusToListeners(String.format("Training vector quantizer from plane %d.", planeIndex)); reportStatusToListeners(String.format("Training vector quantizer from plane %d.", planeIndex));
quantizer = trainVectorQuantizerFromPlaneVectors(planeVectors); quantizer = trainVectorQuantizerFromPlaneVectors(planeVectors);
huffman = createHuffmanCoder(huffmanSymbols, quantizer.getFrequencies()); huffman = createHuffmanCoder(huffmanSymbols, quantizer.getFrequencies());
if (!streamMode) writeQuantizerToCompressStream(quantizer, compressStream);
writeQuantizerToCompressStream(quantizer, compressStream);
} }
assert (quantizer != null);
// Use BestBinFirst KDTree for codebook lookup. // Use BestBinFirst KDTree for codebook lookup.
// final int[] indices = quantizer.quantizeIntoIndicesUsingKDTree(planeVectors, options.getWorkerCount()); // final int[] indices = quantizer.quantizeIntoIndicesUsingKDTree(planeVectors, options.getWorkerCount());
// Use BruteForce for codebook lookup. // Use BruteForce for codebook lookup.
final int[] indices = quantizer.quantizeIntoIndices(planeVectors, options.getWorkerCount()); final int[] indices = quantizer.quantizeIntoIndices(planeVectors, options.getWorkerCount());
...@@ -276,7 +275,6 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm ...@@ -276,7 +275,6 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm
return (datasetPlaneCount / voxelDepth); return (datasetPlaneCount / voxelDepth);
} }
// TODO(Moravec): Remove dependencies on instance variables to enable multi-thread usage.
public long[] compressVoxels(final DataOutputStream compressStream, public long[] compressVoxels(final DataOutputStream compressStream,
final boolean streamMode, final boolean streamMode,
final InputData inputData) throws ImageCompressionException { final InputData inputData) throws ImageCompressionException {
...@@ -325,10 +323,7 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm ...@@ -325,10 +323,7 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm
final int toZ = (voxelLayerIndex == voxelLayerCount - 1) final int toZ = (voxelLayerIndex == voxelLayerCount - 1)
? inputData.getDimensions().getZ() ? inputData.getDimensions().getZ()
: (voxelLayerDepth + (voxelLayerIndex * voxelLayerDepth)); : (voxelLayerDepth + (voxelLayerIndex * voxelLayerDepth));
assert (toZ >= fromZ);
if (toZ < fromZ) {
System.err.println("@Wrong range");
}
final Range<Integer> voxelLayerRange = new Range<>(fromZ, toZ); final Range<Integer> voxelLayerRange = new Range<>(fromZ, toZ);
......
package azgracompress.compression; package azgracompress.compression;
import azgracompress.cache.ICacheFile; import azgracompress.cache.ICacheFile;
import azgracompress.cache.SQCacheFile;
import azgracompress.cache.VQCacheFile; import azgracompress.cache.VQCacheFile;
import azgracompress.compression.exception.ImageDecompressionException; import azgracompress.compression.exception.ImageDecompressionException;
import azgracompress.data.*; import azgracompress.data.*;
...@@ -194,6 +193,49 @@ public class VQImageDecompressor extends CompressorDecompressorBase implements I ...@@ -194,6 +193,49 @@ public class VQImageDecompressor extends CompressorDecompressorBase implements I
} }
} }
@SuppressWarnings("DuplicatedCode")
public void decompressStreamModelImpl(DataInputStream compressedStream,
QCMPFileHeader header,
DecompressCallback callback) throws ImageDecompressionException {
assert (cachedCodebook != null && cachedHuffman != null);
assert (header.getVectorSizeZ() == 1);
final int planeCountForDecompression = header.getImageSizeZ();
final long planeVectorCount = calculatePlaneVectorCount(header);
final V2i qVector = new V2i(header.getVectorSizeX(), header.getVectorSizeY());
final int vectorSize = qVector.multiplyTogether();
for (int planeIndex = 0; planeIndex < planeCountForDecompression; planeIndex++) {
final int planeDataSize = (int) header.getPlaneDataSizes()[planeIndex];
try (InBitStream inBitStream = new InBitStream(compressedStream,
header.getBitsPerCodebookIndex(),
planeDataSize)) {
inBitStream.readToBuffer();
inBitStream.setAllowReadFromUnderlyingStream(false);
int[][] decompressedVectors = new int[(int) planeVectorCount][vectorSize];
int huffmanIndex;
for (int vecIndex = 0; vecIndex < planeVectorCount; vecIndex++) {
huffmanIndex = decodeHuffmanSymbol(cachedHuffman, inBitStream);
System.arraycopy(cachedCodebook.getVectors()[huffmanIndex], 0, decompressedVectors[vecIndex], 0, vectorSize);
}
final Block decompressedPlane = reconstructImageFromQuantizedVectors(decompressedVectors, qVector, header.getImageDims());
callback.process(decompressedPlane, planeIndex);
} catch (Exception ex) {
throw new ImageDecompressionException("VQImageDecompressor::decompressToBuffer() - Unable to read indices from " +
"InBitStream.",
ex);
}
reportProgressToListeners(planeIndex, planeCountForDecompression,
"Decompressed plane %d.", planeIndex);
}
}
@Override @Override
public void decompressToBuffer(DataInputStream compressedStream, public void decompressToBuffer(DataInputStream compressedStream,
...@@ -370,21 +412,34 @@ public class VQImageDecompressor extends CompressorDecompressorBase implements I ...@@ -370,21 +412,34 @@ public class VQImageDecompressor extends CompressorDecompressorBase implements I
@Override @Override
public short[] decompressStreamMode(final DataInputStream compressedStream, public short[] decompressStreamMode(final DataInputStream compressedStream,
final QCMPFileHeader header) throws ImageDecompressionException { final QCMPFileHeader header) throws ImageDecompressionException {
// TODO(Moravec): Implement missing quantization type.
assert (header.getQuantizationType() == QuantizationType.Vector3D);
final short[] buffer = new short[(int) header.getImageDims().multiplyTogether()]; final short[] buffer = new short[(int) header.getImageDims().multiplyTogether()];
final V3i voxelDim = new V3i(header.getVectorSizeX(), header.getVectorSizeY(), header.getVectorSizeZ()); if (header.getQuantizationType() == QuantizationType.Vector3D) {
final V3i voxelDim = new V3i(header.getVectorSizeX(), header.getVectorSizeY(), header.getVectorSizeZ());
decompressVoxelsStreamModeImpl(compressedStream, header, (voxel, voxelData, planeOffset) -> { decompressVoxelsStreamModeImpl(compressedStream, header, (voxel, voxelData, planeOffset) -> {
final ImageU16Dataset currentVoxelLayer = voxel.reconstructFromVoxelsToDataset(voxelDim, voxelData); final ImageU16Dataset currentVoxelLayer = voxel.reconstructFromVoxelsToDataset(voxelDim, voxelData);
int offset = planeOffset * (voxelDim.getX() * voxelDim.getY()); int offset = planeOffset * (voxelDim.getX() * voxelDim.getY());
for (int layer = 0; layer < voxel.getDims().getZ(); layer++) { for (int layer = 0; layer < voxel.getDims().getZ(); layer++) {
final short[] voxelLayerData = currentVoxelLayer.getPlaneData(layer); final short[] voxelLayerData = currentVoxelLayer.getPlaneData(layer);
System.arraycopy(voxelLayerData, 0, buffer, offset, voxelLayerData.length); System.arraycopy(voxelLayerData, 0, buffer, offset, voxelLayerData.length);
offset += voxelLayerData.length; offset += voxelLayerData.length;
} }
}); });
return buffer;
} else {
final int planePixelCount = header.getImageDims().toV2i().multiplyTogether();
decompressStreamModelImpl(compressedStream, header, (imageBlock, planeIndex) -> {
final int offset = planePixelCount * planeIndex;
final int[] data = imageBlock.getData();
for (int i = 0; i < planePixelCount; i++) {
buffer[offset + i] = (short) data[i];
}
});
}
return buffer; return buffer;
} }
} }
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment