Skip to content
Snippets Groups Projects
Commit 13724779 authored by Vojtech Moravec's avatar Vojtech Moravec
Browse files

Support stream compression/decompression of block vectors.

parent 4a227e80
Branches
No related tags found
No related merge requests found
......@@ -102,14 +102,12 @@ public abstract class CompressorDecompressorBase {
return huffman;
}
protected int[] getPlaneIndicesForCompression() {
final InputData ifi = options.getInputDataInfo();
if (ifi.isPlaneIndexSet()) {
return new int[]{ifi.getPlaneIndex()};
} else if (ifi.isPlaneRangeSet()) {
final int from = ifi.getPlaneRange().getFrom();
final int count = ifi.getPlaneRange().getTo() - from;
protected int[] getPlaneIndicesForCompression(final InputData inputData) {
if (inputData.isPlaneIndexSet()) {
return new int[]{inputData.getPlaneIndex()};
} else if (inputData.isPlaneRangeSet()) {
final int from = inputData.getPlaneRange().getFrom();
final int count = inputData.getPlaneRange().getTo() - from;
int[] indices = new int[count + 1];
for (int i = 0; i <= count; i++) {
......@@ -117,7 +115,7 @@ public abstract class CompressorDecompressorBase {
}
return indices;
} else {
return generateAllPlaneIndices(ifi.getDimensions().getZ());
return generateAllPlaneIndices(inputData.getDimensions().getZ());
}
}
......
......@@ -146,7 +146,7 @@ public class SQImageCompressor extends CompressorDecompressorBase implements IIm
reportStatusToListeners("Middle plane codebook with huffman coder created in: " + stopwatch.getElapsedTimeString());
}
final int[] planeIndices = getPlaneIndicesForCompression();
final int[] planeIndices = getPlaneIndicesForCompression(options.getInputDataInfo());
long[] planeDataSizes = new long[planeIndices.length];
int planeCounter = 0;
for (final int planeIndex : planeIndices) {
......@@ -201,7 +201,7 @@ public class SQImageCompressor extends CompressorDecompressorBase implements IIm
}
} else if (inputDataInfo.isPlaneRangeSet()) {
reportStatusToListeners("Loading plane range data.");
final int[] planes = getPlaneIndicesForCompression();
final int[] planes = getPlaneIndicesForCompression(options.getInputDataInfo());
try {
trainData = planeLoader.loadPlanesU16Data(planes);
} catch (IOException e) {
......
......@@ -117,7 +117,7 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm
return compressVoxels(compressStream, false, options.getInputDataInfo());
}
assert (options.getQuantizationVector().getZ() == 1);
return compress1D2DVectors(compressStream, false);
return compress1D2DVectors(compressStream, false, options.getInputDataInfo());
}
@Override
......@@ -126,52 +126,54 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm
return compressVoxels(compressStream, true, inputData);
}
assert (options.getQuantizationVector().getZ() == 1);
return compress1D2DVectors(compressStream, true);
return compress1D2DVectors(compressStream, true, inputData);
}
@NotNull
private long[] compress1D2DVectors(final DataOutputStream compressStream, final boolean streamMode) throws ImageCompressionException {
private long[] compress1D2DVectors(final DataOutputStream compressStream,
final boolean streamMode,
final InputData inputData) throws ImageCompressionException {
final InputData inputDataInfo = options.getInputDataInfo();
Stopwatch stopwatch = new Stopwatch();
final boolean hasGeneralQuantizer = options.getCodebookType() != CompressionOptions.CodebookType.Individual;
final IPlaneLoader planeLoader;
final int[] huffmanSymbols = createHuffmanSymbols(getCodebookSize());
try {
planeLoader = PlaneLoaderFactory.getPlaneLoaderForInputFile(inputDataInfo);
planeLoader = PlaneLoaderFactory.getPlaneLoaderForInputFile(inputData);
} catch (Exception e) {
throw new ImageCompressionException("Unable to create plane reader. " + e.getMessage());
}
VectorQuantizer quantizer = null;
Huffman huffman = null;
if (options.getCodebookType() == CompressionOptions.CodebookType.Global) {
reportStatusToListeners("Loading codebook from cache file.");
quantizer = loadQuantizerFromCache();
huffman = createHuffmanCoder(huffmanSymbols, quantizer.getFrequencies());
reportStatusToListeners("Cached quantizer with huffman coder created.");
if (!streamMode)
writeQuantizerToCompressStream(quantizer, compressStream);
} else if (options.getCodebookType() == CompressionOptions.CodebookType.MiddlePlane) {
stopwatch.restart();
reportStatusToListeners("Training vector quantizer from middle plane.");
final int[][] refPlaneVectors = planeLoader.loadVectorsFromPlaneRange(options, Utils.singlePlaneRange(getMiddlePlaneIndex()));
quantizer = trainVectorQuantizerFromPlaneVectors(refPlaneVectors);
huffman = createHuffmanCoder(huffmanSymbols, quantizer.getFrequencies());
if (!streamMode)
writeQuantizerToCompressStream(quantizer, compressStream);
stopwatch.stop();
reportStatusToListeners("Middle plane codebook created in: " + stopwatch.getElapsedTimeString());
VectorQuantizer quantizer = cachedQuantizer;
Huffman huffman = cachedHuffman;
assert (!streamMode || ((quantizer != null) && (huffman != null)));
if (!streamMode) {
if (options.getCodebookType() == CompressionOptions.CodebookType.Global) {
reportStatusToListeners("Loading codebook from cache file.");
quantizer = loadQuantizerFromCache();
huffman = createHuffmanCoder(huffmanSymbols, quantizer.getFrequencies());
reportStatusToListeners("Cached quantizer with huffman coder created.");
} else if (options.getCodebookType() == CompressionOptions.CodebookType.MiddlePlane) {
stopwatch.restart();
reportStatusToListeners("Training vector quantizer from middle plane.");
final int[][] refPlaneVectors = planeLoader.loadVectorsFromPlaneRange(options,
Utils.singlePlaneRange(getMiddlePlaneIndex()));
quantizer = trainVectorQuantizerFromPlaneVectors(refPlaneVectors);
huffman = createHuffmanCoder(huffmanSymbols, quantizer.getFrequencies());
stopwatch.stop();
reportStatusToListeners("Middle plane codebook created in: " + stopwatch.getElapsedTimeString());
}
writeQuantizerToCompressStream(quantizer, compressStream);
}
final int[] planeIndices = getPlaneIndicesForCompression();
final int[] planeIndices = getPlaneIndicesForCompression(inputData);
if (streamMode) {
try {
final V3i imageDims = options.getInputDataInfo().getDimensions();
// Image dimensions
compressStream.writeShort(imageDims.getX());
compressStream.writeShort(imageDims.getY());
compressStream.writeShort(imageDims.getZ());
compressStream.writeShort(inputData.getDimensions().getX());
compressStream.writeShort(inputData.getDimensions().getY());
compressStream.writeShort(inputData.getDimensions().getZ());
// Write voxel layer in stream mode.
compressStream.writeShort(planeIndices.length);
......@@ -189,18 +191,15 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm
final int[][] planeVectors = planeLoader.loadVectorsFromPlaneRange(options, Utils.singlePlaneRange(planeIndex));
if (!hasGeneralQuantizer) {
if (!streamMode && !hasGeneralQuantizer) {
reportStatusToListeners(String.format("Training vector quantizer from plane %d.", planeIndex));
quantizer = trainVectorQuantizerFromPlaneVectors(planeVectors);
huffman = createHuffmanCoder(huffmanSymbols, quantizer.getFrequencies());
if (!streamMode)
writeQuantizerToCompressStream(quantizer, compressStream);
writeQuantizerToCompressStream(quantizer, compressStream);
}
assert (quantizer != null);
// Use BestBinFirst KDTree for codebook lookup.
// final int[] indices = quantizer.quantizeIntoIndicesUsingKDTree(planeVectors, options.getWorkerCount());
// final int[] indices = quantizer.quantizeIntoIndicesUsingKDTree(planeVectors, options.getWorkerCount());
// Use BruteForce for codebook lookup.
final int[] indices = quantizer.quantizeIntoIndices(planeVectors, options.getWorkerCount());
......@@ -276,7 +275,6 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm
return (datasetPlaneCount / voxelDepth);
}
// TODO(Moravec): Remove dependencies on instance variables to enable multi-thread usage.
public long[] compressVoxels(final DataOutputStream compressStream,
final boolean streamMode,
final InputData inputData) throws ImageCompressionException {
......@@ -325,10 +323,7 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm
final int toZ = (voxelLayerIndex == voxelLayerCount - 1)
? inputData.getDimensions().getZ()
: (voxelLayerDepth + (voxelLayerIndex * voxelLayerDepth));
if (toZ < fromZ) {
System.err.println("@Wrong range");
}
assert (toZ >= fromZ);
final Range<Integer> voxelLayerRange = new Range<>(fromZ, toZ);
......
package azgracompress.compression;
import azgracompress.cache.ICacheFile;
import azgracompress.cache.SQCacheFile;
import azgracompress.cache.VQCacheFile;
import azgracompress.compression.exception.ImageDecompressionException;
import azgracompress.data.*;
......@@ -194,6 +193,49 @@ public class VQImageDecompressor extends CompressorDecompressorBase implements I
}
}
@SuppressWarnings("DuplicatedCode")
public void decompressStreamModelImpl(DataInputStream compressedStream,
QCMPFileHeader header,
DecompressCallback callback) throws ImageDecompressionException {
assert (cachedCodebook != null && cachedHuffman != null);
assert (header.getVectorSizeZ() == 1);
final int planeCountForDecompression = header.getImageSizeZ();
final long planeVectorCount = calculatePlaneVectorCount(header);
final V2i qVector = new V2i(header.getVectorSizeX(), header.getVectorSizeY());
final int vectorSize = qVector.multiplyTogether();
for (int planeIndex = 0; planeIndex < planeCountForDecompression; planeIndex++) {
final int planeDataSize = (int) header.getPlaneDataSizes()[planeIndex];
try (InBitStream inBitStream = new InBitStream(compressedStream,
header.getBitsPerCodebookIndex(),
planeDataSize)) {
inBitStream.readToBuffer();
inBitStream.setAllowReadFromUnderlyingStream(false);
int[][] decompressedVectors = new int[(int) planeVectorCount][vectorSize];
int huffmanIndex;
for (int vecIndex = 0; vecIndex < planeVectorCount; vecIndex++) {
huffmanIndex = decodeHuffmanSymbol(cachedHuffman, inBitStream);
System.arraycopy(cachedCodebook.getVectors()[huffmanIndex], 0, decompressedVectors[vecIndex], 0, vectorSize);
}
final Block decompressedPlane = reconstructImageFromQuantizedVectors(decompressedVectors, qVector, header.getImageDims());
callback.process(decompressedPlane, planeIndex);
} catch (Exception ex) {
throw new ImageDecompressionException("VQImageDecompressor::decompressToBuffer() - Unable to read indices from " +
"InBitStream.",
ex);
}
reportProgressToListeners(planeIndex, planeCountForDecompression,
"Decompressed plane %d.", planeIndex);
}
}
@Override
public void decompressToBuffer(DataInputStream compressedStream,
......@@ -370,21 +412,34 @@ public class VQImageDecompressor extends CompressorDecompressorBase implements I
@Override
public short[] decompressStreamMode(final DataInputStream compressedStream,
final QCMPFileHeader header) throws ImageDecompressionException {
// TODO(Moravec): Implement missing quantization type.
assert (header.getQuantizationType() == QuantizationType.Vector3D);
final short[] buffer = new short[(int) header.getImageDims().multiplyTogether()];
final V3i voxelDim = new V3i(header.getVectorSizeX(), header.getVectorSizeY(), header.getVectorSizeZ());
if (header.getQuantizationType() == QuantizationType.Vector3D) {
final V3i voxelDim = new V3i(header.getVectorSizeX(), header.getVectorSizeY(), header.getVectorSizeZ());
decompressVoxelsStreamModeImpl(compressedStream, header, (voxel, voxelData, planeOffset) -> {
final ImageU16Dataset currentVoxelLayer = voxel.reconstructFromVoxelsToDataset(voxelDim, voxelData);
int offset = planeOffset * (voxelDim.getX() * voxelDim.getY());
decompressVoxelsStreamModeImpl(compressedStream, header, (voxel, voxelData, planeOffset) -> {
final ImageU16Dataset currentVoxelLayer = voxel.reconstructFromVoxelsToDataset(voxelDim, voxelData);
int offset = planeOffset * (voxelDim.getX() * voxelDim.getY());
for (int layer = 0; layer < voxel.getDims().getZ(); layer++) {
final short[] voxelLayerData = currentVoxelLayer.getPlaneData(layer);
System.arraycopy(voxelLayerData, 0, buffer, offset, voxelLayerData.length);
offset += voxelLayerData.length;
}
});
for (int layer = 0; layer < voxel.getDims().getZ(); layer++) {
final short[] voxelLayerData = currentVoxelLayer.getPlaneData(layer);
System.arraycopy(voxelLayerData, 0, buffer, offset, voxelLayerData.length);
offset += voxelLayerData.length;
}
});
return buffer;
} else {
final int planePixelCount = header.getImageDims().toV2i().multiplyTogether();
decompressStreamModelImpl(compressedStream, header, (imageBlock, planeIndex) -> {
final int offset = planePixelCount * planeIndex;
final int[] data = imageBlock.getData();
for (int i = 0; i < planePixelCount; i++) {
buffer[offset + i] = (short) data[i];
}
});
}
return buffer;
}
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment