From bf10676a67ca6bc0f684dbbaec0e08ab90f7b63d Mon Sep 17 00:00:00 2001
From: Vojtech Moravec <vojtech.moravec.st@vsb.cz>
Date: Mon, 21 Sep 2020 16:03:33 +0200
Subject: [PATCH] Pass InputData object to stream decompression.
Stream compression is in its nature compressing different data every time. That's why we shouldn't work with cached instance options which includes InputData object.
This was a problem when multiple threads tried to compress different data, but were overriding each other InputData info.
Because of this we changed compressVoxels() signature to access InputData.
---
.../compression/IImageCompressor.java | 6 +++-
.../compression/ImageCompressor.java | 8 ++++--
.../compression/SQImageCompressor.java | 3 +-
.../compression/VQImageCompressor.java | 28 +++++++++++--------
4 files changed, 29 insertions(+), 16 deletions(-)
diff --git a/src/main/java/azgracompress/compression/IImageCompressor.java b/src/main/java/azgracompress/compression/IImageCompressor.java
index 82cc364..a7610fa 100644
--- a/src/main/java/azgracompress/compression/IImageCompressor.java
+++ b/src/main/java/azgracompress/compression/IImageCompressor.java
@@ -2,6 +2,8 @@ package azgracompress.compression;
import azgracompress.cache.ICacheFile;
import azgracompress.compression.exception.ImageCompressionException;
+import azgracompress.data.V3i;
+import azgracompress.io.InputData;
import java.io.DataOutputStream;
@@ -22,10 +24,12 @@ public interface IImageCompressor extends IListenable {
* META information.
*
* @param compressStream Compressed data stream.
+ * @param inputData Chunk input data.
* @return Size of compressed chunks.
* @throws ImageCompressionException when compression fails
*/
- long[] compressStreamMode(DataOutputStream compressStream) throws ImageCompressionException;
+ long[] compressStreamChunk(final DataOutputStream compressStream,
+ final InputData inputData) throws ImageCompressionException;
/**
* Train codebook from selected frames and save the learned codebook to cache file.
diff --git a/src/main/java/azgracompress/compression/ImageCompressor.java b/src/main/java/azgracompress/compression/ImageCompressor.java
index 6ca9c14..4c23d1d 100644
--- a/src/main/java/azgracompress/compression/ImageCompressor.java
+++ b/src/main/java/azgracompress/compression/ImageCompressor.java
@@ -6,7 +6,6 @@ import azgracompress.compression.exception.ImageCompressionException;
import azgracompress.data.Range;
import azgracompress.fileformat.QCMPFileHeader;
import azgracompress.io.InputData;
-import azgracompress.utilities.Utils;
import java.io.*;
import java.util.Arrays;
@@ -33,6 +32,9 @@ public class ImageCompressor extends CompressorDecompressorBase {
*/
public void setInputData(final InputData inputData) {
options.setInputDataInfo(inputData);
+ if ((imageCompressor != null) && (imageCompressor instanceof CompressorDecompressorBase)) {
+ ((CompressorDecompressorBase) imageCompressor).options.setInputDataInfo(inputData);
+ }
}
/**
@@ -86,11 +88,11 @@ public class ImageCompressor extends CompressorDecompressorBase {
return true;
}
- public int streamCompressChunk(final OutputStream outputStream) {
+ public int streamCompressChunk(final OutputStream outputStream, final InputData inputData) {
assert (imageCompressor != null);
try (DataOutputStream compressStream = new DataOutputStream(new BufferedOutputStream(outputStream, 8192))) {
- final long[] chunkSizes = imageCompressor.compressStreamMode(compressStream);
+ final long[] chunkSizes = imageCompressor.compressStreamChunk(compressStream, inputData);
for (final long chunkSize : chunkSizes) {
assert (chunkSize < U16.Max);
compressStream.writeShort((int) chunkSize);
diff --git a/src/main/java/azgracompress/compression/SQImageCompressor.java b/src/main/java/azgracompress/compression/SQImageCompressor.java
index 3a20b61..7604543 100644
--- a/src/main/java/azgracompress/compression/SQImageCompressor.java
+++ b/src/main/java/azgracompress/compression/SQImageCompressor.java
@@ -5,6 +5,7 @@ import azgracompress.cache.ICacheFile;
import azgracompress.cache.QuantizationCacheManager;
import azgracompress.cache.SQCacheFile;
import azgracompress.compression.exception.ImageCompressionException;
+import azgracompress.data.V3i;
import azgracompress.huffman.Huffman;
import azgracompress.io.InputData;
import azgracompress.io.loader.IPlaneLoader;
@@ -243,7 +244,7 @@ public class SQImageCompressor extends CompressorDecompressorBase implements IIm
}
@Override
- public long[] compressStreamMode(DataOutputStream compressStream) throws ImageCompressionException {
+ public long[] compressStreamChunk(DataOutputStream compressStream, InputData inputData) throws ImageCompressionException {
throw new ImageCompressionException("Not implemented yet");
}
}
diff --git a/src/main/java/azgracompress/compression/VQImageCompressor.java b/src/main/java/azgracompress/compression/VQImageCompressor.java
index f690819..c6ea153 100644
--- a/src/main/java/azgracompress/compression/VQImageCompressor.java
+++ b/src/main/java/azgracompress/compression/VQImageCompressor.java
@@ -114,16 +114,16 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm
@Override
public long[] compress(DataOutputStream compressStream) throws ImageCompressionException {
if (options.getQuantizationType() == QuantizationType.Vector3D) {
- return compressVoxels(compressStream, false);
+ return compressVoxels(compressStream, false, options.getInputDataInfo());
}
assert (options.getQuantizationVector().getZ() == 1);
return compress1D2DVectors(compressStream, false);
}
@Override
- public long[] compressStreamMode(DataOutputStream compressStream) throws ImageCompressionException {
+ public long[] compressStreamChunk(DataOutputStream compressStream, final InputData inputData) throws ImageCompressionException {
if (options.getQuantizationType() == QuantizationType.Vector3D) {
- return compressVoxels(compressStream, true);
+ return compressVoxels(compressStream, true, inputData);
}
assert (options.getQuantizationVector().getZ() == 1);
return compress1D2DVectors(compressStream, true);
@@ -276,26 +276,28 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm
return (datasetPlaneCount / voxelDepth);
}
- public long[] compressVoxels(final DataOutputStream compressStream, final boolean streamMode) throws ImageCompressionException {
+ // TODO(Moravec): Remove dependencies on instance variables to enable multi-thread usage.
+ public long[] compressVoxels(final DataOutputStream compressStream,
+ final boolean streamMode,
+ final InputData inputData) throws ImageCompressionException {
assert (options.getCodebookType() == CompressionOptions.CodebookType.Global);
final IPlaneLoader planeLoader;
final int[] huffmanSymbols = createHuffmanSymbols(getCodebookSize());
try {
- planeLoader = PlaneLoaderFactory.getPlaneLoaderForInputFile(options.getInputDataInfo());
+ planeLoader = PlaneLoaderFactory.getPlaneLoaderForInputFile(inputData);
planeLoader.setWorkerCount(options.getWorkerCount());
} catch (Exception e) {
throw new ImageCompressionException("Unable to create plane reader. " + e.getMessage());
}
final int voxelLayerDepth = options.getQuantizationVector().getZ();
- final int voxelLayerCount = calculateVoxelLayerCount(options.getInputDataInfo().getDimensions().getZ(), voxelLayerDepth);
+ final int voxelLayerCount = calculateVoxelLayerCount(inputData.getDimensions().getZ(), voxelLayerDepth);
if (streamMode) {
try {
- final V3i imageDims = options.getInputDataInfo().getDimensions();
// Image dimensions
- compressStream.writeShort(imageDims.getX());
- compressStream.writeShort(imageDims.getY());
- compressStream.writeShort(imageDims.getZ());
+ compressStream.writeShort(inputData.getDimensions().getX());
+ compressStream.writeShort(inputData.getDimensions().getY());
+ compressStream.writeShort(inputData.getDimensions().getZ());
// Write voxel layer in stream mode.
compressStream.writeShort(voxelLayerCount);
@@ -321,9 +323,13 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm
// Those stupid voxels have only one or two layers of actual data and the rest are zeros.
// This ends up increasing the file size because they have quite long Huffman codes.
final int toZ = (voxelLayerIndex == voxelLayerCount - 1)
- ? options.getInputDataInfo().getDimensions().getZ()
+ ? inputData.getDimensions().getZ()
: (voxelLayerDepth + (voxelLayerIndex * voxelLayerDepth));
+ if (toZ < fromZ) {
+ System.err.println("@Wrong range");
+ }
+
final Range<Integer> voxelLayerRange = new Range<>(fromZ, toZ);
try {
--
GitLab