Skip to content
Snippets Groups Projects
Commit b936ded3 authored by Vojtech Moravec's avatar Vojtech Moravec
Browse files

Support codebook from cache files.

parent bd3bc91c
Branches
Tags
No related merge requests found
...@@ -131,10 +131,12 @@ public class DataCompressor { ...@@ -131,10 +131,12 @@ public class DataCompressor {
options.addOptionGroup(methodGroup); options.addOptionGroup(methodGroup);
options.addOptionGroup(compressionMethodGroup); options.addOptionGroup(compressionMethodGroup);
options.addOption(CliConstants.BITS_SHORT, CliConstants.BITS_LONG, true, "Bit count per pixel [Default 8]"); options.addOption(CliConstants.BITS_SHORT, CliConstants.BITS_LONG, true, "Bit count per pixel [Default 8]");
options.addOption(CliConstants.REFERENCE_PLANE_SHORT, options.addOption(CliConstants.REFERENCE_PLANE_SHORT,
CliConstants.REFERENCE_PLANE_LONG, CliConstants.REFERENCE_PLANE_LONG,
true, true,
"Reference plane index"); "Reference plane index");
options.addOption(new Option(CliConstants.VERBOSE_SHORT, options.addOption(new Option(CliConstants.VERBOSE_SHORT,
CliConstants.VERBOSE_LONG, CliConstants.VERBOSE_LONG,
false, false,
...@@ -145,6 +147,11 @@ public class DataCompressor { ...@@ -145,6 +147,11 @@ public class DataCompressor {
true, true,
"Number of worker threads")); "Number of worker threads"));
options.addOption(new Option(CliConstants.CODEBOOK_CACHE_FOLDER_SHORT,
CliConstants.CODEBOOK_CACHE_FOLDER_LONG,
true,
"Folder of codebook caches"));
options.addOption(CliConstants.OUTPUT_SHORT, CliConstants.OUTPUT_LONG, true, "Custom output file"); options.addOption(CliConstants.OUTPUT_SHORT, CliConstants.OUTPUT_LONG, true, "Custom output file");
return options; return options;
} }
......
...@@ -34,6 +34,9 @@ public class CliConstants { ...@@ -34,6 +34,9 @@ public class CliConstants {
public static final String WORKER_COUNT_SHORT = "wc"; public static final String WORKER_COUNT_SHORT = "wc";
public static final String WORKER_COUNT_LONG = "worker-count"; public static final String WORKER_COUNT_LONG = "worker-count";
public static final String CODEBOOK_CACHE_FOLDER_SHORT = "wc";
public static final String CODEBOOK_CACHE_FOLDER_LONG = "worker-count";
public static final String SCALAR_QUANTIZATION_SHORT = "sq"; public static final String SCALAR_QUANTIZATION_SHORT = "sq";
public static final String SCALAR_QUANTIZATION_LONG = "scalar-quantization"; public static final String SCALAR_QUANTIZATION_LONG = "scalar-quantization";
......
...@@ -18,6 +18,7 @@ public class ParsedCliOptions { ...@@ -18,6 +18,7 @@ public class ParsedCliOptions {
private String inputFile; private String inputFile;
private String outputFile; private String outputFile;
private String codebookCacheFolder = null;
private int bitsPerPixel; private int bitsPerPixel;
private V2i vectorDimension = new V2i(0); private V2i vectorDimension = new V2i(0);
...@@ -107,6 +108,8 @@ public class ParsedCliOptions { ...@@ -107,6 +108,8 @@ public class ParsedCliOptions {
} }
} }
codebookCacheFolder = cmd.getOptionValue(CliConstants.CODEBOOK_CACHE_FOLDER_LONG, null);
if (!errorOccurred) { if (!errorOccurred) {
outputFile = cmd.getOptionValue(CliConstants.OUTPUT_LONG, getDefaultOutputFilePath(inputFile)); outputFile = cmd.getOptionValue(CliConstants.OUTPUT_LONG, getDefaultOutputFilePath(inputFile));
} }
...@@ -404,6 +407,14 @@ public class ParsedCliOptions { ...@@ -404,6 +407,14 @@ public class ParsedCliOptions {
return workerCount; return workerCount;
} }
public String getCodebookCacheFolder() {
return codebookCacheFolder;
}
public boolean hasCodebookCacheFolder() {
return (codebookCacheFolder != null);
}
public String report() { public String report() {
StringBuilder sb = new StringBuilder(); StringBuilder sb = new StringBuilder();
...@@ -443,6 +454,9 @@ public class ParsedCliOptions { ...@@ -443,6 +454,9 @@ public class ParsedCliOptions {
sb.append("BitsPerPixel: ").append(bitsPerPixel).append('\n'); sb.append("BitsPerPixel: ").append(bitsPerPixel).append('\n');
sb.append("Output: ").append(outputFile).append('\n'); sb.append("Output: ").append(outputFile).append('\n');
sb.append("InputFile: ").append(inputFile).append('\n'); sb.append("InputFile: ").append(inputFile).append('\n');
if (hasCodebookCacheFolder()) {
sb.append("CodebookCacheFolder: ").append(codebookCacheFolder).append('\n');
}
if (hasQuantizationType(method)) { if (hasQuantizationType(method)) {
sb.append("Input image dims: ").append(imageDimension.toString()).append('\n'); sb.append("Input image dims: ").append(imageDimension.toString()).append('\n');
...@@ -462,7 +476,6 @@ public class ParsedCliOptions { ...@@ -462,7 +476,6 @@ public class ParsedCliOptions {
sb.append("Verbose: ").append(verbose).append('\n'); sb.append("Verbose: ").append(verbose).append('\n');
sb.append("ThreadWorkerCount: ").append(workerCount).append('\n'); sb.append("ThreadWorkerCount: ").append(workerCount).append('\n');
return sb.toString(); return sb.toString();
} }
......
...@@ -54,6 +54,22 @@ public class SQImageCompressor extends CompressorDecompressorBase implements IIm ...@@ -54,6 +54,22 @@ public class SQImageCompressor extends CompressorDecompressorBase implements IIm
} }
} }
/**
* Load quantization codebook from cache file.
*
* @return Scalar quantizer with cached codebook.
* @throws ImageCompressionException when fails to read cached codebook.
*/
private ScalarQuantizer loadQuantizerFromCache() throws ImageCompressionException {
QuantizationValueCache cache = new QuantizationValueCache(options.getCodebookCacheFolder());
try {
final int[] quantizationValues = cache.readCachedValues(options.getInputFile(), codebookSize);
return new ScalarQuantizer(U16.Min, U16.Max, quantizationValues);
} catch (IOException e) {
throw new ImageCompressionException("Failed to read quantization values from cache file.", e);
}
}
/** /**
* Compress the image file specified by parsed CLI options using scalar quantization. * Compress the image file specified by parsed CLI options using scalar quantization.
* *
...@@ -61,9 +77,16 @@ public class SQImageCompressor extends CompressorDecompressorBase implements IIm ...@@ -61,9 +77,16 @@ public class SQImageCompressor extends CompressorDecompressorBase implements IIm
* @throws ImageCompressionException When compress process fails. * @throws ImageCompressionException When compress process fails.
*/ */
public void compress(DataOutputStream compressStream) throws ImageCompressionException { public void compress(DataOutputStream compressStream) throws ImageCompressionException {
ScalarQuantizer quantizer = null;
Stopwatch stopwatch = new Stopwatch(); Stopwatch stopwatch = new Stopwatch();
if (options.hasReferencePlaneIndex()) { final boolean hasGeneralQuantizer = options.hasCodebookCacheFolder() || options.hasReferencePlaneIndex();
ScalarQuantizer quantizer = null;
if (options.hasCodebookCacheFolder()) {
Log("Loading codebook from cache file.");
quantizer = loadQuantizerFromCache();
Log("Cached quantizer created.");
} else if (options.hasReferencePlaneIndex()) {
// TODO(Moravec): Reference plane will be deprecated in favor of 'middle' plane.
stopwatch.restart(); stopwatch.restart();
ImageU16 referencePlane = null; ImageU16 referencePlane = null;
try { try {
...@@ -99,7 +122,7 @@ public class SQImageCompressor extends CompressorDecompressorBase implements IIm ...@@ -99,7 +122,7 @@ public class SQImageCompressor extends CompressorDecompressorBase implements IIm
throw new ImageCompressionException("Unable to load plane data.", ex); throw new ImageCompressionException("Unable to load plane data.", ex);
} }
if (!options.hasReferencePlaneIndex()) { if (!hasGeneralQuantizer) {
Log(String.format("Training scalar quantizer from plane %d.", planeIndex)); Log(String.format("Training scalar quantizer from plane %d.", planeIndex));
quantizer = trainScalarQuantizerFromData(plane.getData()); quantizer = trainScalarQuantizerFromData(plane.getData());
writeCodebookToOutputStream(quantizer, compressStream); writeCodebookToOutputStream(quantizer, compressStream);
......
...@@ -69,6 +69,26 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm ...@@ -69,6 +69,26 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm
} }
} }
/**
* Load quantizer from cached codebook.
*
* @return Vector quantizer with cached codebook.
* @throws ImageCompressionException when fails to read cached codebook.
*/
private VectorQuantizer loadQuantizerFromCache() throws ImageCompressionException {
QuantizationValueCache cache = new QuantizationValueCache(options.getCodebookCacheFolder());
try {
final CodebookEntry[] codebook = cache.readCachedValues(options.getInputFile(),
codebookSize,
options.getVectorDimension().getX(),
options.getVectorDimension().getY());
return new VectorQuantizer(codebook);
} catch (IOException e) {
throw new ImageCompressionException("Failed to read quantization vectors from cache.", e);
}
}
/** /**
* Compress the image file specified by parsed CLI options using vector quantization. * Compress the image file specified by parsed CLI options using vector quantization.
* *
...@@ -76,9 +96,15 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm ...@@ -76,9 +96,15 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm
* @throws ImageCompressionException When compress process fails. * @throws ImageCompressionException When compress process fails.
*/ */
public void compress(DataOutputStream compressStream) throws ImageCompressionException { public void compress(DataOutputStream compressStream) throws ImageCompressionException {
VectorQuantizer quantizer = null;
Stopwatch stopwatch = new Stopwatch(); Stopwatch stopwatch = new Stopwatch();
if (options.hasReferencePlaneIndex()) { final boolean hasGeneralQuantizer = options.hasCodebookCacheFolder() || options.hasReferencePlaneIndex();
VectorQuantizer quantizer = null;
if (options.hasCodebookCacheFolder()) {
Log("Loading codebook from cache file.");
quantizer = loadQuantizerFromCache();
Log("Cached quantizer created.");
} else if (options.hasReferencePlaneIndex()) {
stopwatch.restart(); stopwatch.restart();
ImageU16 referencePlane = null; ImageU16 referencePlane = null;
...@@ -116,7 +142,7 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm ...@@ -116,7 +142,7 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm
final int[][] planeVectors = getPlaneVectors(plane); final int[][] planeVectors = getPlaneVectors(plane);
if (!options.hasReferencePlaneIndex()) { if (!hasGeneralQuantizer) {
Log(String.format("Training vector quantizer from plane %d.", planeIndex)); Log(String.format("Training vector quantizer from plane %d.", planeIndex));
quantizer = trainVectorQuantizerFromPlaneVectors(planeVectors); quantizer = trainVectorQuantizerFromPlaneVectors(planeVectors);
writeQuantizerToCompressStream(quantizer, compressStream); writeQuantizerToCompressStream(quantizer, compressStream);
...@@ -139,6 +165,7 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm ...@@ -139,6 +165,7 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm
} }
} }
/** /**
* Load plane and convert the plane into quantization vectors. * Load plane and convert the plane into quantization vectors.
* *
......
...@@ -26,7 +26,10 @@ public class QuantizationValueCache { ...@@ -26,7 +26,10 @@ public class QuantizationValueCache {
final int entryHeight) { final int entryHeight) {
final File inputFile = new File(trainFile); final File inputFile = new File(trainFile);
final File cacheFile = new File(cacheFolder, String.format("%s_%d_%dx%d.qvc", final File cacheFile = new File(cacheFolder, String.format("%s_%d_%dx%d.qvc",
inputFile.getName(), codebookSize, entryWidth, entryHeight)); inputFile.getName(),
codebookSize,
entryWidth,
entryHeight));
return cacheFile; return cacheFile;
} }
...@@ -72,7 +75,7 @@ public class QuantizationValueCache { ...@@ -72,7 +75,7 @@ public class QuantizationValueCache {
} }
} }
public int[] readCachedValues(final String trainFile, final int quantizationValueCount) { public int[] readCachedValues(final String trainFile, final int quantizationValueCount) throws IOException {
final File cacheFile = getCacheFileForScalarValues(trainFile, quantizationValueCount); final File cacheFile = getCacheFileForScalarValues(trainFile, quantizationValueCount);
int[] values = new int[quantizationValueCount]; int[] values = new int[quantizationValueCount];
...@@ -82,10 +85,6 @@ public class QuantizationValueCache { ...@@ -82,10 +85,6 @@ public class QuantizationValueCache {
for (int i = 0; i < quantizationValueCount; i++) { for (int i = 0; i < quantizationValueCount; i++) {
values[i] = dis.readInt(); values[i] = dis.readInt();
} }
} catch (IOException ioEx) {
System.err.println("Failed to read scalar quantization values from cache.");
ioEx.printStackTrace();
return new int[0];
} }
return values; return values;
} }
...@@ -93,7 +92,7 @@ public class QuantizationValueCache { ...@@ -93,7 +92,7 @@ public class QuantizationValueCache {
public CodebookEntry[] readCachedValues(final String trainFile, public CodebookEntry[] readCachedValues(final String trainFile,
final int codebookSize, final int codebookSize,
final int entryWidth, final int entryWidth,
final int entryHeight) { final int entryHeight) throws IOException {
final File cacheFile = getCacheFileForVectorValues(trainFile, codebookSize, entryWidth, entryHeight); final File cacheFile = getCacheFileForVectorValues(trainFile, codebookSize, entryWidth, entryHeight);
CodebookEntry[] codebook = new CodebookEntry[codebookSize]; CodebookEntry[] codebook = new CodebookEntry[codebookSize];
...@@ -115,10 +114,6 @@ public class QuantizationValueCache { ...@@ -115,10 +114,6 @@ public class QuantizationValueCache {
} }
codebook[i] = new CodebookEntry(vector); codebook[i] = new CodebookEntry(vector);
} }
} catch (IOException ioEx) {
System.err.println("Failed to read quantization vectors from cache.");
ioEx.printStackTrace();
return new CodebookEntry[0];
} }
return codebook; return codebook;
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment