diff --git a/src/main/java/azgracompress/benchmark/VectorQuantizationBenchmark.java b/src/main/java/azgracompress/benchmark/VectorQuantizationBenchmark.java
index 29ff4dc920fb22f8f92d94ae19b93872ac4ef40e..1c34ee22611baa3730624531cf98c76d0273b144 100644
--- a/src/main/java/azgracompress/benchmark/VectorQuantizationBenchmark.java
+++ b/src/main/java/azgracompress/benchmark/VectorQuantizationBenchmark.java
@@ -85,7 +85,7 @@ public class VectorQuantizationBenchmark extends BenchmarkBase {
}
final int[][] refPlaneData = getPlaneVectors(plane, qVector);
- LBGVectorQuantizer vqInitializer = new LBGVectorQuantizer(refPlaneData, codebookSize);
+ LBGVectorQuantizer vqInitializer = new LBGVectorQuantizer(refPlaneData, codebookSize, 1);
final LBGResult vqResult = vqInitializer.findOptimalCodebook();
quantizer = new VectorQuantizer(vqResult.getCodebook());
System.out.println("Created reference quantizer.");
@@ -106,7 +106,7 @@ public class VectorQuantizationBenchmark extends BenchmarkBase {
if (!hasGeneralQuantizer) {
- LBGVectorQuantizer vqInitializer = new LBGVectorQuantizer(planeData, codebookSize);
+ LBGVectorQuantizer vqInitializer = new LBGVectorQuantizer(planeData, codebookSize,1);
LBGResult vqResult = vqInitializer.findOptimalCodebook();
quantizer = new VectorQuantizer(vqResult.getCodebook());
System.out.println("Created plane quantizer.");
diff --git a/src/main/java/azgracompress/compression/VQImageCompressor.java b/src/main/java/azgracompress/compression/VQImageCompressor.java
index fe3d027130aafc6d51284e3302220e90a0c7036f..ffb6c22ea763ffcc270cc61e09251e3d2b4c6c78 100644
--- a/src/main/java/azgracompress/compression/VQImageCompressor.java
+++ b/src/main/java/azgracompress/compression/VQImageCompressor.java
@@ -29,7 +29,7 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm
* @return Trained vector quantizer with codebook of set size.
*/
private VectorQuantizer trainVectorQuantizerFromPlaneVectors(final int[][] planeVectors) {
- LBGVectorQuantizer vqInitializer = new LBGVectorQuantizer(planeVectors, codebookSize);
+ LBGVectorQuantizer vqInitializer = new LBGVectorQuantizer(planeVectors, codebookSize, options.getWorkerCount());
LBGResult vqResult = vqInitializer.findOptimalCodebook(false);
return new VectorQuantizer(vqResult.getCodebook());
}
@@ -185,7 +185,7 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm
throw new ImageCompressionException("Failed to load reference image data.", e);
}
} else {
- Log(options.hasPlaneRangeSet() ? "Loading plane range data." : "Loading all planes data.");
+ Log(options.hasPlaneRangeSet() ? "VQ: Loading plane range data." : "VQ: Loading all planes data.");
final int[] planeIndices = getPlaneIndicesForCompression();
final int chunkCountPerPlane = Chunk2D.calculateRequiredChunkCountPerPlane(
@@ -198,7 +198,6 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm
int[][] planeVectors;
int planeCounter = 0;
for (final int planeIndex : planeIndices) {
- Log("Loading plane %d vectors", planeIndex);
try {
planeVectors = loadPlaneQuantizationVectors(planeIndex);
assert (planeVectors.length == chunkCountPerPlane) : "Wrong chunk count per plane";
@@ -220,7 +219,7 @@ public class VQImageCompressor extends CompressorDecompressorBase implements IIm
public void trainAndSaveCodebook() throws ImageCompressionException {
final int[][] trainingData = loadConfiguredPlanesData();
- LBGVectorQuantizer vqInitializer = new LBGVectorQuantizer(trainingData, codebookSize);
+ LBGVectorQuantizer vqInitializer = new LBGVectorQuantizer(trainingData, codebookSize, options.getWorkerCount());
Log("Starting LBG optimization.");
LBGResult lbgResult = vqInitializer.findOptimalCodebook(options.isVerbose());
Log("Learned the optimal codebook.");
diff --git a/src/main/java/azgracompress/quantization/scalar/LloydMaxU16ScalarQuantization.java b/src/main/java/azgracompress/quantization/scalar/LloydMaxU16ScalarQuantization.java
index b850a9e6e17df4ccab37bbe2b3e5349a63065aa5..09675021c21898252690de7332e412b83ea389e8 100644
--- a/src/main/java/azgracompress/quantization/scalar/LloydMaxU16ScalarQuantization.java
+++ b/src/main/java/azgracompress/quantization/scalar/LloydMaxU16ScalarQuantization.java
@@ -122,13 +122,8 @@ public class LloydMaxU16ScalarQuantization {
boundaryPoints,
codebookSize);
workers[wId] = new Thread(runnables[wId]);
- }
-
- for (int wId = 0; wId < workerCount; wId++) {
workers[wId].start();
}
-
-
try {
for (int wId = 0; wId < workerCount; wId++) {
workers[wId].join();
diff --git a/src/main/java/azgracompress/quantization/vector/LBGVectorQuantizer.java b/src/main/java/azgracompress/quantization/vector/LBGVectorQuantizer.java
index d851c6e5e2575f2614a9e9954ad17a4adc1b55ce..3c6892e785927074b682ef9dce3915354bdc4630 100644
--- a/src/main/java/azgracompress/quantization/vector/LBGVectorQuantizer.java
+++ b/src/main/java/azgracompress/quantization/vector/LBGVectorQuantizer.java
@@ -17,14 +17,16 @@ public class LBGVectorQuantizer {
private final VectorDistanceMetric metric = VectorDistanceMetric.Euclidean;
boolean verbose = false;
+ private final int workerCount;
- public LBGVectorQuantizer(final int[][] trainingVectors, final int codebookSize) {
+ public LBGVectorQuantizer(final int[][] trainingVectors, final int codebookSize, final int workerCount) {
assert (trainingVectors.length > 0) : "No training vectors provided";
this.trainingVectors = trainingVectors;
this.vectorSize = trainingVectors[0].length;
this.codebookSize = codebookSize;
+ this.workerCount = workerCount;
}
public LBGResult findOptimalCodebook() {
@@ -212,7 +214,6 @@ public class LBGVectorQuantizer {
double previousDistortion = Double.POSITIVE_INFINITY;
- int iteration = 1;
Stopwatch innerLoopStopwatch = new Stopwatch("LBG inner loop");
Stopwatch findingClosestEntryStopwatch = new Stopwatch("FindingClosestEntry");
Stopwatch distCalcStopwatch = new Stopwatch("DistortionCalc");
@@ -222,37 +223,20 @@ public class LBGVectorQuantizer {
innerLoopStopwatch.restart();
// Step 1
- // Speedup - speed the finding of the closest codebook entry.
- findingClosestEntryStopwatch.restart();
- for (final int[] trainingVec : trainingVectors) {
- double minDist = Double.POSITIVE_INFINITY;
- LearningCodebookEntry closestEntry = null;
- for (LearningCodebookEntry entry : codebook) {
- double entryDistance = VectorQuantizer.distanceBetweenVectors(entry.getVector(),
- trainingVec,
- metric);
- if (entryDistance < minDist) {
- minDist = entryDistance;
- closestEntry = entry;
- }
- }
+ findingClosestEntryStopwatch.restart();
+
+ assignVectorsToClosestEntry(codebook);
- if (closestEntry != null) {
- closestEntry.addTrainingVector(trainingVec, minDist);
- } else {
- assert (false) : "Did not found closest entry.";
- System.err.println("Did not found closest entry.");
- }
- }
findingClosestEntryStopwatch.stop();
+
System.out.println(findingClosestEntryStopwatch);
- fixEmptyStopwatch.restart();
+ // fixEmptyStopwatch.restart();
fixEmptyEntries(codebook, verbose);
- fixEmptyStopwatch.stop();
- System.out.println(fixEmptyStopwatch);
+ // fixEmptyStopwatch.stop();
+ // System.out.println(fixEmptyStopwatch);
// Step 2
distCalcStopwatch.restart();
@@ -285,7 +269,7 @@ public class LBGVectorQuantizer {
}
innerLoopStopwatch.stop();
- System.out.println(innerLoopStopwatch);
+ // System.out.println(innerLoopStopwatch);
System.out.println("================");
}
@@ -293,6 +277,83 @@ public class LBGVectorQuantizer {
System.out.println(totalLbgFun);
}
+ private void assignVectorsToClosestEntry(ArrayList<LearningCodebookEntry> codebook) {
+ if (workerCount > 1) {
+ Thread[] workers = new Thread[workerCount];
+
+ final int workSize = trainingVectors.length / workerCount;
+
+ for (int wId = 0; wId < workerCount; wId++) {
+ final int fromIndex = wId * workSize;
+ final int toIndex = (wId == workerCount - 1) ? trainingVectors.length : (workSize + (wId * workSize));
+
+ workers[wId] = new Thread(() -> {
+ double minimalDistance, entryDistance;
+ for (int vecIndex = fromIndex; vecIndex < toIndex; vecIndex++) {
+ minimalDistance = Double.POSITIVE_INFINITY;
+ LearningCodebookEntry closestEntry = null;
+
+ for (LearningCodebookEntry entry : codebook) {
+ entryDistance = VectorQuantizer.distanceBetweenVectors(entry.getVector(),
+ trainingVectors[vecIndex],
+ metric);
+
+ if (entryDistance < minimalDistance) {
+ minimalDistance = entryDistance;
+ closestEntry = entry;
+ }
+ }
+
+ if (closestEntry != null) {
+ closestEntry.addTrainingVector(trainingVectors[vecIndex],
+ minimalDistance);
+ } else {
+ assert (false) : "Did not found closest entry.";
+ System.err.println("Did not found closest entry.");
+ }
+ }
+ });
+ workers[wId].start();
+ }
+
+ try {
+ for (int wId = 0; wId < workerCount; wId++) {
+ workers[wId].join();
+ }
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ assert (false) : "Failed parallel join";
+ }
+
+ } else {
+
+ //////////////////////////////////////////////////////////////////////////
+ // Speedup - speed the finding of the closest codebook entry.
+ for (final int[] trainingVec : trainingVectors) {
+ double minDist = Double.POSITIVE_INFINITY;
+ LearningCodebookEntry closestEntry = null;
+
+ for (LearningCodebookEntry entry : codebook) {
+ double entryDistance = VectorQuantizer.distanceBetweenVectors(entry.getVector(),
+ trainingVec,
+ metric);
+
+ if (entryDistance < minDist) {
+ minDist = entryDistance;
+ closestEntry = entry;
+ }
+ }
+
+ if (closestEntry != null) {
+ closestEntry.addTrainingVector(trainingVec, minDist);
+ } else {
+ assert (false) : "Did not found closest entry.";
+ System.err.println("Did not found closest entry.");
+ }
+ }
+ }
+ }
+
private void fixEmptyEntries(ArrayList<LearningCodebookEntry> codebook, final boolean verbose) {
LearningCodebookEntry emptyEntry = null;
@@ -318,9 +379,10 @@ public class LBGVectorQuantizer {
private void fixSingleEmptyEntry(ArrayList<LearningCodebookEntry> codebook,
final LearningCodebookEntry emptyEntry,
final boolean verbose) {
- if (verbose) {
- System.out.println("******** FOUND EMPTY ENTRY ********");
- }
+ // if (verbose) {
+ // System.out.println("******** FOUND EMPTY ENTRY ********");
+ // }
+
// Remove empty entry from codebook.
codebook.remove(emptyEntry);
diff --git a/src/main/java/azgracompress/quantization/vector/LearningCodebookEntry.java b/src/main/java/azgracompress/quantization/vector/LearningCodebookEntry.java
index decb5c7bd536cc6559d9ac430dfe18fa02300390..38ed138d3d2a6a137efa214a06fc2c75a5e7e360 100644
--- a/src/main/java/azgracompress/quantization/vector/LearningCodebookEntry.java
+++ b/src/main/java/azgracompress/quantization/vector/LearningCodebookEntry.java
@@ -42,7 +42,7 @@ public class LearningCodebookEntry extends CodebookEntry {
this.trainingVectors = trainingVectors;
}
- public void addTrainingVector(final int[] trainingVec, final double vecDist) {
+ public synchronized void addTrainingVector(final int[] trainingVec, final double vecDist) {
trainingVectors.add(trainingVec);
trainingVectorsDistances.add(vecDist);
}