From 45c3b2b984744ef2592fe5435e990ced65262d97 Mon Sep 17 00:00:00 2001
From: Vojtech Moravec <vojtech.moravec.st@vsb.cz>
Date: Thu, 3 Sep 2020 14:26:24 +0200
Subject: [PATCH] Support cache file reading from path.
Until now we could only read cache file by providing cache hint and
quantization type. This commit adds static method which can read any
valid cache file from file.
---
.../java/azgracompress/cache/ICacheFile.java | 2 ++
.../cache/QuantizationCacheManager.java | 23 +++++++++++++++----
.../java/azgracompress/cache/SQCacheFile.java | 6 ++---
.../java/azgracompress/cache/VQCacheFile.java | 5 +++-
.../quantization/vector/VQCodebook.java | 10 ++++++++
5 files changed, 38 insertions(+), 8 deletions(-)
diff --git a/src/main/java/azgracompress/cache/ICacheFile.java b/src/main/java/azgracompress/cache/ICacheFile.java
index 1d75409..5cf5b8d 100644
--- a/src/main/java/azgracompress/cache/ICacheFile.java
+++ b/src/main/java/azgracompress/cache/ICacheFile.java
@@ -10,6 +10,8 @@ public interface ICacheFile {
void readFromStream(DataInputStream inputStream) throws IOException;
+ void readFromStream(DataInputStream inputStream, CacheFileHeader header) throws IOException;
+
CacheFileHeader getHeader();
void report(StringBuilder builder);
diff --git a/src/main/java/azgracompress/cache/QuantizationCacheManager.java b/src/main/java/azgracompress/cache/QuantizationCacheManager.java
index 73858a7..77a3427 100644
--- a/src/main/java/azgracompress/cache/QuantizationCacheManager.java
+++ b/src/main/java/azgracompress/cache/QuantizationCacheManager.java
@@ -36,7 +36,7 @@ public class QuantizationCacheManager {
private File getCacheFilePathForSQ(final String trainFile, final int codebookSize) {
final File inputFile = new File(trainFile);
return new File(cacheFolder, String.format("%s_%d_bits.qvc",
- inputFile.getName(), codebookSize));
+ inputFile.getName(), codebookSize));
}
/**
@@ -52,7 +52,7 @@ public class QuantizationCacheManager {
final V3i vDim) {
final File inputFile = new File(trainFile);
return new File(cacheFolder, String.format("%s_%d_%dx%dx%d.qvc", inputFile.getName(), codebookSize,
- vDim.getX(), vDim.getY(), vDim.getZ()));
+ vDim.getX(), vDim.getY(), vDim.getZ()));
}
@@ -143,8 +143,8 @@ public class QuantizationCacheManager {
*/
public String saveCodebook(final String trainFile, final VQCodebook codebook) throws IOException {
final String fileName = getCacheFilePathForVQ(trainFile,
- codebook.getCodebookSize(),
- codebook.getVectorDims()).getAbsolutePath();
+ codebook.getCodebookSize(),
+ codebook.getVectorDims()).getAbsolutePath();
final CacheFileHeader header = createHeaderForVQ(new File(trainFile).getName(), codebook);
final VQCacheFile cacheFile = new VQCacheFile(header, codebook);
@@ -282,6 +282,21 @@ public class QuantizationCacheManager {
return null;
}
+ public static ICacheFile readCacheFile(final String path) {
+ try (FileInputStream fis = new FileInputStream(path);
+ DataInputStream dis = new DataInputStream(fis)) {
+ CacheFileHeader header = new CacheFileHeader();
+ header.readFromStream(dis);
+
+ ICacheFile cacheFile = getCacheFile(header.getQuantizationType());
+ assert (cacheFile != null);
+ cacheFile.readFromStream(dis, header);
+ return cacheFile;
+ } catch (IOException e) {
+ return null;
+ }
+ }
+
/**
* Inspect cache file specified by the path.
*
diff --git a/src/main/java/azgracompress/cache/SQCacheFile.java b/src/main/java/azgracompress/cache/SQCacheFile.java
index b92428e..f7ffe5a 100644
--- a/src/main/java/azgracompress/cache/SQCacheFile.java
+++ b/src/main/java/azgracompress/cache/SQCacheFile.java
@@ -5,8 +5,6 @@ import azgracompress.quantization.scalar.SQCodebook;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
-import java.util.Arrays;
-import java.util.stream.Collectors;
public class SQCacheFile implements ICacheFile {
private CacheFileHeader header;
@@ -36,9 +34,11 @@ public class SQCacheFile implements ICacheFile {
public void readFromStream(DataInputStream inputStream) throws IOException {
header = new CacheFileHeader();
-
header.readFromStream(inputStream);
+ readFromStream(inputStream, header);
+ }
+ public void readFromStream(DataInputStream inputStream, CacheFileHeader header) throws IOException {
final int codebookSize = header.getCodebookSize();
final int[] centroids = new int[codebookSize];
final long[] frequencies = new long[codebookSize];
diff --git a/src/main/java/azgracompress/cache/VQCacheFile.java b/src/main/java/azgracompress/cache/VQCacheFile.java
index 6e6f54d..3710810 100644
--- a/src/main/java/azgracompress/cache/VQCacheFile.java
+++ b/src/main/java/azgracompress/cache/VQCacheFile.java
@@ -38,9 +38,12 @@ public class VQCacheFile implements ICacheFile {
public void readFromStream(DataInputStream inputStream) throws IOException {
header = new CacheFileHeader();
-
header.readFromStream(inputStream);
+ readFromStream(inputStream, header);
+ }
+ @Override
+ public void readFromStream(DataInputStream inputStream, CacheFileHeader header) throws IOException {
final int codebookSize = header.getCodebookSize();
final CodebookEntry[] vectors = new CodebookEntry[codebookSize];
final long[] frequencies = new long[codebookSize];
diff --git a/src/main/java/azgracompress/quantization/vector/VQCodebook.java b/src/main/java/azgracompress/quantization/vector/VQCodebook.java
index c2a53f6..1afb8d2 100644
--- a/src/main/java/azgracompress/quantization/vector/VQCodebook.java
+++ b/src/main/java/azgracompress/quantization/vector/VQCodebook.java
@@ -43,6 +43,16 @@ public class VQCodebook {
return vectors;
}
+ public int[][] getRawVectors() {
+ assert (codebookSize == vectors.length);
+ assert (vectors[0].getVector().length == (int) vectorDims.multiplyTogether());
+ final int[][] rawCodebook = new int[vectors.length][(int) vectorDims.multiplyTogether()];
+ for (int i = 0; i < codebookSize; i++) {
+ rawCodebook[i] = vectors[i].getVector();
+ }
+ return rawCodebook;
+ }
+
/**
* Get frequencies of codebook vectors at indices.
*
--
GitLab