diff --git a/src/main/java/cz/it4i/qcmp/cache/IQvcFile.java b/src/main/java/cz/it4i/qcmp/cache/IQvcFile.java
index 7238c77afd80540091ed26bd6ba6342dde39ad26..908361a25832bbbc38b04c3cfc311938f3215823 100644
--- a/src/main/java/cz/it4i/qcmp/cache/IQvcFile.java
+++ b/src/main/java/cz/it4i/qcmp/cache/IQvcFile.java
@@ -1,24 +1,18 @@
package cz.it4i.qcmp.cache;
import cz.it4i.qcmp.fileformat.IQvcHeader;
-import cz.it4i.qcmp.fileformat.QvcHeaderV1;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
-// TODO(Moravec): Rename to IQvcFile
-public interface ICacheFile {
+public interface IQvcFile {
void writeToStream(DataOutputStream outputStream) throws IOException;
void readFromStream(DataInputStream inputStream, IQvcHeader header) throws IOException;
- void readFromStream(DataInputStream inputStream, QvcHeaderV1 header) throws IOException;
-
IQvcHeader getHeader();
void report(StringBuilder builder);
-
- String klass();
}
diff --git a/src/main/java/cz/it4i/qcmp/cache/QvcFileReader.java b/src/main/java/cz/it4i/qcmp/cache/QvcFileReader.java
new file mode 100644
index 0000000000000000000000000000000000000000..834397d074cbae8acdc11aab2d9f7e96428366ca
--- /dev/null
+++ b/src/main/java/cz/it4i/qcmp/cache/QvcFileReader.java
@@ -0,0 +1,105 @@
+package cz.it4i.qcmp.cache;
+
+import cz.it4i.qcmp.fileformat.IQvcHeader;
+import cz.it4i.qcmp.fileformat.QuantizationType;
+import cz.it4i.qcmp.fileformat.QvcHeaderV1;
+import cz.it4i.qcmp.fileformat.QvcHeaderV2;
+import cz.it4i.qcmp.io.RawDataIO;
+
+import java.io.DataInputStream;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+public class QvcFileReader {
+ private static final int QVC_HEADER_MAGIC_VALUE_SIZE = 9;
+
+ /**
+ * Read cache file from file.
+ *
+ * @param path File path.
+ * @return Cache file or null if reading fails.
+ */
+ public static IQvcFile readCacheFile(final String path) {
+ try (final FileInputStream fis = new FileInputStream(path)) {
+ return readCacheFileImpl(fis);
+ } catch (final IOException e) {
+ return null;
+ }
+ }
+
+ /**
+ * Make DataInputStream from InputStream.
+ *
+ * @param inputStream Some input stream.
+ * @return DataInputStream.
+ */
+ private static DataInputStream asDataInputStream(final InputStream inputStream) {
+ if (inputStream instanceof DataInputStream) {
+ return (DataInputStream) inputStream;
+ } else {
+ return new DataInputStream(inputStream);
+ }
+ }
+
+
+ /**
+ * Create correct Qvc header version by analyzing the magic value.
+ *
+ * @param magicValue Magic value of the qvc file.
+ * @return Correct version of QVC header.
+ * @throws IOException when the magic value is unknown.
+ */
+ private static IQvcHeader getCorrectQvcHeader(final String magicValue) throws IOException {
+ switch (magicValue) {
+ case QvcHeaderV1.MAGIC_VALUE:
+ return new QvcHeaderV1();
+ case QvcHeaderV2.MAGIC_VALUE:
+ return new QvcHeaderV2();
+ default:
+ throw new IOException("Invalid QVC file. Unknown QVC magic value: " + magicValue);
+ }
+ }
+
+ /**
+ * Create correct Qvc file by analyzing the quantization type.
+ *
+ * @param quantizationType Quantization type of codebook.
+ * @return Correct version of QVC file.
+ * @throws IOException when the quantization type is unknown.
+ */
+ private static IQvcFile getCorrectQvcFile(final QuantizationType quantizationType) throws IOException {
+ switch (quantizationType) {
+ case Scalar:
+ return new SqQvcFile();
+ case Vector1D:
+ case Vector2D:
+ case Vector3D:
+ return new VqQvcFile();
+ default:
+ throw new IOException("Invalid quantization type. Unable to create qvc file impl.");
+ }
+ }
+
+ /**
+ * Read cache file by DataInputStream.
+ *
+ * @param inputStream Input stream.
+ * @return Cache file or null, if exception occurs.
+ */
+ private static IQvcFile readCacheFileImpl(final InputStream inputStream) throws IOException {
+ final DataInputStream dis = asDataInputStream(inputStream);
+
+ final byte[] magicValueBuffer = new byte[QVC_HEADER_MAGIC_VALUE_SIZE];
+ RawDataIO.readFullBuffer(dis, magicValueBuffer);
+ final String magicValue = new String(magicValueBuffer);
+
+ final IQvcHeader header = getCorrectQvcHeader(magicValue);
+ header.readFromStream(dis);
+
+ final IQvcFile cacheFile = getCorrectQvcFile(header.getQuantizationType());
+ cacheFile.readFromStream(dis, header);
+
+ return cacheFile;
+ }
+}
diff --git a/src/main/java/cz/it4i/qcmp/cache/SqQvcFile.java b/src/main/java/cz/it4i/qcmp/cache/SqQvcFile.java
index 1565ef34167e9e579fb338a8fd7c8732b092421e..5071ef2369755ab6c6f3ce948ad945b17f898e43 100644
--- a/src/main/java/cz/it4i/qcmp/cache/SqQvcFile.java
+++ b/src/main/java/cz/it4i/qcmp/cache/SqQvcFile.java
@@ -1,26 +1,28 @@
package cz.it4i.qcmp.cache;
-import cz.it4i.qcmp.fileformat.QvcHeaderV1;
+import cz.it4i.qcmp.fileformat.IQvcHeader;
import cz.it4i.qcmp.quantization.scalar.SQCodebook;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
-public class SQCacheFile implements IQvcFile {
- private QvcHeaderV1 header;
+public class SqQvcFile implements IQvcFile {
+ private IQvcHeader header;
private SQCodebook codebook;
- public SQCacheFile() {
+ public SqQvcFile() {
}
- public SQCacheFile(final QvcHeaderV1 header, final SQCodebook codebook) {
+ public SqQvcFile(final IQvcHeader header, final SQCodebook codebook) {
this.header = header;
this.codebook = codebook;
assert (header.getCodebookSize() == codebook.getCodebookSize());
}
+ @Override
public void writeToStream(final DataOutputStream outputStream) throws IOException {
+ // TODO
header.writeToStream(outputStream);
final int[] quantizationValues = codebook.getCentroids();
final long[] frequencies = codebook.getSymbolFrequencies();
@@ -33,13 +35,9 @@ public class SQCacheFile implements IQvcFile {
}
}
- public void readFromStream(final DataInputStream inputStream) throws IOException {
- header = new QvcHeaderV1();
- header.readFromStream(inputStream);
- readFromStream(inputStream, header);
- }
-
- public void readFromStream(final DataInputStream inputStream, final QvcHeaderV1 header) throws IOException {
+ @Override
+ public void readFromStream(final DataInputStream inputStream, final IQvcHeader header) throws IOException {
+ // TODO
this.header = header;
final int codebookSize = header.getCodebookSize();
final int[] centroids = new int[codebookSize];
@@ -54,7 +52,8 @@ public class SQCacheFile implements IQvcFile {
codebook = new SQCodebook(centroids, frequencies);
}
- public QvcHeaderV1 getHeader() {
+ @Override
+ public IQvcHeader getHeader() {
return header;
}
diff --git a/src/main/java/cz/it4i/qcmp/cache/VqQvcFile.java b/src/main/java/cz/it4i/qcmp/cache/VqQvcFile.java
index 1eb135964c69040541b80d19e096ca50a30248d0..13104c43a9f12e76d1cbd9a0f88b3737206b3dd4 100644
--- a/src/main/java/cz/it4i/qcmp/cache/VqQvcFile.java
+++ b/src/main/java/cz/it4i/qcmp/cache/VqQvcFile.java
@@ -1,26 +1,28 @@
package cz.it4i.qcmp.cache;
-import cz.it4i.qcmp.fileformat.QvcHeaderV1;
+import cz.it4i.qcmp.fileformat.IQvcHeader;
import cz.it4i.qcmp.quantization.vector.VQCodebook;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
-public class VQCacheFile implements IQvcFile {
- private QvcHeaderV1 header;
+public class VqQvcFile implements IQvcFile {
+ private IQvcHeader header;
private VQCodebook codebook;
- public VQCacheFile() {
+ public VqQvcFile() {
}
- public VQCacheFile(final QvcHeaderV1 header, final VQCodebook codebook) {
+ public VqQvcFile(final IQvcHeader header, final VQCodebook codebook) {
this.header = header;
this.codebook = codebook;
assert (header.getCodebookSize() == codebook.getCodebookSize());
}
+ @Override
public void writeToStream(final DataOutputStream outputStream) throws IOException {
+ // TODO
header.writeToStream(outputStream);
final int[][] entries = codebook.getVectors();
@@ -36,18 +38,13 @@ public class VQCacheFile implements IQvcFile {
}
}
- public void readFromStream(final DataInputStream inputStream) throws IOException {
- header = new QvcHeaderV1();
- header.readFromStream(inputStream);
- readFromStream(inputStream, header);
- }
-
@Override
- public void readFromStream(final DataInputStream inputStream, final QvcHeaderV1 header) throws IOException {
+ public void readFromStream(final DataInputStream inputStream, final IQvcHeader header) throws IOException {
+ // TODO
this.header = header;
final int codebookSize = header.getCodebookSize();
- final int entrySize = header.getVectorSizeX() * header.getVectorSizeY() * header.getVectorSizeZ();
+ final int entrySize = header.getVectorDim().multiplyTogether();
final int[][] vectors = new int[codebookSize][entrySize];
final long[] frequencies = new long[codebookSize];
@@ -64,7 +61,8 @@ public class VQCacheFile implements IQvcFile {
codebook = new VQCodebook(header.getVectorDim(), vectors, frequencies);
}
- public QvcHeaderV1 getHeader() {
+ @Override
+ public IQvcHeader getHeader() {
return header;
}
diff --git a/src/main/java/cz/it4i/qcmp/fileformat/IQvcHeader.java b/src/main/java/cz/it4i/qcmp/fileformat/IQvcHeader.java
new file mode 100644
index 0000000000000000000000000000000000000000..b1a752f690a7a1b05ec7c3e245229a859a2d23db
--- /dev/null
+++ b/src/main/java/cz/it4i/qcmp/fileformat/IQvcHeader.java
@@ -0,0 +1,13 @@
+package cz.it4i.qcmp.fileformat;
+
+import cz.it4i.qcmp.data.V3i;
+
+public interface IQvcHeader extends IFileHeader {
+ QuantizationType getQuantizationType();
+
+ int getBitsPerCodebookIndex();
+
+ int getCodebookSize();
+
+ V3i getVectorDim();
+}
diff --git a/src/main/java/cz/it4i/qcmp/fileformat/QvcHeaderV1.java b/src/main/java/cz/it4i/qcmp/fileformat/QvcHeaderV1.java
index cc2ab909dd57f86e4b9ad001fd9d1c658bf35059..5fc30acf6c935edfb49894853e71b2213f0d4df2 100644
--- a/src/main/java/cz/it4i/qcmp/fileformat/QvcHeaderV1.java
+++ b/src/main/java/cz/it4i/qcmp/fileformat/QvcHeaderV1.java
@@ -9,21 +9,22 @@ import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
-public class QvcHeaderV1 implements IFileHeader {
+public class QvcHeaderV1 implements IQvcHeader {
//region Constants
private static final int VERSION = 1;
- private static final String MAGIC_VALUE = "QCMPCACHE";
+ public static final String MAGIC_VALUE = "QCMPCACHE";
+ public static final int BASE_HEADER_SIZE = 20;
//endregion
//region Header fields.
- private String magicValue;
- private QuantizationType quantizationType;
- private int codebookSize;
- private int trainFileNameSize;
- private String trainFileName;
- private int vectorSizeX;
- private int vectorSizeY;
- private int vectorSizeZ;
+ protected String magicValue;
+ protected QuantizationType quantizationType;
+ protected int codebookSize;
+ protected int trainFileNameSize;
+ protected String trainFileName;
+ protected int vectorSizeX;
+ protected int vectorSizeY;
+ protected int vectorSizeZ;
//endregion
//region IFileHeader implementation
@@ -49,18 +50,6 @@ public class QvcHeaderV1 implements IFileHeader {
*/
@Override
public void readFromStream(final DataInputStream inputStream) throws IOException {
- final int MIN_AVAIL = 9;
- if (inputStream.available() < MIN_AVAIL) {
- throw new IOException("Provided file is not QCMP cache file.");
- }
-
- final byte[] magicValueBuffer = new byte[MAGIC_VALUE.length()];
- RawDataIO.readFullBuffer(inputStream, magicValueBuffer);
-
- magicValue = new String(magicValueBuffer);
- if (!magicValue.equals(MAGIC_VALUE)) {
- throw new IOException("Invalid QCMP cache file. Wrong magic value.");
- }
quantizationType = QuantizationType.fromByte(inputStream.readByte());
codebookSize = inputStream.readUnsignedShort();
@@ -68,7 +57,6 @@ public class QvcHeaderV1 implements IFileHeader {
final byte[] fileNameBuffer = new byte[trainFileNameSize];
RawDataIO.readFullBuffer(inputStream, fileNameBuffer);
-
trainFileName = new String(fileNameBuffer);
vectorSizeX = inputStream.readUnsignedShort();
@@ -84,7 +72,7 @@ public class QvcHeaderV1 implements IFileHeader {
*/
@Override
public void writeToStream(final DataOutputStream outputStream) throws IOException {
- outputStream.writeBytes(MAGIC_VALUE);
+ outputStream.writeBytes(getMagicValue());
outputStream.writeByte(quantizationType.getValue());
outputStream.writeShort(codebookSize);
@@ -98,16 +86,16 @@ public class QvcHeaderV1 implements IFileHeader {
@Override
public long getExpectedDataSize() {
- long expectedFileSize = 20 + trainFileNameSize; // Base header size
- expectedFileSize += (codebookSize * 8); // Frequency values
+ long expectedFileSize = BASE_HEADER_SIZE + trainFileNameSize;
+ expectedFileSize += codebookSize * 8L; // Frequency values
switch (quantizationType) {
case Scalar:
- expectedFileSize += (codebookSize * 2); // Scalar quantization values
+ expectedFileSize += codebookSize * 2L; // Scalar quantization values
break;
case Vector1D:
case Vector2D:
case Vector3D:
- expectedFileSize += ((vectorSizeX * vectorSizeY * vectorSizeZ) * codebookSize * 2); // Quantization vectors
+ expectedFileSize += (((long) vectorSizeX * vectorSizeY * vectorSizeZ) * codebookSize * 2L); // Quantization vectors
break;
case Invalid:
return -1;
@@ -117,8 +105,8 @@ public class QvcHeaderV1 implements IFileHeader {
@Override
public void report(final StringBuilder sb, final String inputFile) {
- sb.append("HeaderVersion: ").append(VERSION).append('\n');
- sb.append("Magic: ").append(magicValue).append('\n');
+ sb.append("HeaderVersion: ").append(getHeaderVersion()).append('\n');
+ sb.append("Magic: ").append(getMagicValue()).append('\n');
sb.append("CodebookType: ");
switch (quantizationType) {
@@ -155,18 +143,6 @@ public class QvcHeaderV1 implements IFileHeader {
this.trainFileNameSize = this.trainFileName.length();
}
- public void setVectorSizeX(final int vectorSizeX) {
- this.vectorSizeX = vectorSizeX;
- }
-
- public void setVectorSizeY(final int vectorSizeY) {
- this.vectorSizeY = vectorSizeY;
- }
-
- public void setVectorSizeZ(final int vectorSizeZ) {
- this.vectorSizeZ = vectorSizeZ;
- }
-
public QuantizationType getQuantizationType() {
return quantizationType;
}
@@ -179,10 +155,6 @@ public class QvcHeaderV1 implements IFileHeader {
return (int) Utils.log2(codebookSize);
}
- public int getTrainFileNameSize() {
- return trainFileNameSize;
- }
-
public String getTrainFileName() {
return trainFileName;
}
diff --git a/src/main/java/cz/it4i/qcmp/fileformat/QvcHeaderV2.java b/src/main/java/cz/it4i/qcmp/fileformat/QvcHeaderV2.java
new file mode 100644
index 0000000000000000000000000000000000000000..c5d759c06717b2cfcdb1ecb02cef0bc57b120ac8
--- /dev/null
+++ b/src/main/java/cz/it4i/qcmp/fileformat/QvcHeaderV2.java
@@ -0,0 +1,68 @@
+package cz.it4i.qcmp.fileformat;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+
+public class QvcHeaderV2 extends QvcHeaderV1 {
+ //region Constants
+ private static final int VERSION = 2;
+ public static final String MAGIC_VALUE = "QVCFILEV2";
+ public static final int RESERVED_BYTES_SIZE = 10;
+ public static final int BASE_HEADER_SIZE = 32;
+ //endregion
+
+ //region Header fields.
+ int huffmanDataSize;
+ //endregion
+
+ //region IFileHeader implementation
+ @Override
+ public String getMagicValue() {
+ return MAGIC_VALUE;
+ }
+
+ @Override
+ public int getHeaderVersion() {
+ return VERSION;
+ }
+
+ @Override
+ public boolean validateHeader() {
+ return (magicValue != null && magicValue.equals(MAGIC_VALUE));
+ }
+
+ @Override
+ public void readFromStream(final DataInputStream inputStream) throws IOException {
+ super.readFromStream(inputStream);
+ huffmanDataSize = inputStream.readUnsignedShort();
+ final int skipped = inputStream.skipBytes(RESERVED_BYTES_SIZE);
+ if (skipped != RESERVED_BYTES_SIZE)
+ throw new IOException("Unable to read QvcHeaderV2. Unable to skip reserved bytes.");
+ }
+
+ @Override
+ public long getExpectedDataSize() {
+ long expectedFileSize = BASE_HEADER_SIZE + trainFileNameSize + huffmanDataSize;
+
+ switch (quantizationType) {
+ case Scalar:
+ expectedFileSize += codebookSize * 2L; // Scalar quantization values
+ break;
+ case Vector1D:
+ case Vector2D:
+ case Vector3D:
+ expectedFileSize += (((long) vectorSizeX * vectorSizeY * vectorSizeZ) * codebookSize * 2L); // Quantization vectors
+ break;
+ case Invalid:
+ return -1;
+ }
+ return expectedFileSize;
+ }
+
+ @Override
+ public void report(final StringBuilder sb, final String inputFile) {
+ super.report(sb, inputFile);
+ sb.append("HuffmanDataSize: ").append(huffmanDataSize).append('\n');
+ }
+ //endregion
+}