diff --git a/src/main/java/cz/it4i/qcmp/cache/ICacheFile.java b/src/main/java/cz/it4i/qcmp/cache/ICacheFile.java
index ac77259775965e2ece828e0e9bad9b9323586edc..555175588b53b52744e09f1be636a92ec3803adb 100644
--- a/src/main/java/cz/it4i/qcmp/cache/ICacheFile.java
+++ b/src/main/java/cz/it4i/qcmp/cache/ICacheFile.java
@@ -1,6 +1,6 @@
package cz.it4i.qcmp.cache;
-import cz.it4i.qcmp.fileformat.CacheFileHeader;
+import cz.it4i.qcmp.fileformat.cache.CacheFileHeaderV1;
import java.io.DataInputStream;
import java.io.DataOutputStream;
@@ -12,9 +12,9 @@ public interface ICacheFile {
void readFromStream(DataInputStream inputStream) throws IOException;
- void readFromStream(DataInputStream inputStream, CacheFileHeader header) throws IOException;
+ void readFromStream(DataInputStream inputStream, CacheFileHeaderV1 header) throws IOException;
- CacheFileHeader getHeader();
+ CacheFileHeaderV1 getHeader();
void report(StringBuilder builder);
diff --git a/src/main/java/cz/it4i/qcmp/cache/QuantizationCacheManager.java b/src/main/java/cz/it4i/qcmp/cache/QuantizationCacheManager.java
index 8903a91635b239149e85168e943074b6889555ee..2cb096855733b517823834102416721a45dd94b5 100644
--- a/src/main/java/cz/it4i/qcmp/cache/QuantizationCacheManager.java
+++ b/src/main/java/cz/it4i/qcmp/cache/QuantizationCacheManager.java
@@ -2,8 +2,8 @@ package cz.it4i.qcmp.cache;
import cz.it4i.qcmp.compression.CompressionOptions;
import cz.it4i.qcmp.data.V3i;
-import cz.it4i.qcmp.fileformat.CacheFileHeader;
import cz.it4i.qcmp.fileformat.QuantizationType;
+import cz.it4i.qcmp.fileformat.cache.CacheFileHeaderV1;
import cz.it4i.qcmp.quantization.scalar.SQCodebook;
import cz.it4i.qcmp.quantization.vector.VQCodebook;
@@ -68,8 +68,8 @@ public class QuantizationCacheManager {
* @param codebook Final SQ codebook.
* @return SQ cache file header.
*/
- private CacheFileHeader createHeaderForSQ(final String trainFile, final SQCodebook codebook) {
- final CacheFileHeader header = new CacheFileHeader();
+ private CacheFileHeaderV1 createHeaderForSQ(final String trainFile, final SQCodebook codebook) {
+ final CacheFileHeaderV1 header = new CacheFileHeaderV1();
header.setQuantizationType(QuantizationType.Scalar);
header.setCodebookSize(codebook.getCodebookSize());
header.setTrainFileName(trainFile);
@@ -105,8 +105,8 @@ public class QuantizationCacheManager {
* @param codebook Final VQ codebook.
* @return VQ cache file header.
*/
- private CacheFileHeader createHeaderForVQ(final String trainFile, final VQCodebook codebook) {
- final CacheFileHeader header = new CacheFileHeader();
+ private CacheFileHeaderV1 createHeaderForVQ(final String trainFile, final VQCodebook codebook) {
+ final CacheFileHeaderV1 header = new CacheFileHeaderV1();
header.setQuantizationType(getQuantizationTypeFromVectorDimensions(codebook.getVectorDims()));
header.setCodebookSize(codebook.getCodebookSize());
header.setTrainFileName(trainFile);
@@ -125,7 +125,7 @@ public class QuantizationCacheManager {
public String saveCodebook(final String trainFile, final SQCodebook codebook) throws IOException {
final String fileName = getCacheFilePathForSQ(trainFile, codebook.getCodebookSize()).getAbsolutePath();
- final CacheFileHeader header = createHeaderForSQ(new File(trainFile).getName(), codebook);
+ final CacheFileHeaderV1 header = createHeaderForSQ(new File(trainFile).getName(), codebook);
final SQCacheFile cacheFile = new SQCacheFile(header, codebook);
try (final FileOutputStream fos = new FileOutputStream(fileName, false);
@@ -152,7 +152,7 @@ public class QuantizationCacheManager {
codebook.getCodebookSize(),
codebook.getVectorDims()).getAbsolutePath();
- final CacheFileHeader header = createHeaderForVQ(new File(trainFile).getName(), codebook);
+ final CacheFileHeaderV1 header = createHeaderForVQ(new File(trainFile).getName(), codebook);
final VQCacheFile cacheFile = new VQCacheFile(header, codebook);
try (final FileOutputStream fos = new FileOutputStream(fileName, false);
@@ -357,7 +357,7 @@ public class QuantizationCacheManager {
dis = new DataInputStream(inputStream);
}
- final CacheFileHeader header = new CacheFileHeader();
+ final CacheFileHeaderV1 header = new CacheFileHeaderV1();
try {
header.readFromStream(dis);
} catch (final IOException e) {
@@ -409,12 +409,12 @@ public class QuantizationCacheManager {
* @param path Path to cache file.
*/
public static void inspectCacheFile(final String path, final boolean verbose) {
- CacheFileHeader header = null;
+ CacheFileHeaderV1 header = null;
final long fileSize;
try (final FileInputStream fis = new FileInputStream(path);
final DataInputStream dis = new DataInputStream(fis)) {
fileSize = fis.getChannel().size();
- header = new CacheFileHeader();
+ header = new CacheFileHeaderV1();
header.readFromStream(dis);
} catch (final IOException e) {
e.printStackTrace();
diff --git a/src/main/java/cz/it4i/qcmp/cache/SQCacheFile.java b/src/main/java/cz/it4i/qcmp/cache/SQCacheFile.java
index b16e402af4038766bdec5d9e4582bf9f37c7c356..1847f5e75d6d6c805d335170544269dbbf91e8cc 100644
--- a/src/main/java/cz/it4i/qcmp/cache/SQCacheFile.java
+++ b/src/main/java/cz/it4i/qcmp/cache/SQCacheFile.java
@@ -1,6 +1,6 @@
package cz.it4i.qcmp.cache;
-import cz.it4i.qcmp.fileformat.CacheFileHeader;
+import cz.it4i.qcmp.fileformat.cache.CacheFileHeaderV1;
import cz.it4i.qcmp.quantization.scalar.SQCodebook;
import java.io.DataInputStream;
@@ -8,13 +8,13 @@ import java.io.DataOutputStream;
import java.io.IOException;
public class SQCacheFile implements ICacheFile {
- private CacheFileHeader header;
+ private CacheFileHeaderV1 header;
private SQCodebook codebook;
public SQCacheFile() {
}
- public SQCacheFile(final CacheFileHeader header, final SQCodebook codebook) {
+ public SQCacheFile(final CacheFileHeaderV1 header, final SQCodebook codebook) {
this.header = header;
this.codebook = codebook;
assert (header.getCodebookSize() == codebook.getCodebookSize());
@@ -34,12 +34,12 @@ public class SQCacheFile implements ICacheFile {
}
public void readFromStream(final DataInputStream inputStream) throws IOException {
- header = new CacheFileHeader();
+ header = new CacheFileHeaderV1();
header.readFromStream(inputStream);
readFromStream(inputStream, header);
}
- public void readFromStream(final DataInputStream inputStream, final CacheFileHeader header) throws IOException {
+ public void readFromStream(final DataInputStream inputStream, final CacheFileHeaderV1 header) throws IOException {
this.header = header;
final int codebookSize = header.getCodebookSize();
final int[] centroids = new int[codebookSize];
@@ -54,7 +54,7 @@ public class SQCacheFile implements ICacheFile {
codebook = new SQCodebook(centroids, frequencies);
}
- public CacheFileHeader getHeader() {
+ public CacheFileHeaderV1 getHeader() {
return header;
}
diff --git a/src/main/java/cz/it4i/qcmp/cache/VQCacheFile.java b/src/main/java/cz/it4i/qcmp/cache/VQCacheFile.java
index 8f4414b178704df91991a96f5cec5e204dbae1bd..6f062a5e13d9d234cf80a61542e8071f166c4241 100644
--- a/src/main/java/cz/it4i/qcmp/cache/VQCacheFile.java
+++ b/src/main/java/cz/it4i/qcmp/cache/VQCacheFile.java
@@ -1,6 +1,6 @@
package cz.it4i.qcmp.cache;
-import cz.it4i.qcmp.fileformat.CacheFileHeader;
+import cz.it4i.qcmp.fileformat.cache.CacheFileHeaderV1;
import cz.it4i.qcmp.quantization.vector.VQCodebook;
import java.io.DataInputStream;
@@ -8,13 +8,13 @@ import java.io.DataOutputStream;
import java.io.IOException;
public class VQCacheFile implements ICacheFile {
- private CacheFileHeader header;
+ private CacheFileHeaderV1 header;
private VQCodebook codebook;
public VQCacheFile() {
}
- public VQCacheFile(final CacheFileHeader header, final VQCodebook codebook) {
+ public VQCacheFile(final CacheFileHeaderV1 header, final VQCodebook codebook) {
this.header = header;
this.codebook = codebook;
assert (header.getCodebookSize() == codebook.getCodebookSize());
@@ -37,13 +37,13 @@ public class VQCacheFile implements ICacheFile {
}
public void readFromStream(final DataInputStream inputStream) throws IOException {
- header = new CacheFileHeader();
+ header = new CacheFileHeaderV1();
header.readFromStream(inputStream);
readFromStream(inputStream, header);
}
@Override
- public void readFromStream(final DataInputStream inputStream, final CacheFileHeader header) throws IOException {
+ public void readFromStream(final DataInputStream inputStream, final CacheFileHeaderV1 header) throws IOException {
this.header = header;
final int codebookSize = header.getCodebookSize();
@@ -64,7 +64,7 @@ public class VQCacheFile implements ICacheFile {
codebook = new VQCodebook(header.getVectorDim(), vectors, frequencies);
}
- public CacheFileHeader getHeader() {
+ public CacheFileHeaderV1 getHeader() {
return header;
}
diff --git a/src/main/java/cz/it4i/qcmp/fileformat/IFileHeader.java b/src/main/java/cz/it4i/qcmp/fileformat/IFileHeader.java
new file mode 100644
index 0000000000000000000000000000000000000000..5fc5cb2f746120de71bdb662e6dd699a0cd8540a
--- /dev/null
+++ b/src/main/java/cz/it4i/qcmp/fileformat/IFileHeader.java
@@ -0,0 +1,21 @@
+package cz.it4i.qcmp.fileformat;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+public interface IFileHeader {
+ String getMagicValue();
+
+ int getHeaderVersion();
+
+ boolean validateHeader();
+
+ void writeToStream(final DataOutputStream stream) throws IOException;
+
+ void readFromStream(final DataInputStream stream) throws IOException;
+
+ void report(final StringBuilder builder);
+
+ long getExpectedFileSize();
+}
diff --git a/src/main/java/cz/it4i/qcmp/fileformat/CacheFileHeader.java b/src/main/java/cz/it4i/qcmp/fileformat/cache/CacheFileHeaderV1.java
similarity index 78%
rename from src/main/java/cz/it4i/qcmp/fileformat/CacheFileHeader.java
rename to src/main/java/cz/it4i/qcmp/fileformat/cache/CacheFileHeaderV1.java
index ef775618ab74af0657f0ba76ace2beb1c0d091c4..4538418ac61f48d576574ee40eb9ae18d55277ac 100644
--- a/src/main/java/cz/it4i/qcmp/fileformat/CacheFileHeader.java
+++ b/src/main/java/cz/it4i/qcmp/fileformat/cache/CacheFileHeaderV1.java
@@ -1,87 +1,81 @@
-package cz.it4i.qcmp.fileformat;
+package cz.it4i.qcmp.fileformat.cache;
import cz.it4i.qcmp.data.V2i;
import cz.it4i.qcmp.data.V3i;
+import cz.it4i.qcmp.fileformat.IFileHeader;
+import cz.it4i.qcmp.fileformat.QuantizationType;
+import cz.it4i.qcmp.io.RawDataIO;
import cz.it4i.qcmp.utilities.Utils;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
-public class CacheFileHeader {
- public static final String QCMP_CACHE_MAGIC_VALUE = "QCMPCACHE";
+public class CacheFileHeaderV1 implements IFileHeader {
+ //region Constants
+ private static final int VERSION = 1;
+ private static final String MAGIC_VALUE = "QCMPCACHE";
+ //endregion
+
+ //region Header fields.
private String magicValue;
private QuantizationType quantizationType;
-
private int codebookSize;
-
private int trainFileNameSize;
private String trainFileName;
-
private int vectorSizeX;
private int vectorSizeY;
private int vectorSizeZ;
+ //endregion
- public void setQuantizationType(final QuantizationType quantizationType) {
- this.quantizationType = quantizationType;
+ //region IFileHeader implementation
+ @Override
+ public String getMagicValue() {
+ return MAGIC_VALUE;
}
- public void setCodebookSize(final int codebookSize) {
- this.codebookSize = codebookSize;
+ @Override
+ public int getHeaderVersion() {
+ return VERSION;
}
-
- public void setTrainFileName(final String trainFileName) {
- this.trainFileName = trainFileName;
- this.trainFileNameSize = this.trainFileName.length();
+ @Override
+ public boolean validateHeader() {
+ return (magicValue != null && magicValue.equals(MAGIC_VALUE));
}
- public void setVectorSizeX(final int vectorSizeX) {
- this.vectorSizeX = vectorSizeX;
- }
-
- public void setVectorSizeY(final int vectorSizeY) {
- this.vectorSizeY = vectorSizeY;
- }
-
- public void setVectorSizeZ(final int vectorSizeZ) {
- this.vectorSizeZ = vectorSizeZ;
- }
-
- public QuantizationType getQuantizationType() {
- return quantizationType;
- }
-
- public int getCodebookSize() {
- return codebookSize;
- }
+ /**
+ * Read header from the stream.
+ *
+ * @param inputStream Data input stream.
+ */
+ @Override
+ public void readFromStream(final DataInputStream inputStream) throws IOException {
+ final int MIN_AVAIL = 9;
+ if (inputStream.available() < MIN_AVAIL) {
+ throw new IOException("Provided file is not QCMP cache file.");
+ }
- public int getBitsPerCodebookIndex() {
- return (int) Utils.log2(codebookSize);
- }
+ final byte[] magicValueBuffer = new byte[MAGIC_VALUE.length()];
+ RawDataIO.readFullBuffer(inputStream, magicValueBuffer);
- public int getTrainFileNameSize() {
- return trainFileNameSize;
- }
-
- public String getTrainFileName() {
- return trainFileName;
- }
+ magicValue = new String(magicValueBuffer);
+ if (!magicValue.equals(MAGIC_VALUE)) {
+ throw new IOException("Invalid QCMP cache file. Wrong magic value.");
+ }
+ quantizationType = QuantizationType.fromByte(inputStream.readByte());
+ codebookSize = inputStream.readUnsignedShort();
- public int getVectorSizeX() {
- return vectorSizeX;
- }
+ trainFileNameSize = inputStream.readUnsignedShort();
- public int getVectorSizeY() {
- return vectorSizeY;
- }
+ final byte[] fileNameBuffer = new byte[trainFileNameSize];
+ RawDataIO.readFullBuffer(inputStream, fileNameBuffer);
- public int getVectorSizeZ() {
- return vectorSizeZ;
- }
+ trainFileName = new String(fileNameBuffer);
- public V3i getVectorDim() {
- return new V3i(vectorSizeX, vectorSizeY, vectorSizeZ);
+ vectorSizeX = inputStream.readUnsignedShort();
+ vectorSizeY = inputStream.readUnsignedShort();
+ vectorSizeZ = inputStream.readUnsignedShort();
}
/**
@@ -90,8 +84,9 @@ public class CacheFileHeader {
* @param outputStream Data output stream.
* @throws IOException when fails to write the header to stream.
*/
+ @Override
public void writeToStream(final DataOutputStream outputStream) throws IOException {
- outputStream.writeBytes(QCMP_CACHE_MAGIC_VALUE);
+ outputStream.writeBytes(MAGIC_VALUE);
outputStream.writeByte(quantizationType.getValue());
outputStream.writeShort(codebookSize);
@@ -103,6 +98,7 @@ public class CacheFileHeader {
outputStream.writeShort(vectorSizeZ);
}
+ @Override
public long getExpectedFileSize() {
long expectedFileSize = 20 + trainFileNameSize; // Base header size
expectedFileSize += (codebookSize * 8); // Frequency values
@@ -121,59 +117,9 @@ public class CacheFileHeader {
return expectedFileSize;
}
- /**
- * Read header from the stream.
- *
- * @param inputStream Data input stream.
- */
- public void readFromStream(final DataInputStream inputStream) throws IOException {
- final int MIN_AVAIL = 9;
- if (inputStream.available() < MIN_AVAIL) {
- throw new IOException("Invalid file. File too small.");
- }
-
- final byte[] magicBuffer = new byte[QCMP_CACHE_MAGIC_VALUE.length()];
-
- int toRead = QCMP_CACHE_MAGIC_VALUE.length();
- while (toRead > 0) {
- final int read = inputStream.read(magicBuffer, QCMP_CACHE_MAGIC_VALUE.length() - toRead, toRead);
- if (read < 0) {
- throw new IOException("Invalid file type. Unable to read magic value");
- }
- toRead -= read;
- }
-
- magicValue = new String(magicBuffer);
- if (!magicValue.equals(QCMP_CACHE_MAGIC_VALUE)) {
- throw new IOException("Invalid file type. Wrong magic value.");
- }
- quantizationType = QuantizationType.fromByte(inputStream.readByte());
- codebookSize = inputStream.readUnsignedShort();
-
- trainFileNameSize = inputStream.readUnsignedShort();
- final byte[] fileNameBuffer = new byte[trainFileNameSize];
-
-
- toRead = trainFileNameSize;
- while (toRead > 0) {
- toRead -= inputStream.read(fileNameBuffer, trainFileNameSize - toRead, toRead);
- }
-
-
- trainFileName = new String(fileNameBuffer);
-
- vectorSizeX = inputStream.readUnsignedShort();
- vectorSizeY = inputStream.readUnsignedShort();
- vectorSizeZ = inputStream.readUnsignedShort();
- }
-
- public void setVectorDims(final V3i v3i) {
- this.vectorSizeX = v3i.getX();
- this.vectorSizeY = v3i.getY();
- this.vectorSizeZ = v3i.getZ();
- }
-
+ @Override
public void report(final StringBuilder sb) {
+ sb.append("HeaderVersion: ").append(VERSION).append('\n');
sb.append("Magic: ").append(magicValue).append('\n');
sb.append("CodebookType: ");
@@ -194,4 +140,77 @@ public class CacheFileHeader {
sb.append("CodebookSize: ").append(codebookSize).append('\n');
sb.append("TrainFile: ").append(trainFileName).append('\n');
}
+ //endregion
+
+ //region Getters and Setters
+ public void setQuantizationType(final QuantizationType quantizationType) {
+ this.quantizationType = quantizationType;
+ }
+
+ public void setCodebookSize(final int codebookSize) {
+ this.codebookSize = codebookSize;
+ }
+
+
+ public void setTrainFileName(final String trainFileName) {
+ this.trainFileName = trainFileName;
+ this.trainFileNameSize = this.trainFileName.length();
+ }
+
+ public void setVectorSizeX(final int vectorSizeX) {
+ this.vectorSizeX = vectorSizeX;
+ }
+
+ public void setVectorSizeY(final int vectorSizeY) {
+ this.vectorSizeY = vectorSizeY;
+ }
+
+ public void setVectorSizeZ(final int vectorSizeZ) {
+ this.vectorSizeZ = vectorSizeZ;
+ }
+
+ public QuantizationType getQuantizationType() {
+ return quantizationType;
+ }
+
+ public int getCodebookSize() {
+ return codebookSize;
+ }
+
+ public int getBitsPerCodebookIndex() {
+ return (int) Utils.log2(codebookSize);
+ }
+
+ public int getTrainFileNameSize() {
+ return trainFileNameSize;
+ }
+
+ public String getTrainFileName() {
+ return trainFileName;
+ }
+
+ public int getVectorSizeX() {
+ return vectorSizeX;
+ }
+
+ public int getVectorSizeY() {
+ return vectorSizeY;
+ }
+
+ public int getVectorSizeZ() {
+ return vectorSizeZ;
+ }
+
+ public V3i getVectorDim() {
+ return new V3i(vectorSizeX, vectorSizeY, vectorSizeZ);
+ }
+
+
+ public void setVectorDims(final V3i v3i) {
+ this.vectorSizeX = v3i.getX();
+ this.vectorSizeY = v3i.getY();
+ this.vectorSizeZ = v3i.getZ();
+ }
+
+ //endregion
}
diff --git a/src/main/java/cz/it4i/qcmp/io/RawDataIO.java b/src/main/java/cz/it4i/qcmp/io/RawDataIO.java
index d1af8b3ea50310f4579877025ce21f4e4fa4517f..6a16d014ebdd1dbd246300cfa9e37b246ef697b0 100644
--- a/src/main/java/cz/it4i/qcmp/io/RawDataIO.java
+++ b/src/main/java/cz/it4i/qcmp/io/RawDataIO.java
@@ -5,6 +5,7 @@ import cz.it4i.qcmp.utilities.TypeConverter;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.InputStream;
public class RawDataIO {
@@ -30,6 +31,24 @@ public class RawDataIO {
fileStream.close();
}
+ /**
+ * Read exactly N bytes from stream into the buffer.
+ *
+ * @param stream Input stream.
+ * @param buffer Memory buffer.
+ * @throws IOException when unable to read buffer.length bytes.
+ */
+ public static void readFullBuffer(final InputStream stream, final byte[] buffer) throws IOException {
+ int toRead = buffer.length;
+ while (toRead > 0) {
+ final int read = stream.read(buffer, (buffer.length - toRead), toRead);
+ if (read < 0) {
+ throw new IOException("Unable to read requested number of bytes.");
+ }
+ toRead -= read;
+ }
+ }
+
public static void write(final String outFile, final int[] sliceData, final boolean littleEndian) throws IOException {
writeBytesToFile(outFile, TypeConverter.unsignedShortArrayToByteArray(sliceData, littleEndian));