Skip to content
Snippets Groups Projects
Commit a580de0a authored by Vojtech Moravec's avatar Vojtech Moravec
Browse files

Add IFileHeader interface and implement it in CacheFileHeaderV1.

parent 9f43685e
No related branches found
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
Showing
with 191 additions and 132 deletions
src/main/java/cz/it4i/qcmp/cache/ICacheFile.java
+ 3
− 3
View file @ a580de0a
package cz.it4i.qcmp.cache;
import cz.it4i.qcmp.fileformat.CacheFileHeader;
import cz.it4i.qcmp.fileformat.cache.CacheFileHeaderV1;
import java.io.DataInputStream;
import java.io.DataOutputStream;
......@@ -12,9 +12,9 @@ public interface ICacheFile {
void readFromStream(DataInputStream inputStream) throws IOException;
void readFromStream(DataInputStream inputStream, CacheFileHeader header) throws IOException;
void readFromStream(DataInputStream inputStream, CacheFileHeaderV1 header) throws IOException;
CacheFileHeader getHeader();
CacheFileHeaderV1 getHeader();
void report(StringBuilder builder);
......
src/main/java/cz/it4i/qcmp/cache/QuantizationCacheManager.java
+ 10
− 10
View file @ a580de0a
......@@ -2,8 +2,8 @@ package cz.it4i.qcmp.cache;
import cz.it4i.qcmp.compression.CompressionOptions;
import cz.it4i.qcmp.data.V3i;
import cz.it4i.qcmp.fileformat.CacheFileHeader;
import cz.it4i.qcmp.fileformat.QuantizationType;
import cz.it4i.qcmp.fileformat.cache.CacheFileHeaderV1;
import cz.it4i.qcmp.quantization.scalar.SQCodebook;
import cz.it4i.qcmp.quantization.vector.VQCodebook;
......@@ -68,8 +68,8 @@ public class QuantizationCacheManager {
* @param codebook Final SQ codebook.
* @return SQ cache file header.
*/
private CacheFileHeader createHeaderForSQ(final String trainFile, final SQCodebook codebook) {
final CacheFileHeader header = new CacheFileHeader();
private CacheFileHeaderV1 createHeaderForSQ(final String trainFile, final SQCodebook codebook) {
final CacheFileHeaderV1 header = new CacheFileHeaderV1();
header.setQuantizationType(QuantizationType.Scalar);
header.setCodebookSize(codebook.getCodebookSize());
header.setTrainFileName(trainFile);
......@@ -105,8 +105,8 @@ public class QuantizationCacheManager {
* @param codebook Final VQ codebook.
* @return VQ cache file header.
*/
private CacheFileHeader createHeaderForVQ(final String trainFile, final VQCodebook codebook) {
final CacheFileHeader header = new CacheFileHeader();
private CacheFileHeaderV1 createHeaderForVQ(final String trainFile, final VQCodebook codebook) {
final CacheFileHeaderV1 header = new CacheFileHeaderV1();
header.setQuantizationType(getQuantizationTypeFromVectorDimensions(codebook.getVectorDims()));
header.setCodebookSize(codebook.getCodebookSize());
header.setTrainFileName(trainFile);
......@@ -125,7 +125,7 @@ public class QuantizationCacheManager {
public String saveCodebook(final String trainFile, final SQCodebook codebook) throws IOException {
final String fileName = getCacheFilePathForSQ(trainFile, codebook.getCodebookSize()).getAbsolutePath();
final CacheFileHeader header = createHeaderForSQ(new File(trainFile).getName(), codebook);
final CacheFileHeaderV1 header = createHeaderForSQ(new File(trainFile).getName(), codebook);
final SQCacheFile cacheFile = new SQCacheFile(header, codebook);
try (final FileOutputStream fos = new FileOutputStream(fileName, false);
......@@ -152,7 +152,7 @@ public class QuantizationCacheManager {
codebook.getCodebookSize(),
codebook.getVectorDims()).getAbsolutePath();
final CacheFileHeader header = createHeaderForVQ(new File(trainFile).getName(), codebook);
final CacheFileHeaderV1 header = createHeaderForVQ(new File(trainFile).getName(), codebook);
final VQCacheFile cacheFile = new VQCacheFile(header, codebook);
try (final FileOutputStream fos = new FileOutputStream(fileName, false);
......@@ -357,7 +357,7 @@ public class QuantizationCacheManager {
dis = new DataInputStream(inputStream);
}
final CacheFileHeader header = new CacheFileHeader();
final CacheFileHeaderV1 header = new CacheFileHeaderV1();
try {
header.readFromStream(dis);
} catch (final IOException e) {
......@@ -409,12 +409,12 @@ public class QuantizationCacheManager {
* @param path Path to cache file.
*/
public static void inspectCacheFile(final String path, final boolean verbose) {
CacheFileHeader header = null;
CacheFileHeaderV1 header = null;
final long fileSize;
try (final FileInputStream fis = new FileInputStream(path);
final DataInputStream dis = new DataInputStream(fis)) {
fileSize = fis.getChannel().size();
header = new CacheFileHeader();
header = new CacheFileHeaderV1();
header.readFromStream(dis);
} catch (final IOException e) {
e.printStackTrace();
......
src/main/java/cz/it4i/qcmp/cache/SQCacheFile.java
+ 6
− 6
View file @ a580de0a
package cz.it4i.qcmp.cache;
import cz.it4i.qcmp.fileformat.CacheFileHeader;
import cz.it4i.qcmp.fileformat.cache.CacheFileHeaderV1;
import cz.it4i.qcmp.quantization.scalar.SQCodebook;
import java.io.DataInputStream;
......@@ -8,13 +8,13 @@ import java.io.DataOutputStream;
import java.io.IOException;
public class SQCacheFile implements ICacheFile {
private CacheFileHeader header;
private CacheFileHeaderV1 header;
private SQCodebook codebook;
public SQCacheFile() {
}
public SQCacheFile(final CacheFileHeader header, final SQCodebook codebook) {
public SQCacheFile(final CacheFileHeaderV1 header, final SQCodebook codebook) {
this.header = header;
this.codebook = codebook;
assert (header.getCodebookSize() == codebook.getCodebookSize());
......@@ -34,12 +34,12 @@ public class SQCacheFile implements ICacheFile {
}
public void readFromStream(final DataInputStream inputStream) throws IOException {
header = new CacheFileHeader();
header = new CacheFileHeaderV1();
header.readFromStream(inputStream);
readFromStream(inputStream, header);
}
public void readFromStream(final DataInputStream inputStream, final CacheFileHeader header) throws IOException {
public void readFromStream(final DataInputStream inputStream, final CacheFileHeaderV1 header) throws IOException {
this.header = header;
final int codebookSize = header.getCodebookSize();
final int[] centroids = new int[codebookSize];
......@@ -54,7 +54,7 @@ public class SQCacheFile implements ICacheFile {
codebook = new SQCodebook(centroids, frequencies);
}
public CacheFileHeader getHeader() {
public CacheFileHeaderV1 getHeader() {
return header;
}
......
src/main/java/cz/it4i/qcmp/cache/VQCacheFile.java
+ 6
− 6
View file @ a580de0a
package cz.it4i.qcmp.cache;
import cz.it4i.qcmp.fileformat.CacheFileHeader;
import cz.it4i.qcmp.fileformat.cache.CacheFileHeaderV1;
import cz.it4i.qcmp.quantization.vector.VQCodebook;
import java.io.DataInputStream;
......@@ -8,13 +8,13 @@ import java.io.DataOutputStream;
import java.io.IOException;
public class VQCacheFile implements ICacheFile {
private CacheFileHeader header;
private CacheFileHeaderV1 header;
private VQCodebook codebook;
public VQCacheFile() {
}
public VQCacheFile(final CacheFileHeader header, final VQCodebook codebook) {
public VQCacheFile(final CacheFileHeaderV1 header, final VQCodebook codebook) {
this.header = header;
this.codebook = codebook;
assert (header.getCodebookSize() == codebook.getCodebookSize());
......@@ -37,13 +37,13 @@ public class VQCacheFile implements ICacheFile {
}
public void readFromStream(final DataInputStream inputStream) throws IOException {
header = new CacheFileHeader();
header = new CacheFileHeaderV1();
header.readFromStream(inputStream);
readFromStream(inputStream, header);
}
@Override
public void readFromStream(final DataInputStream inputStream, final CacheFileHeader header) throws IOException {
public void readFromStream(final DataInputStream inputStream, final CacheFileHeaderV1 header) throws IOException {
this.header = header;
final int codebookSize = header.getCodebookSize();
......@@ -64,7 +64,7 @@ public class VQCacheFile implements ICacheFile {
codebook = new VQCodebook(header.getVectorDim(), vectors, frequencies);
}
public CacheFileHeader getHeader() {
public CacheFileHeaderV1 getHeader() {
return header;
}
......
src/main/java/cz/it4i/qcmp/fileformat/IFileHeader.java 0 → 100644
+ 21
− 0
View file @ a580de0a
package cz.it4i.qcmp.fileformat;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
public interface IFileHeader {
String getMagicValue();
int getHeaderVersion();
boolean validateHeader();
void writeToStream(final DataOutputStream stream) throws IOException;
void readFromStream(final DataInputStream stream) throws IOException;
void report(final StringBuilder builder);
long getExpectedFileSize();
}
src/main/java/cz/it4i/qcmp/fileformat/CacheFileHeader.java → src/main/java/cz/it4i/qcmp/fileformat/cache/CacheFileHeaderV1.java
+ 126
− 107
View file @ a580de0a
package cz.it4i.qcmp.fileformat;
package cz.it4i.qcmp.fileformat.cache;
import cz.it4i.qcmp.data.V2i;
import cz.it4i.qcmp.data.V3i;
import cz.it4i.qcmp.fileformat.IFileHeader;
import cz.it4i.qcmp.fileformat.QuantizationType;
import cz.it4i.qcmp.io.RawDataIO;
import cz.it4i.qcmp.utilities.Utils;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
public class CacheFileHeader {
public static final String QCMP_CACHE_MAGIC_VALUE = "QCMPCACHE";
public class CacheFileHeaderV1 implements IFileHeader {
//region Constants
private static final int VERSION = 1;
private static final String MAGIC_VALUE = "QCMPCACHE";
//endregion
//region Header fields.
private String magicValue;
private QuantizationType quantizationType;
private int codebookSize;
private int trainFileNameSize;
private String trainFileName;
private int vectorSizeX;
private int vectorSizeY;
private int vectorSizeZ;
//endregion
public void setQuantizationType(final QuantizationType quantizationType) {
this.quantizationType = quantizationType;
}
public void setCodebookSize(final int codebookSize) {
this.codebookSize = codebookSize;
}
public void setTrainFileName(final String trainFileName) {
this.trainFileName = trainFileName;
this.trainFileNameSize = this.trainFileName.length();
//region IFileHeader implementation
@Override
public String getMagicValue() {
return MAGIC_VALUE;
}
public void setVectorSizeX(final int vectorSizeX) {
this.vectorSizeX = vectorSizeX;
@Override
public int getHeaderVersion() {
return VERSION;
}
public void setVectorSizeY(final int vectorSizeY) {
this.vectorSizeY = vectorSizeY;
@Override
public boolean validateHeader() {
return (magicValue != null && magicValue.equals(MAGIC_VALUE));
}
public void setVectorSizeZ(final int vectorSizeZ) {
this.vectorSizeZ = vectorSizeZ;
}
public QuantizationType getQuantizationType() {
return quantizationType;
}
public int getCodebookSize() {
return codebookSize;
}
public int getBitsPerCodebookIndex() {
return (int) Utils.log2(codebookSize);
/**
* Read header from the stream.
*
* @param inputStream Data input stream.
*/
@Override
public void readFromStream(final DataInputStream inputStream) throws IOException {
final int MIN_AVAIL = 9;
if (inputStream.available() < MIN_AVAIL) {
throw new IOException("Provided file is not QCMP cache file.");
}
public int getTrainFileNameSize() {
return trainFileNameSize;
}
final byte[] magicValueBuffer = new byte[MAGIC_VALUE.length()];
RawDataIO.readFullBuffer(inputStream, magicValueBuffer);
public String getTrainFileName() {
return trainFileName;
magicValue = new String(magicValueBuffer);
if (!magicValue.equals(MAGIC_VALUE)) {
throw new IOException("Invalid QCMP cache file. Wrong magic value.");
}
quantizationType = QuantizationType.fromByte(inputStream.readByte());
codebookSize = inputStream.readUnsignedShort();
public int getVectorSizeX() {
return vectorSizeX;
}
trainFileNameSize = inputStream.readUnsignedShort();
public int getVectorSizeY() {
return vectorSizeY;
}
final byte[] fileNameBuffer = new byte[trainFileNameSize];
RawDataIO.readFullBuffer(inputStream, fileNameBuffer);
public int getVectorSizeZ() {
return vectorSizeZ;
}
trainFileName = new String(fileNameBuffer);
public V3i getVectorDim() {
return new V3i(vectorSizeX, vectorSizeY, vectorSizeZ);
vectorSizeX = inputStream.readUnsignedShort();
vectorSizeY = inputStream.readUnsignedShort();
vectorSizeZ = inputStream.readUnsignedShort();
}
/**
......@@ -90,8 +84,9 @@ public class CacheFileHeader {
* @param outputStream Data output stream.
* @throws IOException when fails to write the header to stream.
*/
@Override
public void writeToStream(final DataOutputStream outputStream) throws IOException {
outputStream.writeBytes(QCMP_CACHE_MAGIC_VALUE);
outputStream.writeBytes(MAGIC_VALUE);
outputStream.writeByte(quantizationType.getValue());
outputStream.writeShort(codebookSize);
......@@ -103,6 +98,7 @@ public class CacheFileHeader {
outputStream.writeShort(vectorSizeZ);
}
@Override
public long getExpectedFileSize() {
long expectedFileSize = 20 + trainFileNameSize; // Base header size
expectedFileSize += (codebookSize * 8); // Frequency values
......@@ -121,77 +117,100 @@ public class CacheFileHeader {
return expectedFileSize;
}
/**
* Read header from the stream.
*
* @param inputStream Data input stream.
*/
public void readFromStream(final DataInputStream inputStream) throws IOException {
final int MIN_AVAIL = 9;
if (inputStream.available() < MIN_AVAIL) {
throw new IOException("Invalid file. File too small.");
@Override
public void report(final StringBuilder sb) {
sb.append("HeaderVersion: ").append(VERSION).append('\n');
sb.append("Magic: ").append(magicValue).append('\n');
sb.append("CodebookType: ");
switch (quantizationType) {
case Scalar:
sb.append("Scalar\n");
break;
case Vector1D:
sb.append(String.format("Vector1D [%sx1]\n", vectorSizeX));
break;
case Vector2D:
sb.append(String.format("Vector2D %s\n", new V2i(vectorSizeX, vectorSizeY).toString()));
break;
case Vector3D:
sb.append(String.format("Vector3D %s\n", new V3i(vectorSizeX, vectorSizeY, vectorSizeZ).toString()));
break;
}
sb.append("CodebookSize: ").append(codebookSize).append('\n');
sb.append("TrainFile: ").append(trainFileName).append('\n');
}
//endregion
final byte[] magicBuffer = new byte[QCMP_CACHE_MAGIC_VALUE.length()];
//region Getters and Setters
public void setQuantizationType(final QuantizationType quantizationType) {
this.quantizationType = quantizationType;
}
int toRead = QCMP_CACHE_MAGIC_VALUE.length();
while (toRead > 0) {
final int read = inputStream.read(magicBuffer, QCMP_CACHE_MAGIC_VALUE.length() - toRead, toRead);
if (read < 0) {
throw new IOException("Invalid file type. Unable to read magic value");
public void setCodebookSize(final int codebookSize) {
this.codebookSize = codebookSize;
}
toRead -= read;
public void setTrainFileName(final String trainFileName) {
this.trainFileName = trainFileName;
this.trainFileNameSize = this.trainFileName.length();
}
magicValue = new String(magicBuffer);
if (!magicValue.equals(QCMP_CACHE_MAGIC_VALUE)) {
throw new IOException("Invalid file type. Wrong magic value.");
public void setVectorSizeX(final int vectorSizeX) {
this.vectorSizeX = vectorSizeX;
}
quantizationType = QuantizationType.fromByte(inputStream.readByte());
codebookSize = inputStream.readUnsignedShort();
trainFileNameSize = inputStream.readUnsignedShort();
final byte[] fileNameBuffer = new byte[trainFileNameSize];
public void setVectorSizeY(final int vectorSizeY) {
this.vectorSizeY = vectorSizeY;
}
public void setVectorSizeZ(final int vectorSizeZ) {
this.vectorSizeZ = vectorSizeZ;
}
toRead = trainFileNameSize;
while (toRead > 0) {
toRead -= inputStream.read(fileNameBuffer, trainFileNameSize - toRead, toRead);
public QuantizationType getQuantizationType() {
return quantizationType;
}
public int getCodebookSize() {
return codebookSize;
}
trainFileName = new String(fileNameBuffer);
public int getBitsPerCodebookIndex() {
return (int) Utils.log2(codebookSize);
}
vectorSizeX = inputStream.readUnsignedShort();
vectorSizeY = inputStream.readUnsignedShort();
vectorSizeZ = inputStream.readUnsignedShort();
public int getTrainFileNameSize() {
return trainFileNameSize;
}
public String getTrainFileName() {
return trainFileName;
}
public int getVectorSizeX() {
return vectorSizeX;
}
public int getVectorSizeY() {
return vectorSizeY;
}
public int getVectorSizeZ() {
return vectorSizeZ;
}
public V3i getVectorDim() {
return new V3i(vectorSizeX, vectorSizeY, vectorSizeZ);
}
public void setVectorDims(final V3i v3i) {
this.vectorSizeX = v3i.getX();
this.vectorSizeY = v3i.getY();
this.vectorSizeZ = v3i.getZ();
}
public void report(final StringBuilder sb) {
sb.append("Magic: ").append(magicValue).append('\n');
sb.append("CodebookType: ");
switch (quantizationType) {
case Scalar:
sb.append("Scalar\n");
break;
case Vector1D:
sb.append(String.format("Vector1D [%sx1]\n", vectorSizeX));
break;
case Vector2D:
sb.append(String.format("Vector2D %s\n", new V2i(vectorSizeX, vectorSizeY).toString()));
break;
case Vector3D:
sb.append(String.format("Vector3D %s\n", new V3i(vectorSizeX, vectorSizeY, vectorSizeZ).toString()));
break;
}
sb.append("CodebookSize: ").append(codebookSize).append('\n');
sb.append("TrainFile: ").append(trainFileName).append('\n');
}
//endregion
}
src/main/java/cz/it4i/qcmp/io/RawDataIO.java
+ 19
− 0
View file @ a580de0a
......@@ -5,6 +5,7 @@ import cz.it4i.qcmp.utilities.TypeConverter;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
public class RawDataIO {
......@@ -30,6 +31,24 @@ public class RawDataIO {
fileStream.close();
}
/**
* Read exactly N bytes from stream into the buffer.
*
* @param stream Input stream.
* @param buffer Memory buffer.
* @throws IOException when unable to read buffer.length bytes.
*/
public static void readFullBuffer(final InputStream stream, final byte[] buffer) throws IOException {
int toRead = buffer.length;
while (toRead > 0) {
final int read = stream.read(buffer, (buffer.length - toRead), toRead);
if (read < 0) {
throw new IOException("Unable to read requested number of bytes.");
}
toRead -= read;
}
}
public static void write(final String outFile, final int[] sliceData, final boolean littleEndian) throws IOException {
writeBytesToFile(outFile, TypeConverter.unsignedShortArrayToByteArray(sliceData, littleEndian));
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment