Skip to content
Snippets Groups Projects
Commit fcc04ca7 authored by Vojtech Moravec's avatar Vojtech Moravec
Browse files

Write the new qvc format and read both formats.

parent 2757703c
Branches
No related tags found
No related merge requests found
package cz.it4i.qcmp.cache;
import cz.it4i.qcmp.fileformat.IQvcHeader;
import cz.it4i.qcmp.fileformat.QvcHeaderV2;
import cz.it4i.qcmp.huffman.HuffmanNode;
import cz.it4i.qcmp.huffman.HuffmanTreeBuilder;
import cz.it4i.qcmp.io.InBitStream;
import cz.it4i.qcmp.io.MemoryOutputStream;
import cz.it4i.qcmp.io.OutBitStream;
import cz.it4i.qcmp.quantization.scalar.SQCodebook;
import java.io.DataInputStream;
......@@ -22,34 +28,66 @@ public class SqQvcFile implements IQvcFile {
@Override
public void writeToStream(final DataOutputStream outputStream) throws IOException {
// TODO
assert (header instanceof QvcHeaderV2) : "Only the latest header is supporter when writing qvc file.";
final int huffmanTreeBinaryRepresentationSize;
final MemoryOutputStream bufferStream = new MemoryOutputStream(256);
try (final OutBitStream bitStream = new OutBitStream(bufferStream, header.getBitsPerCodebookIndex(), 32)) {
codebook.getHuffmanTreeRoot().writeToBinaryStream(bitStream);
huffmanTreeBinaryRepresentationSize = (int) bitStream.getBytesWritten();
}
assert (huffmanTreeBinaryRepresentationSize == bufferStream.getCurrentBufferLength());
((QvcHeaderV2) header).setHuffmanDataSize(huffmanTreeBinaryRepresentationSize);
header.writeToStream(outputStream);
final int[] quantizationValues = codebook.getCentroids();
final long[] frequencies = codebook.getSymbolFrequencies();
for (final int qV : quantizationValues) {
outputStream.writeShort(qV);
}
for (final long sF : frequencies) {
outputStream.writeLong(sF);
}
outputStream.write(bufferStream.getBuffer(), 0, huffmanTreeBinaryRepresentationSize);
}
/**
* Read codebook from file based on format version.
*
* @param inputStream Input stream.
* @param header File header.
* @throws IOException when fails to read from input stream.
*/
@Override
public void readFromStream(final DataInputStream inputStream, final IQvcHeader header) throws IOException {
// TODO
this.header = header;
final int headerVersion = header.getHeaderVersion();
final int codebookSize = header.getCodebookSize();
final int[] centroids = new int[codebookSize];
final long[] frequencies = new long[codebookSize];
final int[] centroids = new int[codebookSize];
for (int i = 0; i < codebookSize; i++) {
centroids[i] = inputStream.readUnsignedShort();
}
for (int i = 0; i < codebookSize; i++) {
frequencies[i] = inputStream.readLong();
final HuffmanNode huffmanRoot;
if (headerVersion == 1) { // First version of qvc file.
final long[] frequencies = new long[codebookSize];
for (int i = 0; i < codebookSize; i++) {
frequencies[i] = inputStream.readLong();
}
final HuffmanTreeBuilder builder = new HuffmanTreeBuilder(codebookSize, frequencies);
builder.buildHuffmanTree();
huffmanRoot = builder.getRoot();
} else if (headerVersion == 2) { // Second version of qvc file.
final InBitStream bitStream = new InBitStream(inputStream,
header.getBitsPerCodebookIndex(),
((QvcHeaderV2) header).getHuffmanDataSize());
bitStream.fillEntireBuffer();
bitStream.setAllowReadFromUnderlyingStream(false);
huffmanRoot = HuffmanNode.readFromStream(bitStream);
} else {
throw new IOException("Unable to read SqQvcFile of version: " + headerVersion);
}
codebook = new SQCodebook(centroids, frequencies);
codebook = new SQCodebook(centroids, huffmanRoot);
}
@Override
......
package cz.it4i.qcmp.cache;
import cz.it4i.qcmp.fileformat.IQvcHeader;
import cz.it4i.qcmp.fileformat.QvcHeaderV2;
import cz.it4i.qcmp.huffman.HuffmanNode;
import cz.it4i.qcmp.huffman.HuffmanTreeBuilder;
import cz.it4i.qcmp.io.InBitStream;
import cz.it4i.qcmp.io.MemoryOutputStream;
import cz.it4i.qcmp.io.OutBitStream;
import cz.it4i.qcmp.quantization.vector.VQCodebook;
import java.io.DataInputStream;
......@@ -22,7 +28,17 @@ public class VqQvcFile implements IQvcFile {
@Override
public void writeToStream(final DataOutputStream outputStream) throws IOException {
// TODO
assert (header instanceof QvcHeaderV2) : "Only the latest header is supporter when writing qvc file.";
final int huffmanTreeBinaryRepresentationSize;
final MemoryOutputStream bufferStream = new MemoryOutputStream(256);
try (final OutBitStream bitStream = new OutBitStream(bufferStream, header.getBitsPerCodebookIndex(), 32)) {
codebook.getHuffmanTreeRoot().writeToBinaryStream(bitStream);
huffmanTreeBinaryRepresentationSize = (int) bitStream.getBytesWritten();
}
assert (huffmanTreeBinaryRepresentationSize == bufferStream.getCurrentBufferLength());
((QvcHeaderV2) header).setHuffmanDataSize(huffmanTreeBinaryRepresentationSize);
header.writeToStream(outputStream);
final int[][] entries = codebook.getVectors();
......@@ -32,33 +48,45 @@ public class VqQvcFile implements IQvcFile {
}
}
final long[] frequencies = codebook.getVectorFrequencies();
for (final long vF : frequencies) {
outputStream.writeLong(vF);
}
outputStream.write(bufferStream.getBuffer(), 0, huffmanTreeBinaryRepresentationSize);
}
@Override
public void readFromStream(final DataInputStream inputStream, final IQvcHeader header) throws IOException {
// TODO
this.header = header;
final int codebookSize = header.getCodebookSize();
final int entrySize = header.getVectorDim().multiplyTogether();
final int[][] vectors = new int[codebookSize][entrySize];
final long[] frequencies = new long[codebookSize];
for (int i = 0; i < codebookSize; i++) {
//int[] vector = new int[entrySize];
for (int j = 0; j < entrySize; j++) {
vectors[i][j] = inputStream.readUnsignedShort();
}
}
for (int i = 0; i < codebookSize; i++) {
frequencies[i] = inputStream.readLong();
final HuffmanNode huffmanRoot;
final int headerVersion = header.getHeaderVersion();
if (headerVersion == 1) {
final long[] frequencies = new long[codebookSize];
for (int i = 0; i < codebookSize; i++) {
frequencies[i] = inputStream.readLong();
}
final HuffmanTreeBuilder builder = new HuffmanTreeBuilder(codebookSize, frequencies);
builder.buildHuffmanTree();
huffmanRoot = builder.getRoot();
} else if (headerVersion == 2) {
final InBitStream bitStream = new InBitStream(inputStream,
header.getBitsPerCodebookIndex(),
((QvcHeaderV2) header).getHuffmanDataSize());
bitStream.fillEntireBuffer();
bitStream.setAllowReadFromUnderlyingStream(false);
huffmanRoot = HuffmanNode.readFromStream(bitStream);
} else {
throw new IOException("Unable to read VqQvcFile of version: " + headerVersion);
}
codebook = new VQCodebook(header.getVectorDim(), vectors, frequencies);
codebook = new VQCodebook(header.getVectorDim(), vectors, huffmanRoot);
}
@Override
......
package cz.it4i.qcmp.fileformat;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
public class QvcHeaderV2 extends QvcHeaderV1 {
......@@ -12,7 +13,7 @@ public class QvcHeaderV2 extends QvcHeaderV1 {
//endregion
//region Header fields.
int huffmanDataSize;
private int huffmanDataSize;
//endregion
//region IFileHeader implementation
......@@ -40,6 +41,17 @@ public class QvcHeaderV2 extends QvcHeaderV1 {
throw new IOException("Unable to read QvcHeaderV2. Unable to skip reserved bytes.");
}
@Override
public void writeToStream(final DataOutputStream outputStream) throws IOException {
super.writeToStream(outputStream);
outputStream.writeShort(huffmanDataSize);
for (int i = 0; i < RESERVED_BYTES_SIZE; i++) {
outputStream.writeByte(0);
}
}
@Override
public long getExpectedDataSize() {
long expectedFileSize = BASE_HEADER_SIZE + trainFileNameSize + huffmanDataSize;
......@@ -65,4 +77,13 @@ public class QvcHeaderV2 extends QvcHeaderV1 {
sb.append("HuffmanDataSize: ").append(huffmanDataSize).append('\n');
}
//endregion
public int getHuffmanDataSize() {
return huffmanDataSize;
}
public void setHuffmanDataSize(final int n) {
huffmanDataSize = n;
}
}
......@@ -47,4 +47,13 @@ public class Codebook {
huffmanEncoder = new HuffmanEncoder(huffmanRoot, HuffmanTreeBuilder.createSymbolCodes(huffmanRoot));
return huffmanEncoder;
}
/**
* Get root of the huffman tree coder.
*
* @return Root of the huffman tree.
*/
public HuffmanNode getHuffmanTreeRoot() {
return huffmanRoot;
}
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment