diff --git a/src/main/java/cz/it4i/qcmp/cache/QuantizationCacheManager.java b/src/main/java/cz/it4i/qcmp/cache/QuantizationCacheManager.java index 6fb310ede0a72cf0ba719c627df93ccca251d48a..bd769e831764f30175ec08400619accc832298f1 100644 --- a/src/main/java/cz/it4i/qcmp/cache/QuantizationCacheManager.java +++ b/src/main/java/cz/it4i/qcmp/cache/QuantizationCacheManager.java @@ -421,7 +421,7 @@ public class QuantizationCacheManager { return; } final StringBuilder reportBuilder = new StringBuilder(); - final long expectedFileSize = header.getExpectedFileSize(); + final long expectedFileSize = header.getExpectedDataSize(); if (expectedFileSize == fileSize) { reportBuilder.append("\u001B[32mCache file is VALID ").append(fileSize).append(" bytes\u001B[0m\n"); } else { @@ -429,7 +429,7 @@ public class QuantizationCacheManager { .append(fileSize).append(" bytes instead of expected ") .append(expectedFileSize).append(" bytes.\n"); } - header.report(reportBuilder); + header.report(reportBuilder, path); if (verbose) { diff --git a/src/main/java/cz/it4i/qcmp/compression/IImageDecompressor.java b/src/main/java/cz/it4i/qcmp/compression/IImageDecompressor.java index 175d9e58c2b8d00d5d07ccc4a8e427b636f31fe0..81817f35a600dc160ee195f6d98cc35c6b0beadf 100644 --- a/src/main/java/cz/it4i/qcmp/compression/IImageDecompressor.java +++ b/src/main/java/cz/it4i/qcmp/compression/IImageDecompressor.java @@ -8,14 +8,6 @@ import java.io.DataInputStream; import java.io.DataOutputStream; public interface IImageDecompressor extends IListenable { - /** - * Get correct size of data block. - * - * @param header QCMPFile header with information about compressed file. - * @return Expected size of data. - */ - long getExpectedDataSize(final QCMPFileHeader header); - /** * Decompress the image planes to decompress stream. * diff --git a/src/main/java/cz/it4i/qcmp/compression/ImageCompressor.java b/src/main/java/cz/it4i/qcmp/compression/ImageCompressor.java index a7cd2ddac5a717c6007d7f0fe1e00ce166035544..92140eb3820d996a65434779e4402273000db67c 100644 --- a/src/main/java/cz/it4i/qcmp/compression/ImageCompressor.java +++ b/src/main/java/cz/it4i/qcmp/compression/ImageCompressor.java @@ -137,7 +137,7 @@ public class ImageCompressor extends CompressorDecompressorBase { final DataOutputStream compressStream = new DataOutputStream(new BufferedOutputStream(fos, 8192))) { final QCMPFileHeader header = createHeader(); - header.writeHeader(compressStream); + header.writeToStream(compressStream); planeDataSizes = imageCompressor.compress(compressStream); diff --git a/src/main/java/cz/it4i/qcmp/compression/ImageDecompressor.java b/src/main/java/cz/it4i/qcmp/compression/ImageDecompressor.java index 74bea24b64a633547663bef3388561a91b791c97..7240b7cdcdec57b7d863e9110753248fe464d7e0 100644 --- a/src/main/java/cz/it4i/qcmp/compression/ImageDecompressor.java +++ b/src/main/java/cz/it4i/qcmp/compression/ImageDecompressor.java @@ -46,10 +46,7 @@ public class ImageDecompressor extends CompressorDecompressorBase { */ private QCMPFileHeader readQCMPFileHeader(final DataInputStream inputStream) throws IOException { final QCMPFileHeader header = new QCMPFileHeader(); - if (!header.readHeader(inputStream)) { - // Invalid QCMPFile header. - return null; - } + header.readFromStream(inputStream); return header; } @@ -102,89 +99,15 @@ public class ImageDecompressor extends CompressorDecompressorBase { return ""; } - if (header == null) { - logBuilder.append("Input file is not valid QCMPFile\n"); + if (!header.validateHeader()) { + logBuilder.append("Input file is not valid QCMP file\n"); validFile = false; } else { - - - final boolean validHeader = header.validateHeader(); - logBuilder.append("Header is:\t\t").append(validHeader ? "valid" : "invalid").append('\n'); - - logBuilder.append("Magic value:\t\t").append(header.getMagicValue()).append('\n'); - logBuilder.append("Quantization type\t"); - switch (header.getQuantizationType()) { - case Scalar: - logBuilder.append("Scalar\n"); - break; - case Vector1D: - logBuilder.append("Vector1D\n"); - break; - case Vector2D: - logBuilder.append("Vector2D\n"); - break; - case Vector3D: - logBuilder.append("Vector3D\n"); - break; - case Invalid: - logBuilder.append("INVALID\n"); - break; - } - logBuilder.append("Bits per pixel:\t\t").append(header.getBitsPerCodebookIndex()).append('\n'); - - logBuilder.append("Codebook:\t\t").append(header.isCodebookPerPlane() ? "one per plane\n" : "one for " + - "all\n"); - - final int codebookSize = (int) Math.pow(2, header.getBitsPerCodebookIndex()); - logBuilder.append("Codebook size:\t\t").append(codebookSize).append('\n'); - - logBuilder.append("Image size X:\t\t").append(header.getImageSizeX()).append('\n'); - logBuilder.append("Image size Y:\t\t").append(header.getImageSizeY()).append('\n'); - logBuilder.append("Image size Z:\t\t").append(header.getImageSizeZ()).append('\n'); - - logBuilder.append("Vector size X:\t\t").append(header.getVectorSizeX()).append('\n'); - logBuilder.append("Vector size Y:\t\t").append(header.getVectorSizeY()).append('\n'); - logBuilder.append("Vector size Z:\t\t").append(header.getVectorSizeZ()).append('\n'); - - final long headerSize = header.getHeaderSize(); - final long fileSize = new File(options.getInputDataInfo().getFilePath()).length(); - final long dataSize = fileSize - header.getHeaderSize(); - - final IImageDecompressor decompressor = getImageDecompressor(header.getQuantizationType()); - - if (decompressor != null) { - final long expectedDataSize = decompressor.getExpectedDataSize(header); - validFile = (dataSize == expectedDataSize); - - logBuilder.append("File size:\t\t").append(fileSize).append(" B"); - - final long KB = (fileSize / 1000); - if (KB > 0) { - logBuilder.append(" (").append(KB).append(" KB)"); - final long MB = (KB / 1000); - if (MB > 0) { - logBuilder.append(" (").append(MB).append(" MB)"); - } - } - logBuilder.append('\n'); - - logBuilder.append("Header size:\t\t").append(headerSize).append(" Bytes\n"); - logBuilder.append("Data size:\t\t").append(dataSize).append(" Bytes ") - .append(dataSize == expectedDataSize ? "(correct)\n" : "(INVALID)\n"); - - final long pixelCount = header.getImageDims().multiplyTogether(); - final long uncompressedSize = 2 * pixelCount; // We assert 16 bit (2 byte) pixel. - final double compressionRatio = (double) fileSize / (double) uncompressedSize; - logBuilder.append(String.format("Compression ratio:\t%.4f\n", compressionRatio)); - - final double BPP = ((double) fileSize * 8.0) / (double) pixelCount; - logBuilder.append(String.format("Bits Per Pixel (BPP):\t%.4f\n", BPP)); - } + header.report(logBuilder, options.getInputDataInfo().getFilePath()); } - logBuilder.append("\n=== Input file is ").append(validFile ? "VALID" : "INVALID").append(" ===\n"); - if (header != null && options.isVerbose()) { + if (validFile && options.isVerbose()) { final String prefix = header.getQuantizationType() != QuantizationType.Vector3D ? "Plane" : "Voxel layer"; final long[] planeDataSizes = header.getPlaneDataSizes(); long planeIndex = 0; @@ -244,7 +167,7 @@ public class ImageDecompressor extends CompressorDecompressorBase { private boolean checkInputFileSize(final QCMPFileHeader header, final IImageDecompressor imageDecompressor) { final long fileSize = new File(options.getInputDataInfo().getFilePath()).length(); final long dataSize = fileSize - header.getHeaderSize(); - final long expectedDataSize = imageDecompressor.getExpectedDataSize(header); + final long expectedDataSize = header.getExpectedDataSize(); if (dataSize != expectedDataSize) { reportStatusToListeners("Invalid file size."); return false; diff --git a/src/main/java/cz/it4i/qcmp/compression/SQImageDecompressor.java b/src/main/java/cz/it4i/qcmp/compression/SQImageDecompressor.java index f18481225ffca453d564bc370474af622b7a324c..2cdfda862681de250c7cc022ef34def6c925a443 100644 --- a/src/main/java/cz/it4i/qcmp/compression/SQImageDecompressor.java +++ b/src/main/java/cz/it4i/qcmp/compression/SQImageDecompressor.java @@ -40,25 +40,6 @@ public class SQImageDecompressor extends CompressorDecompressorBase implements I return new SQCodebook(quantizationValues, symbolFrequencies); } - @Override - public long getExpectedDataSize(final QCMPFileHeader header) { - // Quantization value count. - final int codebookSize = (int) Math.pow(2, header.getBitsPerCodebookIndex()); - - // Total codebook size in bytes. Also symbol frequencies for Huffman. - final long codebookDataSize = ((2 * codebookSize) + (LONG_BYTES * codebookSize)) * - (header.isCodebookPerPlane() ? header.getImageSizeZ() : 1); - - // Indices are encoded using huffman. Plane data size is written in the header. - final long[] planeDataSizes = header.getPlaneDataSizes(); - long totalPlaneDataSize = 0; - for (final long planeDataSize : planeDataSizes) { - totalPlaneDataSize += planeDataSize; - } - - return (codebookDataSize + totalPlaneDataSize); - } - @Override public void decompress(final DataInputStream compressedStream, final DataOutputStream decompressStream, diff --git a/src/main/java/cz/it4i/qcmp/compression/VQImageDecompressor.java b/src/main/java/cz/it4i/qcmp/compression/VQImageDecompressor.java index d3b3e40e5215b936166e4002fc81cb0a4377dbdf..6be7248fdd550eee8f00f3626195c6149ec389e1 100644 --- a/src/main/java/cz/it4i/qcmp/compression/VQImageDecompressor.java +++ b/src/main/java/cz/it4i/qcmp/compression/VQImageDecompressor.java @@ -90,28 +90,7 @@ public class VQImageDecompressor extends CompressorDecompressorBase implements I } return reconstructedChunk; } - - @Override - public long getExpectedDataSize(final QCMPFileHeader header) { - // Vector count in codebook - final int codebookSize = (int) Math.pow(2, header.getBitsPerCodebookIndex()); - - // Single vector size in bytes. - final int vectorDataSize = 2 * header.getVectorSizeX() * header.getVectorSizeY() * header.getVectorSizeZ(); - - // Total codebook size in bytes. - final long codebookDataSize = ((codebookSize * vectorDataSize) + (codebookSize * LONG_BYTES)) * - (header.isCodebookPerPlane() ? header.getImageSizeZ() : 1); - - // Indices are encoded using huffman. Plane data size is written in the header. - final long[] planeDataSizes = header.getPlaneDataSizes(); - long totalPlaneDataSize = 0; - for (final long planeDataSize : planeDataSizes) { - totalPlaneDataSize += planeDataSize; - } - return (codebookDataSize + totalPlaneDataSize); - } - + @Override public void decompress(final DataInputStream compressedStream, final DataOutputStream decompressStream, diff --git a/src/main/java/cz/it4i/qcmp/fileformat/CacheFileHeaderV1.java b/src/main/java/cz/it4i/qcmp/fileformat/CacheFileHeaderV1.java index 988479bd62c8ba0b706908aef238f4032634364c..167b0f780ca2211072dbe95beb7f832b150e0782 100644 --- a/src/main/java/cz/it4i/qcmp/fileformat/CacheFileHeaderV1.java +++ b/src/main/java/cz/it4i/qcmp/fileformat/CacheFileHeaderV1.java @@ -97,7 +97,7 @@ public class CacheFileHeaderV1 implements IFileHeader { } @Override - public long getExpectedFileSize() { + public long getExpectedDataSize() { long expectedFileSize = 20 + trainFileNameSize; // Base header size expectedFileSize += (codebookSize * 8); // Frequency values switch (quantizationType) { @@ -116,7 +116,7 @@ public class CacheFileHeaderV1 implements IFileHeader { } @Override - public void report(final StringBuilder sb) { + public void report(final StringBuilder sb, final String inputFile) { sb.append("HeaderVersion: ").append(VERSION).append('\n'); sb.append("Magic: ").append(magicValue).append('\n'); diff --git a/src/main/java/cz/it4i/qcmp/fileformat/IFileHeader.java b/src/main/java/cz/it4i/qcmp/fileformat/IFileHeader.java index 5fc5cb2f746120de71bdb662e6dd699a0cd8540a..d35e948d6a1d50263b323a8147326accdf992498 100644 --- a/src/main/java/cz/it4i/qcmp/fileformat/IFileHeader.java +++ b/src/main/java/cz/it4i/qcmp/fileformat/IFileHeader.java @@ -15,7 +15,7 @@ public interface IFileHeader { void readFromStream(final DataInputStream stream) throws IOException; - void report(final StringBuilder builder); + void report(final StringBuilder builder, final String inputFile); - long getExpectedFileSize(); + long getExpectedDataSize(); } diff --git a/src/main/java/cz/it4i/qcmp/fileformat/QCMPFileHeader.java b/src/main/java/cz/it4i/qcmp/fileformat/QCMPFileHeader.java index f613977f33ebf23b46361d2860b92f9d1f430502..e9522b44fc23b32eaf463623c295d3d17289936c 100644 --- a/src/main/java/cz/it4i/qcmp/fileformat/QCMPFileHeader.java +++ b/src/main/java/cz/it4i/qcmp/fileformat/QCMPFileHeader.java @@ -3,16 +3,23 @@ package cz.it4i.qcmp.fileformat; import cz.it4i.qcmp.U16; import cz.it4i.qcmp.compression.VQImageCompressor; import cz.it4i.qcmp.data.V3i; +import cz.it4i.qcmp.io.RawDataIO; +import cz.it4i.qcmp.utilities.Utils; import java.io.DataInputStream; import java.io.DataOutputStream; +import java.io.File; import java.io.IOException; -public class QCMPFileHeader implements Cloneable { - public static final int BASE_QCMP_HEADER_SIZE = 23; - public static final String QCMP_MAGIC_VALUE = "QCMPFILE"; +public class QCMPFileHeader implements IFileHeader, Cloneable { + //region Constants + private static final int VERSION = 1; + private static final int BASE_QCMP_HEADER_SIZE = 23; + private static final String MAGIC_VALUE = "QCMPFILE"; + //endregion - private String magicValue = QCMP_MAGIC_VALUE; + //region Header fields + private String magicValue = MAGIC_VALUE; private QuantizationType quantizationType; private byte bitsPerCodebookIndex; private boolean codebookPerPlane; @@ -26,15 +33,18 @@ public class QCMPFileHeader implements Cloneable { private int vectorSizeZ; private long[] planeDataSizes; + //endregion + //region IFileHeader implementation /** * Validate that all header values are in their valid range. * * @return True if this is valid QCMPFILE header. */ + @Override public boolean validateHeader() { - if (!magicValue.equals(QCMP_MAGIC_VALUE)) + if (!magicValue.equals(MAGIC_VALUE)) return false; if (bitsPerCodebookIndex == 0) @@ -51,27 +61,12 @@ public class QCMPFileHeader implements Cloneable { return false; if (!U16.isInRange(vectorSizeY)) return false; - if (!U16.isInRange(vectorSizeZ)) - return false; - - return true; + return U16.isInRange(vectorSizeZ); } @Override - protected Object clone() throws CloneNotSupportedException { - return super.clone(); - } - - public QCMPFileHeader copyOf() { - try { - return (QCMPFileHeader) this.clone(); - } catch (final CloneNotSupportedException e) { - return null; - } - } - - public void writeHeader(final DataOutputStream outputStream) throws IOException { - outputStream.writeBytes(QCMP_MAGIC_VALUE); + public void writeToStream(final DataOutputStream outputStream) throws IOException { + outputStream.writeBytes(MAGIC_VALUE); outputStream.writeByte(quantizationType.getValue()); outputStream.writeByte(bitsPerCodebookIndex); @@ -95,26 +90,18 @@ public class QCMPFileHeader implements Cloneable { } } - public boolean readHeader(final DataInputStream inputStream) throws IOException { + @Override + public void readFromStream(final DataInputStream inputStream) throws IOException { if (inputStream.available() < BASE_QCMP_HEADER_SIZE) { - return false; + throw new IOException("Provided file is not QCMP file. The file is too small."); } - final byte[] magicBuffer = new byte[QCMP_MAGIC_VALUE.length()]; + final byte[] magicValueBuffer = new byte[MAGIC_VALUE.length()]; + RawDataIO.readFullBuffer(inputStream, magicValueBuffer); - int toRead = QCMP_MAGIC_VALUE.length(); - while (toRead > 0) { - final int read = inputStream.read(magicBuffer, QCMP_MAGIC_VALUE.length() - toRead, toRead); - if (read < 0) { - // Invalid magic value. - return false; - } - toRead -= read; - } - - magicValue = new String(magicBuffer); - if (!magicValue.equals(QCMP_MAGIC_VALUE)) { - return false; + magicValue = new String(magicValueBuffer); + if (!magicValue.equals(MAGIC_VALUE)) { + throw new IOException("Provided file is not QCMP file. Magic value is invalid."); } quantizationType = QuantizationType.fromByte(inputStream.readByte()); @@ -139,10 +126,153 @@ public class QCMPFileHeader implements Cloneable { final long readValue = inputStream.readInt(); planeDataSizes[i] = (readValue & 0x00000000FFFFFFFFL); } + } - return true; + @Override + public int getHeaderVersion() { + return VERSION; } + @Override + public void report(final StringBuilder builder, final String inputFile) { + if (!validateHeader()) { + builder.append("Header is:\t\t invalid\n"); + return; + } + builder.append("HeaderVersion\t\t: ").append(VERSION).append('\n'); + builder.append("Magic value\t\t: ").append(magicValue).append('\n'); + builder.append("Quantization type\t: "); + switch (quantizationType) { + case Scalar: + builder.append("Scalar\n"); + break; + case Vector1D: + builder.append("Vector1D\n"); + break; + case Vector2D: + builder.append("Vector2D\n"); + break; + case Vector3D: + builder.append("Vector3D\n"); + break; + case Invalid: + builder.append("INVALID\n"); + break; + } + builder.append("Bits per pixel\t\t: ").append(bitsPerCodebookIndex).append('\n'); + + builder.append("Codebook\t\t: ").append(codebookPerPlane ? "one per plane\n" : "one for all\n"); + + final int codebookSize = (int) Math.pow(2, bitsPerCodebookIndex); + builder.append("Codebook size\t\t: ").append(codebookSize).append('\n'); + + builder.append("Image stack size\t: ") + .append(imageSizeX).append('x') + .append(imageSizeY).append('x') + .append(imageSizeZ).append('\n'); + + builder.append("Quantization vector\t: ") + .append(vectorSizeX).append('x') + .append(vectorSizeY).append('x') + .append(vectorSizeZ).append('\n'); + + final long headerSize = getHeaderSize(); + final long fileSize = new File(inputFile).length(); + final long dataSize = fileSize - headerSize; + + final long expectedDataSize = getExpectedDataSize(); + final boolean correctFileSize = (dataSize == expectedDataSize); + + builder.append("File size\t\t: "); + Utils.prettyPrintFileSize(builder, fileSize).append('\n'); + + builder.append("Header size\t\t: ").append(headerSize).append(" Bytes\n"); + builder.append("Data size\t\t: "); + Utils.prettyPrintFileSize(builder, dataSize).append(correctFileSize ? "(correct)\n" : "(INVALID)\n"); + + final long pixelCount = imageSizeX * imageSizeY * imageSizeZ; + final long uncompressedSize = 2 * pixelCount; // We assert 16 bit (2 byte) pixel. + final double compressionRatio = (double) fileSize / (double) uncompressedSize; + builder.append(String.format("Compression ratio\t: %.4f\n", compressionRatio)); + + final double BPP = ((double) fileSize * 8.0) / (double) pixelCount; + builder.append(String.format("Bits Per Pixel (BPP)\t: %.4f\n", BPP)); + + builder.append("\n=== Input file is ").append(correctFileSize ? "VALID" : "INVALID").append(" ===\n"); + } + + private long calculateDataSizeForSq() { + final int LONG_BYTES = 8; + // Quantization value count. + final int codebookSize = (int) Math.pow(2, bitsPerCodebookIndex); + + // Total codebook size in bytes. Also symbol frequencies for Huffman. + final long codebookDataSize = ((2 * codebookSize) + (LONG_BYTES * codebookSize)) * (codebookPerPlane ? imageSizeZ : 1); + + // Indices are encoded using huffman. Plane data size is written in the header. + long totalPlaneDataSize = 0; + for (final long planeDataSize : planeDataSizes) { + totalPlaneDataSize += planeDataSize; + } + + return (codebookDataSize + totalPlaneDataSize); + } + + private long calculateDataSizeForVq() { + final int LONG_BYTES = 8; + // Vector count in codebook + final int codebookSize = (int) Math.pow(2, bitsPerCodebookIndex); + + // Single vector size in bytes. + final int vectorDataSize = 2 * vectorSizeX * vectorSizeY * vectorSizeZ; + + // Total codebook size in bytes. + final long codebookDataSize = ((codebookSize * vectorDataSize) + (codebookSize * LONG_BYTES)) * (codebookPerPlane ? imageSizeZ : 1); + + // Indices are encoded using huffman. Plane data size is written in the header. + long totalPlaneDataSize = 0; + for (final long planeDataSize : planeDataSizes) { + totalPlaneDataSize += planeDataSize; + } + return (codebookDataSize + totalPlaneDataSize); + } + + @Override + public long getExpectedDataSize() { + switch (quantizationType) { + case Scalar: + return calculateDataSizeForSq(); + case Vector1D: + case Vector2D: + case Vector3D: + return calculateDataSizeForVq(); + } + return -1; + } + + @Override + public String getMagicValue() { + return magicValue; + } + + //endregion + + //region Cloneable implementation + @Override + protected Object clone() throws CloneNotSupportedException { + return super.clone(); + } + + public QCMPFileHeader copyOf() { + try { + return (QCMPFileHeader) this.clone(); + } catch (final CloneNotSupportedException e) { + return null; + } + } + //endregion + + //region Getters and Setters public QuantizationType getQuantizationType() { return quantizationType; } @@ -219,10 +349,6 @@ public class QCMPFileHeader implements Cloneable { this.vectorSizeZ = vectorSizeZ; } - public String getMagicValue() { - return magicValue; - } - public void setImageDimension(final V3i imageDims) { imageSizeX = imageDims.getX(); imageSizeY = imageDims.getY(); @@ -250,4 +376,5 @@ public class QCMPFileHeader implements Cloneable { return BASE_QCMP_HEADER_SIZE + (chunkCount * 4); } + //endregion } \ No newline at end of file diff --git a/src/main/java/cz/it4i/qcmp/utilities/Utils.java b/src/main/java/cz/it4i/qcmp/utilities/Utils.java index 6d71079b747518e5bbd098cb6d5f6ca55be7a1f9..e4320137ec1f14996308399f8820f1515de2af99 100644 --- a/src/main/java/cz/it4i/qcmp/utilities/Utils.java +++ b/src/main/java/cz/it4i/qcmp/utilities/Utils.java @@ -192,4 +192,18 @@ public class Utils { } return result; } + + public static StringBuilder prettyPrintFileSize(final StringBuilder builder, final long fileSize) { + builder.append(fileSize).append(" B"); + + final long KB = (fileSize / 1000); + if (KB > 0) { + builder.append(" (").append(KB).append(" KB)"); + final long MB = (KB / 1000); + if (MB > 0) { + builder.append(" (").append(MB).append(" MB)"); + } + } + return builder; + } } \ No newline at end of file