Fix reading from input stream to buffer.

We ignored a case when read would return different read bytes than requested, we fixed it in this commit.

src/main/java/azgracompress/cache/CacheFileHeader.java

@@ -3,6 +3,7 @@ package azgracompress.cache;
 import azgracompress.data.V2i;
 import azgracompress.data.V3i;
 import azgracompress.fileformat.QuantizationType;
+import azgracompress.utilities.Utils;
 
 import java.io.DataInputStream;
 import java.io.DataOutputStream;
@@ -22,29 +23,29 @@ public class CacheFileHeader {
     private int vectorSizeY;
     private int vectorSizeZ;
 
-    public void setQuantizationType(QuantizationType quantizationType) {
+    public void setQuantizationType(final QuantizationType quantizationType) {
         this.quantizationType = quantizationType;
     }
 
-    public void setCodebookSize(int codebookSize) {
+    public void setCodebookSize(final int codebookSize) {
         this.codebookSize = codebookSize;
     }
 
 
-    public void setTrainFileName(String trainFileName) {
+    public void setTrainFileName(final String trainFileName) {
         this.trainFileName = trainFileName;
         this.trainFileNameSize = this.trainFileName.length();
     }
 
-    public void setVectorSizeX(int vectorSizeX) {
+    public void setVectorSizeX(final int vectorSizeX) {
         this.vectorSizeX = vectorSizeX;
     }
 
-    public void setVectorSizeY(int vectorSizeY) {
+    public void setVectorSizeY(final int vectorSizeY) {
         this.vectorSizeY = vectorSizeY;
     }
 
-    public void setVectorSizeZ(int vectorSizeZ) {
+    public void setVectorSizeZ(final int vectorSizeZ) {
         this.vectorSizeZ = vectorSizeZ;
     }
 
@@ -56,6 +57,10 @@ public class CacheFileHeader {
         return codebookSize;
     }
 
+    public int getBitsPerCodebookIndex() {
+        return (int) Utils.log2(codebookSize);
+    }
+
     public int getTrainFileNameSize() {
         return trainFileNameSize;
     }
@@ -86,7 +91,7 @@ public class CacheFileHeader {
      * @param outputStream Data output stream.
      * @throws IOException when fails to write the header to stream.
      */
-    public void writeToStream(DataOutputStream outputStream) throws IOException {
+    public void writeToStream(final DataOutputStream outputStream) throws IOException {
         outputStream.writeBytes(QCMP_CACHE_MAGIC_VALUE);
         outputStream.writeByte(quantizationType.getValue());
         outputStream.writeShort(codebookSize);
@@ -122,18 +127,23 @@ public class CacheFileHeader {
      *
      * @param inputStream Data input stream.
      */
-    public void readFromStream(DataInputStream inputStream) throws IOException {
+    public void readFromStream(final DataInputStream inputStream) throws IOException {
         final int MIN_AVAIL = 9;
         if (inputStream.available() < MIN_AVAIL) {
             throw new IOException("Invalid file. File too small.");
         }
 
-        byte[] magicBuffer = new byte[QCMP_CACHE_MAGIC_VALUE.length()];
+        final byte[] magicBuffer = new byte[QCMP_CACHE_MAGIC_VALUE.length()];
 
-        final int readFromMagic = inputStream.read(magicBuffer, 0, QCMP_CACHE_MAGIC_VALUE.length());
-        if (readFromMagic != QCMP_CACHE_MAGIC_VALUE.length()) {
-            throw new IOException("Invalid file type. Unable to read magic value");
+        int toRead = QCMP_CACHE_MAGIC_VALUE.length();
+        while (toRead > 0) {
+            final int read = inputStream.read(magicBuffer, QCMP_CACHE_MAGIC_VALUE.length() - toRead, toRead);
+            if (read < 0) {
+                throw new IOException("Invalid file type. Unable to read magic value");
+            }
+            toRead -= read;
         }
+
         magicValue = new String(magicBuffer);
         if (!magicValue.equals(QCMP_CACHE_MAGIC_VALUE)) {
             throw new IOException("Invalid file type. Wrong magic value.");
@@ -142,8 +152,15 @@ public class CacheFileHeader {
         codebookSize = inputStream.readUnsignedShort();
 
         trainFileNameSize = inputStream.readUnsignedShort();
-        byte[] fileNameBuffer = new byte[trainFileNameSize];
-        final int readBytes = inputStream.read(fileNameBuffer, 0, trainFileNameSize);
+        final byte[] fileNameBuffer = new byte[trainFileNameSize];
+
+
+        toRead = trainFileNameSize;
+        while (toRead > 0) {
+            toRead -= inputStream.read(fileNameBuffer, trainFileNameSize - toRead, toRead);
+        }
+
+
         trainFileName = new String(fileNameBuffer);
 
         vectorSizeX = inputStream.readUnsignedShort();

src/main/java/azgracompress/compression/ImageCompressor.java

@@ -15,7 +15,7 @@ public class ImageCompressor extends CompressorDecompressorBase {
 
     private final IImageCompressor imageCompressor;
 
-    public ImageCompressor(CompressionOptions options) {
+    public ImageCompressor(final CompressionOptions options) {
         super(options);
         imageCompressor = getImageCompressor();
     }
@@ -25,6 +25,10 @@ public class ImageCompressor extends CompressorDecompressorBase {
         imageCompressor.preloadGlobalCodebook(codebookCacheFile);
     }
 
+    public int getBitsPerCodebookIndex() {
+        return this.options.getBitsPerCodebookIndex();
+    }
+
     /**
      * Set InputData object for compressor.
      *
@@ -43,7 +47,7 @@ public class ImageCompressor extends CompressorDecompressorBase {
      * @return Correct implementation of image compressor or null if configuration is not valid.
      */
     private IImageCompressor getImageCompressor() {
-        IImageCompressor compressor;
+        final IImageCompressor compressor;
         switch (options.getQuantizationType()) {
             case Scalar:
                 compressor = new SQImageCompressor(options);
@@ -80,7 +84,7 @@ public class ImageCompressor extends CompressorDecompressorBase {
         }
         try {
             imageCompressor.trainAndSaveCodebook();
-        } catch (ImageCompressionException e) {
+        } catch (final ImageCompressionException e) {
             System.err.println(e.getMessage());
             e.printStackTrace();
             return false;
@@ -91,7 +95,7 @@ public class ImageCompressor extends CompressorDecompressorBase {
     public int streamCompressChunk(final OutputStream outputStream, final InputData inputData) {
         assert (imageCompressor != null);
 
-        try (DataOutputStream compressStream = new DataOutputStream(new BufferedOutputStream(outputStream, 8192))) {
+        try (final DataOutputStream compressStream = new DataOutputStream(new BufferedOutputStream(outputStream, 8192))) {
             final long[] chunkSizes = imageCompressor.compressStreamChunk(compressStream, inputData);
             for (final long chunkSize : chunkSizes) {
                 assert (chunkSize < U16.Max);
@@ -99,11 +103,11 @@ public class ImageCompressor extends CompressorDecompressorBase {
             }
 
             return (4 * 2) + ((int) Arrays.stream(chunkSizes).sum()) + (chunkSizes.length * 2);
-        } catch (ImageCompressionException ice) {
+        } catch (final ImageCompressionException ice) {
             System.err.println(ice.getMessage());
             return -1;
 
-        } catch (Exception e) {
+        } catch (final Exception e) {
             System.err.println(e.getMessage());
             e.printStackTrace();
             return -1;
@@ -118,8 +122,8 @@ public class ImageCompressor extends CompressorDecompressorBase {
 
         long[] planeDataSizes = null;
 
-        try (FileOutputStream fos = new FileOutputStream(options.getOutputFilePath(), false);
-             DataOutputStream compressStream = new DataOutputStream(new BufferedOutputStream(fos, 8192))) {
+        try (final FileOutputStream fos = new FileOutputStream(options.getOutputFilePath(), false);
+             final DataOutputStream compressStream = new DataOutputStream(new BufferedOutputStream(fos, 8192))) {
 
             final QCMPFileHeader header = createHeader();
             header.writeHeader(compressStream);
@@ -129,10 +133,10 @@ public class ImageCompressor extends CompressorDecompressorBase {
             if (options.isVerbose()) {
                 reportCompressionRatio(header, compressStream.size());
             }
-        } catch (ImageCompressionException ex) {
+        } catch (final ImageCompressionException ex) {
             System.err.println(ex.getMessage());
             return false;
-        } catch (Exception e) {
+        } catch (final Exception e) {
             e.printStackTrace();
             return false;
         }
@@ -142,10 +146,10 @@ public class ImageCompressor extends CompressorDecompressorBase {
             return false;
         }
 
-        try (RandomAccessFile raf = new RandomAccessFile(options.getOutputFilePath(), "rw")) {
+        try (final RandomAccessFile raf = new RandomAccessFile(options.getOutputFilePath(), "rw")) {
             raf.seek(PLANE_DATA_SIZES_OFFSET);
             writePlaneDataSizes(raf, planeDataSizes);
-        } catch (IOException ex) {
+        } catch (final IOException ex) {
             ex.printStackTrace();
             return false;
         }
@@ -160,7 +164,7 @@ public class ImageCompressor extends CompressorDecompressorBase {
      * @param planeDataSizes Written compressed plane sizes.
      * @throws IOException when fails to write plane data size.
      */
-    private void writePlaneDataSizes(RandomAccessFile outStream, final long[] planeDataSizes) throws IOException {
+    private void writePlaneDataSizes(final RandomAccessFile outStream, final long[] planeDataSizes) throws IOException {
         for (final long planeDataSize : planeDataSizes) {
             outStream.writeInt((int) planeDataSize);
         }
@@ -189,7 +193,7 @@ public class ImageCompressor extends CompressorDecompressorBase {
      * @return Valid QCMPFile header for compressed file.
      */
     private QCMPFileHeader createHeader() {
-        QCMPFileHeader header = new QCMPFileHeader();
+        final QCMPFileHeader header = new QCMPFileHeader();
 
 
         header.setQuantizationType(options.getQuantizationType());

src/main/java/azgracompress/fileformat/QCMPFileHeader.java

@@ -65,12 +65,12 @@ public class QCMPFileHeader implements Cloneable {
     public QCMPFileHeader copyOf() {
         try {
             return (QCMPFileHeader) this.clone();
-        } catch (CloneNotSupportedException e) {
+        } catch (final CloneNotSupportedException e) {
             return null;
         }
     }
 
-    public void writeHeader(DataOutputStream outputStream) throws IOException {
+    public void writeHeader(final DataOutputStream outputStream) throws IOException {
         outputStream.writeBytes(QCMP_MAGIC_VALUE);
 
         outputStream.writeByte(quantizationType.getValue());
@@ -95,16 +95,21 @@ public class QCMPFileHeader implements Cloneable {
         }
     }
 
-    public boolean readHeader(DataInputStream inputStream) throws IOException {
+    public boolean readHeader(final DataInputStream inputStream) throws IOException {
         if (inputStream.available() < BASE_QCMP_HEADER_SIZE) {
             return false;
         }
 
-        byte[] magicBuffer = new byte[QCMP_MAGIC_VALUE.length()];
-        final int readFromMagic = inputStream.read(magicBuffer, 0, QCMP_MAGIC_VALUE.length());
-        if (readFromMagic != QCMP_MAGIC_VALUE.length()) {
-            // Invalid magic value.
-            return false;
+        final byte[] magicBuffer = new byte[QCMP_MAGIC_VALUE.length()];
+
+        int toRead = QCMP_MAGIC_VALUE.length();
+        while (toRead > 0) {
+            final int read = inputStream.read(magicBuffer, QCMP_MAGIC_VALUE.length() - toRead, toRead);
+            if (read < 0) {
+                // Invalid magic value.
+                return false;
+            }
+            toRead -= read;
         }
 
         magicValue = new String(magicBuffer);
@@ -142,7 +147,7 @@ public class QCMPFileHeader implements Cloneable {
         return quantizationType;
     }
 
-    public void setQuantizationType(QuantizationType quantizationType) {
+    public void setQuantizationType(final QuantizationType quantizationType) {
         this.quantizationType = quantizationType;
     }
 
@@ -150,7 +155,7 @@ public class QCMPFileHeader implements Cloneable {
         return bitsPerCodebookIndex;
     }
 
-    public void setBitsPerCodebookIndex(byte bitsPerCodebookIndex) {
+    public void setBitsPerCodebookIndex(final byte bitsPerCodebookIndex) {
         this.bitsPerCodebookIndex = bitsPerCodebookIndex;
     }
 
@@ -158,7 +163,7 @@ public class QCMPFileHeader implements Cloneable {
         return codebookPerPlane;
     }
 
-    public void setCodebookPerPlane(boolean codebookPerPlane) {
+    public void setCodebookPerPlane(final boolean codebookPerPlane) {
         this.codebookPerPlane = codebookPerPlane;
     }
 
@@ -166,7 +171,7 @@ public class QCMPFileHeader implements Cloneable {
         return imageSizeX;
     }
 
-    public void setImageSizeX(int imageSizeX) {
+    public void setImageSizeX(final int imageSizeX) {
         this.imageSizeX = imageSizeX;
     }
 
@@ -174,7 +179,7 @@ public class QCMPFileHeader implements Cloneable {
         return imageSizeY;
     }
 
-    public void setImageSizeY(int imageSizeY) {
+    public void setImageSizeY(final int imageSizeY) {
         this.imageSizeY = imageSizeY;
     }
 
@@ -186,7 +191,7 @@ public class QCMPFileHeader implements Cloneable {
         return new V3i(imageSizeX, imageSizeY, imageSizeZ);
     }
 
-    public void setImageSizeZ(int imageSizeZ) {
+    public void setImageSizeZ(final int imageSizeZ) {
         this.imageSizeZ = imageSizeZ;
     }
 
@@ -194,7 +199,7 @@ public class QCMPFileHeader implements Cloneable {
         return vectorSizeX;
     }
 
-    public void setVectorSizeX(int vectorSizeX) {
+    public void setVectorSizeX(final int vectorSizeX) {
         this.vectorSizeX = vectorSizeX;
     }
 
@@ -202,7 +207,7 @@ public class QCMPFileHeader implements Cloneable {
         return vectorSizeY;
     }
 
-    public void setVectorSizeY(int vectorSizeY) {
+    public void setVectorSizeY(final int vectorSizeY) {
         this.vectorSizeY = vectorSizeY;
     }
 
@@ -210,7 +215,7 @@ public class QCMPFileHeader implements Cloneable {
         return vectorSizeZ;
     }
 
-    public void setVectorSizeZ(int vectorSizeZ) {
+    public void setVectorSizeZ(final int vectorSizeZ) {
         this.vectorSizeZ = vectorSizeZ;
     }
 

src/main/java/azgracompress/io/InBitStream.java

@@ -5,8 +5,8 @@ import java.io.InputStream;
 
 public class InBitStream implements AutoCloseable {
 
-    private InputStream inputStream;
-    private byte[] buffer;
+    private final InputStream inputStream;
+    private final byte[] buffer;
     private int bufferPosition;
     private int bytesAvailable;
 
@@ -17,7 +17,7 @@ public class InBitStream implements AutoCloseable {
 
     private boolean allowReadFromUnderlyingStream = true;
 
-    public InBitStream(InputStream inputStream, final int bitsPerValue, final int bufferSize) {
+    public InBitStream(final InputStream inputStream, final int bitsPerValue, final int bufferSize) {
         this.inputStream = inputStream;
         this.bitsPerValue = bitsPerValue;
 
@@ -29,8 +29,17 @@ public class InBitStream implements AutoCloseable {
         bitBufferSize = 0;
     }
 
+    /**
+     * Read whole buffer from input stream.
+     *
+     * @throws IOException when unable to read from input stream.
+     */
     public void readToBuffer() throws IOException {
-        bytesAvailable = inputStream.read(buffer, 0, buffer.length);
+        int toRead = buffer.length;
+        while (toRead > 0) {
+            toRead -= inputStream.read(buffer, buffer.length - toRead, toRead);
+        }
+        bytesAvailable = buffer.length;
         bufferPosition = 0;
     }
 
@@ -62,7 +71,7 @@ public class InBitStream implements AutoCloseable {
             readByteToBitBuffer();
         }
         --bitBufferSize;
-        int bit = bitBuffer & (1 << bitBufferSize);
+        final int bit = bitBuffer & (1 << bitBufferSize);
         return (bit > 0 ? 1 : 0);
     }
 
@@ -79,7 +88,7 @@ public class InBitStream implements AutoCloseable {
     }
 
     public int[] readNValues(final int n) throws IOException {
-        int[] values = new int[n];
+        final int[] values = new int[n];
         for (int i = 0; i < n; i++) {
             values[i] = readValue();
         }

src/main/java/azgracompress/io/loader/RawDataLoader.java

@@ -22,9 +22,9 @@ public final class RawDataLoader extends BasicLoader implements IPlaneLoader {
 
     @Override
     public int[] loadPlaneData(final int plane) throws IOException {
-        byte[] buffer;
+        final byte[] buffer;
 
-        try (FileInputStream fileStream = new FileInputStream(inputDataInfo.getFilePath())) {
+        try (final FileInputStream fileStream = new FileInputStream(inputDataInfo.getFilePath())) {
             final long planeSize = (long) dims.getX() * (long) dims.getY() * 2;
             final long expectedFileSize = planeSize * dims.getZ();
             final long fileSize = fileStream.getChannel().size();
@@ -41,15 +41,21 @@ public final class RawDataLoader extends BasicLoader implements IPlaneLoader {
             if (fileStream.skip(planeOffset) != planeOffset) {
                 throw new IOException("Failed to skip.");
             }
-            if (fileStream.read(buffer, 0, (int) planeSize) != planeSize) {
-                throw new IOException("Read wrong number of bytes.");
+
+            int toRead = (int) planeSize;
+            while (toRead > 0) {
+                final int read = fileStream.read(buffer, (int) planeSize - toRead, toRead);
+                if (read < 0) {
+                    throw new IOException("Read wrong number of bytes.");
+                }
+                toRead -= read;
             }
         }
         return TypeConverter.unsignedShortBytesToIntArray(buffer);
     }
 
     @Override
-    public int[] loadPlanesU16Data(int[] planes) throws IOException {
+    public int[] loadPlanesU16Data(final int[] planes) throws IOException {
         if (planes.length < 1) {
             return new int[0];
         } else if (planes.length == 1) {
@@ -65,12 +71,12 @@ public final class RawDataLoader extends BasicLoader implements IPlaneLoader {
             throw new IOException("Integer count is too big.");
         }
 
-        int[] values = new int[(int) totalValueCount];
+        final int[] values = new int[(int) totalValueCount];
 
         Arrays.sort(planes);
 
-        try (FileInputStream fileStream = new FileInputStream(inputDataInfo.getFilePath());
-             DataInputStream dis = new DataInputStream(new BufferedInputStream(fileStream, 8192))) {
+        try (final FileInputStream fileStream = new FileInputStream(inputDataInfo.getFilePath());
+             final DataInputStream dis = new DataInputStream(new BufferedInputStream(fileStream, 8192))) {
 
             int lastIndex = 0;
             int valIndex = 0;
@@ -104,10 +110,10 @@ public final class RawDataLoader extends BasicLoader implements IPlaneLoader {
             throw new IOException("RawFile size is too big.");
         }
 
-        int[] values = new int[(int) dataSize];
+        final int[] values = new int[(int) dataSize];
 
-        try (FileInputStream fileStream = new FileInputStream(inputDataInfo.getFilePath());
-             DataInputStream dis = new DataInputStream(new BufferedInputStream(fileStream, 8192))) {
+        try (final FileInputStream fileStream = new FileInputStream(inputDataInfo.getFilePath());
+             final DataInputStream dis = new DataInputStream(new BufferedInputStream(fileStream, 8192))) {
 
             for (int i = 0; i < (int) dataSize; i++) {
                 values[i] = dis.readUnsignedShort();
@@ -123,7 +129,7 @@ public final class RawDataLoader extends BasicLoader implements IPlaneLoader {
     }
 
     @Override
-    public int[][] loadBlocks(V2i blockDim, Range<Integer> planeRange) throws IOException {
+    public int[][] loadBlocks(final V2i blockDim, final Range<Integer> planeRange) throws IOException {
         return loadBlocksImplByLoadPlaneData(blockDim, planeRange);
     }