Skip to content
Snippets Groups Projects
Commit 84cd8992 authored by Vojtech Moravec's avatar Vojtech Moravec
Browse files

Fix reading from input stream to buffer. 

We ignored a case when read would return different read bytes than requested, we fixed it in this commit.
parent 2cf90ad3
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
src/main/java/azgracompress/cache/CacheFileHeader.java
+ 31
− 14
View file @ 84cd8992
......@@ -3,6 +3,7 @@ package azgracompress.cache;
import azgracompress.data.V2i;
import azgracompress.data.V3i;
import azgracompress.fileformat.QuantizationType;
import azgracompress.utilities.Utils;
import java.io.DataInputStream;
import java.io.DataOutputStream;
......@@ -22,29 +23,29 @@ public class CacheFileHeader {
private int vectorSizeY;
private int vectorSizeZ;
public void setQuantizationType(QuantizationType quantizationType) {
public void setQuantizationType(final QuantizationType quantizationType) {
this.quantizationType = quantizationType;
}
public void setCodebookSize(int codebookSize) {
public void setCodebookSize(final int codebookSize) {
this.codebookSize = codebookSize;
}
public void setTrainFileName(String trainFileName) {
public void setTrainFileName(final String trainFileName) {
this.trainFileName = trainFileName;
this.trainFileNameSize = this.trainFileName.length();
}
public void setVectorSizeX(int vectorSizeX) {
public void setVectorSizeX(final int vectorSizeX) {
this.vectorSizeX = vectorSizeX;
}
public void setVectorSizeY(int vectorSizeY) {
public void setVectorSizeY(final int vectorSizeY) {
this.vectorSizeY = vectorSizeY;
}
public void setVectorSizeZ(int vectorSizeZ) {
public void setVectorSizeZ(final int vectorSizeZ) {
this.vectorSizeZ = vectorSizeZ;
}
......@@ -56,6 +57,10 @@ public class CacheFileHeader {
return codebookSize;
}
public int getBitsPerCodebookIndex() {
return (int) Utils.log2(codebookSize);
}
public int getTrainFileNameSize() {
return trainFileNameSize;
}
......@@ -86,7 +91,7 @@ public class CacheFileHeader {
* @param outputStream Data output stream.
* @throws IOException when fails to write the header to stream.
*/
public void writeToStream(DataOutputStream outputStream) throws IOException {
public void writeToStream(final DataOutputStream outputStream) throws IOException {
outputStream.writeBytes(QCMP_CACHE_MAGIC_VALUE);
outputStream.writeByte(quantizationType.getValue());
outputStream.writeShort(codebookSize);
......@@ -122,18 +127,23 @@ public class CacheFileHeader {
*
* @param inputStream Data input stream.
*/
public void readFromStream(DataInputStream inputStream) throws IOException {
public void readFromStream(final DataInputStream inputStream) throws IOException {
final int MIN_AVAIL = 9;
if (inputStream.available() < MIN_AVAIL) {
throw new IOException("Invalid file. File too small.");
}
byte[] magicBuffer = new byte[QCMP_CACHE_MAGIC_VALUE.length()];
final byte[] magicBuffer = new byte[QCMP_CACHE_MAGIC_VALUE.length()];
final int readFromMagic = inputStream.read(magicBuffer, 0, QCMP_CACHE_MAGIC_VALUE.length());
if (readFromMagic != QCMP_CACHE_MAGIC_VALUE.length()) {
throw new IOException("Invalid file type. Unable to read magic value");
int toRead = QCMP_CACHE_MAGIC_VALUE.length();
while (toRead > 0) {
final int read = inputStream.read(magicBuffer, QCMP_CACHE_MAGIC_VALUE.length() - toRead, toRead);
if (read < 0) {
throw new IOException("Invalid file type. Unable to read magic value");
}
toRead -= read;
}
magicValue = new String(magicBuffer);
if (!magicValue.equals(QCMP_CACHE_MAGIC_VALUE)) {
throw new IOException("Invalid file type. Wrong magic value.");
......@@ -142,8 +152,15 @@ public class CacheFileHeader {
codebookSize = inputStream.readUnsignedShort();
trainFileNameSize = inputStream.readUnsignedShort();
byte[] fileNameBuffer = new byte[trainFileNameSize];
final int readBytes = inputStream.read(fileNameBuffer, 0, trainFileNameSize);
final byte[] fileNameBuffer = new byte[trainFileNameSize];
toRead = trainFileNameSize;
while (toRead > 0) {
toRead -= inputStream.read(fileNameBuffer, trainFileNameSize - toRead, toRead);
}
trainFileName = new String(fileNameBuffer);
vectorSizeX = inputStream.readUnsignedShort();
......
src/main/java/azgracompress/compression/ImageCompressor.java
+ 18
− 14
View file @ 84cd8992
......@@ -15,7 +15,7 @@ public class ImageCompressor extends CompressorDecompressorBase {
private final IImageCompressor imageCompressor;
public ImageCompressor(CompressionOptions options) {
public ImageCompressor(final CompressionOptions options) {
super(options);
imageCompressor = getImageCompressor();
}
......@@ -25,6 +25,10 @@ public class ImageCompressor extends CompressorDecompressorBase {
imageCompressor.preloadGlobalCodebook(codebookCacheFile);
}
public int getBitsPerCodebookIndex() {
return this.options.getBitsPerCodebookIndex();
}
/**
* Set InputData object for compressor.
*
......@@ -43,7 +47,7 @@ public class ImageCompressor extends CompressorDecompressorBase {
* @return Correct implementation of image compressor or null if configuration is not valid.
*/
private IImageCompressor getImageCompressor() {
IImageCompressor compressor;
final IImageCompressor compressor;
switch (options.getQuantizationType()) {
case Scalar:
compressor = new SQImageCompressor(options);
......@@ -80,7 +84,7 @@ public class ImageCompressor extends CompressorDecompressorBase {
}
try {
imageCompressor.trainAndSaveCodebook();
} catch (ImageCompressionException e) {
} catch (final ImageCompressionException e) {
System.err.println(e.getMessage());
e.printStackTrace();
return false;
......@@ -91,7 +95,7 @@ public class ImageCompressor extends CompressorDecompressorBase {
public int streamCompressChunk(final OutputStream outputStream, final InputData inputData) {
assert (imageCompressor != null);
try (DataOutputStream compressStream = new DataOutputStream(new BufferedOutputStream(outputStream, 8192))) {
try (final DataOutputStream compressStream = new DataOutputStream(new BufferedOutputStream(outputStream, 8192))) {
final long[] chunkSizes = imageCompressor.compressStreamChunk(compressStream, inputData);
for (final long chunkSize : chunkSizes) {
assert (chunkSize < U16.Max);
......@@ -99,11 +103,11 @@ public class ImageCompressor extends CompressorDecompressorBase {
}
return (4 * 2) + ((int) Arrays.stream(chunkSizes).sum()) + (chunkSizes.length * 2);
} catch (ImageCompressionException ice) {
} catch (final ImageCompressionException ice) {
System.err.println(ice.getMessage());
return -1;
} catch (Exception e) {
} catch (final Exception e) {
System.err.println(e.getMessage());
e.printStackTrace();
return -1;
......@@ -118,8 +122,8 @@ public class ImageCompressor extends CompressorDecompressorBase {
long[] planeDataSizes = null;
try (FileOutputStream fos = new FileOutputStream(options.getOutputFilePath(), false);
DataOutputStream compressStream = new DataOutputStream(new BufferedOutputStream(fos, 8192))) {
try (final FileOutputStream fos = new FileOutputStream(options.getOutputFilePath(), false);
final DataOutputStream compressStream = new DataOutputStream(new BufferedOutputStream(fos, 8192))) {
final QCMPFileHeader header = createHeader();
header.writeHeader(compressStream);
......@@ -129,10 +133,10 @@ public class ImageCompressor extends CompressorDecompressorBase {
if (options.isVerbose()) {
reportCompressionRatio(header, compressStream.size());
}
} catch (ImageCompressionException ex) {
} catch (final ImageCompressionException ex) {
System.err.println(ex.getMessage());
return false;
} catch (Exception e) {
} catch (final Exception e) {
e.printStackTrace();
return false;
}
......@@ -142,10 +146,10 @@ public class ImageCompressor extends CompressorDecompressorBase {
return false;
}
try (RandomAccessFile raf = new RandomAccessFile(options.getOutputFilePath(), "rw")) {
try (final RandomAccessFile raf = new RandomAccessFile(options.getOutputFilePath(), "rw")) {
raf.seek(PLANE_DATA_SIZES_OFFSET);
writePlaneDataSizes(raf, planeDataSizes);
} catch (IOException ex) {
} catch (final IOException ex) {
ex.printStackTrace();
return false;
}
......@@ -160,7 +164,7 @@ public class ImageCompressor extends CompressorDecompressorBase {
* @param planeDataSizes Written compressed plane sizes.
* @throws IOException when fails to write plane data size.
*/
private void writePlaneDataSizes(RandomAccessFile outStream, final long[] planeDataSizes) throws IOException {
private void writePlaneDataSizes(final RandomAccessFile outStream, final long[] planeDataSizes) throws IOException {
for (final long planeDataSize : planeDataSizes) {
outStream.writeInt((int) planeDataSize);
}
......@@ -189,7 +193,7 @@ public class ImageCompressor extends CompressorDecompressorBase {
* @return Valid QCMPFile header for compressed file.
*/
private QCMPFileHeader createHeader() {
QCMPFileHeader header = new QCMPFileHeader();
final QCMPFileHeader header = new QCMPFileHeader();
header.setQuantizationType(options.getQuantizationType());
......
src/main/java/azgracompress/fileformat/QCMPFileHeader.java
+ 22
− 17
View file @ 84cd8992
......@@ -65,12 +65,12 @@ public class QCMPFileHeader implements Cloneable {
public QCMPFileHeader copyOf() {
try {
return (QCMPFileHeader) this.clone();
} catch (CloneNotSupportedException e) {
} catch (final CloneNotSupportedException e) {
return null;
}
}
public void writeHeader(DataOutputStream outputStream) throws IOException {
public void writeHeader(final DataOutputStream outputStream) throws IOException {
outputStream.writeBytes(QCMP_MAGIC_VALUE);
outputStream.writeByte(quantizationType.getValue());
......@@ -95,16 +95,21 @@ public class QCMPFileHeader implements Cloneable {
}
}
public boolean readHeader(DataInputStream inputStream) throws IOException {
public boolean readHeader(final DataInputStream inputStream) throws IOException {
if (inputStream.available() < BASE_QCMP_HEADER_SIZE) {
return false;
}
byte[] magicBuffer = new byte[QCMP_MAGIC_VALUE.length()];
final int readFromMagic = inputStream.read(magicBuffer, 0, QCMP_MAGIC_VALUE.length());
if (readFromMagic != QCMP_MAGIC_VALUE.length()) {
// Invalid magic value.
return false;
final byte[] magicBuffer = new byte[QCMP_MAGIC_VALUE.length()];
int toRead = QCMP_MAGIC_VALUE.length();
while (toRead > 0) {
final int read = inputStream.read(magicBuffer, QCMP_MAGIC_VALUE.length() - toRead, toRead);
if (read < 0) {
// Invalid magic value.
return false;
}
toRead -= read;
}
magicValue = new String(magicBuffer);
......@@ -142,7 +147,7 @@ public class QCMPFileHeader implements Cloneable {
return quantizationType;
}
public void setQuantizationType(QuantizationType quantizationType) {
public void setQuantizationType(final QuantizationType quantizationType) {
this.quantizationType = quantizationType;
}
......@@ -150,7 +155,7 @@ public class QCMPFileHeader implements Cloneable {
return bitsPerCodebookIndex;
}
public void setBitsPerCodebookIndex(byte bitsPerCodebookIndex) {
public void setBitsPerCodebookIndex(final byte bitsPerCodebookIndex) {
this.bitsPerCodebookIndex = bitsPerCodebookIndex;
}
......@@ -158,7 +163,7 @@ public class QCMPFileHeader implements Cloneable {
return codebookPerPlane;
}
public void setCodebookPerPlane(boolean codebookPerPlane) {
public void setCodebookPerPlane(final boolean codebookPerPlane) {
this.codebookPerPlane = codebookPerPlane;
}
......@@ -166,7 +171,7 @@ public class QCMPFileHeader implements Cloneable {
return imageSizeX;
}
public void setImageSizeX(int imageSizeX) {
public void setImageSizeX(final int imageSizeX) {
this.imageSizeX = imageSizeX;
}
......@@ -174,7 +179,7 @@ public class QCMPFileHeader implements Cloneable {
return imageSizeY;
}
public void setImageSizeY(int imageSizeY) {
public void setImageSizeY(final int imageSizeY) {
this.imageSizeY = imageSizeY;
}
......@@ -186,7 +191,7 @@ public class QCMPFileHeader implements Cloneable {
return new V3i(imageSizeX, imageSizeY, imageSizeZ);
}
public void setImageSizeZ(int imageSizeZ) {
public void setImageSizeZ(final int imageSizeZ) {
this.imageSizeZ = imageSizeZ;
}
......@@ -194,7 +199,7 @@ public class QCMPFileHeader implements Cloneable {
return vectorSizeX;
}
public void setVectorSizeX(int vectorSizeX) {
public void setVectorSizeX(final int vectorSizeX) {
this.vectorSizeX = vectorSizeX;
}
......@@ -202,7 +207,7 @@ public class QCMPFileHeader implements Cloneable {
return vectorSizeY;
}
public void setVectorSizeY(int vectorSizeY) {
public void setVectorSizeY(final int vectorSizeY) {
this.vectorSizeY = vectorSizeY;
}
......@@ -210,7 +215,7 @@ public class QCMPFileHeader implements Cloneable {
return vectorSizeZ;
}
public void setVectorSizeZ(int vectorSizeZ) {
public void setVectorSizeZ(final int vectorSizeZ) {
this.vectorSizeZ = vectorSizeZ;
}
......
src/main/java/azgracompress/io/InBitStream.java
+ 15
− 6
View file @ 84cd8992
......@@ -5,8 +5,8 @@ import java.io.InputStream;
public class InBitStream implements AutoCloseable {
private InputStream inputStream;
private byte[] buffer;
private final InputStream inputStream;
private final byte[] buffer;
private int bufferPosition;
private int bytesAvailable;
......@@ -17,7 +17,7 @@ public class InBitStream implements AutoCloseable {
private boolean allowReadFromUnderlyingStream = true;
public InBitStream(InputStream inputStream, final int bitsPerValue, final int bufferSize) {
public InBitStream(final InputStream inputStream, final int bitsPerValue, final int bufferSize) {
this.inputStream = inputStream;
this.bitsPerValue = bitsPerValue;
......@@ -29,8 +29,17 @@ public class InBitStream implements AutoCloseable {
bitBufferSize = 0;
}
/**
* Read whole buffer from input stream.
*
* @throws IOException when unable to read from input stream.
*/
public void readToBuffer() throws IOException {
bytesAvailable = inputStream.read(buffer, 0, buffer.length);
int toRead = buffer.length;
while (toRead > 0) {
toRead -= inputStream.read(buffer, buffer.length - toRead, toRead);
}
bytesAvailable = buffer.length;
bufferPosition = 0;
}
......@@ -62,7 +71,7 @@ public class InBitStream implements AutoCloseable {
readByteToBitBuffer();
}
--bitBufferSize;
int bit = bitBuffer & (1 << bitBufferSize);
final int bit = bitBuffer & (1 << bitBufferSize);
return (bit > 0 ? 1 : 0);
}
......@@ -79,7 +88,7 @@ public class InBitStream implements AutoCloseable {
}
public int[] readNValues(final int n) throws IOException {
int[] values = new int[n];
final int[] values = new int[n];
for (int i = 0; i < n; i++) {
values[i] = readValue();
}
......
src/main/java/azgracompress/io/loader/RawDataLoader.java
+ 18
− 12
View file @ 84cd8992
......@@ -22,9 +22,9 @@ public final class RawDataLoader extends BasicLoader implements IPlaneLoader {
@Override
public int[] loadPlaneData(final int plane) throws IOException {
byte[] buffer;
final byte[] buffer;
try (FileInputStream fileStream = new FileInputStream(inputDataInfo.getFilePath())) {
try (final FileInputStream fileStream = new FileInputStream(inputDataInfo.getFilePath())) {
final long planeSize = (long) dims.getX() * (long) dims.getY() * 2;
final long expectedFileSize = planeSize * dims.getZ();
final long fileSize = fileStream.getChannel().size();
......@@ -41,15 +41,21 @@ public final class RawDataLoader extends BasicLoader implements IPlaneLoader {
if (fileStream.skip(planeOffset) != planeOffset) {
throw new IOException("Failed to skip.");
}
if (fileStream.read(buffer, 0, (int) planeSize) != planeSize) {
throw new IOException("Read wrong number of bytes.");
int toRead = (int) planeSize;
while (toRead > 0) {
final int read = fileStream.read(buffer, (int) planeSize - toRead, toRead);
if (read < 0) {
throw new IOException("Read wrong number of bytes.");
}
toRead -= read;
}
}
return TypeConverter.unsignedShortBytesToIntArray(buffer);
}
@Override
public int[] loadPlanesU16Data(int[] planes) throws IOException {
public int[] loadPlanesU16Data(final int[] planes) throws IOException {
if (planes.length < 1) {
return new int[0];
} else if (planes.length == 1) {
......@@ -65,12 +71,12 @@ public final class RawDataLoader extends BasicLoader implements IPlaneLoader {
throw new IOException("Integer count is too big.");
}
int[] values = new int[(int) totalValueCount];
final int[] values = new int[(int) totalValueCount];
Arrays.sort(planes);
try (FileInputStream fileStream = new FileInputStream(inputDataInfo.getFilePath());
DataInputStream dis = new DataInputStream(new BufferedInputStream(fileStream, 8192))) {
try (final FileInputStream fileStream = new FileInputStream(inputDataInfo.getFilePath());
final DataInputStream dis = new DataInputStream(new BufferedInputStream(fileStream, 8192))) {
int lastIndex = 0;
int valIndex = 0;
......@@ -104,10 +110,10 @@ public final class RawDataLoader extends BasicLoader implements IPlaneLoader {
throw new IOException("RawFile size is too big.");
}
int[] values = new int[(int) dataSize];
final int[] values = new int[(int) dataSize];
try (FileInputStream fileStream = new FileInputStream(inputDataInfo.getFilePath());
DataInputStream dis = new DataInputStream(new BufferedInputStream(fileStream, 8192))) {
try (final FileInputStream fileStream = new FileInputStream(inputDataInfo.getFilePath());
final DataInputStream dis = new DataInputStream(new BufferedInputStream(fileStream, 8192))) {
for (int i = 0; i < (int) dataSize; i++) {
values[i] = dis.readUnsignedShort();
......@@ -123,7 +129,7 @@ public final class RawDataLoader extends BasicLoader implements IPlaneLoader {
}
@Override
public int[][] loadBlocks(V2i blockDim, Range<Integer> planeRange) throws IOException {
public int[][] loadBlocks(final V2i blockDim, final Range<Integer> planeRange) throws IOException {
return loadBlocksImplByLoadPlaneData(blockDim, planeRange);
}
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment