czi_file.cpp 14.97 KiB
#include "czi_file.h"
bool CziFile::is_master_file() const
{
return ((header.filePart == 0) && (header.fileGuid == header.masterFileGuid));
}
void CziFile::print_segment_header(const SegmentHeader &segmentHeader) const
{
printf("---------SegmentHeader---------\n");
printf("%-25s %15s\n", "SID", segmentHeader.sId.c_str());
printf("%-25s %15li\n", "AllocatedSize", segmentHeader.allocatedSize);
printf("%-25s %15li\n", "UsedSize", segmentHeader.usedSize);
printf("-------------------------------\n");
}
const char *CziFile::pixel_type_str(const PixelType px) const
{
switch (px)
{
case PixelType_Gray8:
return "Gray8";
case PixelType_Gray16:
return "Gray16";
case PixelType_Gray32Float:
return "Gray32Float";
case PixelType_Bgr24:
return "Bgr24";
case PixelType_Bgr48:
return "Bgr48";
case PixelType_Bgr96Float:
return "Bgr96Float";
case PixelType_Bgra32:
return "Bgra32";
case PixelType_Gray64ComplexFloat:
return "Gray64ComplexFloat";
case PixelType_Bgr192ComplexFloat:
return "Bgr192ComplexFloat";
case PixelType_Gray32:
return "Gray32";
case PixelType_Gray64:
return "Gray64";
default:
always_assert("Bad pixel type." && false);
break;
}
return nullptr;
}
const char *CziFile::pyramid_type_str(const PyramidType pt) const
{
switch (pt)
{
case PyramidType_None:
return "None";
case PyramidType_SingleSubBlock:
return "SingleSubBlock";
case PyramidType_MultiSubBlock:
return "MultiSubBlock";
break;
default:
{
always_assert("Bad pyramid type." && false);
break;
}
}
return nullptr;
}
const char *CziFile::compression_type_str(const CompressionType ct) const
{
switch (ct)
{
case CompressionType_Uncompressed:
return "Uncompressed";
case CompressionType_LZW:
return "LZW";
case CompressionType_JpgFile:
return "JpgFile";
case CompressionType_JpegXrFile:
return "JpegXrFile";
case CompressionType_Camera:
return "Camera";
case CompressionType_System:
return "System";
break;
default:
{
always_assert("Bad compression type." && false);
break;
}
}
return nullptr;
}
const char *CziFile::dimension_type_str(const Dimension &d) const
{
switch (d)
{
case Dimension_X:
return "X (width)";
case Dimension_Y:
return "Y (height)";
case Dimension_C:
return "C (channels)";
case Dimension_Z:
return "Z (Z-slices)";
case Dimension_T:
return "T (timestamps)";
case Dimension_R:
return "R (rotation)";
case Dimension_S:
return "S (scene)";
case Dimension_I:
return "I (illumination)";
case Dimension_B:
return "B (block index)";
case Dimension_M:
return "M (mosaic)";
case Dimension_H:
return "H (phase)";
case Dimension_V:
return "V (views)";
break;
default:
{
always_assert("Bad dimension type." && false);
break;
}
}
return nullptr;
}
const char *CziFile::dimension_char(const Dimension &d) const
{
std::string result;
switch (d) {
case Dimension_X:
return "X";
break;
case Dimension_Y:
return "Y";
break;
case Dimension_C:
return "C";
case Dimension_Z:
return "Z";
case Dimension_T:
return "T";
case Dimension_R:
return "R";
case Dimension_S:
return "S";
case Dimension_I:
return "I";
case Dimension_B:
return "B";
case Dimension_M:
return "M";
case Dimension_H:
return "H";
case Dimension_V:
return "V";
default:
{
always_assert("Bad dimension type." && false);
break;
}
}
return nullptr;
}
std::string CziFile::dimension_stack_str(const std::vector<DimensionEntryDV1> &dims, bool includeSize) const
{
std::string result;
for (const DimensionEntryDV1 &entry : dims)
{
result.append(dimension_char(entry.dimension));
// + std::string(includeSize ? () : ""));
if (includeSize)
result.append("(" + std::to_string(entry.size) + ")-");
}
if (includeSize)
{
result = result.substr(0, result.length() - 1);
}
return result;
// return (includeSize ?: result).c_str();
}
void CziFile::report_verbose() const
{
report();
printf("==========SubBlock directory==========\n");
print_segment_header(subBlockDirectory.header);
printf("%-25s %15i\n", "EntryCount", subBlockDirectory.entryCount);
for (size_t entry = 0; entry < subBlockDirectory.entryCount; entry++)
{
printf("SubBlockId: %i\tPixelType: %s\tCompression: %s\tDimensionStack: %s\tPyramidType: %s\n",
(int)entry,
pixel_type_str(subBlockDirectory.entries[entry].pixelType), compression_type_str(subBlockDirectory.entries[entry].compression),
dimension_stack_str(subBlockDirectory.entries[entry].dimensions, true).c_str(),
pyramid_type_str(subBlockDirectory.entries[entry].pyramidType));
}
printf("======================================\n");
}
std::string byte_array_str(const std::vector<byte> &bytes)
{
std::string result = "";
for (size_t i = 0; i < bytes.size(); i++)
{
result.append(std::to_string(bytes[i]));
if (i % 4 == 0 && i != bytes.size() - 1)
result.append("-");
}
return result;
}
void CziFile::report() const
{
printf("Loaded CZI FILE: %s\n", fileName.c_str());
printf("==========Summary report==========\n");
print_segment_header(header.header);
printf("%-25s %13i.%i\n", "FileVersion", header.fileVersion.major, header.fileVersion.minor);
printf("%-25s %15i\n", "FilePart", header.filePart);
printf("%-25s %30s\n", "MasterGUID", byte_array_str(header.masterFileGuid).c_str());
printf("%-25s %30s\n", "FileGUID", byte_array_str(header.masterFileGuid).c_str());
//TODO: Replace with GUID print.
printf("%-25s %15i\n", "GUIDs are matching", vecUtil::vector_eq(header.masterFileGuid, header.fileGuid));
printf("%-25s %15i\n", "UpdatePending", header.updatePending);
printf("%-25s %15li\n", "SubblockDirPosition", header.subBlockDirectoryPosition);
printf("%-25s %15li\n", "MetadataPosition", header.metadataPosition);
printf("%-25s %15li\n", "AttachmentDirPosition", header.attachmentDirectoryPosition);
printf("%-25s %15i\n", "SubBlockCount", subBlockDirectory.entryCount);
printf("%-25s %15i\n", "AttachmentCount", attachmentDirectory.entryCount);
if (subBlockDirectory.entryCount > 0)
{
{
std::map<PixelType, int> pixelTypeMap;
std::map<CompressionType, int> compressionMap;
std::map<std::string, int> dimensionMap;
for (size_t sbId = 0; sbId < subBlockDirectory.entryCount; sbId++)
{
const DirectoryEntryDV entry = subBlockDirectory.entries[sbId];
if (pixelTypeMap.count(entry.pixelType))
pixelTypeMap[entry.pixelType]++;
else
pixelTypeMap.emplace(std::make_pair(entry.pixelType, 1));
if (compressionMap.count(entry.compression))
compressionMap[entry.compression]++;
else
compressionMap.emplace(std::make_pair(entry.compression, 1));
std::string dimStackStr = dimension_stack_str(entry.dimensions, false);
if (dimensionMap.count(dimStackStr))
dimensionMap[dimStackStr]++;
else
dimensionMap.emplace(std::make_pair(dimStackStr, 1)); }
printf("Pixel type distribution:\n");
for (auto &&ptPair : pixelTypeMap)
{
printf("\t%s %ix\n", pixel_type_str(ptPair.first), ptPair.second);
}
printf("Compression type distribution:\n");
for (auto &&ctPair : compressionMap)
{
printf("\t%s %ix\n", compression_type_str(ctPair.first), ctPair.second);
}
printf("Dimension stack distribution:\n");
for (auto &&dimPair : dimensionMap)
{
printf("\t%s %ix\n", dimPair.first.c_str(), dimPair.second);
}
}
//printf("%-25s %15i\n", "DimensionCount", subBlockDirectory.entries[0].dimensionCount);
//printf("%-25s %15s\n", "PixelType", pixel_type_str(subBlockDirectory.entries[0].pixelType));
}
printf("==================================\n");
}
ImageMatrix CziFile::get_image(const uint subblockId) const
{
// // subblockId is in bounds.
// always_assert((uint)subblockId < subBlockDirectory.entries.size());
// // Image is not empty.
// always_assert(entry.width > 0 && entry.height > 0);
// // Read image bytes.
// std::vector<byte> imageBytes;
// {
// BinaryFileStream cziStream(fileName);
// imageBytes = cziStream.move_and_consume_bytes(entry.subBlock.dataLocation, entry.subBlock.dataSize);
// }
auto imageBytes = get_image_data(subblockId, false);
auto entry = subBlockDirectory.entries[subblockId];
ImageMatrix image = ImageMatrix((uint)entry.width, (uint)entry.height, entry.pixelType, imageBytes);
return image;
}
std::vector<byte> CziFile::get_image_data(const uint subblockId, const bool ZCurveOrdered) const
{
// subblockId is in bounds.
always_assert((uint)subblockId < subBlockDirectory.entries.size());
auto entry = subBlockDirectory.entries[subblockId];
// Image is not empty.
always_assert(entry.width > 0 && entry.height > 0);
// Read image bytes.
std::vector<byte> imageBytes;
{
BinaryFileStream cziStream(fileName);
imageBytes = cziStream.move_and_consume_bytes(entry.subBlock.dataLocation, entry.subBlock.dataSize);
}
// Check if read data are correct and not corrupted.
int bytesPerPixel = get_bytes_per_pixel_type(entry.pixelType);
ulong expectedSize = entry.width * entry.height * bytesPerPixel;
always_assert(expectedSize == imageBytes.size());
if (ZCurveOrdered)
{
always_assert((uint)entry.width <= UINT_MAX && (uint)entry.height <= UINT_MAX);
uint componentCount = get_count_of_components_of_pixel(entry.pixelType);
uint componentSize = get_component_size_of_pixel(entry.pixelType);
auto zIndices = generate_ordered_z_order_indices(entry.width, entry.height, componentCount);
always_assert(zIndices.size() == (ulong)(componentCount * entry.width * entry.height));
std::vector<byte> zOrderedBytes = reoder_bytes_to_z_order(imageBytes, zIndices, componentSize);
auto back = reoder_bytes_from_z_order(zOrderedBytes, zIndices, componentSize);
bool eq = vecUtil::vector_eq(imageBytes, back);
always_assert(eq);
return zOrderedBytes;
}
else
{
return imageBytes;
}
}
void CziFile::differences_between_next(const std::string baseName) const
{
if (subBlockDirectory.entries.size() <= 1)
{
printf("There aren't 2 different images in this CZI file.\n");
return;
}
std::string fName;
ImageMatrix ref = get_image(0);
for (size_t i = 1; i < subBlockDirectory.entries.size(); i++)
{
fName = baseName + "diff_" + std::to_string(i - 1) + "_" + std::to_string(i) + ".ppm";
ImageMatrix next = get_image(i);
ImageMatrix diff_ref_next = ref.create_difference_matrix(next);
diff_ref_next.save_as_ppm(fName.c_str());
ref = next;
}
}
void CziFile::extract_images(const std::string &baseName) const
{
std::string fName;
for (size_t i = 0; i < subBlockDirectory.entries.size(); i++)
{
auto subblock = get_image(i);
fName = baseName + "subblock_" + std::to_string(i) + ".ppm";
subblock.save_as_ppm(fName.c_str());
}
}
void CziFile::dump_image_data(const std::string &baseName) const
{
BinaryFileStream cziStream(fileName);
int imageIndex = 0;
for (const DirectoryEntryDV &entry : subBlockDirectory.entries)
{
always_assert(entry.width > 0 && entry.height > 0);
std::string binaryFileName = baseName + std::to_string(imageIndex++) + ".bin";
auto imageBytes = cziStream.move_and_consume_bytes(entry.subBlock.dataLocation, entry.subBlock.dataSize);
write_bytes_to_file(imageBytes, binaryFileName.c_str());
printf("Wrote %s\n", binaryFileName.c_str());
}
}
void CziFile::test_compression(CompressionMethod method, bool verbose) const
{
switch (method)
{
case CompressionMethod_GZIP:
printf("Selected compression: GZIP (zlib)\n");
break;
case CompressionMethod_LZMA:
printf("Selected compression: LZMA (2?)\n");
break;
case CompressionMethod_BZIP2:
printf("Selected compression: BZIP2 \n");
break;
default:
INVALID_CASE;
return;
}
CompressionResult overallN;
CompressionResult overallZ;
Stopwatch stopwatch;
//TODO: Use settings.
CompressionSettings settings;
for (size_t i = 0; i < subBlockDirectory.entries.size(); i++)
{
auto data = get_image_data(i, false);
auto dataZ = get_image_data(i, true);
CompressionResult nResult = test_compression_method(data, method, settings, stopwatch);
CompressionResult zResult = test_compression_method(dataZ, method, settings, stopwatch);
overallN.compressionRatio += nResult.compressionRatio;
overallZ.compressionRatio += zResult.compressionRatio;
overallN.originalSize += data.size();
overallN.compressedSize += nResult.compressedSize;
overallZ.compressedSize += zResult.compressedSize;
overallN.percentageOfOriginalSize += nResult.percentageOfOriginalSize;
overallZ.percentageOfOriginalSize += zResult.percentageOfOriginalSize;
if (verbose)
{
printf("Subblock %-3i Compression ratios: Normal: %8f Z-Ordered: %8f; Size(N): %5.3f%%; Size(Z): %5.3f%%; Time: %8.1f ms\n",
(int)i, nResult.compressionRatio, zResult.compressionRatio, nResult.percentageOfOriginalSize,
zResult.percentageOfOriginalSize, stopwatch.last_lap_milliseconds());
}
}
float dataCount = (float)subBlockDirectory.entries.size();
overallN.divide(dataCount);
overallZ.divide(dataCount);
printf("Overall compression ratios: Normal %8f Z-Ordered: %8f\n", overallN.compressionRatio, overallZ.compressionRatio);
printf("Original size: %8lu B Compressed size: %8lu B Compressed Z-Order size: %8lu B\n", overallN.originalSize, overallN.compressedSize, overallZ.compressedSize);
printf("Original size (%%): 100%% Compressed: %5.5f%% Compressed Z-Order: %5.5f%%\n", overallN.percentageOfOriginalSize, overallZ.percentageOfOriginalSize);
printf("Average compression time: %.1f ms\n", stopwatch.average_lap_time_in_milliseconds());
}