diff --git a/czi-format/Links to Download.txt b/czi-format/Links to Download.txt
index bc4196e2eb233a5a1e3df85c2b1257ebc752d5e8..51d5cfbd5638492149ac470b08aa6f42e7ead38f 100644
--- a/czi-format/Links to Download.txt
+++ b/czi-format/Links to Download.txt
@@ -1,36 +1,47 @@
Basic testing
-------------
-X http://czisamples.blob.core.windows.net/czi-samples/TESTIMAGES.zip
+X
Advanced testing
----------------
-X http://czisamples.blob.core.windows.net/czi-samples/Legend.txt
-http://czisamples.blob.core.windows.net/czi-samples/ApoTome/40x075_Artemia-Flash-AT-1Ch-Z-sect.czi
http://czisamples.blob.core.windows.net/czi-samples/ApoTome/40x14_GMMe_AT-4ch-Z_Dapi-Vimentin488_Phalloidine568-Lamin647(WF).czi
http://czisamples.blob.core.windows.net/czi-samples/ApoTome/40x14_GMMe_AT-4ch-Z_Dapi-Vimentin488_Phalloidine568-Lamin647-dcv.czi
http://czisamples.blob.core.windows.net/czi-samples/ApoTome/40x14_GMMe_AT-4ch-Z_Dapi-Vimentin488_Phalloidine568-Lamin647-raw.czi
http://czisamples.blob.core.windows.net/czi-samples/ApoTome/40x14_GMMe_AT-4ch-Z_Dapi-Vimentin488_Phalloidine568-Lamin647-sect.czi
-http://czisamples.blob.core.windows.net/czi-samples/ApoTome/40x_RatBrain-AT-2ch-Z-7ph-raw.czi
-http://czisamples.blob.core.windows.net/czi-samples/ApoTome/40x_RatBrain-AT-2ch-Z-DCV.czi
-http://czisamples.blob.core.windows.net/czi-samples/ApoTome/40x_RatBrain-AT-2ch-Z-sect.czi
-http://czisamples.blob.core.windows.net/czi-samples/ApoTome/40x_RatBrain-AT-2ch-Z-wf.czi
-http://czisamples.blob.core.windows.net/czi-samples/ApoTome/AxioZoom_Artemia_AT-1Ch-Z_sect.czi
-http://czisamples.blob.core.windows.net/czi-samples/ApoTome/Kidney-FS44_FS45_ApoTome_2ch_Tiles_sect.czi
+http://czisamples.blob.core.windows.net/czi-samples/ApoTome/Kidney-FS44_FS45_ApoTome_2ch_Tiles_sect
http://czisamples.blob.core.windows.net/czi-samples/ApoTome/Kidney-FS44_FS45_ApoTome_2ch_raw_Tiles.czi
http://czisamples.blob.core.windows.net/czi-samples/ApoTome/Kidney-FS44_FS45_ApoTome_2ch_sect_Tiles-stitched.czi
-http://czisamples.blob.core.windows.net/czi-samples/Axio Scan.Z1/20x_HE.czi
-X http://czisamples.blob.core.windows.net/czi-samples/Axio Scan.Z1/Intestine_3color_RAC.czi
-http://czisamples.blob.core.windows.net/czi-samples/Axio Scan.Z1/Kidney_40x_z_stack.czi
-http://czisamples.blob.core.windows.net/czi-samples/Axio Scan.Z1/Kidney_RAC_3color.czi
-http://czisamples.blob.core.windows.net/czi-samples/Axio Scan.Z1/Young_mouse.czi
-X http://czisamples.blob.core.windows.net/czi-samples/CellObserver SD/EB3-microtubules-GFP_H2BmCherry_2chT(SD).czi
-http://czisamples.blob.core.windows.net/czi-samples/CellObserver SD/LLC-PK1_2chTZ-triggered-20msec(SD).czi
http://czisamples.blob.core.windows.net/czi-samples/CellObserver SD/LLC-PK1_TubX-emerald_H2B-mCherry-2chZ(SD).czi
-http://czisamples.blob.core.windows.net/czi-samples/CellObserver SD/Micrasterias fimbriata#1_2chTZ(SD).czi
http://czisamples.blob.core.windows.net/czi-samples/Multiscene/Mouse-kidney-3-scene_VivaTome_1chZS.czi
-http://czisamples.blob.core.windows.net/czi-samples/Tiles/Mouse_stomach_20x_ROI_3chZTiles(WF).czi
+
+WANT TO DOWNLOAD
+http://czisamples.blob.core.windows.net/czi-samples/Axio Scan.Z1/Kidney_40x_z_stack.czi
+http://czisamples.blob.core.windows.net/czi-samples/Axio Scan.Z1/Young_mouse.czi
+http://czisamples.blob.core.windows.net/czi-samples/Axio Scan.Z1/20x_HE.czi
+
+
+
+
+
+ALREADY DOWNLOADED
+http://czisamples.blob.core.windows.net/czi-samples/TESTIMAGES.zip
+http://czisamples.blob.core.windows.net/czi-samples/Legend.txt
+http://czisamples.blob.core.windows.net/czi-samples/ApoTome/40x075_Artemia-Flash-AT-1Ch-Z-sect.czi
+http://czisamples.blob.core.windows.net/czi-samples/Widefield/BPAE-Cells_63x_oversampled-3chZ(WF).czi
+http://czisamples.blob.core.windows.net/czi-samples/Widefield/BPAE-cells-bin2x2_3chTZ(WF).czi
+http://czisamples.blob.core.windows.net/czi-samples/Widefield/BPAE-cells_mitosis_3chZ(WF).czi
+http://czisamples.blob.core.windows.net/czi-samples/CellObserver SD/EB3-microtubules-GFP_H2BmCherry_2chT(SD).czi
+http://czisamples.blob.core.windows.net/czi-samples/ApoTome/40x_RatBrain-AT-2ch-Z-7ph-raw.czi
http://czisamples.blob.core.windows.net/czi-samples/VivaTome/Mouse_kidney_VivaTome-1chTZ.czi
-X http://czisamples.blob.core.windows.net/czi-samples/Widefield/BPAE-Cells_63x_oversampled-3chZ(WF).czi
-X http://czisamples.blob.core.windows.net/czi-samples/Widefield/BPAE-cells-bin2x2_3chTZ(WF).czi
-X http://czisamples.blob.core.windows.net/czi-samples/Widefield/BPAE-cells_mitosis_3chZ(WF).czi
+
+http://czisamples.blob.core.windows.net/czi-samples/ApoTome/AxioZoom_Artemia_AT-1Ch-Z_sect.czi
+
+
+NOT USING IN BENCHMARK
+http://czisamples.blob.core.windows.net/czi-samples/Axio Scan.Z1/Intestine_3color_RAC.czi
+http://czisamples.blob.core.windows.net/czi-samples/ApoTome/40x_RatBrain-AT-2ch-Z-DCV.czi
+http://czisamples.blob.core.windows.net/czi-samples/ApoTome/40x_RatBrain-AT-2ch-Z-wf.czi
+http://czisamples.blob.core.windows.net/czi-samples/ApoTome/40x_RatBrain-AT-2ch-Z-sect.czi
+http://czisamples.blob.core.windows.net/czi-samples/Axio Scan.Z1/Kidney_RAC_3color.czi BROKEN?
+http://czisamples.blob.core.windows.net/czi-samples/CellObserver SD/LLC-PK1_2chTZ-triggered-20msec(SD).czi `QUITE BORING`
diff --git a/czi-format/czi-parser/compression/benchmark.h b/czi-format/czi-parser/compression/benchmark.h
new file mode 100644
index 0000000000000000000000000000000000000000..6c88d38526a218f96ec67987604c1fa0da4f1e39
--- /dev/null
+++ b/czi-format/czi-parser/compression/benchmark.h
@@ -0,0 +1,58 @@
+#pragma once
+#include "compressors.h"
+#include "../czi_file.h"
+#include "../file_system.h"
+#include <iomanip>
+
+struct BenchmarkRecord
+{
+ const char *fileName;
+ size_t subblockId;
+ const char *pixelType;
+ size_t width;
+ size_t height;
+ const char *compressionMethod;
+ int compressionLevel;
+ size_t originalSize;
+ size_t compressedSize;
+ size_t zOrderCompressedSize;
+ float compressionRation;
+ float zOrderCompressionRation;
+ size_t compressionTime;
+ size_t zOrderCompressionTime;
+ //filename;subblockId;pixelType;width;height;compression_method;compression_level;original_size;compressed_size;z_order_compressed_size;compression_ration;z_order_compression_ration;compression_time;z_order_compression_time
+};
+
+void write_report(const std::vector<BenchmarkRecord> &results, const std::string &reportFile)
+{
+ std::ofstream csvFile = std::ofstream(reportFile, std::ios::out);
+ always_assert(csvFile.is_open());
+
+ csvFile << std::fixed << std::setprecision(5);
+ // CSV header.
+ csvFile << "filename;subblockId;pixelType;width;height;compression_method;compression_level;original_size;compressed_size;z_order_compressed_size;compression_ration;z_order_compression_ration;compression_time;z_order_compression_time" << std::endl;
+ const char sep = ';';
+
+ for (const BenchmarkRecord &record : results)
+ {
+ csvFile << record.fileName << sep << record.subblockId << sep << record.pixelType << sep << record.width << sep << record.height << sep << record.compressionMethod
+ << sep << record.compressionLevel << sep << record.originalSize << sep << record.compressedSize << sep << record.zOrderCompressedSize
+ << sep << record.compressionRation << sep << record.zOrderCompressionRation << sep << record.compressionTime
+ << sep << record.zOrderCompressionTime << std::endl;
+ }
+}
+
+void benchmark_compression(const std::string &sampleFolder, const std::string &reportFile, bool verbose)
+{
+ always_assert(fs_wrapper::is_directory(sampleFolder) && "sampleFolder is not directory!");
+ auto cziFiles = fs_wrapper::get_files_in_directory(sampleFolder, true);
+ printf("Found files:\n");
+ for (auto &&f : cziFiles)
+ {
+ printf("\t%s\n", f.path.c_str());
+ }
+
+ std::vector<BenchmarkRecord> results;
+
+ write_report(results, reportFile);
+}
\ No newline at end of file
diff --git a/czi-format/czi-parser/custom_types.h b/czi-format/czi-parser/custom_types.h
index 20fb6b06489bcd68416b291165d2a539927ee31f..c2996969f64d2755e260ccc191b73c3217325044 100644
--- a/czi-format/czi-parser/custom_types.h
+++ b/czi-format/czi-parser/custom_types.h
@@ -23,6 +23,7 @@ typedef unsigned long ulong;
constexpr uint UINT_MAX = std::numeric_limits<uint>::max();
constexpr ulong ULONG_MAX = std::numeric_limits<ulong>::max();
+constexpr short SHORT_MAX = std::numeric_limits<short>::max();
#define TagType std::string, std::string
typedef std::pair<TagType> Tag;
diff --git a/czi-format/czi-parser/czi_file.cpp b/czi-format/czi-parser/czi_file.cpp
index dcfc6ebcf63e38ec7548b62c3832caab0b79626c..32bf02632d3e3ab166c96fdba2dcfdd83f4acf54 100644
--- a/czi-format/czi-parser/czi_file.cpp
+++ b/czi-format/czi-parser/czi_file.cpp
@@ -389,18 +389,18 @@ void CziFile::differences_between_next(const std::string baseName) const
}
}
-void CziFile::extract_images(const std::string &baseName) const
+void CziFile::dump_images(const std::string &baseName) const
{
- std::string fName;
+ //#pragma omp parallel for
for (size_t i = 0; i < subBlockDirectory.entries.size(); i++)
{
auto subblock = get_image(i);
- fName = baseName + "subblock_" + std::to_string(i) + ".ppm";
+ std::string fName = baseName + "subblock_" + std::to_string(i) + ".ppm";
subblock.save_as_ppm(fName.c_str());
}
}
-void CziFile::dump_image_data(const std::string &baseName) const
+void CziFile::dump_data(const std::string &baseName) const
{
BinaryFileStream cziStream(fileName);
int imageIndex = 0;
diff --git a/czi-format/czi-parser/czi_file.h b/czi-format/czi-parser/czi_file.h
index 621f0bb3a78b4edc0716525594880182cc3151a4..a02ca0d9c9c0ea18339ff2ba09e0b9e1e2f29cb4 100644
--- a/czi-format/czi-parser/czi_file.h
+++ b/czi-format/czi-parser/czi_file.h
@@ -43,9 +43,9 @@ public:
// Get image data by subblock id.
std::vector<byte> get_image_data(const uint subblockId, const bool ZCurveOrdered) const;
// Dump all images to binary files.
- void dump_image_data(const std::string &baseName) const;
+ void dump_data(const std::string &baseName) const;
// Dump all images to graphics files.
- void extract_images(const std::string &baseName) const;
+ void dump_images(const std::string &baseName) const;
// Save differences between next images, so 0x1;1x2;...
void differences_between_next(const std::string baseName) const;
// Report compression ratios of images in normal and Z order.
diff --git a/czi-format/czi-parser/file_system.h b/czi-format/czi-parser/file_system.h
index d833dcd8422fc78c5bce323c04c11e8a863fdc78..9bd13a491647956776eb4d7592556ab8841b5ae4 100644
--- a/czi-format/czi-parser/file_system.h
+++ b/czi-format/czi-parser/file_system.h
@@ -8,6 +8,9 @@
#include "boost/filesystem.hpp"
#include "boost/algorithm/string.hpp"
+namespace fs_wrapper
+{
+
namespace fs = boost::filesystem;
// CZI file extension.
@@ -70,20 +73,16 @@ std::vector<SmallFileInfo> get_files_with_same_prefix(const std::vector<SmallFil
return result;
}
-
-// Get all regular files from parent directory of given file.
-std::vector<SmallFileInfo> get_files_in_parent_directory(const std::string &filePath, const bool onlyCziFiles)
+// Get all regular files from directory.
+std::vector<SmallFileInfo> get_files_in_directory(const std::string &directoryPath, const bool onlyCziFiles)
{
- fs::path refFile(filePath);
- always_assert(fs::is_regular_file(refFile));
- always_assert(refFile.has_parent_path());
-
- fs::path dir = refFile.parent_path();
+ fs::path refDirectory(directoryPath);
+ always_assert(fs::is_directory(refDirectory));
std::vector<SmallFileInfo> files;
fs::path entryPath;
- for (fs::directory_entry &entry : fs::directory_iterator(dir))
+ for (fs::directory_entry &entry : fs::directory_iterator(refDirectory))
{
entryPath = entry.path();
if (fs::is_regular_file(entryPath))
@@ -99,4 +98,17 @@ std::vector<SmallFileInfo> get_files_in_parent_directory(const std::string &file
}
return files;
-}
\ No newline at end of file
+}
+
+// Get all regular files from parent directory of given file.
+std::vector<SmallFileInfo> get_files_in_parent_directory(const std::string &filePath, const bool onlyCziFiles)
+{
+ fs::path refFile(filePath);
+ always_assert(fs::is_regular_file(refFile));
+ always_assert(refFile.has_parent_path());
+
+ fs::path dir = refFile.parent_path();
+
+ return get_files_in_directory(dir.string(), onlyCziFiles);
+}
+}; // namespace fs_wrapper
\ No newline at end of file
diff --git a/czi-format/czi-parser/image/image_matrix.cpp b/czi-format/czi-parser/image/image_matrix.cpp
index 4b149096f505ee2a5f190d776d293c425507d867..33c8acd6a622f2fdec059ab31db3e5ac989ea059 100644
--- a/czi-format/czi-parser/image/image_matrix.cpp
+++ b/czi-format/czi-parser/image/image_matrix.cpp
@@ -167,19 +167,21 @@ void ImageMatrix::save_as_ppm(const char *fName) const
}
else if (this->pixelType == PixelType_Gray16)
{
- printf(RED "Gray16 values will be normalized to Gray8 values!\n" RESET);
-
- auto bwImage = cimg_wrapper::create_grayscale_image(colCount, rowCount);
- ulong index = 0;
- for (uint row = 0; row < this->rowCount; row++)
- {
- for (uint col = 0; col < this->colCount; col++)
- {
- cimg_wrapper::set_grayscale_pixel(bwImage, row, col, std::static_pointer_cast<Gray16Pixel>(data[index++])->normalize_to_Gray8Pixel());
- }
- }
- bwImage.save_pnm(fName);
- printf("Saved %s\n", fName);
+ printf(RED "Better normalization method must be provided! Won't save image.\n" RESET);
+ // printf(RED "Gray16 values will be normalized to Gray8 values!\n" RESET);
+
+ // auto bwImage = cimg_wrapper::create_grayscale_image(colCount, rowCount);
+ // ulong index = 0;
+
+ // for (uint row = 0; row < this->rowCount; row++)
+ // {
+ // for (uint col = 0; col < this->colCount; col++)
+ // {
+ // cimg_wrapper::set_grayscale_pixel(bwImage, row, col, std::static_pointer_cast<Gray16Pixel>(data[index++])->normalize_to_Gray8Pixel(max));
+ // }
+ // }
+ // bwImage.save_pnm(fName);
+ // printf("Saved %s\n", fName);
}
else if (this->pixelType == PixelType_Bgr24)
{
diff --git a/czi-format/czi-parser/image/pixel_structures.h b/czi-format/czi-parser/image/pixel_structures.h
index 505a5bb07cc5d28bbf439f4c0f164b9b06df692d..4c771bdb6205c7b088e16a81bb416f1fff9e2d1e 100644
--- a/czi-format/czi-parser/image/pixel_structures.h
+++ b/czi-format/czi-parser/image/pixel_structures.h
@@ -190,9 +190,11 @@ struct Gray16Pixel : BasePixel
}
// Normalize value of this pixel to byte.
- inline byte normalize_to_Gray8Pixel() const
+ inline byte normalize_to_Gray8Pixel(short max) const
{
- return normalize_to_byte(this->value);
+ // byte n = (byte)(((float)value / (float)SHORT_MAX) * 255.0f);
+ // printf("%i to %i\n", this->value, n);
+ return (byte)(((float)value / (float)max) * 255.0f);
}
};
/*--------------------------------------------------------------------------------------------------------------------*/
diff --git a/czi-format/czi-parser/main.cpp b/czi-format/czi-parser/main.cpp
index fbafb3968892aaca40b533139ebc553a17e64534..e992c7a41d9dc1a2b6df86d96c32149e2d371199 100644
--- a/czi-format/czi-parser/main.cpp
+++ b/czi-format/czi-parser/main.cpp
@@ -1,6 +1,7 @@
#include "czi_parser.h"
-#include "file_system.h"
+#include "compression/benchmark.h"
#include "utilities/args.hxx"
+
int main(int argc, char **argv)
{
args::ArgumentParser argParser("CZI file tools", "Optional arguments are in `[]` necessary in `<>`.");
@@ -17,11 +18,15 @@ int main(int argc, char **argv)
args::HelpFlag helpMethod(mainMethodGroup, "help", "Print help", {'h', "help"});
args::Flag reportMethod(mainMethodGroup, "report", "Basic information about CZI file. [verbose]", {"report"});
args::Flag compressionTestMethod(mainMethodGroup, "compression-test", "Compress subblocks and output compression ratios <compression method>", {"compression-test"});
+ args::Flag benchmarkMethod(mainMethodGroup, "Compression benchmark", "Start compression benchmark for all files in folder. Folder is selected using <input> flag. Save result in report file.", {"benchmark"});
+ args::Flag exportDataMethod(mainMethodGroup, "Export binary dat", "Export raw image data <folder>", {"dump-data"});
+ args::Flag exportImagesMethod(mainMethodGroup, "Export images", "Export images to ppm files <folder>", {"dump-images"});
// Options
args::Flag dontParseMetadataOption(optionsGroup, "no metadata", "Dont read metadata byte", {"no-metadata"});
args::Flag verboseOption(optionsGroup, "verbose", "Extend output of method", {'v', "verbose"});
args::ValueFlag<int> compressionLevelOption(optionsGroup, "Compression level", "Compression level", {'l', "level"});
+ args::ValueFlag<std::string> folderOption(optionsGroup, "Folder", "Folder to which save exported items.", {'f', "folder"});
args::Flag gzipCompressionOption(compressionGroup, "GZIP", "GZIP (zlib) compression", {"gzip"});
args::Flag lzmaCompressionOption(compressionGroup, "LZMA", "LZMA (2?) compression", {"lzma"});
args::Flag bzip2CompressionOption(compressionGroup, "BZIP2", "BZip2 compression", {"bzip2"});
@@ -47,6 +52,12 @@ int main(int argc, char **argv)
return 1;
}
+ if (benchmarkMethod)
+ {
+ benchmark_compression(cziFile.Get(), "benchmarkResult.csv", verboseOption.Matched());
+ return 0;
+ }
+
CziParser parser(dontParseMetadataOption.Get());
auto parsedFile = parser.parse_czi_file(cziFile.Get());
@@ -69,6 +80,7 @@ int main(int argc, char **argv)
printf("Using default compression level: %i\n", level);
parsedFile.test_compression(cm, verboseOption.Matched(), level);
+ return 0;
}
if (reportMethod)
@@ -77,62 +89,31 @@ int main(int argc, char **argv)
parsedFile.report_verbose();
else
parsedFile.report();
- }
- /*
- std::string cziFile = (argc > 1) ? argv[1] : "/home/mor0146/gitlab/data_project/czi-format/data/CZT-Stack-Anno.czi"; //"/home/mor0146/gitlab/data_project/czi-format/data/m2/exampleSingleChannel.czi";
- if (cziFile == "-v" || cziFile == "--version")
- {
-#if DEBUG
- printf("Running `Debug` version\n.");
-#else
- printf("Running `Release` version\n.");
-#endif
return 0;
}
- always_assert(is_file(cziFile));
-
- std::string method = argc > 2 ? argv[2] : "";
- std::string dumpName = argc > 3 ? argv[3] : "";
-
- bool report = method == "--report";
- bool reportAll = method == "--report-verbose";
- bool dumpRawImageData = method == "--dump-raw-image-data";
- bool dumpImageData = method == "--dump-images";
- bool nextImageDiff = method == "--diff-next";
- bool testRle = method == "--rle-report";
-
- auto name = get_filename_without_extension(cziFile);
- auto x = get_files_in_parent_directory(cziFile, true);
- auto y = get_files_with_same_prefix(x, name);
-
- CziParser parser;
- auto parseResult = parser.parse_czi_file(cziFile);
-
- if (reportAll)
- parseResult.report_verbose();
- else if (report)
- parseResult.report();
- else if (dumpRawImageData)
- {
- parseResult.dump_image_data(dumpName);
- }
- else if (dumpImageData)
- {
- parseResult.extract_images(dumpName);
- }
- else if (nextImageDiff)
+ if (exportDataMethod || exportImagesMethod)
{
- always_assert(dumpName != "");
- parseResult.differences_between_next(dumpName);
- }
- else if (testRle)
- {
- parseResult.test_rle_encode();
+ if (!folderOption.Matched())
+ {
+ printf(RED "Folder must be specified!\n" RESET);
+ return 1;
+ }
+
+ fs_wrapper::create_directory_path(folderOption.Get());
+
+ if (exportDataMethod)
+ {
+ // Raw data.
+ parsedFile.dump_data(folderOption.Get());
+ }
+ else
+ {
+ // Images.
+ parsedFile.dump_images(folderOption.Get());
+ }
+ return 0;
}
-
- printf("Finished.\n");
-*/
return 0;
}
\ No newline at end of file
diff --git a/czi-format/data/Legend.txt b/czi-format/data/Legend.txt
new file mode 100644
index 0000000000000000000000000000000000000000..9369905dae5ca7f5bb4212e42225426b9043a656
--- /dev/null
+++ b/czi-format/data/Legend.txt
@@ -0,0 +1,10 @@
+(WF) = widefield
+(SD) = Spinning Disc
+"n"ch = number of channels
+T = Timelapse
+Z= Z-stack
+S = Scenes (=positions)
+Tiles = tile imaging creating mosaic images
+(LS_MV) = "lightsheet MultiView"
+AT = ApoTome
+Raw = with ApoTome phase images
\ No newline at end of file