Skip to content
Snippets Groups Projects
main.cpp 18.3 KiB
Newer Older
  • Learn to ignore specific revisions
  • #include <compression/benchmark.h>
    
    #include <azgra/cli/cli_arguments.h>
    
    theazgra's avatar
    theazgra committed
    #include <iostream>
    
    #include <azgra/utilities/histogram.h>
    
    #include <azgra/stream/in_binary_file_stream.h>
    
    
    CompressionMethod get_matched_compressor(const azgra::cli::CliFlag gzip,
                                             const azgra::cli::CliFlag lzma,
                                             const azgra::cli::CliFlag bzip2,
                                             const azgra::cli::CliFlag &b3d)
    
        CompressionMethod result;
    
        if (gzip.is_matched())
            result = CompressionMethod::CompressionMethod_GZIP;
        else if (lzma.is_matched())
            result = CompressionMethod::CompressionMethod_LZMA;
        else if (bzip2.is_matched())
            result = CompressionMethod::CompressionMethod_BZIP2;
        else if (b3d.is_matched())
            result = CompressionMethod::CompressionMethod_B3D;
        else
            result = CompressionMethod::CompressionMethod_None;
    
        return result;
    
    void show_version()
    {
    
        bool b3d = false;
        bool debug = false;
        bool ompEnabled = true;
    
    theazgra's avatar
    theazgra committed
    
    
    theazgra's avatar
    theazgra committed
    #if DEBUG
    
        debug = true;
    
    theazgra's avatar
    theazgra committed
    #endif
    
        ompEnabled = false;
    
        fprintf(stdout, "%s version, with\nB3D - %s\nOMP - %s\n",
                debug ? "DEBUG" : "RELEASE",
                b3d ? "Enabled" : "Disabled", ompEnabled ? "Enabled" : "Disabled");
    
    #if 0 // Routine used to retrive data for original paper.
    void save_czi_frame_histogram(const CziFile & czi, const azgra::u32 frame, const std::string & reportFile)
    
    theazgra's avatar
    theazgra committed
    {
    
        ByteArray bytes = czi.get_image_data(frame, false);
        std::vector<azgra::u16> frameValues = azgra::bytes_to_ushort_array(bytes);
        auto histogramBins = azgra::create_histogram_from_numbers(frameValues, 10);
        azgra::save_histogram_bins(histogramBins, reportFile.c_str());
    
    theazgra's avatar
    theazgra committed
    }
    
    theazgra's avatar
    theazgra committed
    void paper_benchmarks()
    {
    
        std::vector<CziFile> czis;
        std::vector<std::vector<azgra::u32>> cziFrames;
        std::vector<std::string> reportTemplates;
    
        const size_t repeatCount = 3;
        fprintf(stdout, "Entered paper benchmark routine\n");
        CziParser parser(false);
        CziFile czi = parser.parse_czi_file("/home/mor0146/gitlab/data_project/czi/benchmark/BPAE-Cells_63x_oversampled-3chZ(WF).czi");
        std::string reportTemplate = "cells";
        std::vector<std::pair<Dimension, azgra::i32>> q1 = { std::make_pair(Dimension_C, 0), std::make_pair(Dimension_Z, 24) };
        std::vector<std::pair<Dimension, azgra::i32>> q2 = { std::make_pair(Dimension_C, 1), std::make_pair(Dimension_Z, 24) };
        std::vector<std::pair<Dimension, azgra::i32>> q3 = { std::make_pair(Dimension_C, 2), std::make_pair(Dimension_Z, 24) };
        azgra::u32 f_c1_z24 = czi.get_subblock_id(q1);
        azgra::u32 f_c2_z24 = czi.get_subblock_id(q2);
        azgra::u32 f_c3_z24 = czi.get_subblock_id(q3);
        std::vector<azgra::u32> frames = { f_c1_z24, f_c2_z24, f_c3_z24 };
    
        CziFile czi2 = parser.parse_czi_file("/home/mor0146/gitlab/data_project/czi/benchmark/2015-04-21_LZ2_Stock32.czi");
        std::string reportTemplate2 = "lz2";
        std::vector<azgra::u32> frames2 = { 50 };
    
        CziFile czi3 = parser.parse_czi_file("/home/mor0146/gitlab/data_project/czi/benchmark/40x075_Artemia-Flash-AT-1Ch-Z-sect.czi");
        std::string reportTemplate3 = "artemia";
        std::vector<azgra::u32> frames3 = { czi.get_subblock_id({std::make_pair(Dimension_Z, 19)}) };
    
        CziFile czi4 = parser.parse_czi_file("/home/mor0146/gitlab/data_project/czi/benchmark/LLC-PK1_TubX-emerald_H2B-mCherry-2chZ(SD).czi");
        std::string reportTemplate4 = "llc";
        azgra::u32 a = czi.get_subblock_id({ std::make_pair(Dimension_C, 0), std::make_pair(Dimension_Z, 27) });
        azgra::u32 b = czi.get_subblock_id({ std::make_pair(Dimension_C, 1), std::make_pair(Dimension_Z, 27) });
        std::vector<azgra::u32> frames4 = { a, b };
    
        czis.push_back(czi);
        czis.push_back(czi2);
        czis.push_back(czi3);
        czis.push_back(czi4);
    
        cziFrames.push_back(frames);
        cziFrames.push_back(frames2);
        cziFrames.push_back(frames3);
        cziFrames.push_back(frames4);
    
        reportTemplates.push_back(reportTemplate);
        reportTemplates.push_back(reportTemplate2);
        reportTemplates.push_back(reportTemplate3);
        reportTemplates.push_back(reportTemplate4);
    
        for (size_t i = 0; i < czis.size(); i++)
        {
            for (const azgra::u32& frame : cziFrames[i])
            {
                save_czi_frame_histogram(czi, frame,
                    "/home/mor0146/gitlab/data_project/czi/benchmark/result/histogram_" +
                    reportTemplates[i] + "_" + std::to_string(frame) + ".csv");
            }
        }
    
        // value_diff_histogram(czi, "/home/mor0146/gitlab/data_project/czi/benchmark/result/diff_histogram_" + reportTemplate + ".csv", 50);
        // return;
        return;
    
        // Compression benchmark for all 3 types of compression and levels from 1 to 9
        for (const azgra::u32& frame : frames)
        {
            benchmark_compression(czi, "/home/mor0146/gitlab/data_project/czi/benchmark/result/compression_" + reportTemplate + ".csv", true, -1, frame, repeatCount);
        }
        fprintf(stdout, "======================\n\tFinished compression benchmark...\n======================\n");
    
        // Frame difference using all 3 types of compression and selected level of compression
        for (const azgra::u32& frame : frames)
        {
            value_diff_benchmark_for_frame(czi, "/home/mor0146/gitlab/data_project/czi/benchmark/result/diff_" + reportTemplate + ".csv", frame, true, true, 6, repeatCount);
            value_diff_benchmark_for_frame(czi, "/home/mor0146/gitlab/data_project/czi/benchmark/result/diff_" + reportTemplate + ".csv", frame, true, false, 6, repeatCount);
        }
        fprintf(stdout, "======================\n\tFinished frame difference...\n======================\n");
    
        // Frame difference using bsdiff tool and all 3 types of compression.
    
        for (const azgra::u32& frame : frames)
        {
            bsdiff_benchmark_for_frame(czi, "/home/mor0146/gitlab/data_project/czi/benchmark/result/bsdiff_" + reportTemplate + ".csv", true, 6, frame, repeatCount);
        }
        fprintf(stdout, "======================\n\tFinished bsdiff run...\n======================\n");
    
        // B3D losslesss compression
        for (const azgra::u32& frame : frames)
        {
            benchmark_b3d(czi, "/home/mor0146/gitlab/data_project/czi/benchmark/result/b3d_" + reportTemplate + ".csv", true, false, 0.0f, frame, 10);
        }
        fprintf(stdout, "======================\n\tFinished B3D lossless...\n======================\n");
    
        // B3D lossy compression with different quantization steps.
        for (const azgra::u32& frame : frames)
        {
            b3d_test_quantization_step(czi, "/home/mor0146/gitlab/data_project/czi/benchmark/result/b3d_lossy_" + reportTemplate + ".csv", frame);
        }
        fprintf(stdout, "======================\n\tFinished B3D lossy test...\n======================\n");
    
    theazgra's avatar
    theazgra committed
    }
    
    theazgra's avatar
    theazgra committed
    
    
    int main(azgra::i32 argc, const char **argv)
    
        azgra::cli::CliArguments args("CZI file tools", "Tools for benchmarking compression of images inside CZI files. Optional arguments "
                                                        "are in [] brackets.", 80);
    
    
        // Options flag.
        azgra::cli::CliValueFlag<std::string> flagCziFile("CziFile", "Czi file to load", 'i', "input", true);
        azgra::cli::CliValueFlag<std::string> flagFolder("Folder", "Export folder path.", 'f', "folder");
        azgra::cli::CliValueFlag<std::string> flagReportFile("ReportFile", "Report file path.", 'r', "report");
    
        azgra::cli::CliValueFlag<int> valFlagSubblock("Subblock", "Selected subblock index.", 's', "subblock", false, -1);
        azgra::cli::CliValueFlag<int> valFlagCompressionLevel("CompressionLevel", "Compression level.", 'l', "level", false, -1);
        azgra::cli::CliValueFlag<int> valFlagHistogramBinCount("HistogramBinCount", "Histogram bin count, default is 10.",
                                                               'b', "bins", false, 10);
    
        azgra::cli::CliFlag flagVerbose("Verbose", "Make program output verbose.", 'v', "verbose");
    
        azgra::cli::CliFlag flagGzipCompressionOption("GZIP", "gzip (zlib) compression", '\0', "gzip");
        azgra::cli::CliFlag flagLzmaCompressionOption("LZMA", "lzma compression", '\0', "lzma");
        azgra::cli::CliFlag flagBzip2CompressionOption("BZIP2", "bzip2 compression", '\0', "bzip2");
        azgra::cli::CliFlag flagB3dCompressionOption("B3D", "B3D cuda compression", '\0', "b3d");
    
    
        azgra::cli::CliFlagGroup compressorGroup("CompressorGroup",
                                                 {&flagGzipCompressionOption, &flagLzmaCompressionOption,
                                                  &flagBzip2CompressionOption, &flagB3dCompressionOption},
                                                 azgra::cli::CliGroupMatchPolicy_AtLeastOneIfGroupIsRequired);
    
    
        // Methods
        azgra::cli::CliMethod methodVersion("version",
                                            "Show program version info.");
    
        azgra::cli::CliMethod methodReport("report",
                                           "Report information about loaded czi file.");
    
        azgra::cli::CliMethod methodDump("dump",
    
                                         "Dump raw image data.",
                                         {&flagFolder}, {&valFlagSubblock});
    
    
        azgra::cli::CliMethod methodCompressionBenchmark("compressors-benchmark",
    
                                                         "Compressors benchmark for subblocks in czi file.",
                                                         {&flagReportFile}, {&valFlagCompressionLevel, &valFlagSubblock, &compressorGroup});
    
    
        azgra::cli::CliMethod methodFrameDiffCompression("frame-diff-compression",
    
                                                         "Compress frames by their difference.",
                                                         {&flagReportFile}, {&valFlagCompressionLevel, &compressorGroup});
    
    
        azgra::cli::CliMethod methodFrameDiffHistogram("frame-diff-histogram",
    
                                                       "Save histograms of frame differences.",
                                                       {&flagFolder}, {&valFlagCompressionLevel, &valFlagHistogramBinCount});
    
    
        azgra::cli::CliMethod methodHistogram("histogram",
    
                                              "Save histograms of all frames or just one specific frame.",
                                              {&flagFolder}, {&valFlagSubblock, &valFlagHistogramBinCount});
    
    
        azgra::cli::CliMethod methodCudaCompress("b3d",
    
                                                 "Test only b3d compression.",
                                                 {&flagReportFile}, {&valFlagSubblock});
    
    
        azgra::cli::CliMethod methodCudaCompressLossyTest("b3d-quantization-test",
    
                                                          "Test quantization steps of lossy compression.",
    
                                                          {&flagFolder, &flagReportFile});
    
        args.flags = {&flagCziFile, &flagFolder, &flagReportFile, &valFlagSubblock, &valFlagCompressionLevel, &valFlagHistogramBinCount,
    
                      &flagVerbose};
        args.add_group(compressorGroup);
    
        args.methods = {&methodVersion, &methodReport, &methodDump, &methodCompressionBenchmark, &methodFrameDiffCompression,
                        &methodFrameDiffHistogram, &methodHistogram, &methodCudaCompress,
                        &methodCudaCompressLossyTest};
    
        if (!args.parse(argc, argv))
        {
            fprintf(stderr, "%s\n", args.get_error().c_str());
            return 1;
        }
    
        if (!args.is_any_method_matched())
        {
            fprintf(stdout, "No method was selected.\n");
            return 0;
        }
    
        if (methodVersion)
        {
            show_version();
            return 0;
        }
    
        cziCpp::CziFile parsedCziFile(flagCziFile.value());
    
        cziCpp::ParserConfiguration parseConfig = {};
        parseConfig.loadAllPixelDataToMemory = false;
        parseConfig.parseInMemory = false;
        parseConfig.verbose = flagVerbose.is_matched();
    
        try
        {
            parsedCziFile.parse(parseConfig);
        }
        catch (const cziCpp::DeserializationError &parseError)
        {
            fprintf(stderr, "Unable to parse czi file %s:\n", flagCziFile.value().c_str());
            fprintf(stderr, "%s\n", parseError.what());
        }
    
        if (methodReport)
        {
            parsedCziFile.report();
            return 0;
        }
        else if (methodDump)
        {
            azgra::i32 maxSubblockCount = parsedCziFile.subblockDirectory.entryCount - 1;
    
            parsedCziFile.dump_image_data(flagFolder.value(), valFlagSubblock.value(), flagVerbose.is_matched());
    
            return 0;
        }
        else if (methodCompressionBenchmark)
        {
            auto compressor = get_matched_compressor(flagGzipCompressionOption, flagLzmaCompressionOption, flagBzip2CompressionOption,
                                                     flagB3dCompressionOption);
            if (compressor != CompressionMethod::CompressionMethod_None)
            {
    
                benchmark_compressor(compressor, parsedCziFile, flagReportFile.value(), flagVerbose.is_matched(),
                                     valFlagCompressionLevel.value(),
                                     valFlagSubblock.value());
            }
            else
            {
                benchmark_compression(parsedCziFile, flagReportFile.value(), flagVerbose.is_matched(), valFlagCompressionLevel.value(),
                                      valFlagSubblock.value());
            }
            return 0;
        }
    
        //args::Command cmd_FrameDiffCompression(commandGroup, "frame-diff-compression", "Compress frames by their difference. <report-file> [compression-level] [compressor]");
        //args::Command cmd_FrameDiffHistogram(commandGroup, "frame-diff-histogram", "Save histograms of frame differences. <folder> [compression-level] [bin-count]");
        //args::Command cmd_Histogram(commandGroup, "histogram", "Save histograms of all frames or just one specific frame. <folder> [subblock] [bin-count]");
        //args::Command cmd_cudaCompress(commandGroup, "b3d", "Test only b3d compression. <report-file> [subblock]");
        //args::Command cmd_cudaCompressLossyTest(commandGroup, "b3d-quantization-test", "Test quantization steps of lossy compression. <report-file> <subblock>");
    
    
        /*
    
        CziParser parser(dontParseMetadataOption.Get());
        auto parsedFile = parser.parse_czi_file(cziFile.Get());
    
        if (benchmarkMethod)
        {
            if (!reportFileOption.Matched())
            {
                printf(RED "Report file wasn't specified!\n" RESET);
                return 1;
            }
    
            azgra::i32 level = -1;
            if (compressionLevelOption.Matched())
                level = compressionLevelOption.Get();
    
            if (continuousCompressionOption.Matched())
            {
                benchmark_continuos_compression(parsedFile, reportFileOption.Get(), verboseOption.Matched(), level);
            }
            else
            {
                benchmark_compression(parsedFile, reportFileOption.Get(), verboseOption.Matched(), level);
            }
    
            return 0;
        }
    
        if (histogramMethod)
        {
            if (!folderOption.Matched())
            {
                printf(RED "Folder wasn't specified!\n" RESET);
                return 1;
            }
            fs_wrapper::create_directory_path(folderOption.Get());
            azgra::i32 bins = binCountOption.Matched() ? binCountOption.Get() : 10;
            parsedFile.save_histogram(folderOption.Get(), !frameOption.Matched(), frameOption.Get(), bins, verboseOption.Matched());
            return 0;
        }
    
        if (frameDiffHistogramMethod)
        {
            if (!folderOption.Matched())
            {
                printf(RED "Folder wasn't specified!\n" RESET);
                return 1;
            }
    
            fs_wrapper::create_directory_path(folderOption.Get());
    
            value_diff_histogram(parsedFile, folderOption.Get().c_str(), 1);
            return 0;
        }
    
        if (frameDiffMethod || bsFrameDiffMethod)
        {
            if (!reportFileOption.Matched())
            {
                printf(RED "Report file wasn't specified!\n" RESET);
                return 1;
            }
    
            azgra::i32 level = 6;
            if (compressionLevelOption.Matched())
                level = compressionLevelOption.Get();
    
            CompressionMethod cm = get_chosen_compression_method(gzipCompressionOption, lzmaCompressionOption, bzip2CompressionOption);
    
            if (frameDiffMethod)
            {
                value_diff_by_prev_frame_benchmark(parsedFile, reportFileOption.Get(), verboseOption.Matched(), level, cm, variableBitCount.Matched());
            }
            else if (bsFrameDiffMethod)
            {
                bsdiff_benchmark(parsedFile, reportFileOption.Get(), verboseOption.Matched(), cm, level, diffToFirstFrame.Matched());
            }
    
            return 0;
        }
    
    #ifdef B3D
    
        if (b3dMethod)
        {
            if (!reportFileOption.Matched())
            {
                printf(RED "Report file wasn't specified!\n" RESET);
                return 1;
            }
            if (continuousCompressionOption.Matched())
            {
                benchmark_b3d_continuos(parsedFile, reportFileOption.Get(), verboseOption.Matched(), quantizationOption.Matched(), quantizationOption.Get());
            }
            else
            {
                benchmark_b3d(parsedFile, reportFileOption.Get(), verboseOption.Matched(), quantizationOption.Matched(), quantizationOption.Get());
            }
    
            return 0;
        }
        else if (b3dQuantTest)
        {
            if (!reportFileOption.Matched())
            {
                printf(RED "Report file wasn't specified!\n" RESET);
                return 1;
            }
            b3d_test_quantization_step(parsedFile, reportFileOption.Get(), parsedFile.subBlockDirectory.entryCount / 2);
            return 0;
        }
    
        // // Test compression method.
        // if (compressionTestMethod)
        // {
        //     CompressionMethod cm = get_chosen_compression_method(gzipCompressionOption, lzmaCompressionOption, bzip2CompressionOption);
    
        //     azgra::i32 level = 6;
        //     if (compressionLevelOption.Matched())
        //         level = compressionLevelOption.Get();
        //     else
        //         printf("Using default compression level: %i\n", level);
    
        //     parsedFile.test_compression(cm, verboseOption.Matched(), level);
        //     return 0;
        // }
    
        if (reportMethod)
        {
            if (verboseOption.Matched())
                parsedFile.report_verbose();
            else
                parsedFile.report();
    
            return 0;
        }
    
        if (exportDataMethod)
        {
            if (!folderOption.Matched())
            {
                printf(RED "Folder must be specified!\n" RESET);
                return 1;
                fs_wrapper::create_directory_path(folderOption.Get());
            }
    
            if (exportDataMethod && continuousCompressionOption)
            {
                parsedFile.dump_continuous_data(folderOption.Get());
                return 0;
            }
    
            if (exportDataMethod) // Raw data.
                parsedFile.dump_data(folderOption.Get());
    
            return 0;
        }
        */
        return 0;