[WIP] Implementing mini-batch gradient computation.

build_scripts/linux/linux_gcc_build_x64_release_local.sh

@@ -18,8 +18,8 @@ BUILD_TESTS=no
 BUILD_LIB=yes
 
 # C++ compiler
-CXX_COMPILER="g++"
-C_COMPILER="gcc"
+#CXX_COMPILER="g++"
+#C_COMPILER="gcc"
 CXX_COMPILER="mpicxx"
 C_COMPILER="mpicc"
 

src/DataSet/DataSet.cpp

 
 #include <algorithm>
 #include <limits>
+#include <random>
 #include <vector>
+#include <numeric>
 
 #include "../mpi_wrapper.h"
 
 #include <armadillo>
 #include <boost/serialization/export.hpp>
+#include <ctime>
 
 #include "DataSetSerialization.h"
 #include "exceptions.h"
@@ -197,6 +200,9 @@ namespace lib4neuro {
         this->data       = *data_ptr;
         this->input_dim  = this->data[0].first.size();
         this->output_dim = this->data[0].second.size();
+        this->permutation_data_inds.resize(this->data.size());
+        this->active_elements.resize(this->data.size());
+        std::iota(this->active_elements.begin(), this->active_elements.end(), 0);
 
         if (ns) {
             std::shared_ptr<NormalizationStrategy> ns_tmp;
@@ -231,6 +237,9 @@ namespace lib4neuro {
         this->n_elements = 0;
         this->input_dim  = 1;
         this->output_dim = 1;
+        this->permutation_data_inds.resize(this->data.size());
+        this->active_elements.resize(this->data.size());
+        std::iota(this->active_elements.begin(), this->active_elements.end(), 0);
 
         if (ns) {
             std::shared_ptr<NormalizationStrategy> ns_tmp(ns);
@@ -253,6 +262,9 @@ namespace lib4neuro {
         this->input_dim  = bounds.size() / 2;
         this->output_dim = output_dim;
         this->n_elements = 0;
+        this->permutation_data_inds.resize(this->data.size());
+        this->active_elements.resize(this->data.size());
+        std::iota(this->active_elements.begin(), this->active_elements.end(), 0);
 
         if (ns) {
             std::shared_ptr<NormalizationStrategy> ns_tmp;
@@ -296,6 +308,9 @@ namespace lib4neuro {
         this->n_elements++;
         this->data.emplace_back(std::make_pair(inputs,
                                                outputs));
+        this->permutation_data_inds.resize(this->data.size());
+        this->active_elements.resize(this->data.size());
+        std::iota(this->active_elements.begin(), this->active_elements.end(), 0);
     }
 
     void DataSet::add_isotropic_data(double lower_bound,
@@ -332,6 +347,10 @@ namespace lib4neuro {
         }
 
         this->n_elements += size;
+
+        this->permutation_data_inds.resize(this->data.size());
+        this->active_elements.resize(this->data.size());
+        std::iota(this->active_elements.begin(), this->active_elements.end(), 0);
     }
 
     void DataSet::add_isotropic_data(std::vector<double>& bounds,
@@ -370,6 +389,10 @@ namespace lib4neuro {
             this->data.emplace_back(std::make_pair(vec,
                                                    output_func(vec)));
         }
+
+        this->permutation_data_inds.resize(this->data.size());
+        this->active_elements.resize(this->data.size());
+        std::iota(this->active_elements.begin(), this->active_elements.end(), 0);
     }
 
     std::vector<std::pair<std::vector<double>, std::vector<double>>>* DataSet::get_data() {
@@ -722,4 +745,88 @@ namespace lib4neuro {
 
         return this->stability_matrix;
     }
+
+    void DataSet::activate_next_data_batch(size_t batch_size) {
+
+//        if (lib4neuro::mpi_rank == 0) {  // TODO shouldn't be parallelized?
+
+            /* Activate the whole DataSet */
+            this->active_elements.clear();
+
+            if (batch_size > 0) {
+
+                if (this->current_batch_vector_ind == 0) {
+                    /* If the DataSet has not been permuted yet or it was completely "used up", shuffle it randomly */
+                    this->permutation_data_inds.resize(this->data.size());
+                    std::iota(this->permutation_data_inds.begin(), this->permutation_data_inds.end(), 0);
+                    std::shuffle(this->permutation_data_inds.begin(),
+                                 this->permutation_data_inds.end(),
+                                 std::mt19937(std::random_device()()));
+                }
+
+                /* Determine active elements of the DataSet */
+                size_t first_ind = this->current_batch_vector_ind;
+                size_t last_ind  = this->current_batch_vector_ind + batch_size;
+
+                if (last_ind >= this->data.size()) {
+                    this->current_batch_vector_ind = 0;
+                    last_ind = this->data.size();
+                } else {
+                    this->current_batch_vector_ind = last_ind;
+                }
+
+//                std::cout << first_ind << " " << last_ind << " ";
+
+                for (size_t i = first_ind; i < last_ind; i++) {
+                    this->active_elements.emplace_back(i);
+                }
+            } else {
+                std::iota(this->active_elements.begin(), this->active_elements.end(), 0);
+            }
+
+//            std::cout << this->active_elements.size() << std::endl;
+
+//            for(auto e : this->active_elements) {
+//                std::cout << e << "[" << this->permutation_data_inds[e] << "] ";
+//            }
+//            std::cout << std::endl;
+
+            this->num_active_elements = this->active_elements.size();
+//        }
+
+//        MPI_Bcast(&this->num_active_elements, 1, MPI_UNSIGNED, 0, lib4neuro::mpi_active_comm);
+//        if(lib4neuro::mpi_rank > 0) {
+////            std::cout << "resize " << lib4neuro::mpi_rank << " " << this->num_active_elements << std::endl;
+//            this->active_elements.resize(this->num_active_elements);
+////            std::cout << "size " << lib4neuro::mpi_rank << " " << this->active_elements.size() << std::endl;
+//        }
+    }
+
+    std::vector<unsigned int>* DataSet::get_active_elements_indices() {
+        return &this->active_elements;
+    }
+
+    std::vector<unsigned int>* DataSet::get_permutation_data_indices() {
+        return &this->permutation_data_inds;
+    }
+
+    unsigned int DataSet::get_current_batch_vector_index() {
+        return this->current_batch_vector_ind;
+    }
+
+    unsigned int* DataSet::get_current_batch_vector_index_ptr() {
+        return &this->current_batch_vector_ind;
+    }
+
+    unsigned int* DataSet::get_permutation_data_indices_content() {
+        return &this->permutation_data_inds[0];
+    }
+
+    unsigned int* DataSet::get_num_active_elements_ptr() {
+        return &this->num_active_elements;
+    }
+
+    unsigned int* DataSet::get_active_elements_content() {
+        return &this->active_elements[0];
+    }
 }

src/DataSet/DataSet.h

@@ -58,8 +58,7 @@ namespace lib4neuro {
          */
         std::vector<std::pair<std::vector<double>, std::vector<double>>> data;
 
-        std::vector<std::pair<std::vector<double>, std::vector<double>>> permutation_data;
-
+        std::vector<unsigned int> permutation_data_inds;
 
         /**
          *
@@ -82,7 +81,16 @@ namespace lib4neuro {
 
         arma::Mat<double>* stability_matrix;
 
-        unsigned int last_batch_vector_ind = 0;
+        unsigned int current_batch_vector_ind = 0;
+
+        /**
+         * Indices of the active elements in DataSet
+         *
+         * If empty, than the whole DataSet is active!
+         */
+        std::vector<unsigned int> active_elements;
+
+        unsigned int num_active_elements;
 
 
     public:
@@ -343,7 +351,13 @@ namespace lib4neuro {
          * @param max
          * @return
          */
-        LIB4NEURO_API std::vector<std::pair<std::vector<double>, std::vector<double>>> get_next_data_batch(size_t max);
+        [[nodiscard]] LIB4NEURO_API std::vector<std::pair<std::vector<double>, std::vector<double>>> get_next_data_batch(size_t max);
+
+        /**
+         *
+         * @param batch_size
+         */
+        LIB4NEURO_API void activate_next_data_batch(size_t batch_size);
 
 		/**
 		 * Adds a new output column filled with zeros
@@ -368,6 +382,20 @@ namespace lib4neuro {
         [[nodiscard]] LIB4NEURO_API arma::Mat<double>* get_outputs_matrix();
 
         [[nodiscard]] LIB4NEURO_API arma::Mat<double>* get_stability_matrix();
+
+        [[nodiscard]] LIB4NEURO_API std::vector<unsigned int>* get_active_elements_indices();
+
+        [[nodiscard]] LIB4NEURO_API std::vector<unsigned int>* get_permutation_data_indices();
+
+        [[nodiscard]] LIB4NEURO_API unsigned int* get_permutation_data_indices_content();
+
+        [[nodiscard]] LIB4NEURO_API unsigned int* get_active_elements_content();
+
+        [[nodiscard]] LIB4NEURO_API unsigned int get_current_batch_vector_index();
+
+        [[nodiscard]] LIB4NEURO_API unsigned int* get_current_batch_vector_index_ptr();
+
+        [[nodiscard]] LIB4NEURO_API unsigned int* get_num_active_elements_ptr();
     };
 }
 #endif //INC_4NEURO_DATASET_H

src/ErrorFunction/ErrorFunctions.cpp

@@ -407,13 +407,49 @@ namespace lib4neuro {
 
         std::fill(grad.begin(), grad.end(), 0.0);
 
+        batch = (unsigned int)(batch / lib4neuro::mpi_nranks);
+
         if (batch > 0) {
-            *data = this->ds->get_random_data_batch(batch);
-            n_elements = data->size();
+//            if (lib4neuro::mpi_rank == 0) {
+//                *data = this->ds->get_random_data_batch(batch);
+//                n_elements = data->size();
+//            }
+
+            this->ds->activate_next_data_batch(batch);
+
+//            std::cout << lib4neuro::mpi_rank << " " << this->ds->get_permutation_data_indices()->size() << std::endl;
+
+//            MPI_Barrier(lib4neuro::mpi_active_comm);
+
+//            MPI_Bcast(this->ds->get_permutation_data_indices_content(),
+//                      this->ds->get_permutation_data_indices()->size(),
+//                      MPI_UNSIGNED,
+//                      0,
+//                      lib4neuro::mpi_active_comm);
+
+//            MPI_Bcast(this->ds->get_active_elements_content(),
+//                      this->ds->get_active_elements_indices()->size(),
+//                      MPI_UNSIGNED,
+//                      0,
+//                      lib4neuro::mpi_active_comm);
+//            unsigned int batch_ind;
+//            if(lib4neuro::mpi_rank == 0) {
+//                batch_ind = this->ds->get_current_batch_vector_index();
+//            }
+//            MPI_Bcast(this->ds->get_current_batch_vector_index_ptr(),
+//                      1,
+//                      MPI_UNSIGNED,
+//                      0,
+//                      lib4neuro::mpi_active_comm);
         }
+
         std::vector<double> error_derivative(dim_out);
 
-        for (auto el: *data) {  // Iterate through every element in the test set
+//        for (auto el: *data) {  // Iterate through every element in the test set
+
+        for(auto ind : *this->ds->get_active_elements_indices()) {
+
+            auto el = data->at(ind);
 
             this->nets[0]->eval_single(el.first,
                                        error_derivative,

src/examples/dev_sandbox.cpp

@@ -65,12 +65,12 @@ double optimize_via_gradient_descent(l4n::NeuralNetwork& net,
 //    for( double tol = 1e-1; tol > 1e-6; tol *= 1e-1 ){
         l4n::GradientDescentBB gd(1e-4,
                                   1000,
-                                  100000);
+                                  100000, 32);
 
-        l4n::LazyLearning lazy_wrapper( gd, 1e-4 );
-        lazy_wrapper.optimize(ef);
+//        l4n::LazyLearning lazy_wrapper( gd, 1e-4 );
+//        lazy_wrapper.optimize(ef);
 //    }
-//    gd.optimize(ef);
+    gd.optimize(ef);
 
 //    net.copy_parameter_space(gd.get_parameters());
 
@@ -240,8 +240,8 @@ int main() {
 //
 ////        optimize_via_NelderMead(net, mse);
 //
-        double err1 = optimize_via_LBMQ(net, mse);
-//        double err2 = optimize_via_gradient_descent(net, mse);
+//        double err1 = optimize_via_LBMQ(net, mse);
+        double err2 = optimize_via_gradient_descent(net, mse);
 
 //        std::cout << "Weights: " << net.get_min_max_weight().first << " " << net.get_min_max_weight().second << std::endl;
 
@@ -261,6 +261,12 @@ int main() {
 //            std::cout << output.at(0) << std::endl;
 //        }
 
+//        l4n::DataSet ds2(3, 1, 5, 0);
+//        ds2.activate_next_data_batch(2);
+//        ds2.activate_next_data_batch(2);
+//        ds2.activate_next_data_batch(2);
+//        ds2.activate_next_data_batch(2);
+
     } catch (const std::exception& e) {
         std::cerr << e.what() << std::endl;
         exit(EXIT_FAILURE);
@@ -269,8 +275,6 @@ int main() {
 //    arma::Mat<double> m = {{1,2,3}, {4,5,6}, {7,8,9}};
 //    arma::Col<double> v = arma::conv_to<std::vector<double>(m);
 
-
-
 //    std::cout << arma::stddev(m) << std::endl;
 	MPI_FINISH
     return 0;