GradientDescentSingleItem.cpp

/**
 * DESCRIPTION OF THE FILE
 *
 * @author Michal Kravčenko
 * @date 19.2.19 -
 */

#include "../mpi_wrapper.h"

#include "GradientDescentSingleItem.h"

#include <random.hpp>
#include "message.h"

namespace lib4neuro {
    GradientDescentSingleItem::GradientDescentSingleItem(double epsilon,
                                                         size_t n_to_restart,
                                                         int max_iters,
                                                         size_t batch) {
        this->tolerance         = epsilon;
        this->restart_frequency = n_to_restart;
        this->maximum_niters    = max_iters;
        this->batch             = batch;
    }

    GradientDescentSingleItem::~GradientDescentSingleItem() {
        if (this->optimal_parameters) {
            delete this->optimal_parameters;
            this->optimal_parameters = nullptr;
        }
    }


    double GradientDescentSingleItem::get_optimal_step_size(lib4neuro::ErrorFunction& f,
                                                            std::vector<double>& x,
                                                            std::vector<double>& d,
                                                            size_t n_elems) {

        double alpha = 10.0 / n_elems;
        alpha = 1.0;
        double value         = f.eval();
        double value_shifted = value + 1.0;


        std::shared_ptr<std::vector<double>> shifted_x = std::make_shared<std::vector<double>>(std::vector<double>(x));
        while (value_shifted > value) {
            alpha *= 0.5;

            for (size_t i = 0; i < x.size(); ++i) {
                (*shifted_x).at(i) = x.at(i) - alpha * d.at(i);
            }

            value_shifted = f.eval(shifted_x.get());
        }
        return alpha;
    }


    void GradientDescentSingleItem::optimize(lib4neuro::ErrorFunction& ef,
                                             std::ofstream* ofs) {
        double err_ = ef.eval();

        COUT_INFO("Finding a solution via a Gradient Descent [Single Item] method with adaptive step-length..."
                      << std::endl);
        COUT_INFO("Initial error: " << err_ << std::endl);

        size_t total_elements = ef.get_n_data_set(), updated_elements = 0, iter = 0;
        double max_error      = 1.0, error, gamma;
        size_t iter_idx       = this->maximum_niters;
        size_t dim            = ef.get_dimension();

        std::vector<double> parameter_vector = ef.get_parameters();
        std::vector<double> gradient_vector(dim);
        std::vector<double> search_direction(dim);
        std::vector<double> error_vector(ef.get_n_outputs());
        while (max_error >= this->tolerance && iter_idx >= 1) {
            iter_idx--;
            iter++;

            max_error        = 0.0;
            updated_elements = 0;
            std::fill(search_direction.begin(),
                      search_direction.end(),
                      0);
            for (size_t i = 0; i < total_elements; ++i) {
                error = ef.eval_single_item_by_idx(i,
                                                   &parameter_vector,
                                                   error_vector);

                if (error > max_error) {
                    max_error = error;
                }

                if (error > this->tolerance) {
                    updated_elements++;
                    ef.calculate_error_gradient_single(error_vector,
                                                       gradient_vector);

                    for (size_t j = 0; j < dim; ++j) {
                        search_direction[j] += gradient_vector[j];
                    }
                }
            }
            gamma = this->get_optimal_step_size(ef,
                                                parameter_vector,
                                                search_direction,
                                                updated_elements);

            for (size_t j = 0; j < dim; ++j) {
                parameter_vector[j] -= gamma * search_direction[j];
            }

            COUT_DEBUG("Iteration: " << iter << ", Total elements in train set: " << total_elements
                                     << ", # of elements with high error: " << updated_elements << ", max. error: "
                                     << max_error << "\r");
        }
        COUT_DEBUG("Iteration: " << iter << ", Total elements in train set: " << total_elements
                                 << ", # of elements with high error: " << updated_elements << ", max. error: "
                                 << max_error << std::endl);

        this->optimal_parameters = &parameter_vector;
        ef.set_parameters(*this->optimal_parameters);

    }

}