GradientDescentSingleItem.cpp

/**
 * DESCRIPTION OF THE FILE
 *
 * @author Michal Kravčenko
 * @date 19.2.19 -
 */

#include "GradientDescentSingleItem.h"

#include <random.hpp>
#include "message.h"

namespace lib4neuro {
    GradientDescentSingleItem::GradientDescentSingleItem(double epsilon, size_t n_to_restart, int max_iters, size_t batch) {
        this->tolerance = epsilon;
        this->restart_frequency = n_to_restart;
//        this->optimal_parameters = new std::vector<double>(0);
        this->maximum_niters = max_iters;
        this->batch = batch;
    }

    GradientDescentSingleItem::~GradientDescentSingleItem() {
        if (this->optimal_parameters) {
            delete this->optimal_parameters;
            this->optimal_parameters = nullptr;
        }
    }


    double GradientDescentSingleItem::get_optimal_step_size(lib4neuro::ErrorFunction &f, std::vector<double> &x,
                                                   std::vector<double> &d, size_t n_elems) {

        double alpha = 10.0 / n_elems;
        alpha = 1.0;
        double value = f.eval();
        double value_shifted = value + 1.0;


        std::shared_ptr<std::vector<double>> shifted_x = std::make_shared<std::vector<double>>(std::vector<double>(x));
        while( value_shifted > value ){
            alpha *= 0.5;

            for( size_t i = 0; i < x.size(); ++i ){
                (*shifted_x).at(i) = x.at(i) - alpha * d.at(i);
            }

            value_shifted = f.eval( shifted_x.get() );
        }
//        std::cout << "Error reduction: " << value - value_shifted << std::endl;
        return alpha;
    }


    void GradientDescentSingleItem::optimize(lib4neuro::ErrorFunction &ef, std::ofstream* ofs) {

        COUT_INFO("Finding a solution via a Gradient Descent [Single Item] method with adaptive step-length..." << std::endl);
        COUT_INFO("Initial error: " << ef.eval() << std::endl);

        size_t total_elements = ef.get_dataset()->get_n_elements(), updated_elements = 0, iter = 0;
        double max_error = 1.0, error, gamma;
        size_t iter_idx = this->maximum_niters;
        size_t  dim = ef.get_network_instance()->get_n_biases() + ef.get_network_instance()->get_n_weights();

        std::vector<double> parameter_vector = ef.get_parameters();
        std::vector<double> gradient_vector(dim);
        std::vector<double> search_direction(dim);
        std::vector<double> error_vector(ef.get_network_instance()->get_n_outputs());
        while( max_error >= this->tolerance && iter_idx >= 1 ){
            iter_idx--;
            iter++;

            max_error = 0.0;
            updated_elements = 0;
            std::fill(search_direction.begin(), search_direction.end(), 0);
            for( size_t i = 0; i < ef.get_dataset()->get_n_elements(); ++i){
                error = ef.eval_single_item_by_idx( i, &parameter_vector, error_vector );

                if( error > max_error ){
                    max_error = error;
                }

                if( error > this->tolerance ){
                    updated_elements++;
                    ef.calculate_error_gradient_single(error_vector, gradient_vector);

                    for(size_t j = 0; j < dim; ++j ){
                        search_direction[ j ] += gradient_vector[ j ];
                    }
                }
            }
            gamma = this->get_optimal_step_size(ef, parameter_vector, search_direction, updated_elements);

            for( size_t j = 0; j < dim; ++j ){
                parameter_vector[ j ] -= gamma * search_direction[ j ];
            }

            COUT_DEBUG("Iteration: " << iter << ", Total elements in train set: " << total_elements << ", # of elements with high error: " << updated_elements << ", max. error: " << max_error << "\r");
        }
        COUT_DEBUG("Iteration: " << iter << ", Total elements in train set: " << total_elements << ", # of elements with high error: " << updated_elements << ", max. error: " << max_error << std::endl);

        this->optimal_parameters = &parameter_vector;
//        std::shared_ptr<std::vector<double>> opt_params;
//        opt_params.reset(this->optimal_parameters);
        ef.get_network_instance()->copy_parameter_space( this->optimal_parameters );

    }

    std::vector<double> GradientDescentSingleItem::get_parameters() {
//        std::shared_ptr<std::vector<double>> ret;
//        ret.reset(&this->optimal_parameters);
        return std::vector<double>(*this->optimal_parameters);
    }

}