Commit 427b094c authored by Martin Beseda's avatar Martin Beseda
Browse files

MERGE: New version of GD algorithm and using Mock objects for unit-testing -...

MERGE: New version of GD algorithm and using Mock objects for unit-testing - not working completely!
parents 58a652ca 8044f986
......@@ -4,6 +4,7 @@
# UNIT TESTS #
##############
for f in build/unit-tests/*_test; do
${f} || exit -1
echo "Test ${f} starting..."
${f} || exit 1
done
Subproject commit 67c6344e238d9f484dc34ab52c9de78334ec6acf
Subproject commit 6a58c03865b11c2a21229c6e71b8cebf15a9c8fb
Subproject commit e92f52821da312c5e30b447c8046893e576aefca
Subproject commit 4a853f9952708b5f1c57055a6c1944e2c4ad6544
......@@ -3,9 +3,12 @@
//
#include <algorithm>
#include <boost/serialization/export.hpp>
#include "DataSetSerialization.h"
BOOST_CLASS_EXPORT_IMPLEMENT(lib4neuro::DataSet);
namespace lib4neuro {
DataSet::DataSet() {
......@@ -16,13 +19,17 @@ namespace lib4neuro {
DataSet::DataSet(std::string file_path) {
std::ifstream ifs(file_path);
boost::archive::text_iarchive ia(ifs);
ia >> *this;
ifs.close();
if(ifs.is_open()) {
boost::archive::text_iarchive ia(ifs);
ia >> *this;
ifs.close();
} else {
throw std::runtime_error("File couldn't be open!");
}
}
DataSet::DataSet(std::vector<std::pair<std::vector<double>,
std::vector<double>>> *data_ptr,
DataSet::DataSet(std::vector<std::pair<std::vector<double>, std::vector<double>>> *data_ptr,
NormalizationStrategy* ns) {
this->n_elements = data_ptr->size();
this->data = *data_ptr;
......@@ -31,8 +38,8 @@ namespace lib4neuro {
if(ns) {
this->normalization_strategy = ns;
this->max_inp_val = this->normalization_strategy->get_max_value();
this->min_inp_val = this->normalization_strategy->get_min_value();
this->max_min_inp_val.at(0) = this->normalization_strategy->get_max_value();
this->max_min_inp_val.at(1) = this->normalization_strategy->get_min_value();
}
//TODO check the complete data set for input/output dimensions
......@@ -51,8 +58,8 @@ namespace lib4neuro {
if(ns) {
this->normalization_strategy = ns;
this->max_inp_val = this->normalization_strategy->get_max_value();
this->min_inp_val = this->normalization_strategy->get_min_value();
this->max_min_inp_val.at(0) = this->normalization_strategy->get_max_value();
this->max_min_inp_val.at(1) = this->normalization_strategy->get_min_value();
}
this->add_isotropic_data(lower_bound, upper_bound, size, output);
......@@ -71,8 +78,8 @@ namespace lib4neuro {
if(ns) {
this->normalization_strategy = ns;
this->max_inp_val = this->normalization_strategy->get_max_value();
this->min_inp_val = this->normalization_strategy->get_min_value();
this->max_min_inp_val.at(0) = this->normalization_strategy->get_max_value();
this->max_min_inp_val.at(1) = this->normalization_strategy->get_min_value();
}
this->add_isotropic_data(bounds, no_elems_in_one_dim, output_func);
......@@ -220,7 +227,7 @@ namespace lib4neuro {
// }
/* Find maximum and minimum values */
this->max_inp_val = this->min_inp_val = this->data[0].first.at(0);
this->max_min_inp_val.at(0) = this->max_min_inp_val.at(1) = this->data[0].first.at(0);
double tmp, tmp2;
for(auto pair : this->data) {
/* Finding maximum */
......@@ -230,8 +237,9 @@ namespace lib4neuro {
tmp = std::max(tmp, tmp2);
if (tmp > this->max_inp_val) {
this->max_inp_val = tmp;
/* Testing for a new maxima */
if (tmp > this->max_min_inp_val.at(0)) {
this->max_min_inp_val.at(0) = tmp;
}
/* Finding minimum */
......@@ -240,19 +248,20 @@ namespace lib4neuro {
tmp = std::min(tmp, tmp2);
if (tmp < this->min_inp_val) {
this->min_inp_val = tmp;
/* Testing for a new minima */
if (tmp < this->max_min_inp_val.at(1)) {
this->max_min_inp_val.at(1) = tmp;
}
}
/* Normalize every number in the data set */
for(auto& pair : this->data) {
for(auto& v : pair.first) {
v = this->normalization_strategy->normalize(v, this->max_inp_val, this->min_inp_val);
v = this->normalization_strategy->normalize(v, this->max_min_inp_val.at(0), this->max_min_inp_val.at(1));
}
for(auto& v : pair.second) {
v = this->normalization_strategy->normalize(v, this->max_inp_val, this->min_inp_val);
v = this->normalization_strategy->normalize(v, this->max_min_inp_val.at(0), this->max_min_inp_val.at(1));
}
}
......@@ -290,10 +299,40 @@ namespace lib4neuro {
// }
double DataSet::get_max_inp_val() {
return this->max_inp_val;
return this->max_min_inp_val.at(0);
}
double DataSet::get_min_inp_val() {
return this->min_inp_val;
return this->max_min_inp_val.at(1);
}
}
\ No newline at end of file
/**
* Method returning random amount of data pairs between 1-max
*/
std::vector<std::pair<std::vector<double>, std::vector<double>>> DataSet::get_random_data_batch(size_t max) {
if (max <= 0) {
return this->data;
} else {
std::vector<std::pair<std::vector<double>, std::vector<double>>> newData;
srand(time(NULL)); //TODO use Mersen twister from Boost
size_t n_chosen = rand() % std::min(max, this->data.size())+1;
std::vector<size_t> chosens;
size_t chosen;
for (int i = 0; i < n_chosen; i++) {
chosen = rand() % this->data.size();
auto it = std::find(chosens.begin(), chosens.end(), chosen);
if (it != chosens.end()) {
i--;
} else {
newData.push_back(this->data.at(chosen));
}
}
return newData;
}
}
}
......@@ -17,6 +17,8 @@
#include "../Exception/Exceptions.h"
#include "../NormalizationStrategy/NormalizationStrategy.h"
namespace lib4neuro {
/**
* Class representing data, which can be used for training
......@@ -41,15 +43,20 @@ namespace lib4neuro {
*/
size_t output_dim = 0;
/**
* Maximum input value
*/
double max_inp_val = std::numeric_limits<double>::quiet_NaN();
// /**
// * Maximum input value
// */
// double max_inp_val = //std::numeric_limits<double>::quiet_NaN();
//
// /**
// * Minimum input value
// */
// double min_inp_val = std::numeric_limits<double>::quiet_NaN();
/**
* Minimum input value
* Maximum (index 0) and minimum (index 1) input value
*/
double min_inp_val = std::numeric_limits<double>::quiet_NaN();
std::vector<double> max_min_inp_val; //TODO make more efficiently, than by vector!
/**
* Stored data in the format of pairs of corresponding
......@@ -100,8 +107,7 @@ namespace lib4neuro {
* Constructor accepting data vector
* @param data_ptr Pointer to the vector containing data
*/
LIB4NEURO_API DataSet(std::vector<std::pair<std::vector<double>,
std::vector<double>>> *data_ptr,
LIB4NEURO_API DataSet(std::vector<std::pair<std::vector<double>, std::vector<double>>> *data_ptr,
NormalizationStrategy* ns = nullptr);
/**
......@@ -259,6 +265,8 @@ namespace lib4neuro {
* @return
*/
LIB4NEURO_API double get_min_inp_val();
LIB4NEURO_API std::vector<std::pair<std::vector<double>, std::vector<double>>> get_random_data_batch(size_t max);
};
}
#endif //INC_4NEURO_DATASET_H
......@@ -11,9 +11,12 @@
#include <boost/serialization/utility.hpp>
#include <boost/archive/text_oarchive.hpp>
#include <boost/archive/text_iarchive.hpp>
#include <boost/serialization/export.hpp>
#include "DataSet.h"
BOOST_CLASS_EXPORT_KEY(lib4neuro::DataSet);
namespace lib4neuro {
struct DataSet::access {
template<class Archive>
......@@ -22,8 +25,7 @@ namespace lib4neuro {
ar & ds.input_dim;
ar & ds.output_dim;
ar & ds.data;
ar & ds.max_inp_val;
ar & ds.min_inp_val;
ar & ds.max_min_inp_val;
ar & ds.normalization_strategy;
}
};
......
......@@ -95,28 +95,74 @@ namespace lib4neuro {
}
double MSE::eval(std::vector<double> *weights) {
return this->eval_general(this->ds, weights);
size_t dim_out = this->ds->get_output_dim();
// unsigned int dim_in = this->ds->get_input_dim();
size_t n_elements = this->ds->get_n_elements();
double error = 0.0, val;
std::vector<std::pair<std::vector<double>, std::vector<double>>> *data = this->ds->get_data();
// //TODO instead use something smarter
// this->net->copy_weights(weights);
std::vector<double> output(dim_out);
for (auto el: *data) { // Iterate through every element in the test set
this->net->eval_single(el.first, output,
weights); // Compute the net output and store it into 'output' variable
for (size_t j = 0; j < dim_out; ++j) { // Compute difference for every element of the output vector
val = output[j] - el.second[j];
error += val * val;
}
}
return error / n_elements;
}
double MSE::eval_on_test_data(std::vector<double> *weights) {
return this->eval_general(this->ds_test, weights);
}
void MSE::calculate_error_gradient(std::vector<double> &params, std::vector<double> &grad, double alpha) {
// void MSE::calculate_error_gradient(std::vector<double> &params, std::vector<double> &grad, double alpha) {
//
// size_t dim_out = this->ds->get_output_dim();
// size_t n_elements = this->ds->get_n_elements();
//
// std::vector<std::pair<std::vector<double>, std::vector<double>>> *data = this->ds->get_data();
//
// std::vector<double> error_derivative(dim_out);
//
//
// for (auto el: *data) { // Iterate through every element in the test set
//
// this->net->eval_single(el.first, error_derivative,
// &params); // Compute the net output and store it into 'output' variable
//
// for (size_t j = 0; j < dim_out; ++j) {
// error_derivative[j] = 2.0 * (error_derivative[j] - el.second[j]); //real - expected result
// }
//
// this->net->add_to_gradient_single(el.first, error_derivative, alpha / n_elements, grad);
// }
// }
void MSE::calculate_error_gradient(std::vector<double> &params, std::vector<double> &grad, double alpha, size_t batch) {
size_t dim_out = this->ds->get_output_dim();
size_t n_elements = this->ds->get_n_elements();
std::vector<std::pair<std::vector<double>, std::vector<double>>> *data = this->ds->get_data();
if(batch) {
*data = this->ds->get_random_data_batch(batch);
n_elements = data->size();
}
std::vector<double> error_derivative(dim_out);
for (auto el: *data) { // Iterate through every element in the test set
/* Input normalization to [0,1] interval */
//TODO
this->net->eval_single(el.first, error_derivative,
&params); // Compute the net output and store it into 'output' variable
......@@ -193,14 +239,14 @@ namespace lib4neuro {
return output;
}
void ErrorSum::calculate_error_gradient(std::vector<double> &params, std::vector<double> &grad, double alpha) {
void ErrorSum::calculate_error_gradient(std::vector<double> &params, std::vector<double> &grad, double alpha, size_t batch) {
ErrorFunction *ef = nullptr;
for (size_t i = 0; i < this->summand->size(); ++i) {
ef = this->summand->at(i);
if (ef) {
ef->calculate_error_gradient(params, grad, this->summand_coefficient->at(i) * alpha);
ef->calculate_error_gradient(params, grad, this->summand_coefficient->at(i) * alpha, batch);
}
}
}
......@@ -241,4 +287,4 @@ namespace lib4neuro {
return this->summand->at(0)->get_parameters();
}
}
\ No newline at end of file
}
......@@ -25,7 +25,7 @@ namespace lib4neuro {
* @param weights
* @return
*/
virtual double eval(std::vector<double>* weights) = 0;
virtual double eval(std::vector<double> *weights = nullptr) = 0;
/**
*
......@@ -37,9 +37,14 @@ namespace lib4neuro {
*
* @param params
* @param grad
* @param alpha
* @param batch
*/
virtual void
calculate_error_gradient(std::vector<double> &params, std::vector<double> &grad, double alpha=1.0) = 0;
calculate_error_gradient(std::vector<double> &params,
std::vector<double> &grad,
double alpha = 1.0,
size_t batch = 0) = 0;
/**
*
......@@ -125,9 +130,14 @@ namespace lib4neuro {
*
* @param params
* @param grad
* @param alpha
* @param batch
*/
LIB4NEURO_API void
calculate_error_gradient(std::vector<double> &params, std::vector<double> &grad, double alpha = 1.0) override;
calculate_error_gradient(std::vector<double> &params,
std::vector<double> &grad,
double alpha = 1.0,
size_t batch = 0) override;
/**
*
......@@ -188,10 +198,14 @@ namespace lib4neuro {
*
* @param params
* @param grad
* @param alpha
* @param batch
*/
LIB4NEURO_API void
calculate_error_gradient(std::vector<double> &params, std::vector<double> &grad, double alpha = 1.0) override;
calculate_error_gradient(std::vector<double> &params,
std::vector<double> &grad,
double alpha = 1.0,
size_t batch = 0) override;
/**
*
* @return
......
......@@ -8,11 +8,12 @@
#include "GradientDescent.h"
namespace lib4neuro {
GradientDescent::GradientDescent(double epsilon, size_t n_to_restart, long long int max_iters) {
GradientDescent::GradientDescent(double epsilon, size_t n_to_restart, int max_iters, size_t batch) {
this->tolerance = epsilon;
this->restart_frequency = n_to_restart;
this->optimal_parameters = new std::vector<double>(0);
this->maximum_niters = max_iters;
this->batch = batch;
}
GradientDescent::~GradientDescent() {
......@@ -22,8 +23,13 @@ namespace lib4neuro {
}
}
void GradientDescent::eval_step_size_mk(double &gamma, double beta, double &c, double grad_norm_prev,
double grad_norm, double fi, double fim) {
void GradientDescent::eval_step_size_mk(double &gamma,
double beta,
double &c,
double grad_norm_prev,
double grad_norm,
double fi,
double fim) {
if (fi > fim) {
c /= 1.0000005;
......@@ -148,4 +154,4 @@ namespace lib4neuro {
return this->optimal_parameters;
}
}
\ No newline at end of file
}
......@@ -14,6 +14,9 @@
#include "../ErrorFunction/ErrorFunctions.h"
namespace lib4neuro {
/**
*
*/
class GradientDescent : public ILearningMethods {
private:
......@@ -23,10 +26,17 @@ namespace lib4neuro {
*/
double tolerance;
/**
*
*/
double max_error;
/**
* Number of iterations to reset step size to tolerance/10.0
*/
size_t restart_frequency;
size_t batch;
size_t iter_max;
/**
* Maximal number of iterations - optimization will stop after that, even if not converged
......@@ -49,10 +59,14 @@ namespace lib4neuro {
* @param fim
*/
virtual void
eval_step_size_mk(double &gamma, double beta, double &c, double grad_norm_prev, double grad_norm, double fi,
eval_step_size_mk(double &gamma,
double beta,
double &c,
double grad_norm_prev,
double grad_norm,
double fi,
double fim);
public:
/**
......@@ -61,7 +75,7 @@ namespace lib4neuro {
* @param n_to_restart Number of iterations to reset step size to tolerance/10.0
* @param max_iters Maximal number of iterations - optimization will stop after that, even if not converged
*/
LIB4NEURO_API GradientDescent(double epsilon = 1e-3, size_t n_to_restart = 100, long long int max_iters = -1);
LIB4NEURO_API GradientDescent(double epsilon = 1e-3, size_t n_to_restart = 100, int max_iters = 1000, size_t batch = 0);
/**
* Deallocates the instance
......@@ -79,8 +93,6 @@ namespace lib4neuro {
* @return
*/
LIB4NEURO_API virtual std::vector<double> *get_parameters();
};
}
......
......@@ -13,27 +13,27 @@ BOOST_CLASS_EXPORT_IMPLEMENT(NormalizationStrategy);
BOOST_CLASS_EXPORT_IMPLEMENT(DoubleUnitStrategy);
double NormalizationStrategy::get_max_value() {
return this->max_value;
return this->max_min_inp_val.at(0);
}
double NormalizationStrategy::get_min_value() {
return this->min_value;
return this->max_min_inp_val.at(1);
}
DoubleUnitStrategy::DoubleUnitStrategy() {}
double DoubleUnitStrategy::normalize(double n, double max, double min) {
this->max_value = max;
this->min_value = min;
this->max_min_inp_val.at(0) = max;
this->max_min_inp_val.at(1) = min;
return 2*(n - min)/(max - min) - 1;
}
double DoubleUnitStrategy::de_normalize(double n) {
if(std::isnan(this->max_value)) {
if(this->max_min_inp_val.empty()) {
throw std::runtime_error("Data were not normalized, so de-normalization cannot progress!");
}
return 0.5 * (1 + (this->max_value - this->min_value) * n) + this->min_value;
return 0.5 * (1 + (this->get_max_value() - this->get_min_value()) * n) + this->get_min_value();
}
......@@ -6,21 +6,28 @@
#define LIB4NEURO_NORMALIZATIONSTRATEGY_H
#include <limits>
#include <vector>
/**
*
*/
class NormalizationStrategy {
protected:
/**
*
*/
double max_value = std::numeric_limits<double>::quiet_NaN();
// /**
// *
// */
// double max_value = std::numeric_limits<double>::quiet_NaN();
//
// /**
// *
// */
// double min_value = std::numeric_limits<double>::quiet_NaN();
//
/**
*
* Maximum (index 0) and minimum (index 1) input value
*/
double min_value = std::numeric_limits<double>::quiet_NaN();
std::vector<double> max_min_inp_val;
public:
......
......@@ -9,6 +9,7 @@
#include <boost/archive/text_oarchive.hpp>
#include <boost/archive/text_iarchive.hpp>
#include <boost/serialization/export.hpp>
#include <boost/serialization/vector.hpp>
#include "NormalizationStrategy.h"
......@@ -19,8 +20,7 @@ BOOST_CLASS_EXPORT_KEY(DoubleUnitStrategy);
struct NormalizationStrategy::access {
template<class Archive>
static void serialize(Archive &ar, NormalizationStrategy& ns, const unsigned int version) {
ar & ns.max_value;
ar & ns.min_value;
ar & ns.max_min_inp_val;
}
};
......
......@@ -428,7 +428,7 @@ namespace lib4neuro {
NeuralNetwork *nn;
DataSet *ds;
/* DEFINITION OF THE PARTIAL ERROR FUNCTIONS */
///* DEFINITION OF THE PARTIAL ERROR FUNCTIONS */
std::vector<ErrorFunction *> error_functions(this->n_equations);
for (size_t i = 0; i < this->n_equations; ++i) {
nn = this->differential_equations->at(i);
......@@ -448,7 +448,8 @@ namespace lib4neuro {
total_error.add_error_function(error_functions[i], 1.0);
}
return total_error.eval(&weights_and_biases);