diff --git a/build.sh b/build.sh
index fc353b2cd6141c46a3690e154f1873171c644f06..c38d9b18f9403b224b89c6b4401fd2a765b96252 100755
--- a/build.sh
+++ b/build.sh
@@ -5,7 +5,7 @@ export CLEAN_AFTER=no
#./clean.sh
cd build_scripts/linux
-#./download_dependencies.sh
-export DEPENDENCIES_LINK_TYPE=shared
+./download_dependencies.sh
+export DEPENDENCIES_LINK_TYPE=static
./linux_gcc_build_x64_debug_local.sh
cd ../..
diff --git a/src/ErrorFunction/ErrorFunctions.cpp b/src/ErrorFunction/ErrorFunctions.cpp
index e5bc62cfde10f76b87da0e643862afe4f69a24ff..f785a08651211e07fdb69c6a9b502a70640bfa27 100644
--- a/src/ErrorFunction/ErrorFunctions.cpp
+++ b/src/ErrorFunction/ErrorFunctions.cpp
@@ -30,7 +30,8 @@ namespace lib4neuro {
/* Choose random subset of the DataSet for training and the remaining part for validation */
boost::random::mt19937 gen;
- boost::random::uniform_int_distribution<> dist(0, ds_size - 1);
+ boost::random::uniform_int_distribution<> dist(0,
+ ds_size - 1);
size_t test_set_size = ceil(ds_size * percent_test);
@@ -39,7 +40,9 @@ namespace lib4neuro {
for (unsigned int i = 0; i < test_set_size; i++) {
test_indices.emplace_back(dist(gen));
}
- std::sort(test_indices.begin(), test_indices.end(), std::greater<unsigned int>());
+ std::sort(test_indices.begin(),
+ test_indices.end(),
+ std::greater<unsigned int>());
std::vector<std::pair<std::vector<double>, std::vector<double>>> test_data, train_data;
@@ -55,10 +58,12 @@ namespace lib4neuro {
}
/* Re-initialize data set for training */
- this->ds = new DataSet(&train_data, this->ds_full->get_normalization_strategy());
+ this->ds = new DataSet(&train_data,
+ this->ds_full->get_normalization_strategy());
/* Initialize test data */
- this->ds_test = new DataSet(&test_data, this->ds_full->get_normalization_strategy());
+ this->ds_test = new DataSet(&test_data,
+ this->ds_full->get_normalization_strategy());
}
void ErrorFunction::return_full_data_set_for_training() {
@@ -93,16 +98,18 @@ namespace lib4neuro {
return output;
}
- MSE::MSE(NeuralNetwork* net, DataSet* ds) {
+ MSE::MSE(NeuralNetwork* net,
+ DataSet* ds) {
this->net = net;
this->ds = ds;
this->dimension = net->get_n_weights() + net->get_n_biases();
}
- double MSE::eval_on_data_set(lib4neuro::DataSet* data_set, std::ofstream* results_file_path,
- std::vector<double>* weights) {
- //TODO do NOT duplicate code - rewrite the function in a better way
-
+ double MSE::eval_on_data_set(lib4neuro::DataSet* data_set,
+ std::ofstream* results_file_path,
+ std::vector<double>* weights,
+ bool denormalize_data,
+ bool verbose) {
size_t dim_in = data_set->get_input_dim();
size_t dim_out = data_set->get_output_dim();
double error = 0.0, val, output_norm = 0;
@@ -114,26 +121,28 @@ namespace lib4neuro {
std::vector<std::vector<double>> outputs(data->size());
std::vector<double> output(dim_out);
- COUT_DEBUG("Evaluation of the error function MSE on the given data-set" << std::endl);
- COUT_DEBUG(R_ALIGN << "[Element index]" << " "
- << R_ALIGN << "[Input]" << " "
- << R_ALIGN << "[Real output]" << " "
- << R_ALIGN << "[Predicted output]" << " "
- << R_ALIGN << "[Absolute error]" << " "
- << R_ALIGN << "[Relative error]"
- << std::endl);
-
- *results_file_path << R_ALIGN << "[Element index]" << " "
- << R_ALIGN << "[Input]" << " "
- << R_ALIGN << "[Real output]" << " "
- << R_ALIGN << "[Predicted output]" << " "
- << R_ALIGN << "[Abs. error]" << " "
- << R_ALIGN << "[Rel. error]"
- << std::endl;
+ if (verbose) {
+ COUT_DEBUG("Evaluation of the error function MSE on the given data-set" << std::endl);
+ COUT_DEBUG(R_ALIGN << "[Element index]" << " "
+ << R_ALIGN << "[Input]" << " "
+ << R_ALIGN << "[Real output]" << " "
+ << R_ALIGN << "[Predicted output]" << " "
+ << R_ALIGN << "[Absolute error]" << " "
+ << R_ALIGN << "[Relative error %]"
+ << std::endl);
+ }
+
+ if (results_file_path) {
+ *results_file_path << R_ALIGN << "[Element index]" << " "
+ << R_ALIGN << "[Input]" << " "
+ << R_ALIGN << "[Real output]" << " "
+ << R_ALIGN << "[Predicted output]" << " "
+ << R_ALIGN << "[Abs. error]" << " "
+ << R_ALIGN << "[Rel. error %]"
+ << std::endl;
+ }
for (auto i = 0; i < data->size(); i++) { // Iterate through every element in the test set
- error = 0.0;
- output_norm = 0.0;
/* Compute the net output and store it into 'output' variable */
this->net->eval_single(data->at(i).first,
output,
@@ -142,40 +151,64 @@ namespace lib4neuro {
outputs.at(i) = output;
}
- bool denormalize_output = false;
- if(data_set->is_normalized()) {
- data_set->de_normalize();
- denormalize_output = true;
- }
+ double denormalized_output;
+ double denormalized_real_input;
+ double denormalized_real_output;
+
+// bool denormalize_output = false;
+// if (denormalize_data) {
+// if(data_set->is_normalized()) {
+// data_set->de_normalize();
+// }
+// denormalize_output = true;
+// }
for (auto i = 0; i < data->size(); i++) {
/* Compute difference for every element of the output vector */
#ifdef L4N_DEBUG
std::stringstream ss_input;
- for(auto j = 0; j < dim_in-1; j++) {
- ss_input << data->at(i).first.at(j) << ",";
+ std::string separator = "";
+ for (auto j = 0; j < dim_in; j++) {
+ if(denormalize_data) {
+ denormalized_real_input = data_set->get_normalization_strategy()->de_normalize(data->at(i).first.at(j));
+ } else {
+ denormalized_real_input = data->at(i).first.at(j);
+ }
+ ss_input << separator << denormalized_real_input;
+ separator = ",";
+ }
+ if(denormalize_data) {
+ denormalized_real_input = data_set->get_normalization_strategy()->de_normalize(data->at(i).first.back());
+ } else {
+ denormalized_real_input = data->at(i).first.back();
}
- ss_input << data->at(i).first.back();
std::stringstream ss_real_output;
std::stringstream ss_predicted_output;
#endif
- double denormalized_output;
+
+ double loc_error = 0;
+ output_norm = 0;
+ separator = "";
for (size_t j = 0; j < dim_out; ++j) {
- if(denormalize_output) {
+ if (denormalize_data) {
+ denormalized_real_output = data_set->get_normalization_strategy()->de_normalize(data->at(i).second.at(j));
denormalized_output = data_set->get_normalization_strategy()->de_normalize(outputs.at(i).at(j));
} else {
+ denormalized_real_output = data->at(i).second.at(j);
denormalized_output = outputs.at(i).at(j);
}
#ifdef L4N_DEBUG
- ss_real_output << data->at(i).second.at(j);
- ss_predicted_output << denormalized_output;
+ ss_real_output << separator << denormalized_real_output;
+ ss_predicted_output << separator << denormalized_output;
+ separator = ",";
#endif
- val = denormalized_output - data->at(i).second.at(j);
- error += val * val;
+ val = denormalized_output - denormalized_real_output;
+ loc_error += val * val;
+ error += loc_error;
output_norm += denormalized_output * denormalized_output;
}
@@ -184,229 +217,111 @@ namespace lib4neuro {
std::stringstream ss_ind;
ss_ind << "[" << i << "]";
- COUT_DEBUG(R_ALIGN << ss_ind.str() << " "
- << R_ALIGN << ss_input.str() << " "
- << R_ALIGN << ss_real_output.str() << " "
- << R_ALIGN << ss_predicted_output.str() << " "
- << R_ALIGN << std::sqrt(error) << " "
- << R_ALIGN << 2 * std::sqrt(error) / (std::sqrt(error) + std::sqrt(output_norm))
- << std::endl);
-
- *results_file_path << R_ALIGN << ss_ind.str() << " "
- << R_ALIGN << ss_input.str() << " "
- << R_ALIGN << ss_real_output.str() << " "
- << R_ALIGN << ss_predicted_output.str() << " "
- << R_ALIGN << std::sqrt(error) << " "
- << R_ALIGN << 2 * std::sqrt(error) / (std::sqrt(error) + std::sqrt(output_norm))
- << std::endl;
-#endif
- }
-
- double result = std::sqrt(error) / n_elements;
-
- COUT_DEBUG("MSE = " << result << std::endl);
- *results_file_path << "MSE = " << result << std::endl;
-
- return result;
- }
-
- double MSE::eval_on_data_set(DataSet* data_set, std::string results_file_path, std::vector<double>* weights) {
- //TODO do NOT duplicate code - rewrite the function in a better way
-
- size_t dim_out = data_set->get_output_dim();
- size_t n_elements = data_set->get_n_elements();
- double error = 0.0, val;
-
- std::vector<std::pair<std::vector<double>, std::vector<double>>>* data = data_set->get_data();
-
- //TODO instead use something smarter
- std::vector<std::vector<double>> outputs(data->size());
- std::vector<double> output(dim_out);
-
- COUT_DEBUG("Evaluation of the error function MSE on the given data-set" << std::endl);
- COUT_DEBUG(R_ALIGN << "[Input]" << " "
- << R_ALIGN << "[Real output]" << " "
- << R_ALIGN << "[Predicted output]" << " "
- << std::endl);
-
- std::ofstream ofs(results_file_path);
- if (!ofs.is_open()) {
- THROW_RUNTIME_ERROR("File path: " + results_file_path + " was not successfully opened!");
- }
-
- ofs << R_ALIGN << "[Index]" << " "
- << R_ALIGN << "[Input]" << " "
- << R_ALIGN << "[Real output]" << " "
- << R_ALIGN << "[Predicted output]"
- << std::endl;
-
- for (auto i = 0; i < data->size(); i++) { // Iterate through every element in the test set
-
- /* Compute the net output and store it into 'output' variable */
- this->net->eval_single(data->at(i).first,
- output,
- weights);
-
- outputs.at(i) = output;
- }
-
- bool denormalize_output = false;
- if(data_set->is_normalized()) {
- data_set->de_normalize();
- denormalize_output = true;
- }
-
- for(auto i = 0; i < data->size(); i++) {
- /* Compute difference for every element of the output vector */
- double denormalized_output;
- for (size_t j = 0; j < dim_out; ++j) {
- if(denormalize_output) {
- denormalized_output = data_set->get_normalization_strategy()->de_normalize(outputs.at(i).at(j));
- } else {
- denormalized_output = outputs.at(i).at(j);
- }
-
- std::stringstream ss_ind;
- ss_ind << "[" << i << "]";
-
+ if (verbose) {
COUT_DEBUG(R_ALIGN << ss_ind.str() << " "
- << R_ALIGN << data->at(i).first.at(j) << " "
- << R_ALIGN << data->at(i).second.at(j) << " "
- << R_ALIGN << denormalized_output
- << std::endl);
-
- ofs << R_ALIGN << ss_ind.str() << " "
- << R_ALIGN << data->at(i).first.at(j) << " "
- << R_ALIGN << data->at(i).second.at(j) << " "
- << R_ALIGN << denormalized_output
- << std::endl;
-
- val = denormalized_output - data->at(i).second.at(j);
- error += val * val;
+ << R_ALIGN << ss_input.str() << " "
+ << R_ALIGN << ss_real_output.str() << " "
+ << R_ALIGN << ss_predicted_output.str() << " "
+ << R_ALIGN << std::sqrt(loc_error) << " "
+ << R_ALIGN
+ << 200.0 * std::sqrt(loc_error) / (std::sqrt(loc_error) + std::sqrt(output_norm))
+ << std::endl);
}
- ofs << std::endl;
+ if (results_file_path) {
+ *results_file_path << R_ALIGN << ss_ind.str() << " "
+ << R_ALIGN << ss_input.str() << " "
+ << R_ALIGN << ss_real_output.str() << " "
+ << R_ALIGN << ss_predicted_output.str() << " "
+ << R_ALIGN << std::sqrt(loc_error) << " "
+ << R_ALIGN
+ << 200.0 * std::sqrt(loc_error) / (std::sqrt(loc_error) + std::sqrt(output_norm))
+ << std::endl;
+ }
+#endif
}
-
double result = std::sqrt(error) / n_elements;
- ofs << "MSE = " << result << std::endl;
- ofs.close();
-
- COUT_DEBUG("MSE = " << result << std::endl);
-
- return result;
- }
-
- double MSE::eval_on_data_set(DataSet* data_set, std::vector<double>* weights) {
- size_t dim_out = data_set->get_output_dim();
- size_t n_elements = data_set->get_n_elements();
- double error = 0.0, val;
-
- std::vector<std::pair<std::vector<double>, std::vector<double>>>* data = data_set->get_data();
-
- //TODO instead use something smarter
- std::vector<std::vector<double>> outputs(data->size());
- std::vector<double> output(dim_out);
-
- COUT_DEBUG("Evaluation of the error function MSE on the given data-set" << std::endl);
- COUT_DEBUG(R_ALIGN << "[Input]" << " "
- << R_ALIGN << "[Real output]" << " "
- << R_ALIGN << "[Predicted output]" << " "
- << std::endl);
-
- /* Compute predicted outputs */
- for (auto i = 0; i < data->size(); i++) { // Iterate through every element in the test set
-
- /* Compute the net output and store it into 'output' variable */
- this->net->eval_single(data->at(i).first,
- output,
- weights);
-
- outputs.at(i) = output;
+ if (verbose) {
+ COUT_DEBUG("MSE = " << result << std::endl);
}
- /* De-normalize data-set, if it's normalized */
- bool denormalize_output = false;
- if(data_set->is_normalized()) {
- data_set->de_normalize();
- denormalize_output = true;
+ if (results_file_path) {
+ *results_file_path << "MSE = " << result << std::endl;
}
- /* Evaluate the prediction error on de-normalized data */
- for(auto i = 0; i < data->size(); i++) {
-
- /* Compute difference for every element of the output vector */
- double denormalized_output;
- for (auto j = 0; j < dim_out; ++j) {
- if(denormalize_output) {
- denormalized_output = data_set->get_normalization_strategy()->de_normalize(outputs.at(i).at(j));
- } else {
- denormalized_output = outputs.at(i).at(j);
- }
-
- std::stringstream ss_ind;
- ss_ind << "[" << i << "]";
-
- COUT_DEBUG(R_ALIGN << ss_ind.str() << " "
- << R_ALIGN << data->at(i).first.at(j) << " "
- << R_ALIGN << data->at(i).second.at(j) << " "
- << R_ALIGN << denormalized_output
- << std::endl);
-
- val = denormalized_output - data->at(i).second.at(j);
- error += val * val;
- }
- }
-
- double result = std::sqrt(result)/n_elements;
- COUT_DEBUG("MSE = " << result << std::endl);
-
return result;
}
- double MSE::eval(std::vector<double>* weights) {
- size_t dim_out = this->ds->get_output_dim();
-// unsigned int dim_in = this->ds->get_input_dim();
- size_t n_elements = this->ds->get_n_elements();
- double error = 0.0, val;
-
- std::vector<std::pair<std::vector<double>, std::vector<double>>>* data = this->ds->get_data();
-
-// //TODO instead use something smarter
-// this->net->copy_weights(weights);
-
- std::vector<double> output(dim_out);
-
- for (auto el: *data) { // Iterate through every element in the test set
-
- this->net->eval_single(el.first, output,
- weights); // Compute the net output and store it into 'output' variable
+ double MSE::eval_on_data_set(DataSet* data_set,
+ std::string results_file_path,
+ std::vector<double>* weights,
+ bool verbose) {
+ std::ofstream ofs(results_file_path);
+ if (ofs.is_open()) {
+ this->eval_on_data_set(data_set,
+ &ofs,
+ weights,
+ true,
+ verbose);
+ ofs.close();
+ } else {
+ THROW_RUNTIME_ERROR("File " + results_file_path + " couldn't be open!");
+ }
+ }
- for (size_t j = 0; j < dim_out; ++j) { // Compute difference for every element of the output vector
+ double MSE::eval_on_data_set(DataSet* data_set,
+ std::vector<double>* weights,
+ bool verbose) {
+ this->eval_on_data_set(data_set,
+ nullptr,
+ weights,
+ true,
+ verbose);
+ }
- val = output.at(j) - el.second.at(j);
- error += val * val;
- }
- }
- return error / n_elements;
+ double MSE::eval(std::vector<double>* weights,
+ bool denormalize_data,
+ bool verbose) {
+ this->eval_on_data_set(this->ds,
+ nullptr,
+ weights,
+ denormalize_data,
+ verbose);
}
- double MSE::eval_on_test_data(std::vector<double>* weights) {
- return this->eval_on_data_set(this->ds_test, weights);
+ double MSE::eval_on_test_data(std::vector<double>* weights,
+ bool verbose) {
+ return this->eval_on_data_set(this->ds_test,
+ weights,
+ verbose);
}
- double MSE::eval_on_test_data(std::string results_file_path, std::vector<double>* weights) {
- return this->eval_on_data_set(this->ds_test, results_file_path, weights);
+ double MSE::eval_on_test_data(std::string results_file_path,
+ std::vector<double>* weights,
+ bool verbose) {
+ return this->eval_on_data_set(this->ds_test,
+ results_file_path,
+ weights,
+ verbose);
}
- double MSE::eval_on_test_data(std::ofstream* results_file_path, std::vector<double>* weights) {
- return this->eval_on_data_set(this->ds_test, results_file_path, weights);
+ double MSE::eval_on_test_data(std::ofstream* results_file_path,
+ std::vector<double>* weights,
+ bool verbose) {
+ return this->eval_on_data_set(this->ds_test,
+ results_file_path,
+ weights,
+ true,
+ verbose);
}
void
- MSE::calculate_error_gradient(std::vector<double>& params, std::vector<double>& grad, double alpha, size_t batch) {
+ MSE::calculate_error_gradient(std::vector<double>& params,
+ std::vector<double>& grad,
+ double alpha,
+ size_t batch) {
size_t dim_out = this->ds->get_output_dim();
size_t n_elements = this->ds->get_n_elements();
@@ -421,14 +336,18 @@ namespace lib4neuro {
for (auto el: *data) { // Iterate through every element in the test set
- this->net->eval_single(el.first, error_derivative,
+ this->net->eval_single(el.first,
+ error_derivative,
¶ms); // Compute the net output and store it into 'output' variable
for (size_t j = 0; j < dim_out; ++j) {
error_derivative[j] = 2.0 * (error_derivative[j] - el.second[j]); //real - expected result
}
- this->net->add_to_gradient_single(el.first, error_derivative, alpha / n_elements, grad);
+ this->net->add_to_gradient_single(el.first,
+ error_derivative,
+ alpha / n_elements,
+ grad);
}
}
@@ -447,7 +366,8 @@ namespace lib4neuro {
}
}
- double ErrorSum::eval_on_test_data(std::vector<double>* weights) {
+ double ErrorSum::eval_on_test_data(std::vector<double>* weights,
+ bool verbose) {
//TODO take care of the case, when there are no test data
double output = 0.0;
@@ -464,37 +384,50 @@ namespace lib4neuro {
return output;
}
- double ErrorSum::eval_on_test_data(std::string results_file_path, std::vector<double>* weights) {
+ double ErrorSum::eval_on_test_data(std::string results_file_path,
+ std::vector<double>* weights,
+ bool verbose) {
THROW_NOT_IMPLEMENTED_ERROR();
return -1;
}
- double ErrorSum::eval_on_test_data(std::ofstream* results_file_path, std::vector<double>* weights) {
+ double ErrorSum::eval_on_test_data(std::ofstream* results_file_path,
+ std::vector<double>* weights,
+ bool verbose) {
THROW_NOT_IMPLEMENTED_ERROR();
return -1;
}
- double ErrorSum::eval_on_data_set(lib4neuro::DataSet* data_set, std::vector<double>* weights) {
+ double ErrorSum::eval_on_data_set(lib4neuro::DataSet* data_set,
+ std::vector<double>* weights,
+ bool verbose) {
THROW_NOT_IMPLEMENTED_ERROR();
return -1;
}
- double ErrorSum::eval_on_data_set(lib4neuro::DataSet* data_set, std::string results_file_path,
- std::vector<double>* weights) {
+ double ErrorSum::eval_on_data_set(lib4neuro::DataSet* data_set,
+ std::string results_file_path,
+ std::vector<double>* weights,
+ bool verbose) {
THROW_NOT_IMPLEMENTED_ERROR();
return -1;
}
- double ErrorSum::eval_on_data_set(lib4neuro::DataSet* data_set, std::ofstream* results_file_path,
- std::vector<double>* weights) {
+ double ErrorSum::eval_on_data_set(lib4neuro::DataSet* data_set,
+ std::ofstream* results_file_path,
+ std::vector<double>* weights,
+ bool denormalize_data,
+ bool verbose) {
THROW_NOT_IMPLEMENTED_ERROR();
return -1;
}
- double ErrorSum::eval(std::vector<double>* weights) {
+ double ErrorSum::eval(std::vector<double>* weights,
+ bool denormalize_data,
+ bool verbose) {
double output = 0.0;
ErrorFunction* ef = nullptr;
@@ -509,7 +442,9 @@ namespace lib4neuro {
return output;
}
- void ErrorSum::calculate_error_gradient(std::vector<double>& params, std::vector<double>& grad, double alpha,
+ void ErrorSum::calculate_error_gradient(std::vector<double>& params,
+ std::vector<double>& grad,
+ double alpha,
size_t batch) {
ErrorFunction* ef = nullptr;
@@ -517,12 +452,16 @@ namespace lib4neuro {
ef = this->summand->at(i);
if (ef) {
- ef->calculate_error_gradient(params, grad, this->summand_coefficient->at(i) * alpha, batch);
+ ef->calculate_error_gradient(params,
+ grad,
+ this->summand_coefficient->at(i) * alpha,
+ batch);
}
}
}
- void ErrorSum::add_error_function(ErrorFunction* F, double alpha) {
+ void ErrorSum::add_error_function(ErrorFunction* F,
+ double alpha) {
if (!this->summand) {
this->summand = new std::vector<ErrorFunction*>(0);
}
diff --git a/src/ErrorFunction/ErrorFunctions.h b/src/ErrorFunction/ErrorFunctions.h
index 6b1317eede8177259102b1437e0b4b1779c3bad4..b61430643b4244921b3d72638dab61c6098ebf93 100644
--- a/src/ErrorFunction/ErrorFunctions.h
+++ b/src/ErrorFunction/ErrorFunctions.h
@@ -27,7 +27,8 @@ namespace lib4neuro {
* @param weights
* @return
*/
- virtual double eval(std::vector<double>* weights = nullptr) = 0;
+ virtual double eval(std::vector<double>* weights = nullptr, bool denormalize_data=false,
+ bool verbose = false) = 0;
/**
*
@@ -86,7 +87,7 @@ namespace lib4neuro {
/**
*
*/
- virtual double eval_on_test_data(std::vector<double>* weights = nullptr) = 0;
+ virtual double eval_on_test_data(std::vector<double>* weights = nullptr, bool verbose = false) = 0;
/**
*
@@ -94,7 +95,8 @@ namespace lib4neuro {
* @param weights
* @return
*/
- virtual double eval_on_test_data(std::string results_file_path, std::vector<double>* weights = nullptr) = 0;
+ virtual double eval_on_test_data(std::string results_file_path, std::vector<double>* weights = nullptr,
+ bool verbose = false) = 0;
/**
*
@@ -102,7 +104,8 @@ namespace lib4neuro {
* @param weights
* @return
*/
- virtual double eval_on_test_data(std::ofstream* results_file_path, std::vector<double>* weights = nullptr) = 0;
+ virtual double eval_on_test_data(std::ofstream* results_file_path, std::vector<double>* weights = nullptr,
+ bool verbose = false) = 0;
/**
*
@@ -110,7 +113,8 @@ namespace lib4neuro {
* @param weights
* @return
*/
- virtual double eval_on_data_set(DataSet* data_set, std::vector<double>* weights = nullptr) = 0;
+ virtual double eval_on_data_set(DataSet* data_set, std::vector<double>* weights = nullptr,
+ bool verbose = false) = 0;
/**
*
@@ -120,7 +124,8 @@ namespace lib4neuro {
* @return
*/
virtual double
- eval_on_data_set(DataSet* data_set, std::string results_file_path, std::vector<double>* weights = nullptr) = 0;
+ eval_on_data_set(DataSet* data_set, std::string results_file_path, std::vector<double>* weights = nullptr,
+ bool verbose = false) = 0;
/**
*
@@ -129,8 +134,11 @@ namespace lib4neuro {
* @param weights
* @return
*/
- virtual double eval_on_data_set(DataSet* data_set, std::ofstream* results_file_path,
- std::vector<double>* weights = nullptr) = 0;
+ virtual double eval_on_data_set(DataSet* data_set,
+ std::ofstream* results_file_path = nullptr,
+ std::vector<double>* weights = nullptr,
+ bool denormalize_data = true,
+ bool verbose = false) = 0;
protected:
@@ -177,7 +185,9 @@ namespace lib4neuro {
* @param weights
* @return
*/
- LIB4NEURO_API double eval(std::vector<double>* weights = nullptr) override;
+ LIB4NEURO_API double eval(std::vector<double>* weights = nullptr,
+ bool denormalize_data = false,
+ bool verbose = false) override;
/**
*
@@ -197,7 +207,7 @@ namespace lib4neuro {
* @param weights
* @return
*/
- LIB4NEURO_API double eval_on_test_data(std::vector<double>* weights = nullptr) override;
+ LIB4NEURO_API double eval_on_test_data(std::vector<double>* weights = nullptr, bool verbose = false) override;
/**
*
@@ -205,7 +215,9 @@ namespace lib4neuro {
* @param weights
* @return
*/
- LIB4NEURO_API double eval_on_test_data(std::string results_file_path, std::vector<double>* weights = nullptr);
+ LIB4NEURO_API double eval_on_test_data(std::string results_file_path = nullptr,
+ std::vector<double>* weights = nullptr,
+ bool verbose = false);
/**
*
@@ -213,7 +225,9 @@ namespace lib4neuro {
* @param weights
* @return
*/
- LIB4NEURO_API double eval_on_test_data(std::ofstream* results_file_path, std::vector<double>* weights = nullptr) override;
+ LIB4NEURO_API double eval_on_test_data(std::ofstream* results_file_path,
+ std::vector<double>* weights = nullptr,
+ bool verbose = false) override;
/**
*
@@ -224,7 +238,9 @@ namespace lib4neuro {
*/
LIB4NEURO_API double eval_on_data_set(DataSet* data_set,
std::ofstream* results_file_path,
- std::vector<double>* weights = nullptr) override;
+ std::vector<double>* weights = nullptr,
+ bool denormalize_data = false,
+ bool verbose = false) override;
/**
*
@@ -232,7 +248,9 @@ namespace lib4neuro {
* @param weights
* @return
*/
- LIB4NEURO_API double eval_on_data_set(DataSet* data_set, std::vector<double>* weights = nullptr) override;
+ LIB4NEURO_API double eval_on_data_set(DataSet* data_set,
+ std::vector<double>* weights = nullptr,
+ bool verbose = false) override;
/**
*
@@ -241,8 +259,10 @@ namespace lib4neuro {
* @param weights
* @return
*/
- LIB4NEURO_API double eval_on_data_set(DataSet* data_set, std::string results_file_path,
- std::vector<double>* weights = nullptr) override;
+ LIB4NEURO_API double eval_on_data_set(DataSet* data_set,
+ std::string results_file_path,
+ std::vector<double>* weights = nullptr,
+ bool verbose = false) override;
};
@@ -263,14 +283,16 @@ namespace lib4neuro {
* @param weights
* @return
*/
- LIB4NEURO_API double eval(std::vector<double>* weights = nullptr) override;
+ LIB4NEURO_API double eval(std::vector<double>* weights = nullptr,
+ bool denormalize_data = false,
+ bool verbose = false) override;
/**
*
* @param weights
* @return
*/
- LIB4NEURO_API double eval_on_test_data(std::vector<double>* weights = nullptr) override;
+ LIB4NEURO_API double eval_on_test_data(std::vector<double>* weights = nullptr, bool verbose = false) override;
/**
*
@@ -278,7 +300,9 @@ namespace lib4neuro {
* @param weights
* @return
*/
- LIB4NEURO_API double eval_on_test_data(std::string results_file_path, std::vector<double>* weights = nullptr) override;
+ LIB4NEURO_API double eval_on_test_data(std::string results_file_path,
+ std::vector<double>* weights = nullptr,
+ bool verbose = false) override;
/**
*
@@ -286,7 +310,9 @@ namespace lib4neuro {
* @param weights
* @return
*/
- LIB4NEURO_API double eval_on_test_data(std::ofstream* results_file_path, std::vector<double>* weights = nullptr) override;
+ LIB4NEURO_API double eval_on_test_data(std::ofstream* results_file_path,
+ std::vector<double>* weights = nullptr,
+ bool verbose = false) override;
/**
*
@@ -294,7 +320,9 @@ namespace lib4neuro {
* @param weights
* @return
*/
- LIB4NEURO_API double eval_on_data_set(DataSet* data_set, std::vector<double>* weights = nullptr) override;
+ LIB4NEURO_API double eval_on_data_set(DataSet* data_set,
+ std::vector<double>* weights = nullptr,
+ bool verbose = false) override;
/**
*
@@ -303,8 +331,10 @@ namespace lib4neuro {
* @param weights
* @return
*/
- LIB4NEURO_API double eval_on_data_set(DataSet* data_set, std::string results_file_path,
- std::vector<double>* weights = nullptr) override;
+ LIB4NEURO_API double eval_on_data_set(DataSet* data_set,
+ std::string results_file_path,
+ std::vector<double>* weights = nullptr,
+ bool verbose = false) override;
/**
*
@@ -315,7 +345,9 @@ namespace lib4neuro {
*/
LIB4NEURO_API double eval_on_data_set(DataSet* data_set,
std::ofstream* results_file_path,
- std::vector<double>* weights = nullptr) override;
+ std::vector<double>* weights = nullptr,
+ bool denormalize_data = true,
+ bool verbose = false) override;
/**
*
diff --git a/src/LearningMethods/GradientDescent.cpp b/src/LearningMethods/GradientDescent.cpp
index 440a072273e5dd82ac30867988b2166d4e933a69..ee8cec516a588b2baf59f059db6129a7add914e7 100644
--- a/src/LearningMethods/GradientDescent.cpp
+++ b/src/LearningMethods/GradientDescent.cpp
@@ -107,8 +107,8 @@ namespace lib4neuro {
/* step length calculation */
if (iter_counter < 10 || iter_counter % this->restart_frequency == 0) {
/* fixed step length */
- //gamma = 0.1 * this->tolerance;
gamma = 0.1 * this->tolerance;
+ //gamma = 0.001;
} else {
/* angle between two consecutive gradients */
sx = 0.0;
@@ -147,7 +147,7 @@ namespace lib4neuro {
<< ". C: " << c
<< ". Gradient norm: " << grad_norm
<< ". Total error: " << val
- << "." << std::endl );
+ << ".\r" );
WRITE_TO_OFS_DEBUG(ofs, "Iteration: " << (unsigned int)(iter_counter)
<< ". Step size: " << gamma
diff --git a/src/LearningMethods/ParticleSwarm.cpp b/src/LearningMethods/ParticleSwarm.cpp
index 3aadd29d487c16d54e77d6fcb630523e89bef5a5..2574935ea4b1e2849abdf3966bdc95bebc0b66a3 100644
--- a/src/LearningMethods/ParticleSwarm.cpp
+++ b/src/LearningMethods/ParticleSwarm.cpp
@@ -209,30 +209,12 @@ namespace lib4neuro {
"Parameters 'gamma', 'epsilon' and 'delta' must be greater than or equal to zero!");
}
- this->c1 = c1;
-
- this->c2 = c2;
-
- this->c3 = (c1 + c2) / 2.0;
-
- this->w = w;
-
this->gamma = gamma;
-
this->epsilon = epsilon;
-
this->delta = delta;
+ this->pst = PARTICLE_SWARM_TYPE::GENERAL;
- this->n_particles = n_particles;
-
- this->iter_max = iter_max;
-
- this->particle_swarm = new std::vector<Particle *>(this->n_particles);
-
- this->domain_bounds = domain_bounds;
-
- std::fill(this->particle_swarm->begin(), this->particle_swarm->end(), nullptr);
-
+ this->init_constructor(domain_bounds, c1, c2, w, n_particles, iter_max);
}
ParticleSwarm::~ParticleSwarm() {
@@ -260,6 +242,8 @@ namespace lib4neuro {
void ParticleSwarm::optimize(lib4neuro::ErrorFunction &ef, std::ofstream* ofs) {
//TODO add output to the 'ofs'
+ COUT_INFO("Finding optima via Globalized Particle Swarm method..." << std::endl);
+
/* Copy data set max and min values, if it's normalized */
if(ef.get_dataset()->is_normalized()) {
ef.get_network_instance()->set_normalization_strategy_instance(
@@ -302,6 +286,7 @@ namespace lib4neuro {
double max_vel_step = 0;
double prev_max_vel_step;
double euclidean_dist;
+ double current_err = -1;
this->determine_optimal_coordinate_and_value(*this->p_min_glob, optimal_value);
// for(unsigned int i = 0; i < this->n_particles; ++i){
@@ -383,16 +368,41 @@ namespace lib4neuro {
// }
// }
- /* Check if the particles are near to each other AND the maximal velocity is less than 'gamma' */
- if (cluster.size() >= this->delta * this->n_particles && prev_max_vel_step <= this->gamma * max_vel_step) {
- break;
+ current_err = ef.eval(this->p_min_glob);
+
+ COUT_DEBUG(std::string("Iteration: ") << (unsigned int)(outer_it)
+ << ". Total error: " << current_err
+ << ". Objective function value: " << optimal_value
+ << ".\r");
+
+ if(this->err_thresh) {
+
+ /* If the error threshold is given, then check the current error */
+ if(current_err <= this->err_thresh) {
+ break;
+ }
+ } else {
+
+ /* Check if the particles are near to each other AND the maximal velocity is less than 'gamma' */
+ if (cluster.size() >= this->delta * this->n_particles &&
+ prev_max_vel_step <= this->gamma * max_vel_step) {
+ break;
+ }
}
outer_it++;
+
+ //TODO parameter for inertia weight decrease?
// this->w *= 0.99;
+
}
+ COUT_DEBUG(std::string("Iteration: ") << (unsigned int)(outer_it)
+ << ". Total error: " << current_err
+ << ". Objective function value: " << optimal_value
+ << "." << std::endl );
this->determine_optimal_coordinate_and_value(*this->p_min_glob, optimal_value);
+ //TODO rewrite following output using COUT_INFO
if (outer_it < this->iter_max) {
/* Convergence reached */
printf("\nFound optimum in %d iterations. Objective function value: %10.8f\n", (int) outer_it,
@@ -406,13 +416,33 @@ namespace lib4neuro {
// printf("%10.8f \n", (*this->p_min_glob)[i]);
// }
//
-// this->f->eval( this->get_solution() );
ef.eval( this->get_parameters() );
+// ef.eval( this->get_parameters() );
+
delete centroid;
}
+ ParticleSwarm::ParticleSwarm(std::vector<double>* domain_bounds,
+ double err_thresh,
+ PARTICLE_SWARM_TYPE pst,
+ double c1,
+ double c2,
+ double w,
+ size_t n_particles,
+ size_t iter_max) {
+
+ if(err_thresh <= 0 ) {
+ THROW_INVALID_ARGUMENT_ERROR("Error threshold has to be greater then 0!");
+ }
+
+ this->err_thresh = err_thresh;
+ this->pst = pst;
+
+ this->init_constructor(domain_bounds, c1, c2, w, n_particles, iter_max);
+ }
+
Particle *ParticleSwarm::determine_optimal_coordinate_and_value(std::vector<double> &coord, double &val) {
Particle *p;
@@ -468,4 +498,21 @@ namespace lib4neuro {
return this->p_min_glob;
}
+ void ParticleSwarm::init_constructor(std::vector<double>* domain_bounds,
+ double c1,
+ double c2,
+ double w,
+ size_t n_particles,
+ size_t iter_max) {
+ this->c1 = c1;
+ this->c2 = c2;
+ this->c3 = (c1 + c2) / 2.0;
+ this->w = w;
+ this->n_particles = n_particles;
+ this->iter_max = iter_max;
+ this->particle_swarm = new std::vector<Particle *>(this->n_particles);
+ this->domain_bounds = domain_bounds;
+ std::fill(this->particle_swarm->begin(), this->particle_swarm->end(), nullptr);
+ }
+
}
\ No newline at end of file
diff --git a/src/LearningMethods/ParticleSwarm.h b/src/LearningMethods/ParticleSwarm.h
index fd371b73918fb741b05fbb3a6c52ab7728445bec..82a0194b95f5ba0358538790c0b5112d9a725e75 100644
--- a/src/LearningMethods/ParticleSwarm.h
+++ b/src/LearningMethods/ParticleSwarm.h
@@ -12,6 +12,8 @@
#include "../ErrorFunction/ErrorFunctions.h"
#include "ILearningMethods.h"
+
+
/**
*
*/
@@ -92,6 +94,16 @@ public:
};
namespace lib4neuro {
+
+ /**
+ * Particle Swarm method type differentiating between a general version and a version expecting cost function minima
+ * to be 0!
+ */
+ enum PARTICLE_SWARM_TYPE {
+ GENERAL,
+ MIN_ZERO
+ };
+
/**
* Class implementing the Global Particle Swarm Optimization method
*/
@@ -154,6 +166,18 @@ namespace lib4neuro {
*/
double delta;
+ /**
+ * Error threshold - determines a successful convergence
+ *
+ * Must be greater than 0!
+ */
+ double err_thresh = 0;
+
+ /**
+ * Type of particle swarm optimizer
+ */
+ PARTICLE_SWARM_TYPE pst;
+
/**
* Bounds for every optimized parameter (p1_lower, p1_upper, p2_lower, p2_upper...)
*/
@@ -188,6 +212,16 @@ namespace lib4neuro {
*/
LIB4NEURO_API double get_euclidean_distance(std::vector<double> *a, std::vector<double> *b);
+ /**
+ *
+ */
+ void init_constructor(std::vector<double> *domain_bounds,
+ double c1,
+ double c2,
+ double w,
+ size_t n_particles,
+ size_t iter_max);
+
public:
/**
@@ -215,6 +249,31 @@ namespace lib4neuro {
size_t iter_max = 1000
);
+ /**
+ * Creates an instance of the Global Particle Swarm Optimizer
+ *
+ * WARNING: This constructor expects the cost function minimum to be 0!
+ *
+ * @param domain_bounds Bounds for every optimized parameter (p1_lower, p1_upper, p2_lower, p2_upper...)
+ * @param c1 Cognitive parameter
+ * @param c2 Social parameter
+ * @param w Inertia weight
+ * @param n_particles Number of particles in the swarm
+ * @param iter_max Maximal number of iterations - optimization will stop after that, even if not converged
+ * @param err_thresh Error threshold for the method to converge successfully - depending on the used
+ * ErrorFunction
+ */
+ LIB4NEURO_API explicit ParticleSwarm(
+ std::vector<double> *domain_bounds,
+ double err_thresh,
+ PARTICLE_SWARM_TYPE,
+ double c1 = 1.711897,
+ double c2 = 1.711897,
+ double w = 0.711897,
+ size_t n_particles = 50,
+ size_t iter_max = 1000
+ );
+
/**
*
*/
diff --git a/src/examples/net_test_ode_1.cpp b/src/examples/net_test_ode_1.cpp
index f2c7d689a7e4a9c38f2129630ef5830b849ac465..6dfa6e9788211e9136cc436aa175e082809bf3a3 100644
--- a/src/examples/net_test_ode_1.cpp
+++ b/src/examples/net_test_ode_1.cpp
@@ -62,7 +62,7 @@ void optimize_via_particle_swarm( l4n::DESolver &solver, l4n::MultiIndex &alpha,
void optimize_via_gradient_descent(l4n::DESolver &solver, double accuracy ){
printf("Solution via a gradient descent method!\n");
- l4n::GradientDescent gd( accuracy, 1000 , 50);
+ l4n::GradientDescent gd( accuracy, 1000 , 500000);
solver.randomize_parameters( );
solver.solve( gd );
diff --git a/src/examples/simulator.cpp b/src/examples/simulator.cpp
index 8e972703bb7268805896f2a8e1a4b5df336d7500..f527ebf2d26199fd85e4bbc6d2a73af4b90a8880 100644
--- a/src/examples/simulator.cpp
+++ b/src/examples/simulator.cpp
@@ -12,14 +12,12 @@
#include <assert.h>
#include "4neuro.h"
-#include "../CrossValidator/CrossValidator.h"
-
int main(int argc, char** argv) {
try {
/* PHASE 1 - TRAINING DATA LOADING, NETWORK ASSEMBLY AND PARTICLE SWARM OPTIMIZATION */
- l4n::CSVReader reader1("/home/martin/Desktop/lib4neuro_test2/simulator_input.txt", "\t", true); // File, separator, skip 1st line
+ l4n::CSVReader reader1("/home/martin/Desktop/data_BACK_RH_1.csv", ";", true); // File, separator, skip 1st line
reader1.read(); // Read from the file
/* PHASE 1 - NEURAL NETWORK SPECIFICATION */
@@ -31,7 +29,7 @@ int main(int argc, char** argv) {
ds1.normalize(); // Normalization of data to prevent numerical problems
/* Numbers of neurons in layers (including input and output layers) */
- std::vector<unsigned int> neuron_numbers_in_layers = { 1, 3, 1 };
+ std::vector<unsigned int> neuron_numbers_in_layers = { 1, 2, 1 };
/* Fully connected feed-forward network with linear activation functions for input and output */
/* layers and the specified activation fns for the hidden ones (each entry = layer)*/
@@ -58,15 +56,24 @@ int main(int argc, char** argv) {
// 7) delta Amount of particles, which has to be in the cluster for the algorithm to stop (0-1)
// 8) n_particles Number of particles in the swarm
// 9) iter_max Maximal number of iterations - optimization will stop after that, even if not converged
+// l4n::ParticleSwarm ps(&domain_bounds,
+// 0.5,
+// 0.3,
+// 0.7,
+// 1.711897,
+// 1.711897,
+// 0.711897,
+// 150,
+// 200);
+
l4n::ParticleSwarm ps(&domain_bounds,
+ 1e-4,
+ lib4neuro::PARTICLE_SWARM_TYPE::MIN_ZERO,
1.711897,
1.711897,
0.711897,
- 0.5,
- 0.3,
- 0.7,
- 150,
- 1500);
+ 250,
+ 20);
/* Weight and bias randomization in the network accordingly to the uniform distribution */
nn1.randomize_parameters();
@@ -80,7 +87,7 @@ int main(int argc, char** argv) {
l4n::NeuralNetwork nn2("test_net_Particle_Swarm.4n");
/* Training data loading for the second phase */
- l4n::CSVReader reader2("/home/martin/Desktop/lib4neuro_test2/simulator_input.txt", "\t", true); // File, separator, skip 1st line
+ l4n::CSVReader reader2("/home/martin/Desktop/data_BACK_RH_1.csv", ";", true); // File, separator, skip 1st line
reader2.read(); // Read from the file
/* Create data set for both the first training of the neural network */
@@ -97,7 +104,7 @@ int main(int argc, char** argv) {
// 1) Threshold for the successful ending of the optimization - deviation from minima
// 2) Number of iterations to reset step size to tolerance/10.0
// 3) Maximal number of iterations - optimization will stop after that, even if not converged
- l4n::GradientDescent gs(1e-4, 150, 200000);
+ l4n::GradientDescent gs(1e-6, 150, 2000);
/* Gradient Descent Optimization */
gs.optimize(mse2); // Network training
@@ -124,7 +131,7 @@ int main(int argc, char** argv) {
nn3.write_biases(&output_file);
/* Evaluate network on an arbitrary data-set and save results into the file */
- l4n::CSVReader reader3("/home/martin/Desktop/lib4neuro_test2/simulator_input.txt", "\t", true); // File, separator, skip 1st line
+ l4n::CSVReader reader3("/home/martin/Desktop/data_BACK_RH_1.csv", ";", true); // File, separator, skip 1st line
reader3.read(); // Read from the file
/* Create data set for both the testing of the neural network */
@@ -143,7 +150,11 @@ int main(int argc, char** argv) {
/* Error function */
l4n::MSE mse3(&nn3, &ds3); // First parameter - neural network, second parameter - data-set
- mse3.eval_on_data_set(&ds3, &output_file);
+ std::cout << "Network trained only by GPS: " << std::endl;
+ mse1.eval_on_data_set(&ds3, &output_file, nullptr, true, true);
+
+ std::cout << "Network trained only by GS:" << std::endl;
+ mse3.eval_on_data_set(&ds3, &output_file, nullptr, true, true);
/* Close the output file for writing */
output_file.close();