From 664bfecf1424ef629105d73c3b0597515bfb3ba1 Mon Sep 17 00:00:00 2001
From: Michal Kravcenko <michal.kravcenko@vsb.cz>
Date: Thu, 31 Jan 2019 14:07:53 +0100
Subject: [PATCH] FIX: error calculation in error::eval_on_data_set ADD: added
proggress debug output for particle swarm optimization
---
src/ErrorFunction/ErrorFunctions.cpp | 23 ++++++++++++-----------
src/LearningMethods/GradientDescent.cpp | 4 ++--
src/LearningMethods/ParticleSwarm.cpp | 7 +++++++
src/examples/net_test_ode_1.cpp | 2 +-
src/examples/simulator.cpp | 12 ++++++------
5 files changed, 28 insertions(+), 20 deletions(-)
diff --git a/src/ErrorFunction/ErrorFunctions.cpp b/src/ErrorFunction/ErrorFunctions.cpp
index e5bc62cf..4ce84c64 100644
--- a/src/ErrorFunction/ErrorFunctions.cpp
+++ b/src/ErrorFunction/ErrorFunctions.cpp
@@ -120,7 +120,7 @@ namespace lib4neuro {
<< R_ALIGN << "[Real output]" << " "
<< R_ALIGN << "[Predicted output]" << " "
<< R_ALIGN << "[Absolute error]" << " "
- << R_ALIGN << "[Relative error]"
+ << R_ALIGN << "[Relative error %]"
<< std::endl);
*results_file_path << R_ALIGN << "[Element index]" << " "
@@ -128,12 +128,10 @@ namespace lib4neuro {
<< R_ALIGN << "[Real output]" << " "
<< R_ALIGN << "[Predicted output]" << " "
<< R_ALIGN << "[Abs. error]" << " "
- << R_ALIGN << "[Rel. error]"
+ << R_ALIGN << "[Rel. error %]"
<< std::endl;
for (auto i = 0; i < data->size(); i++) { // Iterate through every element in the test set
- error = 0.0;
- output_norm = 0.0;
/* Compute the net output and store it into 'output' variable */
this->net->eval_single(data->at(i).first,
output,
@@ -162,6 +160,8 @@ namespace lib4neuro {
std::stringstream ss_predicted_output;
#endif
double denormalized_output;
+ double loc_error = 0;
+ output_norm = 0;
for (size_t j = 0; j < dim_out; ++j) {
if(denormalize_output) {
denormalized_output = data_set->get_normalization_strategy()->de_normalize(outputs.at(i).at(j));
@@ -175,7 +175,8 @@ namespace lib4neuro {
#endif
val = denormalized_output - data->at(i).second.at(j);
- error += val * val;
+ loc_error += val * val;
+ error += loc_error;
output_norm += denormalized_output * denormalized_output;
}
@@ -188,16 +189,16 @@ namespace lib4neuro {
<< R_ALIGN << ss_input.str() << " "
<< R_ALIGN << ss_real_output.str() << " "
<< R_ALIGN << ss_predicted_output.str() << " "
- << R_ALIGN << std::sqrt(error) << " "
- << R_ALIGN << 2 * std::sqrt(error) / (std::sqrt(error) + std::sqrt(output_norm))
+ << R_ALIGN << std::sqrt(loc_error) << " "
+ << R_ALIGN << 200.0 * std::sqrt(loc_error) / (std::sqrt(loc_error) + std::sqrt(output_norm))
<< std::endl);
*results_file_path << R_ALIGN << ss_ind.str() << " "
<< R_ALIGN << ss_input.str() << " "
<< R_ALIGN << ss_real_output.str() << " "
<< R_ALIGN << ss_predicted_output.str() << " "
- << R_ALIGN << std::sqrt(error) << " "
- << R_ALIGN << 2 * std::sqrt(error) / (std::sqrt(error) + std::sqrt(output_norm))
+ << R_ALIGN << std::sqrt(loc_error) << " "
+ << R_ALIGN << 200.0 * std::sqrt(loc_error) / (std::sqrt(loc_error) + std::sqrt(output_norm))
<< std::endl;
#endif
}
@@ -360,7 +361,7 @@ namespace lib4neuro {
}
}
- double result = std::sqrt(result)/n_elements;
+ double result = std::sqrt(error)/n_elements;
COUT_DEBUG("MSE = " << result << std::endl);
return result;
@@ -390,7 +391,7 @@ namespace lib4neuro {
error += val * val;
}
}
- return error / n_elements;
+ return sqrt(error) / n_elements;
}
double MSE::eval_on_test_data(std::vector<double>* weights) {
diff --git a/src/LearningMethods/GradientDescent.cpp b/src/LearningMethods/GradientDescent.cpp
index 440a0722..ee8cec51 100644
--- a/src/LearningMethods/GradientDescent.cpp
+++ b/src/LearningMethods/GradientDescent.cpp
@@ -107,8 +107,8 @@ namespace lib4neuro {
/* step length calculation */
if (iter_counter < 10 || iter_counter % this->restart_frequency == 0) {
/* fixed step length */
- //gamma = 0.1 * this->tolerance;
gamma = 0.1 * this->tolerance;
+ //gamma = 0.001;
} else {
/* angle between two consecutive gradients */
sx = 0.0;
@@ -147,7 +147,7 @@ namespace lib4neuro {
<< ". C: " << c
<< ". Gradient norm: " << grad_norm
<< ". Total error: " << val
- << "." << std::endl );
+ << ".\r" );
WRITE_TO_OFS_DEBUG(ofs, "Iteration: " << (unsigned int)(iter_counter)
<< ". Step size: " << gamma
diff --git a/src/LearningMethods/ParticleSwarm.cpp b/src/LearningMethods/ParticleSwarm.cpp
index 6d904055..a7698056 100644
--- a/src/LearningMethods/ParticleSwarm.cpp
+++ b/src/LearningMethods/ParticleSwarm.cpp
@@ -383,6 +383,10 @@ namespace lib4neuro {
// }
// }
+ COUT_DEBUG(std::string("Iteration: ") << (unsigned int)(outer_it)
+ << ". Total error: " << optimal_value
+ << ".\r" );
+
/* Check if the particles are near to each other AND the maximal velocity is less than 'gamma' */
if (cluster.size() >= this->delta * this->n_particles && prev_max_vel_step <= this->gamma * max_vel_step) {
break;
@@ -391,6 +395,9 @@ namespace lib4neuro {
outer_it++;
// this->w *= 0.99;
}
+ COUT_DEBUG(std::string("Iteration: ") << (unsigned int)(outer_it)
+ << ". Total error: " << optimal_value
+ << ".\n" );
this->determine_optimal_coordinate_and_value(*this->p_min_glob, optimal_value);
if (outer_it < this->iter_max) {
diff --git a/src/examples/net_test_ode_1.cpp b/src/examples/net_test_ode_1.cpp
index f2c7d689..6dfa6e97 100644
--- a/src/examples/net_test_ode_1.cpp
+++ b/src/examples/net_test_ode_1.cpp
@@ -62,7 +62,7 @@ void optimize_via_particle_swarm( l4n::DESolver &solver, l4n::MultiIndex &alpha,
void optimize_via_gradient_descent(l4n::DESolver &solver, double accuracy ){
printf("Solution via a gradient descent method!\n");
- l4n::GradientDescent gd( accuracy, 1000 , 50);
+ l4n::GradientDescent gd( accuracy, 1000 , 500000);
solver.randomize_parameters( );
solver.solve( gd );
diff --git a/src/examples/simulator.cpp b/src/examples/simulator.cpp
index 8e972703..35fc257b 100644
--- a/src/examples/simulator.cpp
+++ b/src/examples/simulator.cpp
@@ -19,7 +19,7 @@ int main(int argc, char** argv) {
try {
/* PHASE 1 - TRAINING DATA LOADING, NETWORK ASSEMBLY AND PARTICLE SWARM OPTIMIZATION */
- l4n::CSVReader reader1("/home/martin/Desktop/lib4neuro_test2/simulator_input.txt", "\t", true); // File, separator, skip 1st line
+ l4n::CSVReader reader1("/home/fluffymoo/Dropbox/data_BACK_RH_1.csv", ";", true); // File, separator, skip 1st line
reader1.read(); // Read from the file
/* PHASE 1 - NEURAL NETWORK SPECIFICATION */
@@ -31,7 +31,7 @@ int main(int argc, char** argv) {
ds1.normalize(); // Normalization of data to prevent numerical problems
/* Numbers of neurons in layers (including input and output layers) */
- std::vector<unsigned int> neuron_numbers_in_layers = { 1, 3, 1 };
+ std::vector<unsigned int> neuron_numbers_in_layers = { 1, 2, 1 };
/* Fully connected feed-forward network with linear activation functions for input and output */
/* layers and the specified activation fns for the hidden ones (each entry = layer)*/
@@ -66,7 +66,7 @@ int main(int argc, char** argv) {
0.3,
0.7,
150,
- 1500);
+ 200);
/* Weight and bias randomization in the network accordingly to the uniform distribution */
nn1.randomize_parameters();
@@ -80,7 +80,7 @@ int main(int argc, char** argv) {
l4n::NeuralNetwork nn2("test_net_Particle_Swarm.4n");
/* Training data loading for the second phase */
- l4n::CSVReader reader2("/home/martin/Desktop/lib4neuro_test2/simulator_input.txt", "\t", true); // File, separator, skip 1st line
+ l4n::CSVReader reader2("/home/fluffymoo/Dropbox/data_BACK_RH_1.csv", ";", true); // File, separator, skip 1st line
reader2.read(); // Read from the file
/* Create data set for both the first training of the neural network */
@@ -97,7 +97,7 @@ int main(int argc, char** argv) {
// 1) Threshold for the successful ending of the optimization - deviation from minima
// 2) Number of iterations to reset step size to tolerance/10.0
// 3) Maximal number of iterations - optimization will stop after that, even if not converged
- l4n::GradientDescent gs(1e-4, 150, 200000);
+ l4n::GradientDescent gs(1e-6, 150, 50000);
/* Gradient Descent Optimization */
gs.optimize(mse2); // Network training
@@ -124,7 +124,7 @@ int main(int argc, char** argv) {
nn3.write_biases(&output_file);
/* Evaluate network on an arbitrary data-set and save results into the file */
- l4n::CSVReader reader3("/home/martin/Desktop/lib4neuro_test2/simulator_input.txt", "\t", true); // File, separator, skip 1st line
+ l4n::CSVReader reader3("/home/fluffymoo/Dropbox/data_BACK_RH_1.csv", ";", true); // File, separator, skip 1st line
reader3.read(); // Read from the file
/* Create data set for both the testing of the neural network */
--
GitLab