FIX: error calculation in error::eval_on_data_set

ADD: added proggress debug output for particle swarm optimization

src/ErrorFunction/ErrorFunctions.cpp

@@ -120,7 +120,7 @@ namespace lib4neuro {
                    << R_ALIGN << "[Real output]" << " "
                    << R_ALIGN << "[Predicted output]" << " "
                    << R_ALIGN << "[Absolute error]" << " "
-                   << R_ALIGN << "[Relative error]"
+                   << R_ALIGN << "[Relative error %]"
                    << std::endl);
 
         *results_file_path << R_ALIGN << "[Element index]" << " "
@@ -128,12 +128,10 @@ namespace lib4neuro {
                            << R_ALIGN << "[Real output]" << " "
                            << R_ALIGN << "[Predicted output]" << " "
                            << R_ALIGN << "[Abs. error]" << " "
-                           << R_ALIGN << "[Rel. error]"
+                           << R_ALIGN << "[Rel. error %]"
                            << std::endl;
 
         for (auto i = 0; i < data->size(); i++) {  // Iterate through every element in the test set
-            error = 0.0;
-            output_norm = 0.0;
             /* Compute the net output and store it into 'output' variable */
             this->net->eval_single(data->at(i).first,
                                    output,
@@ -162,6 +160,8 @@ namespace lib4neuro {
             std::stringstream ss_predicted_output;
 #endif
             double denormalized_output;
+            double loc_error = 0;
+            output_norm = 0;
             for (size_t j = 0; j < dim_out; ++j) {
                 if(denormalize_output) {
                     denormalized_output = data_set->get_normalization_strategy()->de_normalize(outputs.at(i).at(j));
@@ -175,7 +175,8 @@ namespace lib4neuro {
 #endif
 
                 val = denormalized_output - data->at(i).second.at(j);
-                error += val * val;
+                loc_error += val * val;
+                error += loc_error;
 
                 output_norm += denormalized_output * denormalized_output;
             }
@@ -188,16 +189,16 @@ namespace lib4neuro {
                        << R_ALIGN << ss_input.str() << " "
                        << R_ALIGN << ss_real_output.str() << " "
                        << R_ALIGN << ss_predicted_output.str() << " "
-                       << R_ALIGN << std::sqrt(error) << " "
-                       << R_ALIGN << 2 * std::sqrt(error) / (std::sqrt(error) + std::sqrt(output_norm))
+                       << R_ALIGN << std::sqrt(loc_error) << " "
+                       << R_ALIGN << 200.0 * std::sqrt(loc_error) / (std::sqrt(loc_error) + std::sqrt(output_norm))
                        << std::endl);
 
             *results_file_path << R_ALIGN << ss_ind.str() << " "
                                << R_ALIGN << ss_input.str() << " "
                                << R_ALIGN << ss_real_output.str() << " "
                                << R_ALIGN << ss_predicted_output.str() << " "
-                               << R_ALIGN << std::sqrt(error) << " "
-                               << R_ALIGN << 2 * std::sqrt(error) / (std::sqrt(error) + std::sqrt(output_norm))
+                               << R_ALIGN << std::sqrt(loc_error) << " "
+                               << R_ALIGN << 200.0 * std::sqrt(loc_error) / (std::sqrt(loc_error) + std::sqrt(output_norm))
                                << std::endl;
 #endif
         }
@@ -360,7 +361,7 @@ namespace lib4neuro {
             }
         }
 
-        double result = std::sqrt(result)/n_elements;
+        double result = std::sqrt(error)/n_elements;
         COUT_DEBUG("MSE = " << result << std::endl);
 
         return result;
@@ -390,7 +391,7 @@ namespace lib4neuro {
                 error += val * val;
             }
         }
-        return error / n_elements;
+        return sqrt(error) / n_elements;
     }
 
     double MSE::eval_on_test_data(std::vector<double>* weights) {

src/LearningMethods/GradientDescent.cpp

@@ -107,8 +107,8 @@ namespace lib4neuro {
             /* step length calculation */
             if (iter_counter < 10 || iter_counter % this->restart_frequency == 0) {
                 /* fixed step length */
-                //gamma = 0.1 * this->tolerance;
                 gamma = 0.1 * this->tolerance;
+                //gamma = 0.001;
             } else {
                 /* angle between two consecutive gradients */
                 sx = 0.0;
@@ -147,7 +147,7 @@ namespace lib4neuro {
                                                   << ". C: " << c
                                                   << ". Gradient norm: " << grad_norm
                                                   << ". Total error: " << val
-                                                  << "." << std::endl );
+                                                  << ".\r" );
 
             WRITE_TO_OFS_DEBUG(ofs, "Iteration: " << (unsigned int)(iter_counter)
                                                   << ". Step size: " << gamma

src/LearningMethods/ParticleSwarm.cpp

@@ -383,6 +383,10 @@ namespace lib4neuro {
 //            }
 //        }
 
+            COUT_DEBUG(std::string("Iteration: ") << (unsigned int)(outer_it)
+                                                  << ". Total error: " << optimal_value
+                                                  << ".\r" );
+
             /* Check if the particles are near to each other AND the maximal velocity is less than 'gamma' */
             if (cluster.size() >= this->delta * this->n_particles && prev_max_vel_step <= this->gamma * max_vel_step) {
                 break;
@@ -391,6 +395,9 @@ namespace lib4neuro {
             outer_it++;
 //        this->w *= 0.99;
         }
+        COUT_DEBUG(std::string("Iteration: ") << (unsigned int)(outer_it)
+                                              << ". Total error: " << optimal_value
+                                              << ".\n" );
 
         this->determine_optimal_coordinate_and_value(*this->p_min_glob, optimal_value);
         if (outer_it < this->iter_max) {

src/examples/net_test_ode_1.cpp

@@ -62,7 +62,7 @@ void optimize_via_particle_swarm( l4n::DESolver &solver, l4n::MultiIndex &alpha,
 
 void optimize_via_gradient_descent(l4n::DESolver &solver, double accuracy ){
     printf("Solution via a gradient descent method!\n");
-    l4n::GradientDescent gd( accuracy, 1000 , 50);
+    l4n::GradientDescent gd( accuracy, 1000 , 500000);
 
     solver.randomize_parameters( );
     solver.solve( gd );

src/examples/simulator.cpp

@@ -19,7 +19,7 @@ int main(int argc, char** argv) {
 
     try {
         /* PHASE 1 - TRAINING DATA LOADING, NETWORK ASSEMBLY AND PARTICLE SWARM OPTIMIZATION */
-        l4n::CSVReader reader1("/home/martin/Desktop/lib4neuro_test2/simulator_input.txt", "\t", true);  // File, separator, skip 1st line
+        l4n::CSVReader reader1("/home/fluffymoo/Dropbox/data_BACK_RH_1.csv", ";", true);  // File, separator, skip 1st line
         reader1.read();  // Read from the file
 
         /* PHASE 1 - NEURAL NETWORK SPECIFICATION */
@@ -31,7 +31,7 @@ int main(int argc, char** argv) {
         ds1.normalize();  // Normalization of data to prevent numerical problems
 
         /* Numbers of neurons in layers (including input and output layers) */
-        std::vector<unsigned int> neuron_numbers_in_layers = { 1, 3, 1 };
+        std::vector<unsigned int> neuron_numbers_in_layers = { 1, 2, 1 };
 
         /* Fully connected feed-forward network with linear activation functions for input and output */
         /* layers and the specified activation fns for the hidden ones (each entry = layer)*/
@@ -66,7 +66,7 @@ int main(int argc, char** argv) {
                               0.3,
                               0.7,
                               150,
-                              1500);
+                              200);
 
         /* Weight and bias randomization in the network accordingly to the uniform distribution */
         nn1.randomize_parameters();
@@ -80,7 +80,7 @@ int main(int argc, char** argv) {
 
         l4n::NeuralNetwork nn2("test_net_Particle_Swarm.4n");
         /* Training data loading for the second phase */
-        l4n::CSVReader reader2("/home/martin/Desktop/lib4neuro_test2/simulator_input.txt", "\t", true);  // File, separator, skip 1st line
+        l4n::CSVReader reader2("/home/fluffymoo/Dropbox/data_BACK_RH_1.csv", ";", true);  // File, separator, skip 1st line
         reader2.read();  // Read from the file
 
         /* Create data set for both the first training of the neural network */
@@ -97,7 +97,7 @@ int main(int argc, char** argv) {
         // 1) Threshold for the successful ending of the optimization - deviation from minima
         // 2) Number of iterations to reset step size to tolerance/10.0
         // 3) Maximal number of iterations - optimization will stop after that, even if not converged
-        l4n::GradientDescent gs(1e-4, 150, 200000);
+        l4n::GradientDescent gs(1e-6, 150, 50000);
 
         /* Gradient Descent Optimization */
         gs.optimize(mse2);  // Network training
@@ -124,7 +124,7 @@ int main(int argc, char** argv) {
         nn3.write_biases(&output_file);
 
         /* Evaluate network on an arbitrary data-set and save results into the file */
-        l4n::CSVReader reader3("/home/martin/Desktop/lib4neuro_test2/simulator_input.txt", "\t", true);  // File, separator, skip 1st line
+        l4n::CSVReader reader3("/home/fluffymoo/Dropbox/data_BACK_RH_1.csv", ";", true);  // File, separator, skip 1st line
         reader3.read();  // Read from the file
 
         /* Create data set for both the testing of the neural network */