diff --git a/src/ErrorFunction/ErrorFunctions.cpp b/src/ErrorFunction/ErrorFunctions.cpp
index 5169a76191f99c399c209bcce25c91d8dfcf785b..f785a08651211e07fdb69c6a9b502a70640bfa27 100644
--- a/src/ErrorFunction/ErrorFunctions.cpp
+++ b/src/ErrorFunction/ErrorFunctions.cpp
@@ -151,41 +151,62 @@ namespace lib4neuro {
outputs.at(i) = output;
}
- bool denormalize_output = false;
- if (denormalize_data && data_set->is_normalized()) {
- data_set->de_normalize();
- denormalize_output = true;
- }
+ double denormalized_output;
+ double denormalized_real_input;
+ double denormalized_real_output;
+
+// bool denormalize_output = false;
+// if (denormalize_data) {
+// if(data_set->is_normalized()) {
+// data_set->de_normalize();
+// }
+// denormalize_output = true;
+// }
for (auto i = 0; i < data->size(); i++) {
/* Compute difference for every element of the output vector */
#ifdef L4N_DEBUG
std::stringstream ss_input;
- for (auto j = 0; j < dim_in - 1; j++) {
- ss_input << data->at(i).first.at(j) << ",";
+ std::string separator = "";
+ for (auto j = 0; j < dim_in; j++) {
+ if(denormalize_data) {
+ denormalized_real_input = data_set->get_normalization_strategy()->de_normalize(data->at(i).first.at(j));
+ } else {
+ denormalized_real_input = data->at(i).first.at(j);
+ }
+ ss_input << separator << denormalized_real_input;
+ separator = ",";
+ }
+ if(denormalize_data) {
+ denormalized_real_input = data_set->get_normalization_strategy()->de_normalize(data->at(i).first.back());
+ } else {
+ denormalized_real_input = data->at(i).first.back();
}
- ss_input << data->at(i).first.back();
std::stringstream ss_real_output;
std::stringstream ss_predicted_output;
#endif
- double denormalized_output;
+
double loc_error = 0;
output_norm = 0;
+ separator = "";
for (size_t j = 0; j < dim_out; ++j) {
- if (denormalize_output) {
+ if (denormalize_data) {
+ denormalized_real_output = data_set->get_normalization_strategy()->de_normalize(data->at(i).second.at(j));
denormalized_output = data_set->get_normalization_strategy()->de_normalize(outputs.at(i).at(j));
} else {
+ denormalized_real_output = data->at(i).second.at(j);
denormalized_output = outputs.at(i).at(j);
}
#ifdef L4N_DEBUG
- ss_real_output << data->at(i).second.at(j);
- ss_predicted_output << denormalized_output;
+ ss_real_output << separator << denormalized_real_output;
+ ss_predicted_output << separator << denormalized_output;
+ separator = ",";
#endif
- val = denormalized_output - data->at(i).second.at(j);
+ val = denormalized_output - denormalized_real_output;
loc_error += val * val;
error += loc_error;
diff --git a/src/LearningMethods/ParticleSwarm.cpp b/src/LearningMethods/ParticleSwarm.cpp
index e6c21ced612026fbb0a024206bbf9dcd19bbb84d..2574935ea4b1e2849abdf3966bdc95bebc0b66a3 100644
--- a/src/LearningMethods/ParticleSwarm.cpp
+++ b/src/LearningMethods/ParticleSwarm.cpp
@@ -209,30 +209,12 @@ namespace lib4neuro {
"Parameters 'gamma', 'epsilon' and 'delta' must be greater than or equal to zero!");
}
- this->c1 = c1;
-
- this->c2 = c2;
-
- this->c3 = (c1 + c2) / 2.0;
-
- this->w = w;
-
this->gamma = gamma;
-
this->epsilon = epsilon;
-
this->delta = delta;
+ this->pst = PARTICLE_SWARM_TYPE::GENERAL;
- this->n_particles = n_particles;
-
- this->iter_max = iter_max;
-
- this->particle_swarm = new std::vector<Particle *>(this->n_particles);
-
- this->domain_bounds = domain_bounds;
-
- std::fill(this->particle_swarm->begin(), this->particle_swarm->end(), nullptr);
-
+ this->init_constructor(domain_bounds, c1, c2, w, n_particles, iter_max);
}
ParticleSwarm::~ParticleSwarm() {
@@ -304,6 +286,7 @@ namespace lib4neuro {
double max_vel_step = 0;
double prev_max_vel_step;
double euclidean_dist;
+ double current_err = -1;
this->determine_optimal_coordinate_and_value(*this->p_min_glob, optimal_value);
// for(unsigned int i = 0; i < this->n_particles; ++i){
@@ -385,23 +368,41 @@ namespace lib4neuro {
// }
// }
+ current_err = ef.eval(this->p_min_glob);
+
COUT_DEBUG(std::string("Iteration: ") << (unsigned int)(outer_it)
- << ". Total error: " << optimal_value
- << ".\r" );
+ << ". Total error: " << current_err
+ << ". Objective function value: " << optimal_value
+ << ".\r");
+
+ if(this->err_thresh) {
- /* Check if the particles are near to each other AND the maximal velocity is less than 'gamma' */
- if (cluster.size() >= this->delta * this->n_particles && prev_max_vel_step <= this->gamma * max_vel_step) {
- break;
+ /* If the error threshold is given, then check the current error */
+ if(current_err <= this->err_thresh) {
+ break;
+ }
+ } else {
+
+ /* Check if the particles are near to each other AND the maximal velocity is less than 'gamma' */
+ if (cluster.size() >= this->delta * this->n_particles &&
+ prev_max_vel_step <= this->gamma * max_vel_step) {
+ break;
+ }
}
outer_it++;
+
+ //TODO parameter for inertia weight decrease?
// this->w *= 0.99;
+
}
COUT_DEBUG(std::string("Iteration: ") << (unsigned int)(outer_it)
- << ". Total error: " << optimal_value
- << ".\n" );
+ << ". Total error: " << current_err
+ << ". Objective function value: " << optimal_value
+ << "." << std::endl );
this->determine_optimal_coordinate_and_value(*this->p_min_glob, optimal_value);
+ //TODO rewrite following output using COUT_INFO
if (outer_it < this->iter_max) {
/* Convergence reached */
printf("\nFound optimum in %d iterations. Objective function value: %10.8f\n", (int) outer_it,
@@ -418,11 +419,30 @@ namespace lib4neuro {
ef.eval( this->get_parameters() );
- ef.eval( this->get_parameters() );
+// ef.eval( this->get_parameters() );
delete centroid;
}
+ ParticleSwarm::ParticleSwarm(std::vector<double>* domain_bounds,
+ double err_thresh,
+ PARTICLE_SWARM_TYPE pst,
+ double c1,
+ double c2,
+ double w,
+ size_t n_particles,
+ size_t iter_max) {
+
+ if(err_thresh <= 0 ) {
+ THROW_INVALID_ARGUMENT_ERROR("Error threshold has to be greater then 0!");
+ }
+
+ this->err_thresh = err_thresh;
+ this->pst = pst;
+
+ this->init_constructor(domain_bounds, c1, c2, w, n_particles, iter_max);
+ }
+
Particle *ParticleSwarm::determine_optimal_coordinate_and_value(std::vector<double> &coord, double &val) {
Particle *p;
@@ -478,4 +498,21 @@ namespace lib4neuro {
return this->p_min_glob;
}
+ void ParticleSwarm::init_constructor(std::vector<double>* domain_bounds,
+ double c1,
+ double c2,
+ double w,
+ size_t n_particles,
+ size_t iter_max) {
+ this->c1 = c1;
+ this->c2 = c2;
+ this->c3 = (c1 + c2) / 2.0;
+ this->w = w;
+ this->n_particles = n_particles;
+ this->iter_max = iter_max;
+ this->particle_swarm = new std::vector<Particle *>(this->n_particles);
+ this->domain_bounds = domain_bounds;
+ std::fill(this->particle_swarm->begin(), this->particle_swarm->end(), nullptr);
+ }
+
}
\ No newline at end of file
diff --git a/src/LearningMethods/ParticleSwarm.h b/src/LearningMethods/ParticleSwarm.h
index fd371b73918fb741b05fbb3a6c52ab7728445bec..82a0194b95f5ba0358538790c0b5112d9a725e75 100644
--- a/src/LearningMethods/ParticleSwarm.h
+++ b/src/LearningMethods/ParticleSwarm.h
@@ -12,6 +12,8 @@
#include "../ErrorFunction/ErrorFunctions.h"
#include "ILearningMethods.h"
+
+
/**
*
*/
@@ -92,6 +94,16 @@ public:
};
namespace lib4neuro {
+
+ /**
+ * Particle Swarm method type differentiating between a general version and a version expecting cost function minima
+ * to be 0!
+ */
+ enum PARTICLE_SWARM_TYPE {
+ GENERAL,
+ MIN_ZERO
+ };
+
/**
* Class implementing the Global Particle Swarm Optimization method
*/
@@ -154,6 +166,18 @@ namespace lib4neuro {
*/
double delta;
+ /**
+ * Error threshold - determines a successful convergence
+ *
+ * Must be greater than 0!
+ */
+ double err_thresh = 0;
+
+ /**
+ * Type of particle swarm optimizer
+ */
+ PARTICLE_SWARM_TYPE pst;
+
/**
* Bounds for every optimized parameter (p1_lower, p1_upper, p2_lower, p2_upper...)
*/
@@ -188,6 +212,16 @@ namespace lib4neuro {
*/
LIB4NEURO_API double get_euclidean_distance(std::vector<double> *a, std::vector<double> *b);
+ /**
+ *
+ */
+ void init_constructor(std::vector<double> *domain_bounds,
+ double c1,
+ double c2,
+ double w,
+ size_t n_particles,
+ size_t iter_max);
+
public:
/**
@@ -215,6 +249,31 @@ namespace lib4neuro {
size_t iter_max = 1000
);
+ /**
+ * Creates an instance of the Global Particle Swarm Optimizer
+ *
+ * WARNING: This constructor expects the cost function minimum to be 0!
+ *
+ * @param domain_bounds Bounds for every optimized parameter (p1_lower, p1_upper, p2_lower, p2_upper...)
+ * @param c1 Cognitive parameter
+ * @param c2 Social parameter
+ * @param w Inertia weight
+ * @param n_particles Number of particles in the swarm
+ * @param iter_max Maximal number of iterations - optimization will stop after that, even if not converged
+ * @param err_thresh Error threshold for the method to converge successfully - depending on the used
+ * ErrorFunction
+ */
+ LIB4NEURO_API explicit ParticleSwarm(
+ std::vector<double> *domain_bounds,
+ double err_thresh,
+ PARTICLE_SWARM_TYPE,
+ double c1 = 1.711897,
+ double c2 = 1.711897,
+ double w = 0.711897,
+ size_t n_particles = 50,
+ size_t iter_max = 1000
+ );
+
/**
*
*/
diff --git a/src/examples/simulator.cpp b/src/examples/simulator.cpp
index f37a4eac69843aa05192d8ac2cd12b82960e1153..f527ebf2d26199fd85e4bbc6d2a73af4b90a8880 100644
--- a/src/examples/simulator.cpp
+++ b/src/examples/simulator.cpp
@@ -12,8 +12,6 @@
#include <assert.h>
#include "4neuro.h"
-#include "../CrossValidator/CrossValidator.h"
-
int main(int argc, char** argv) {
@@ -58,15 +56,24 @@ int main(int argc, char** argv) {
// 7) delta Amount of particles, which has to be in the cluster for the algorithm to stop (0-1)
// 8) n_particles Number of particles in the swarm
// 9) iter_max Maximal number of iterations - optimization will stop after that, even if not converged
+// l4n::ParticleSwarm ps(&domain_bounds,
+// 0.5,
+// 0.3,
+// 0.7,
+// 1.711897,
+// 1.711897,
+// 0.711897,
+// 150,
+// 200);
+
l4n::ParticleSwarm ps(&domain_bounds,
+ 1e-4,
+ lib4neuro::PARTICLE_SWARM_TYPE::MIN_ZERO,
1.711897,
1.711897,
0.711897,
- 0.5,
- 0.3,
- 0.7,
- 150,
- 2);
+ 250,
+ 20);
/* Weight and bias randomization in the network accordingly to the uniform distribution */
nn1.randomize_parameters();
@@ -97,7 +104,7 @@ int main(int argc, char** argv) {
// 1) Threshold for the successful ending of the optimization - deviation from minima
// 2) Number of iterations to reset step size to tolerance/10.0
// 3) Maximal number of iterations - optimization will stop after that, even if not converged
- l4n::GradientDescent gs(1e-6, 150, 50000);
+ l4n::GradientDescent gs(1e-6, 150, 2000);
/* Gradient Descent Optimization */
gs.optimize(mse2); // Network training
@@ -115,7 +122,6 @@ int main(int argc, char** argv) {
}
/* Neural network loading */
-// std::cout << "asdf" << std::endl;
l4n::NeuralNetwork nn3("test_net_Gradient_Descent.4n");
/* Check of the saved network - write to the file */
@@ -144,6 +150,10 @@ int main(int argc, char** argv) {
/* Error function */
l4n::MSE mse3(&nn3, &ds3); // First parameter - neural network, second parameter - data-set
+ std::cout << "Network trained only by GPS: " << std::endl;
+ mse1.eval_on_data_set(&ds3, &output_file, nullptr, true, true);
+
+ std::cout << "Network trained only by GS:" << std::endl;
mse3.eval_on_data_set(&ds3, &output_file, nullptr, true, true);
/* Close the output file for writing */