Commit 6dca5886 authored by Michal Kravcenko's avatar Michal Kravcenko

MOD: modified all examples to include some meaningful commentary of what's going on

parent afcae45d
......@@ -178,11 +178,14 @@ void Particle::print_coordinate() {
printf("%10.8f\n", (*this->coordinate)[this->coordinate_dim - 1]);
}
ParticleSwarm::ParticleSwarm(ErrorFunction* ef, double *domain_bounds,
ParticleSwarm::ParticleSwarm(ErrorFunction* ef, std::vector<double> *domain_bounds,
double c1, double c2, double w, size_t n_particles, size_t iter_max) {
srand(time(NULL));
if(domain_bounds->size() < 2 * ef->get_dimension()){
std::cerr << "The supplied domain bounds dimension is too low! It should be at least " << 2 * ef->get_dimension() << "\n" << std::endl;
}
this->f = ef;
this->func_dim = ef->get_dimension();
......@@ -200,14 +203,12 @@ ParticleSwarm::ParticleSwarm(ErrorFunction* ef, double *domain_bounds,
this->iter_max = iter_max;
this->particle_swarm = new Particle*[this->n_particles];
this->domain_bounds = &(domain_bounds->at(0));
for( size_t pi = 0; pi < this->n_particles; ++pi ){
this->particle_swarm[pi] = new Particle(ef, domain_bounds);
this->particle_swarm[pi] = new Particle(ef, this->domain_bounds);
}
this->domain_bounds = domain_bounds;
}
ParticleSwarm::~ParticleSwarm() {
......@@ -337,7 +338,7 @@ void ParticleSwarm::optimize( double gamma, double epsilon, double delta) {
// }
/* Check if the particles are near to each other AND the maximal velocity is less than 'gamma' */
if(cluster.size() > delta * this->n_particles && std::abs(prev_max_vel_step/max_vel_step) > gamma) {
if( cluster.size() > delta * this->n_particles && prev_max_vel_step < gamma * max_vel_step ) {
break;
}
......@@ -348,21 +349,18 @@ void ParticleSwarm::optimize( double gamma, double epsilon, double delta) {
this->determine_optimal_coordinate_and_value(*this->p_min_glob, optimal_value);
if(outer_it < this->iter_max) {
/* Convergence reached */
printf("\nFound optimum in %d iterations: %10.8f at coordinates: \n", (int)outer_it, optimal_value);
for (size_t i = 0; i <= this->func_dim - 1; ++i) {
printf("%10.8f \n", (*this->p_min_glob)[i]);
}
printf("\nFound optimum in %d iterations. Objective function value: %10.8f\n", (int)outer_it, optimal_value);
} else {
/* Maximal number of iterations reached */
printf("\nMax number of iterations reached (%d)! Found value %10.8f at coordinates: \n", (int)outer_it, optimal_value);
for (size_t i = 0; i <= this->func_dim - 1; ++i) {
printf("\t%10.8f \n", (*this->p_min_glob)[i]);
}
printf("\nMax number of iterations reached (%d)! Objective function value: %10.8f\n", (int)outer_it, optimal_value);
}
// for (size_t i = 0; i <= this->func_dim - 1; ++i) {
// printf("%10.8f \n", (*this->p_min_glob)[i]);
// }
//
// this->f->eval( this->get_solution() );
//delete [] p_min_glob; // TODO
delete centroid;
}
......
......@@ -170,7 +170,8 @@ public:
* @param n_particles
* @param iter_max
*/
ParticleSwarm( ErrorFunction* ef, double* domain_bounds, double c1, double c2, double w, size_t n_particles, size_t iter_max = 1 );
//TODO make domain_bounds constant
ParticleSwarm( ErrorFunction* ef, std::vector<double> *domain_bounds, double c1 = 1.711897, double c2 = 1.711897, double w = 0.711897, size_t n_particles = 50, size_t iter_max = 1000 );
/**
*
......
......@@ -319,7 +319,7 @@ void DESolver::set_error_function(size_t equation_idx, ErrorFunctionType F, Data
}
//TODO instead use general method with Optimizer as its argument (create hierarchy of optimizers)
void DESolver::solve_via_particle_swarm(double *domain_bounds, double c1, double c2, double w,
void DESolver::solve_via_particle_swarm(std::vector<double> *domain_bounds, double c1, double c2, double w,
size_t n_particles, size_t max_iters, double gamma,
double epsilon, double delta) {
......
......@@ -135,7 +135,7 @@ public:
void solve_via_particle_swarm(
double * domain_bounds,
std::vector<double> *domain_bounds,
double c1,
double c2,
double w,
......
/**
* Basic example using particle swarm method to train the network
* (result 0, -1/4)
*/
//
......@@ -13,6 +12,13 @@
int main() {
std::cout << "Running lib4neuro example 1: Basic use of the particle swarm method to train a simple network with few linear neurons" << std::endl;
std::cout << "***********************************************************************************************************************" <<std::endl;
std::cout << "The code attempts to find an approximate solution to the system of equations below:" << std::endl;
std::cout << "0 * w1 + 1 * w2 = 0.50" << std::endl;
std::cout << "1 * w1 + 0.5*w2 = 0.75" << std::endl;
std::cout << "***********************************************************************************************************************" <<std::endl;
/* TRAIN DATA DEFINITION */
std::vector<std::pair<std::vector<double>, std::vector<double>>> data_vec;
std::vector<double> inp, out;
......@@ -35,20 +41,15 @@ int main() {
NeuronLinear *i2 = new NeuronLinear( ); //f(x) = x
/* Output neuron */
double b = 1.0;//bias
NeuronLinear *o1 = new NeuronLinear( ); //f(x) = x + 1
NeuronLinear *o1 = new NeuronLinear( ); //f(x) = x
/* Adding neurons to the net */
size_t idx1 = net.add_neuron(i1, BIAS_TYPE::NO_BIAS);
size_t idx2 = net.add_neuron(i2, BIAS_TYPE::NO_BIAS);
size_t idx3 = net.add_neuron(o1, BIAS_TYPE::NEXT_BIAS);
std::vector<double> *bv = net.get_parameter_ptr_biases();
for(size_t i = 0; i < 1; ++i){
bv->at(i) = 1.0;
}
size_t idx3 = net.add_neuron(o1, BIAS_TYPE::NO_BIAS);
//
/* Adding connections */
net.add_connection_simple(idx1, idx3, SIMPLE_CONNECTION_TYPE::NEXT_WEIGHT);
......@@ -70,18 +71,39 @@ int main() {
MSE mse(&net, &ds);
/* TRAINING METHOD SETUP */
unsigned int max_iters = 2000;
double domain_bounds[4] = {-800.0, 800.0, -800.0, 800.0};
double c1 = 0.5, c2 = 1.5, w = 0.8;
std::vector<double> domain_bounds = {-10.0, 10.0, -10.0, 10.0, -10.0, 10.0};
ParticleSwarm swarm_01(&mse, &domain_bounds);
/* if the maximal velocity from the previous step is less than 'gamma' times the current maximal velocity, then one
* terminating criterion is met */
double gamma = 0.5;
/* if 'delta' times 'n' particles are in the centroid neighborhood given by the radius 'epsilon', then the second
* terminating criterion is met ('n' is the total number of particles) */
double epsilon = 0.02;
double delta = 0.7;
swarm_01.optimize(gamma, epsilon, delta);
std::vector<double> *parameters = swarm_01.get_solution();
net.copy_parameter_space(parameters);
printf("w1 = %10.7f\n", parameters->at( 0 ));
printf("w2 = %10.7f\n", parameters->at( 1 ));
std::cout << "***********************************************************************************************************************" <<std::endl;
/* ERROR CALCULATION */
double error = 0.0;
inp = {0, 1};
net.eval_single( inp, out );
error += (0.5 - out[0]) * (0.5 - out[0]);
std::cout << "x = (0, 1), expected output: 0.50, real output: " << out[0] << std::endl;
unsigned int n_particles = 10;
inp = {1, 0.5};
net.eval_single( inp, out );
error += (0.75 - out[0]) * (0.75 - out[0]);
std::cout << "x = (1, 0.5), expected output: 0.75, real output: " << out[0] << std::endl;
std::cout << "Run finished! Error of the network: " << 0.5 * error << std::endl;
ParticleSwarm swarm_01(&mse, domain_bounds, c1, c2, w, n_particles, max_iters);
swarm_01.optimize(0.5, 0.02);
return 0;
}
/**
* Example of a neural network with reused edge weights
* The system of equations associated with the net in this example is not regular
* minimizes the function: ((2y+0.5)^2 + (2x+1)^2 + (2x + y + 0.25)^2 + (2x+1)^2 + 1 + (4.5x + 0.37)^2 ) /3
* minimum [0.705493164] at (x, y) = (-1133/6290, -11193/62900) = (-0.180127186, -0.177949126)
*/
//
......@@ -14,6 +11,13 @@
#include "../include/4neuro.h"
int main() {
std::cout << "Running lib4neuro example 2: Basic use of the particle swarm method to train a network with five linear neurons and repeating edge weights" << std::endl;
std::cout << "********************************************************************************************************************************************" <<std::endl;
std::cout << "The code attempts to find an approximate solution to the system of equations below:" << std::endl;
std::cout << " 0 * w1 + 1 * w2 = 0.50 + b1" << std::endl;
std::cout << " 1 * w1 + 0.5*w2 = 0.75 + b1" << std::endl;
std::cout << "(1.25 + b2) * w2 = 0.63 + b3" << std::endl;
std::cout << "***********************************************************************************************************************" <<std::endl;
/* TRAIN DATA DEFINITION */
std::vector<std::pair<std::vector<double>, std::vector<double>>> data_vec;
......@@ -40,11 +44,11 @@ int main() {
NeuronLinear *i2 = new NeuronLinear( ); //f(x) = x
double b = 1;//bias
NeuronLinear *i3 = new NeuronLinear( ); //f(x) = x + 1
NeuronLinear *i3 = new NeuronLinear( ); //f(x) = x
/* Output neurons */
NeuronLinear *o1 = new NeuronLinear( ); //f(x) = x + 1
NeuronLinear *o2 = new NeuronLinear( ); //f(x) = x + 1
NeuronLinear *o1 = new NeuronLinear( ); //f(x) = x
NeuronLinear *o2 = new NeuronLinear( ); //f(x) = x
......@@ -55,14 +59,7 @@ int main() {
size_t idx4 = net.add_neuron(i3, BIAS_TYPE::NEXT_BIAS);
size_t idx5 = net.add_neuron(o2, BIAS_TYPE::NEXT_BIAS);
std::vector<double> *bv = net.get_parameter_ptr_biases();
for(size_t i = 0; i < 3; ++i){
bv->at(i) = 1.0;
}
/* Adding connections */
//net.add_connection_simple(idx1, idx3, -1, 1.0);
//net.add_connection_simple(idx2, idx3, -1, 1.0);
net.add_connection_simple(idx1, idx3, SIMPLE_CONNECTION_TYPE::NEXT_WEIGHT); // weight index 0
net.add_connection_simple(idx2, idx3, SIMPLE_CONNECTION_TYPE::NEXT_WEIGHT); // weight index 1
net.add_connection_simple(idx4, idx5, SIMPLE_CONNECTION_TYPE::EXISTING_WEIGHT, 0); // AGAIN weight index 0 - same weight!
......@@ -94,21 +91,50 @@ int main() {
// printf("evaluation of error at point (%f, %f) => %f\n", weights[0], weights[1], mse.eval(weights));
/* TRAINING METHOD SETUP */
unsigned int max_iters = 5000;
//must encapsulate each of the partial error functions
double domain_bounds[4] = {-800.0, 800.0, -800.0, 800.0};
std::vector<double> domain_bounds = {-10.0, 10.0, -10.0, 10.0,-10.0, 10.0, -10.0, 10.0,-10.0, 10.0, -10.0, 10.0,-10.0, 10.0, -10.0, 10.0,-10.0, 10.0, -10.0, 10.0,-10.0, 10.0, -10.0, 10.0,-10.0, 10.0, -10.0, 10.0};
ParticleSwarm swarm_01(&mse, &domain_bounds);
/* if the maximal velocity from the previous step is less than 'gamma' times the current maximal velocity, then one
* terminating criterion is met */
double gamma = 0.5;
/* if 'delta' times 'n' particles are in the centroid neighborhood given by the radius 'epsilon', then the second
* terminating criterion is met ('n' is the total number of particles) */
double epsilon = 0.02;
double delta = 0.9;
swarm_01.optimize(gamma, epsilon, delta);
std::vector<double> *parameters = swarm_01.get_solution();
net.copy_parameter_space(parameters);
printf("w1 = %10.7f\n", parameters->at( 0 ));
printf("w2 = %10.7f\n", parameters->at( 1 ));
printf("b1 = %10.7f\n", parameters->at( 2 ));
printf("b2 = %10.7f\n", parameters->at( 3 ));
printf("b3 = %10.7f\n", parameters->at( 4 ));
std::cout << "***********************************************************************************************************************" <<std::endl;
/* ERROR CALCULATION */
double error = 0.0;
inp = {0, 1, 0};
net.eval_single( inp, out );
error += (0.5 - out[0]) * (0.5 - out[0]) + (0.0 - out[1]) * (0.0 - out[1]);
printf("x = (%4.2f, %4.2f, %4.2f), expected output: (%4.2f, %4.2f), real output: (%10.7f, %10.7f)\n", inp[0], inp[1], inp[2], 0.5, 0.0, out[0], out[1]);
double c1 = 0.5, c2 = 1.5, w = 0.8;
inp = {1, 0.5, 0};
net.eval_single( inp, out );
error += (0.75 - out[0]) * (0.75 - out[0]) + (0.0 - out[1]) * (0.0 - out[1]);
printf("x = (%4.2f, %4.2f, %4.2f), expected output: (%4.2f, %4.2f), real output: (%10.7f, %10.7f)\n", inp[0], inp[1], inp[2], 0.75, 0.0, out[0], out[1]);
unsigned int n_particles = 100;
inp = {0, 0, 1.25};
net.eval_single( inp, out );
error += (0.0 - out[0]) * (0.0 - out[0]) + (0.63 - out[1]) * (0.63 - out[1]);
printf("x = (%4.2f, %4.2f, %4.2f), expected output: (%4.2f, %4.2f), real output: (%10.7f, %10.7f)\n", inp[0], inp[1], inp[2], 0.0, 0.63, out[0], out[1]);
ParticleSwarm swarm_01(&mse, domain_bounds, c1, c2, w, n_particles, max_iters);
std::cout << "Run finished! Error of the network: " << error / 3.0 << std::endl;
swarm_01.optimize(0.5, 0.02, 0.9);
printf("evaluation of error: %f\n", mse.eval());
return 0;
}
\ No newline at end of file
......@@ -14,6 +14,8 @@
#include "../include/4neuro.h"
int main() {
std::cout << "Running lib4neuro example 3: Use of the particle swarm method to train a set of networks sharing some edge weights" << std::endl;
std::cout << "********************************************************************************************************************" <<std::endl;
/* TRAIN DATA DEFINITION */
std::vector<std::pair<std::vector<double>, std::vector<double>>> data_vec_01, data_vec_02;
......@@ -42,11 +44,11 @@ int main() {
NeuronLinear *i1 = new NeuronLinear(); //f(x) = x
NeuronLinear *i2 = new NeuronLinear(); //f(x) = x
NeuronLinear *i3 = new NeuronLinear( ); //f(x) = x + 1
NeuronLinear *i3 = new NeuronLinear( ); //f(x) = x
/* Output neurons */
NeuronLinear *o1 = new NeuronLinear( ); //f(x) = x + 1
NeuronLinear *o2 = new NeuronLinear( ); //f(x) = x + 1
NeuronLinear *o1 = new NeuronLinear( ); //f(x) = x
NeuronLinear *o2 = new NeuronLinear( ); //f(x) = x
......@@ -57,11 +59,6 @@ int main() {
size_t idx4 = net.add_neuron(i3, BIAS_TYPE::NEXT_BIAS);
size_t idx5 = net.add_neuron(o2, BIAS_TYPE::NEXT_BIAS);
std::vector<double> *bv = net.get_parameter_ptr_biases();
for(size_t i = 0; i < 3; ++i){
bv->at(i) = 1.0;
}
/* Adding connections */
net.add_connection_simple(idx1, idx3, SIMPLE_CONNECTION_TYPE::NEXT_WEIGHT); // weight index 0
net.add_connection_simple(idx2, idx3, SIMPLE_CONNECTION_TYPE::NEXT_WEIGHT); // weight index 1
......@@ -91,40 +88,50 @@ int main() {
subnet_01_input_neurons.push_back(idx1);
subnet_01_input_neurons.push_back(idx2);
subnet_01_output_neurons.push_back(idx3);
NeuralNetwork *subnet_01 = net.get_subnet(subnet_01_input_neurons, subnet_01_output_neurons);
NeuralNetwork *subnet_01 = net.get_subnet( subnet_01_input_neurons, subnet_01_output_neurons );
subnet_02_input_neurons.push_back(idx4);
subnet_02_output_neurons.push_back(idx5);
NeuralNetwork *subnet_02 = net.get_subnet(subnet_02_input_neurons, subnet_02_output_neurons);
NeuralNetwork *subnet_02 = net.get_subnet( subnet_02_input_neurons, subnet_02_output_neurons );
/* COMPLEX ERROR FUNCTION SPECIFICATION */
MSE mse_01(subnet_01, &ds_01);
MSE mse_02(subnet_02, &ds_02);
if(subnet_01 && subnet_02){
/* COMPLEX ERROR FUNCTION SPECIFICATION */
MSE mse_01(subnet_01, &ds_01);
MSE mse_02(subnet_02, &ds_02);
ErrorSum mse_sum;
mse_sum.add_error_function( &mse_01 );
mse_sum.add_error_function( &mse_02 );
ErrorSum mse_sum;
mse_sum.add_error_function( &mse_01 );
mse_sum.add_error_function( &mse_02 );
/* TRAINING METHOD SETUP */
unsigned int max_iters = 50;
/* TRAINING METHOD SETUP */
std::vector<double> domain_bounds = {-10.0, 10.0, -10.0, 10.0,-10.0, 10.0, -10.0, 10.0,-10.0, 10.0, -10.0, 10.0,-10.0, 10.0, -10.0, 10.0,-10.0, 10.0, -10.0, 10.0,-10.0, 10.0, -10.0, 10.0,-10.0, 10.0, -10.0, 10.0};
ParticleSwarm swarm_01(&mse_sum, &domain_bounds);
/* if the maximal velocity from the previous step is less than 'gamma' times the current maximal velocity, then one
* terminating criterion is met */
double gamma = 0.5;
//must encapsulate each of the partial error functions
double domain_bounds[4] = {-800.0, 800.0, -800.0, 800.0};
/* if 'delta' times 'n' particles are in the centroid neighborhood given by the radius 'epsilon', then the second
* terminating criterion is met ('n' is the total number of particles) */
double epsilon = 0.02;
double delta = 0.9;
swarm_01.optimize(gamma, epsilon, delta);
double c1 = 0.5, c2 = 1.5, w = 0.8;
unsigned int n_particles = 100;
// printf("mse2: %d\n", mse_02.get_dimension());
ParticleSwarm swarm_01(&mse_sum, domain_bounds, c1, c2, w, n_particles, max_iters);
}
else{
std::cout << "We apologize, this example is unfinished as we are in the process of developing methods for efficient subnetwork definition" << std::endl;
}
swarm_01.optimize(0.5, 0.02, 0.9);
if(subnet_01){
delete subnet_01;
subnet_01 = nullptr;
}
// double weights[2] = {0, -0.25};
// printf("evaluation of error at (x, y) = (%f, %f): %f\n", weights[0], weights[1], mse_01.eval(weights));
if(subnet_02){
delete subnet_02;
subnet_02 = nullptr;
}
delete subnet_02;
delete subnet_01;
return 0;
}
\ No newline at end of file
This diff is collapsed.
This diff is collapsed.
......@@ -10,6 +10,17 @@
#include "4neuro.h"
int main() {
std::cout << "Running lib4neuro Serialization example 1" << std::endl;
std::cout << "********************************************************************************************************************************************" <<std::endl;
std::cout << "First, it finds an approximate solution to the system of equations below:" << std::endl;
std::cout << "0 * w1 + 1 * w2 = 0.50 + b" << std::endl;
std::cout << "1 * w1 + 0.5*w2 = 0.75 + b" << std::endl;
std::cout << "********************************************************************************************************************************************" <<std::endl;
std::cout << "Then it stores the network with its weights into a file via serialization" <<std::endl;
std::cout << "Then it loads the network from a file via serialization" <<std::endl;
std::cout << "Finally it tests the loaded network parameters by evaluating the error function" <<std::endl;
std::cout << "********************************************************************************************************************************************" <<std::endl;
/* TRAIN DATA DEFINITION */
std::vector<std::pair<std::vector<double>, std::vector<double>>> data_vec;
std::vector<double> inp, out;
......@@ -32,8 +43,7 @@ int main() {
NeuronLinear *i2 = new NeuronLinear( ); //f(x) = x
/* Output neuron */
double b = 1.0;//bias
NeuronLinear *o1 = new NeuronLinear( ); //f(x) = x + 1
NeuronLinear *o1 = new NeuronLinear( ); //f(x) = x
......@@ -68,26 +78,60 @@ int main() {
MSE mse(&net, &ds);
/* TRAINING METHOD SETUP */
unsigned int max_iters = 20;
double domain_bounds[4] = {-800.0, 800.0, -800.0, 800.0};
std::vector<double> domain_bounds = {-10.0, 10.0, -10.0, 10.0, -10.0, 10.0};
ParticleSwarm swarm_01(&mse, &domain_bounds);
/* if the maximal velocity from the previous step is less than 'gamma' times the current maximal velocity, then one
* terminating criterion is met */
double gamma = 0.5;
double c1 = 0.5, c2 = 1.5, w = 0.8;
/* if 'delta' times 'n' particles are in the centroid neighborhood given by the radius 'epsilon', then the second
* terminating criterion is met ('n' is the total number of particles) */
double epsilon = 0.02;
double delta = 0.9;
swarm_01.optimize(gamma, epsilon, delta);
unsigned int n_particles = 10;
std::vector<double> *parameters = swarm_01.get_solution();
net.copy_parameter_space(parameters);
ParticleSwarm swarm_01(&mse, domain_bounds, c1, c2, w, n_particles, max_iters);
printf("w1 = %10.7f\n", parameters->at( 0 ));
printf("w2 = %10.7f\n", parameters->at( 1 ));
printf(" b = %10.7f\n", parameters->at( 2 ));
swarm_01.optimize(0.5, 0.02);
/* SAVE NETWORK TO THE FILE */
std::cout << "Network 1" << std::endl;
std::cout << "********************************************************************************************************************************************" <<std::endl;
std::cout << "Network generated by the example" << std::endl;
net.print_stats();
net.save_text("saved_network.4nt");
std::cout << "--------------------------------------------------------------------------------------------------------------------------------------------" <<std::endl;
double error = 0.0;
inp = {0, 1};
net.eval_single( inp, out );
error += (0.5 - out[0]) * (0.5 - out[0]);
std::cout << "x = (0, 1), expected output: 0.50, real output: " << out[0] << std::endl;
std::cout << "Network 2" << std::endl;
inp = {1, 0.5};
net.eval_single( inp, out );
error += (0.75 - out[0]) * (0.75 - out[0]);
std::cout << "x = (1, 0.5), expected output: 0.75, real output: " << out[0] << std::endl;
std::cout << "Error of the network: " << 0.5 * error << std::endl;
std::cout << "********************************************************************************************************************************************" <<std::endl;
std::cout << "Network loaded from a file" << std::endl;
NeuralNetwork net2("saved_network.4nt");
net2.print_stats();
std::cout << "--------------------------------------------------------------------------------------------------------------------------------------------" <<std::endl;
error = 0.0;
inp = {0, 1};
net.eval_single( inp, out );
error += (0.5 - out[0]) * (0.5 - out[0]);
std::cout << "x = (0, 1), expected output: 0.50, real output: " << out[0] << std::endl;
inp = {1, 0.5};
net.eval_single( inp, out );
error += (0.75 - out[0]) * (0.75 - out[0]);
std::cout << "x = (1, 0.5), expected output: 0.75, real output: " << out[0] << std::endl;
std::cout << "Error of the network: " << 0.5 * error << std::endl;
std::cout << "********************************************************************************************************************************************" <<std::endl;
return 0;
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment