-Added a new learning algorithm (Particle Swarm) and a corresponding example to main

src/LearningMethods/ParticleSwarm.cpp

+/**
+ * DESCRIPTION OF THE FILE
+ *
+ * @author Michal Kravčenko
+ * @date 2.7.18 -
+ */
+
+#include "ParticleSwarm.h"
+
+Particle::Particle(unsigned int f_dim, double *domain_bounds, double (*F)(double*)) {
+    this->coordinate_dim = f_dim;
+
+    this->coordinate = new double[f_dim];
+
+    this->velocity = new double[f_dim];
+
+
+
+    for(unsigned int i = 0; i < f_dim; ++i){
+        this->velocity[i] = (rand() % 100001 - 50000) / (double) 50000;
+    }
+//    this->r1 = (rand() % 100001) / (double) 100000;
+//    this->r2 = (rand() % 100001) / (double) 100000;
+    this->r1 = 1.0;
+    this->r2 = 1.0;
+
+    this->optimal_coordinate = new double[f_dim];
+
+    this->f = F;
+
+    this->domain_bounds = domain_bounds;
+    for(unsigned int i = 0; i < f_dim; ++i){
+        this->coordinate[i] = (rand() % 100001) / (double)100000 * (domain_bounds[2 * i + 1] - domain_bounds[2 * i]) + domain_bounds[2 * i];
+        this->optimal_coordinate[i] = this->coordinate[i];
+    }
+
+    this->optimal_value = this->f(this->coordinate);
+
+//    this->print_coordinate();
+}
+
+Particle::~Particle() {
+
+    if( this->optimal_coordinate ){
+        delete [] this->optimal_coordinate;
+    }
+
+    if( this->coordinate ){
+        delete [] this->coordinate;
+    }
+
+    if( this->velocity ){
+        delete [] this->velocity;
+    }
+
+}
+
+double Particle::get_optimal_value() {
+    return this->optimal_value;
+}
+
+void Particle::get_optimal_coordinate(double *ref_coordinate) {
+    for( unsigned int i = 0; i < this->coordinate_dim; ++i ){
+        ref_coordinate[i] = this->optimal_coordinate[i];
+    }
+}
+
+double Particle::change_coordinate(double w, double c1, double c2, double *glob_min_coord) {
+
+    /**
+     * v = w * v + c1r1(p_min_loc - x) + c2r2(p_min_glob - x)
+     * x = x + v
+     */
+
+    double vel_mem, output = 0.0;
+    for(unsigned int i = 0; i < this->coordinate_dim; ++i){
+
+        vel_mem = w * this->velocity[i] + c1 * this->r1 * (this->optimal_coordinate[i] - this->coordinate[i]) + c2 * this->r2 * (glob_min_coord[i] - this->coordinate[i]);
+
+        if( this->coordinate[i] + vel_mem > this->domain_bounds[2 * i + 1] ){
+//            vel_mem = vel_mem * (-1.0) * (vel_mem - this->domain_bounds[2 * i + 1]) / (this->domain_bounds[2 * i + 1] - this->velocity[i]);
+            vel_mem = -0.25 * w * vel_mem;
+        }
+        else if( this->coordinate[i] + vel_mem < this->domain_bounds[2 * i] ){
+//            vel_mem = vel_mem * (-1.0) * (this->domain_bounds[2 * i] - vel_mem) / (this->velocity[i] - this->domain_bounds[2 * i]);
+            vel_mem = -0.25 * w * vel_mem;
+        }
+
+        this->velocity[i] = vel_mem;
+        this->coordinate[i] += vel_mem;
+
+        output += vel_mem * vel_mem;
+    }
+
+    vel_mem = this->f(this->coordinate);
+
+    if(vel_mem < this->optimal_value){
+        this->optimal_value = vel_mem;
+        for(unsigned int i = 0; i < this->coordinate_dim; ++i){
+            this->optimal_coordinate[i] = this->coordinate[i];
+        }
+    }
+
+//    this->print_coordinate();
+
+    return output;
+}
+
+void Particle::print_coordinate() {
+    for(unsigned int i = 0; i < this->coordinate_dim - 1; ++i){
+        printf("%10.8f, ", this->coordinate[i]);
+    }
+    printf("%10.8f\n", this->coordinate[this->coordinate_dim - 1]);
+}
+
+ParticleSwarm::ParticleSwarm(double (*F)(double*), unsigned int f_dim, double *domain_bounds, double c1, double c2, double w,
+                             unsigned int n_particles, unsigned int iter_max) {
+
+    srand(time(NULL));
+
+    this->func = F;
+
+    this->func_dim = f_dim;
+
+    this->c1 = c1;
+
+    this->c2 = c2;
+
+    this->w = w;
+
+    this->n_particles = n_particles;
+
+    this->iter_max = iter_max;
+
+    this->particle_swarm = new Particle*[this->n_particles];
+
+    for( unsigned int pi = 0; pi < this->n_particles; ++pi ){
+        this->particle_swarm[pi] = new Particle( f_dim, domain_bounds, F);
+    }
+
+    this->domain_bounds = domain_bounds;
+
+
+}
+
+ParticleSwarm::~ParticleSwarm() {
+
+    if( this->particle_swarm ){
+        for( unsigned int i = 0; i < this->n_particles; ++i ){
+            delete this->particle_swarm[i];
+        }
+
+        delete [] this->particle_swarm;
+    }
+
+}
+
+void ParticleSwarm::optimize( double epsilon ) {
+
+    epsilon *= epsilon;
+
+    unsigned int outer_it = 0;
+    double optimum_step = epsilon * 10;
+    Particle *particle;
+    double *p_min_glob = new double[this->func_dim];
+    double optimal_value;
+
+    this->determine_optimal_coordinate_and_value(p_min_glob, optimal_value);
+    double optimum_prev;
+
+    double max_velocity, mem;
+    printf("%10d: max. velocity^2: %10.8f, optimum: %10.5f\n", outer_it, max_velocity, optimal_value);
+    while( outer_it < this->iter_max && optimum_step > epsilon ){
+
+        optimum_prev = optimal_value;
+
+
+        max_velocity = 0.0;
+        for(unsigned int pi = 0; pi < this->n_particles; ++pi){
+            particle = this->particle_swarm[pi];
+            mem = particle->change_coordinate( this->w, this->c1, this->c2, p_min_glob );
+
+            if(mem > max_velocity){
+                max_velocity = mem;
+            }
+        }
+
+        this->determine_optimal_coordinate_and_value(p_min_glob, optimal_value);
+
+//        optimum_step = std::abs( optimal_value - optimum_prev );
+        optimum_step = max_velocity;
+
+        printf("%10d: max. velocity^2: %10.8f, optimum: %10.5f\n", outer_it, max_velocity, optimal_value);
+
+        outer_it++;
+//        this->w *= 0.999;
+    }
+//    printf("\n");
+
+    printf("Found optimum in %6d iterations: %10.8f at coordinate: ", outer_it, optimal_value);
+    for(unsigned int i = 0; i < this->func_dim - 1; ++i){
+        printf("%10.8f, ", p_min_glob[i]);
+    }
+    printf("%10.8f\n", p_min_glob[this->func_dim - 1]);
+
+    delete [] p_min_glob;
+
+}
+
+void ParticleSwarm::determine_optimal_coordinate_and_value(double *coord, double &val) {
+
+    val = this->particle_swarm[0]->get_optimal_value( );
+    this->particle_swarm[0]->get_optimal_coordinate(coord);
+
+    for(unsigned int i = 1; i < this->n_particles; ++i){
+
+        double val_m = this->particle_swarm[i]->get_optimal_value( );
+
+        if(val_m < val){
+            val = val_m;
+            this->particle_swarm[i]->get_optimal_coordinate(coord);
+        }
+    }
+}
\ No newline at end of file

src/LearningMethods/ParticleSwarm.h

+/**
+ * DESCRIPTION OF THE FILE
+ *
+ * @author Michal Kravčenko
+ * @date 2.7.18 -
+ */
+
+#ifndef INC_4NEURO_PARTICLESWARM_H
+#define INC_4NEURO_PARTICLESWARM_H
+
+#include "stdlib.h"
+#include "time.h"
+
+#include "../Network/NeuralNetwork.h"
+
+
+class Particle{
+private:
+
+    unsigned int coordinate_dim;
+    double *coordinate = nullptr;
+    double *velocity;
+
+    double *optimal_coordinate = nullptr;
+    double optimal_value;
+
+    double r1;
+    double r2;
+
+    double (*f)(double*);
+
+    double *domain_bounds;
+
+    void print_coordinate();
+
+public:
+
+    /**
+     *
+     * @param f_dim
+     */
+    Particle(unsigned int f_dim, double *domain_bounds, double (*F)(double*));
+    ~Particle( );
+
+    /**
+     *
+     * @return
+     */
+    double get_optimal_value();
+
+    /**
+     *
+     * @param ref_coordinate
+     */
+    void get_optimal_coordinate(double *ref_coordinate);
+
+    /**
+     *
+     * @param w
+     * @param c1
+     * @param c2
+     * @param glob_min_coord
+     * @return
+     */
+    double change_coordinate(double w, double c1, double c2, double* glob_min_coord);
+};
+
+
+class ParticleSwarm {
+
+private:
+
+    /**
+     *
+     */
+    Particle** particle_swarm = nullptr;
+
+    /**
+     *
+     */
+    double (*func)(double*);
+
+    unsigned int func_dim;
+
+    unsigned int n_particles;
+
+    unsigned int iter_max;
+
+    double c1;
+
+    double c2;
+
+    double w;
+
+    double global_optimal_value;
+
+    double *domain_bounds;
+
+    /**
+     *
+     * @param coord
+     * @param val
+     */
+    void determine_optimal_coordinate_and_value(double *coord, double &val);
+
+public:
+
+    /**
+     *
+     * @param F
+     * @param f_dim
+     * @param domain_bounds
+     * @param c1
+     * @param c2
+     * @param w
+     * @param n_particles
+     * @param iter_max
+     */
+    ParticleSwarm( double (*F)(double*), unsigned int f_dim, double* domain_bounds, double c1, double c2, double w, unsigned int n_particles, unsigned int iter_max = 1 );
+    ~ParticleSwarm( );
+
+
+    /**
+     *
+     * @param epsilon
+     */
+    void optimize( double epsilon );
+
+
+};
+
+
+#endif //INC_4NEURO_PARTICLESWARM_H

src/Network/NeuralNetwork.cpp

@@ -12,7 +12,6 @@ NeuralNetwork::NeuralNetwork() {
     this->neurons = new std::vector<Neuron*>(0);
     this->connection_weights = new std::vector<double>(0);
 
-    //TODO tady pozor, pri nedostatecne alokaci se pri pridani hrany nad limit reserve prealokuje cele pole a tim padem padaji reference na vahy v ConnectionWeight
     this->connection_weights->reserve(0);
 }
 
@@ -170,6 +169,7 @@ void NeuralNetwork::eval_single(std::vector<double> &input, std::vector<double>
 
         neuron->set_potential(input[i]);
 
+        printf("INPUT NEURON %2d, POTENTIAL: %f\n", i, input[i]);
 
         ++i;
     }
@@ -206,6 +206,17 @@ void NeuralNetwork::eval_single(std::vector<double> &input, std::vector<double>
     for(Neuron* neuron: *this->output_neurons){
 
         output[i] = neuron->get_state();
+
+        printf("OUTPUT NEURON %2d, VALUE: %f\n", i, output[i]);
+
         ++i;
     }
+
+
+}
+
+void NeuralNetwork::copy_weights(double *weights) {
+    for(unsigned int i = 0; i < this->connection_weights->size(); ++i){
+        (*this->connection_weights)[i] = weights[i];
+    }
 }
\ No newline at end of file

src/Network/NeuralNetwork.h

@@ -120,6 +120,11 @@ public:
     void add_connection_general(int n1_idx, int n2_idx, std::function<double(double *, int*, int)> *f, int* weight_indices, double* weight_values, int n_weights);
 
 
+    /**
+     *
+     * @param weights
+     */
+    void copy_weights(double *weights);
 
 };
 

src/Neuron/NeuronLogistic.h

@@ -15,6 +15,7 @@
 #include "../constants.h"
 
 class NeuronLogistic:public Neuron {
+public:
     /**
      * Constructs the object of the Logistic neuron with activation function
      * f(x) = (1 + e^(-x + b))^(-a)

src/main.cpp

@@ -11,16 +11,20 @@
 
 #include "Neuron/NeuronLinear.h"
 
+#include "Neuron/NeuronLogistic.h"
+
 #include "NetConnection/Connection.h"
 
 #include "NetConnection/ConnectionWeightIdentity.h"
 
+#include "LearningMethods/ParticleSwarm.h"
+
 //TODO prepsat tak, aby neuronova sit managovala destruktory vsech potrebnych objektu (kvuli serializaci)
 /**
  * Test of simple neural network
  * Network should evaluate the function f(x) = x + 1
  */
-void test1(){
+void test1( ){
     std::vector<double> in(1);
     std::vector<double> out(1);
 
@@ -57,11 +61,148 @@ void test1(){
     delete u2;
 }
 
+double particle_swarm_test_function(double *x){
+//    return x[0] * x[1] - x[0] * x[0] + x[1] * x[2];
+    return x[0] * x[0];
+}
+
+NeuralNetwork net;
+std::vector<std::vector<double>*> *train_data_input;
+std::vector<std::vector<double>*> *train_data_output;
+double test_particle_swarm_neural_net_error_function(double *weights){
+
+    net.copy_weights(weights);
+
+    unsigned int dim_out = train_data_output->at(0)->size();
+    unsigned int dim_in = train_data_input->at(0)->size();
+
+    double error = 0.0, val;
+
+    std::vector<double> output( dim_out );
+    for(unsigned int i = 0; i < train_data_input->size(); ++i){
+
+        net.eval_single(*train_data_input->at(i), output);
+
+        for(unsigned int j = 0; j < dim_out; ++j){
+            val = output[j] - train_data_output->at(i)->at(j);
+            error += val * val;
+        }
+
+    }
+
+    printf("INPUT: ");
+    for(unsigned int i = 0; i < dim_in; ++i){
+        printf("%f ", weights[i]);
+    }
+    printf(", ERROR: %f\n", 0.5 * error);
+
+    return 0.5 * error;
+}
+
+//TODO proper memory management
+void test_particle_swarm_neural_net(){
+
+    unsigned int dim_in = 2;
+    unsigned int dim_out = 1;
+
+    /* TRAIN DATA DEFINITION */
+    train_data_input = new std::vector<std::vector<double>*>();
+    train_data_output = new std::vector<std::vector<double>*>();
+
+    std::vector<double> *input_01 = new std::vector<double>(dim_in);
+    std::vector<double> *input_02 = new std::vector<double>(dim_in);
+
+    std::vector<double> *output_01 = new std::vector<double>(dim_out);
+    std::vector<double> *output_02 = new std::vector<double>(dim_out);
+
+    (*input_01)[0] = 0.0;
+    (*input_01)[1] = 1.0;
+    (*output_01)[0] = 0.5;
+
+    (*input_02)[0] = 1.0;
+    (*input_02)[1] = 0.5;
+    (*output_02)[0] = 0.75;
+
+    train_data_input->push_back(input_01);
+    train_data_output->push_back(output_01);
+
+    train_data_input->push_back(input_02);
+    train_data_output->push_back(output_02);
+    /* NETWORK DEFINITION */
+
+
+    NeuronLinear* i1 = new NeuronLinear(0.0, 1.0); //f(x) = x
+    NeuronLinear* i2 = new NeuronLinear(0.0, 1.0); //f(x) = x
+
+//    NeuronLogistic* o1 = new NeuronLogistic(1.0, 0.0); //f(x) = (1 + e^(-x + 0.0))^(1.0)
+    NeuronLinear* o1 = new NeuronLinear(1.0, 2.0); //f(x) = 2x + 1
+
+    int idx1 = net.add_neuron(i1);
+    int idx2 = net.add_neuron(i2);
+    int idx3 = net.add_neuron(o1);
+
+    net.add_connection_simple(idx1, idx3, -1, 1.0);
+    net.add_connection_simple(idx2, idx3, -1, 1.0);
+
+    /* PARTICLE SETUP */
+    double (*F)(double*) = &test_particle_swarm_neural_net_error_function;
+
+    unsigned int n_edges = 2;
+    unsigned int dim = n_edges, max_iters = 2;
+
+
+    double domain_bounds [4] = {-800.0, 800.0, -800.0, 800.0};
+
+    double c1 = 0.5, c2 = 1.5, w = 1.0;
+
+    unsigned int n_particles = 2;
+
+    double accuracy = 1e-6;
+
+    ParticleSwarm swarm_01(F, dim, domain_bounds, c1, c2, w, n_particles, max_iters);
+
+    swarm_01.optimize(accuracy);
+
+    /* CLEANUP PHASE */
+    for( std::vector<double> *input: *train_data_input){
+        delete input;
+    }
+    for( std::vector<double> *output: *train_data_output){
+        delete output;
+    }
+
+    delete train_data_output;
+    delete train_data_input;
+
+}
+
+void test_particle_swarm(){
+    double (*F)(double*) = &particle_swarm_test_function;
+
+    unsigned int dim = 3, max_iters = 100;
+
+//    double domain_bounds [2] = {2.0, 3.0};
+    double domain_bounds [6] = {-3.0, 3.0, 2.0, 5.0, 1.0, 15.0};
+
+    double c1 = 0.5, c2 = 1.5, w = 1.0;
+
+    unsigned int n_particles = 1000;
+
+    double accuracy = 1e-6;
+
+    ParticleSwarm swarm_01(F, dim, domain_bounds, c1, c2, w, n_particles, max_iters);
+
+    swarm_01.optimize(accuracy);
+
+}
+
 int main(int argc, char** argv){
 
-    test1();
+//    test1();
+
+//    test_particle_swarm();
 
-    printf("hello biatch\n");
+    test_particle_swarm_neural_net();
 
     return 0;
 }
\ No newline at end of file