Merge branch 'dev_cpp' of code.it4i.cz:bes0030/4Neuro into dev_cpp

src/CMakeLists.txt

+#TODO make atomic libraries AND also one large library containing all others + one large header
 
 add_library(neuron SHARED
         Neuron/Neuron.cpp
@@ -31,6 +32,9 @@ add_library(boost_unit_test SHARED tests/boost_test_lib_dummy.cpp)
 add_library(data_set SHARED DataSet/DataSet.cpp)
 target_link_libraries(data_set boost_serialization)
 
+add_library(error_functions SHARED ErrorFunction/ErrorFunctions.cpp)
+target_link_libraries(error_functions data_set)
+
 ############
 # EXAMPLES #
 ############
@@ -40,6 +44,9 @@ target_link_libraries(test_cases neuron particle_swarm boost_serialization data_
 add_executable(neuron_serialization_example neuron_serialization_example.cpp)
 target_link_libraries(neuron_serialization_example neuron boost_serialization)
 
+add_executable(net_test_1 net_test_1.cpp)
+target_link_libraries(net_test_1 neuron particle_swarm data_set error_functions)
+
 ##############
 # UNIT TESTS #
 ##############

src/ErrorFunction/ErrorFunctions.cpp

@@ -2,4 +2,43 @@
 // Created by martin on 7/15/18.
 //
 
+#include <vector>
+#include <utility>
+
 #include "ErrorFunctions.h"
+
+size_t ErrorFunction::get_dimension() {
+    return this->dimension;
+}
+
+MSE::MSE(NeuralNetwork *net, DataSet *ds) {
+    this->net = net;
+    this->ds = ds;
+    this->dimension = net->get_n_weights();
+}
+
+double MSE::eval(double *weights) {
+    unsigned int dim_out = this->ds->get_output_dim();
+    unsigned int dim_in = this->ds->get_input_dim();
+    size_t n_elements = this->ds->get_n_elements();
+    double error = 0.0, val;
+
+    std::vector<std::pair<std::vector<double>, std::vector<double>>>* data = this->ds->get_data();
+
+    this->net->copy_weights(weights);
+
+    std::vector<double> output( dim_out );
+
+    for(unsigned int i = 0; i < n_elements; ++i){  // Iterate through every element in the test set
+
+        this->net->eval_single(std::get<0>(data->at(i)), output);  // Compute the net output and store it into 'output' variable
+
+        for(unsigned int j = 0; j < dim_out; ++j) {  // Compute difference for every element of the output vector
+            val = output[j] - std::get<1>(data->at(i))[j];
+            error += val * val;
+        }
+
+    }
+
+    return error/n_elements;
+}
\ No newline at end of file

src/ErrorFunction/ErrorFunctions.h

@@ -5,9 +5,60 @@
 #ifndef INC_4NEURO_ERRORFUNCTION_H
 #define INC_4NEURO_ERRORFUNCTION_H
 
+#include "../Network/NeuralNetwork.h"
+#include "../DataSet/DataSet.h"
 
 class ErrorFunction {
+public:
+
+    /**
+     *
+     * @param weights
+     * @return
+     */
+    virtual double eval(double* weights) = 0;
     
+    /**
+     * 
+     * @return 
+     */
+    virtual size_t get_dimension();
+
+protected:
+
+    /**
+     *
+     */
+    size_t dimension;
+};
+
+class MSE : public ErrorFunction {
+
+public:
+    /**
+     * Constructor for single neural network
+     * @param net
+     * @param ds
+     */
+    MSE(NeuralNetwork* net, DataSet* ds);
+
+    /**
+     * Constructor for multiple error functions, which will get summed up
+     * @param func_vec
+     */
+    //MSE(std::vector<ErrorFunction> func_vec);
+
+    /**
+     *
+     * @param weights
+     * @return
+     */
+    virtual double eval(double* weights);
+
+private:
+
+    NeuralNetwork* net;
+    DataSet* ds;
 };
 
 

src/LearningMethods/ParticleSwarm.cpp

@@ -9,6 +9,7 @@
 #include <set>
 #include <stdexcept>
 #include "ParticleSwarm.h"
+#include "../ErrorFunction/ErrorFunctions.h"
 
 /**
  * TODO
@@ -17,16 +18,17 @@
  * @param domain_bounds
  * @param F
  */
-Particle::Particle(unsigned int f_dim, double *domain_bounds, double (*F)(double*)) {
-    this->coordinate_dim = f_dim;
+Particle::Particle(ErrorFunction* ef, double *domain_bounds) {
+    //TODO better generating of random numbers
+    this->coordinate_dim = ef->get_dimension();
 
-    this->coordinate = new double[f_dim];
+    this->coordinate = new double[this->coordinate_dim];
 
-    this->velocity = new double[f_dim];
+    this->velocity = new double[this->coordinate_dim];
 
 
 
-    for(unsigned int i = 0; i < f_dim; ++i){
+    for(unsigned int i = 0; i < this->coordinate_dim; ++i){
         this->velocity[i] = (rand() % 100001 - 50000) / (double) 50000;
     }
 //    this->r1 = (rand() % 100001) / (double) 100000;
@@ -34,17 +36,17 @@ Particle::Particle(unsigned int f_dim, double *domain_bounds, double (*F)(double
     this->r1 = 1.0;
     this->r2 = 1.0;
 
-    this->optimal_coordinate = new double[f_dim];
+    this->optimal_coordinate = new double[this->coordinate_dim];
 
-    this->f = F;
+    this->ef = ef;
 
     this->domain_bounds = domain_bounds;
-    for(unsigned int i = 0; i < f_dim; ++i){
+    for(unsigned int i = 0; i < this->coordinate_dim; ++i){
         this->coordinate[i] = (rand() % 100001) / (double)100000 * (domain_bounds[2 * i + 1] - domain_bounds[2 * i]) + domain_bounds[2 * i];
         this->optimal_coordinate[i] = this->coordinate[i];
     }
 
-    this->optimal_value = this->f(this->coordinate);
+    this->optimal_value = this->ef->eval(this->coordinate);
 
 //    this->print_coordinate();
 }
@@ -114,7 +116,7 @@ double Particle::change_coordinate(double w, double c1, double c2, double *glob_
         output += std::abs(vel_mem);
     }
 
-    vel_mem = this->f(this->coordinate);
+    vel_mem = this->ef->eval(this->coordinate);
 
     if(vel_mem < this->optimal_value){
         this->optimal_value = vel_mem;
@@ -133,14 +135,14 @@ void Particle::print_coordinate() {
     printf("%10.8f\n", this->coordinate[this->coordinate_dim - 1]);
 }
 
-ParticleSwarm::ParticleSwarm(double (*F)(double*), unsigned int f_dim, double *domain_bounds, double c1, double c2, double w,
-                             unsigned int n_particles, unsigned int iter_max) {
+ParticleSwarm::ParticleSwarm(ErrorFunction* ef, double *domain_bounds,
+                             double c1, double c2, double w, unsigned int n_particles, unsigned int iter_max) {
 
     srand(time(NULL));
 
-    this->func = F;
+    //this->func = F;
 
-    this->func_dim = f_dim;
+    this->func_dim = ef->get_dimension();
 
     this->c1 = c1;
 
@@ -155,7 +157,7 @@ ParticleSwarm::ParticleSwarm(double (*F)(double*), unsigned int f_dim, double *d
     this->particle_swarm = new Particle*[this->n_particles];
 
     for( unsigned int pi = 0; pi < this->n_particles; ++pi ){
-        this->particle_swarm[pi] = new Particle( f_dim, domain_bounds, F);
+        this->particle_swarm[pi] = new Particle(ef, domain_bounds);
     }
 
     this->domain_bounds = domain_bounds;

src/LearningMethods/ParticleSwarm.h

@@ -12,12 +12,14 @@
 #include "time.h"
 
 #include "../Network/NeuralNetwork.h"
+#include "../DataSet/DataSet.h"
+#include "../ErrorFunction/ErrorFunctions.h"
 
 
 class Particle{
 private:
 
-    unsigned int coordinate_dim;
+    size_t coordinate_dim;
     double *coordinate = nullptr;
     double *velocity;
 
@@ -27,7 +29,7 @@ private:
     double r1;
     double r2;
 
-    double (*f)(double*);
+    ErrorFunction* ef;
 
     double *domain_bounds;
 
@@ -39,7 +41,7 @@ public:
      *
      * @param f_dim
      */
-    Particle(unsigned int f_dim, double *domain_bounds, double (*F)(double*));
+    Particle(ErrorFunction* ef, double *domain_bounds);
     ~Particle( );
 
     /**
@@ -84,9 +86,9 @@ private:
     /**
      *
      */
-    double (*func)(double*);
+    double (*func)(NeuralNetwork, double*, DataSet);
 
-    unsigned int func_dim;
+    size_t func_dim;
 
     unsigned int n_particles;
 
@@ -130,7 +132,7 @@ public:
 
     /**
      *
-     * @param F
+     * @param ef
      * @param f_dim
      * @param domain_bounds
      * @param c1
@@ -139,7 +141,11 @@ public:
      * @param n_particles
      * @param iter_max
      */
-    ParticleSwarm( double (*F)(double*), unsigned int f_dim, double* domain_bounds, double c1, double c2, double w, unsigned int n_particles, unsigned int iter_max = 1 );
+    ParticleSwarm( ErrorFunction* ef, double* domain_bounds, double c1, double c2, double w, unsigned int n_particles, unsigned int iter_max = 1 );
+
+    /**
+     *
+     */
     ~ParticleSwarm( );
 
 

src/Network/NeuralNetwork.cpp

@@ -52,14 +52,7 @@ void NeuralNetwork::add_connection_simple(int n1_idx, int n2_idx) {
 }
 
 void NeuralNetwork::add_connection_simple(int n1_idx, int n2_idx, int weight_idx) {
-    boost::random::mt19937 gen;
-
-    // Init weight guess ("optimal" for logistic activation functions)
-    double r = 4 * sqrt(6./(this->n_inputs + this->n_outputs));
-
-    boost::random::uniform_real_distribution<> dist(-r, r);
-
-    add_connection_simple(n1_idx, n2_idx, weight_idx, dist(gen));
+    add_connection_simple(n1_idx, n2_idx, weight_idx, 1);
 }
 
 void NeuralNetwork::add_connection_simple(int n1_idx, int n2_idx, int weight_idx, double weight_value) {
@@ -237,4 +230,33 @@ void NeuralNetwork::copy_weights(double *weights) {
     for(unsigned int i = 0; i < this->connection_weights->size(); ++i){
         (*this->connection_weights)[i] = weights[i];
     }
+}
+
+void NeuralNetwork::randomize_weights() {
+    boost::random::mt19937 gen;
+
+    // Init weight guess ("optimal" for logistic activation functions)
+    //TODO catch exception when there are 0 neurons
+    double r = 4 * sqrt(6./(this->n_inputs + this->n_outputs));
+
+    boost::random::uniform_real_distribution<> dist(-r, r);
+
+    for(unsigned int i = 0; i < this->n_weights; i++) {
+        this->connection_weights->at(i) = dist(gen);
+    }
+}
+
+size_t NeuralNetwork::get_n_inputs() {
+    return this->n_inputs;
+}
+
+size_t  NeuralNetwork::get_n_outputs() {
+    return this->n_outputs;
+}
+
+size_t NeuralNetwork::get_n_weights() {
+    if(!this->n_weights) {
+        this->n_weights = this->connection_weights->size();
+    }
+    return this->n_weights;
 }
\ No newline at end of file

src/Network/NeuralNetwork.h

@@ -27,7 +27,6 @@ private:
      */
     NET_TYPE network_type = GENERAL;
 
-
      /**
       *
       */
@@ -48,12 +47,17 @@ private:
      /**
       *
       */
-     int n_inputs = -1;
+     size_t n_inputs = 0;
 
      /**
       *
       */
-     int n_outputs = -1;
+     size_t n_outputs = 0;
+
+     /**
+      *
+      */
+     size_t n_weights = 0;
 
      /**
       *
@@ -135,13 +139,35 @@ public:
      */
     void add_connection_general(int n1_idx, int n2_idx, std::function<double(double *, int*, int)> *f, int* weight_indices, double* weight_values, int n_weights);
 
-
     /**
      *
      * @param weights
      */
     void copy_weights(double *weights);
 
+    /**
+     *
+     */
+    void randomize_weights();
+
+    /**
+     *
+     * @return
+     */
+    size_t get_n_inputs();
+
+    /**
+     *
+     * @return
+     */
+    size_t get_n_outputs();
+
+    /**
+     *
+     * @return
+     */
+    size_t get_n_weights();
+
 };
 
 

src/net_test_1.cpp

+/**
+ * Basic example using particle swarm method to train the network
+ * (result 0, -1/4)
+ */
+
 //
 // Created by martin on 7/16/18.
 //
 
+#include <vector>
+#include <utility>
+
+#include "Network/NeuralNetwork.h"
+#include "DataSet/DataSet.h"
+#include "Neuron/NeuronLinear.h"
+#include "LearningMethods/ParticleSwarm.h"
+#include "ErrorFunction/ErrorFunctions.h"
+
+int main() {
+
+    /* TRAIN DATA DEFINITION */
+    std::vector<std::pair<std::vector<double>, std::vector<double>>> data_vec;
+    std::vector<double> inp, out;
+
+    inp = {0, 1};
+    out = {0.5};
+    data_vec.emplace_back(std::make_pair(inp, out));
+
+    inp = {1, 0.5};
+    out = {0.75};
+    data_vec.emplace_back(std::make_pair(inp, out));
+
+    DataSet ds(&data_vec);
+
+    /* NETWORK DEFINITION */
+    NeuralNetwork net;
+
+    /* Input neurons */
+    NeuronLinear *i1 = new NeuronLinear(0.0, 1.0);  //f(x) = x
+    NeuronLinear *i2 = new NeuronLinear(0.0, 1.0);  //f(x) = x
+
+    /* Output neuron */
+    NeuronLinear *o1 = new NeuronLinear(1.0, 2.0);  //f(x) = 2x + 1
+
+
+
+    /* Adding neurons to the net */
+    int idx1 = net.add_neuron(i1);
+    int idx2 = net.add_neuron(i2);
+    int idx3 = net.add_neuron(o1);
+
+    /* Adding connections */
+    //net.add_connection_simple(idx1, idx3, -1, 1.0);
+    //net.add_connection_simple(idx2, idx3, -1, 1.0);
+    net.add_connection_simple(idx1, idx3);
+    net.add_connection_simple(idx2, idx3);
+
+    //net.randomize_weights();
+
+    /* ERROR FUNCTION SPECIFICATION */
+    MSE mse(&net, &ds);
+
+    /* TRAINING METHOD SETUP */
+    unsigned int n_edges = 2;
+    unsigned int dim = n_edges, max_iters = 2000;
+
+
+    double domain_bounds[4] = {-800.0, 800.0, -800.0, 800.0};
+
+    double c1 = 0.5, c2 = 1.5, w = 0.8;
+
+    unsigned int n_particles = 10;
+
+    ParticleSwarm swarm_01(&mse, domain_bounds, c1, c2, w, n_particles, max_iters);
+
+    swarm_01.optimize(0.5, 0.02);
+
+    return 0;
+}
\ No newline at end of file