Commit before merge

include/4neuro.h

@@ -10,16 +10,13 @@
 #include "../src/DataSet/DataSet.h"
 #include "../src/ErrorFunction/ErrorFunctions.h"
 #include "../src/LearningMethods/ParticleSwarm.h"
-#include "../src/NetConnection/Connection.h"
-#include "../src/NetConnection/ConnectionWeight.h"
-#include "../src/NetConnection/ConnectionWeightIdentity.h"
+#include "../src/NetConnection/ConnectionFunctionGeneral.h"
+#include "../src/NetConnection/ConnectionFunctionIdentity.h"
 #include "../src/Network/NeuralNetwork.h"
 #include "../src/Network/NeuralNetworkSum.h"
 #include "../src/Neuron/Neuron.h"
 #include "../src/Neuron/NeuronBinary.h"
 #include "../src/Neuron/NeuronLinear.h"
 #include "../src/Neuron/NeuronLogistic.h"
-#include "../src/Neuron/NeuronNeuralNet.h"
-#include "../src/Neuron/NeuronTanh.h"
 
 #endif //INC_4NEURO_4NEURO_H

src/CMakeLists.txt

@@ -5,15 +5,12 @@ add_library(4neuro SHARED
         Neuron/NeuronBinary.cpp
         Neuron/NeuronLinear.cpp
         Neuron/NeuronLogistic.cpp
-        Neuron/NeuronTanh.cpp
-        NetConnection/Connection.cpp
         Network/NeuralNetwork.cpp
-        Neuron/NeuronNeuralNet.cpp
-        NetConnection/ConnectionWeight.cpp
-        NetConnection/ConnectionWeightIdentity.cpp
+        NetConnection/ConnectionFunctionGeneral.cpp
+        NetConnection/ConnectionFunctionIdentity.cpp
         LearningMethods/ParticleSwarm.cpp
         DataSet/DataSet.cpp
-        ErrorFunction/ErrorFunctions.cpp Network/NeuralNetworkSum.cpp Network/NeuralNetworkSum.h Solvers/DESolver.cpp Solvers/DESolver.h)
+        ErrorFunction/ErrorFunctions.cpp Network/NeuralNetworkSum.cpp Network/NeuralNetworkSum.h Solvers/DESolver.cpp Solvers/DESolver.h Neuron/NeuronConstant.cpp Neuron/NeuronConstant.h)
 
 target_link_libraries(4neuro boost_serialization)
 
@@ -40,6 +37,9 @@ target_link_libraries(net_test_3 4neuro)
 add_executable(net_test_ode_1 net_test_ode_1.cpp)
 target_link_libraries(net_test_ode_1 4neuro)
 
+add_executable(net_test_pde_1 net_test_pde_1.cpp)
+target_link_libraries(net_test_pde_1 4neuro)
+
 ##############
 # UNIT TESTS #
 ##############
@@ -57,15 +57,10 @@ target_link_libraries(binary_neuron_test boost_unit_test 4neuro)
 add_executable(logistic_neuron_test tests/NeuronLogistic_test.cpp)
 target_link_libraries(logistic_neuron_test boost_unit_test 4neuro)
 
-add_executable(tanh_neuron_test tests/NeuronTanh.cpp)
-target_link_libraries(tanh_neuron_test boost_unit_test 4neuro)
 
 add_executable(connection_weight_test tests/ConnectionWeight_test.cpp)
 target_link_libraries(connection_weight_test boost_unit_test 4neuro)
 
-add_executable(connection_test tests/Connection_test.cpp)
-target_link_libraries(connection_test boost_unit_test 4neuro)
-
 add_executable(neural_network_test tests/NeuralNetwork_test.cpp)
 target_link_libraries(neural_network_test boost_unit_test 4neuro)
 

src/ErrorFunction/ErrorFunctions.cpp

@@ -3,7 +3,6 @@
 //
 
 #include <vector>
-#include <utility>
 
 #include "ErrorFunctions.h"
 
@@ -14,10 +13,10 @@ size_t ErrorFunction::get_dimension() {
 MSE::MSE(NeuralNetwork *net, DataSet *ds) {
     this->net = net;
     this->ds = ds;
-    this->dimension = net->get_n_weights();
+    this->dimension = net->get_n_weights() + net->get_n_biases();
 }
 
-double MSE::eval(double *weights) {
+double MSE::eval(std::vector<double> *weights) {
     unsigned int dim_out = this->ds->get_output_dim();
 //    unsigned int dim_in = this->ds->get_input_dim();
     size_t n_elements = this->ds->get_n_elements();
@@ -66,7 +65,7 @@ ErrorSum::~ErrorSum(){
     }
 }
 
-double ErrorSum::eval(double *weights) {
+double ErrorSum::eval(std::vector<double> *weights) {
     double output = 0.0;
 
     for( unsigned int i = 0; i < this->summand->size(); ++i ){

src/ErrorFunction/ErrorFunctions.h

@@ -20,7 +20,7 @@ public:
      * @param weights
      * @return
      */
-    virtual double eval(double* weights) = 0;
+    virtual double eval(std::vector<double>* weights = nullptr) = 0;
     
     /**
      * 
@@ -51,7 +51,7 @@ public:
      * @param weights
      * @return
      */
-    virtual double eval(double* weights = nullptr);
+    virtual double eval(std::vector<double>* weights = nullptr);
 
 private:
 
@@ -76,7 +76,7 @@ public:
      * @param weights
      * @return
      */
-    virtual double eval(double* weights = nullptr);
+    virtual double eval(std::vector<double>* weights = nullptr);
 
     /**
      *

src/LearningMethods/ParticleSwarm.h

@@ -26,10 +26,10 @@ class Particle{
 private:
 
     size_t coordinate_dim;
-    double *coordinate = nullptr;
-    double *velocity;
+    std::vector<double> *coordinate = nullptr;
+    std::vector<double> *velocity = nullptr;
 
-    double *optimal_coordinate = nullptr;
+    std::vector<double> *optimal_coordinate = nullptr;
     double optimal_value;
 
     double r1;
@@ -42,10 +42,15 @@ private:
 
     double *domain_bounds;
 
-    void print_coordinate();
+
 
 public:
 
+    /**
+     *
+     */
+    void print_coordinate();
+
     /**
      *
      * @param f_dim
@@ -57,7 +62,7 @@ public:
      *
      * @return
      */
-    double* get_coordinate();
+    std::vector<double>* get_coordinate();
 
     /**
      *
@@ -85,7 +90,7 @@ public:
      * @param glob_min_coord
      * @param penalty_coef
      */
-    double change_coordinate(double w, double c1, double c2, std::vector<double> glob_min_coord, std::vector<std::vector<double>> global_min_vec, double penalty_coef=0.25);
+    double change_coordinate(double w, double c1, double c2, std::vector<double> &glob_min_coord, std::vector<std::vector<double>> &global_min_vec, double penalty_coef=0.25);
 };
 
 
@@ -101,13 +106,13 @@ private:
     /**
      *
      */
-    double (*func)(NeuralNetwork, double*, DataSet);
+    ErrorFunction* f;
 
     size_t func_dim;
 
-    unsigned int n_particles;
+    size_t n_particles;
 
-    unsigned int iter_max;
+    size_t iter_max;
 
     double c1;
 
@@ -121,6 +126,8 @@ private:
 
     double *domain_bounds;
 
+    std::vector<double> *p_min_glob = nullptr;
+
 protected:
     /**
      *
@@ -134,7 +141,7 @@ protected:
      *
      * @return
      */
-    double* get_centroid_coordinates();
+    std::vector<double>* get_centroid_coordinates();
 
     /**
      *
@@ -143,7 +150,7 @@ protected:
      * @param n
      * @return
      */
-    double get_euclidean_distance(double* a, double* b, unsigned int n);
+    double get_euclidean_distance(std::vector<double>* a, std::vector<double>* b);
 
 public:
 
@@ -158,7 +165,7 @@ public:
      * @param n_particles
      * @param iter_max
      */
-    ParticleSwarm( ErrorFunction* ef, double* domain_bounds, double c1, double c2, double w, unsigned int n_particles, unsigned int iter_max = 1 );
+    ParticleSwarm( ErrorFunction* ef, double* domain_bounds, double c1, double c2, double w, size_t n_particles, size_t iter_max = 1 );
 
     /**
      *
@@ -174,6 +181,12 @@ public:
      */
     void optimize( double gamma, double epsilon, double delta=0.7 );
 
+    /**
+     *
+     * @return
+     */
+    std::vector<double>* get_solution();
+
 
 };
 

src/NetConnection/ConnectionFunctionGeneral.cpp

@@ -6,58 +6,31 @@
  */
 
 
-#include "ConnectionWeight.h"
+#include "ConnectionFunctionGeneral.h"
 
-ConnectionWeight::ConnectionWeight() {
+ConnectionFunctionGeneral::ConnectionFunctionGeneral() {
 
 
 }
 
-ConnectionWeight::ConnectionWeight(int param_count,  std::vector<double>* w_array) {
-    this->param_indices = new int[param_count];
-    this->n_params = param_count;
+ConnectionFunctionGeneral::ConnectionFunctionGeneral(std::vector<double>* w_array, std::vector<unsigned int> &param_indices, std::string &function_string) {
+    this->param_indices = new std::vector<unsigned int>(param_indices);
     this->weight_array = w_array;
 }
 
-ConnectionWeight::~ConnectionWeight() {
+ConnectionFunctionGeneral::~ConnectionFunctionGeneral() {
     if(this->param_indices){
-        delete [] this->param_indices;
+        delete this->param_indices;
         this->param_indices = nullptr;
     }
 }
 
-void ConnectionWeight::adjust_weights(double *values) {
-    for(int i = 0; i < this->n_params; ++i){
-        this->weight_array->at(this->param_indices[i]) += values[i];
-    }
-}
+double ConnectionFunctionGeneral::eval() {
+    //TODO
 
-std::vector<double> ConnectionWeight::get_weights(){
-    return *this->weight_array;
+    return 0.0;
 }
 
-void ConnectionWeight::set_weights(double *values) {
-    for(int i = 0; i < this->n_params; ++i){
-        this->weight_array->at(this->param_indices[i]) = values[i];
-    }
+void ConnectionFunctionGeneral::eval_partial_derivative(std::vector<double> &weight_gradient, double alpha) {
+    //TODO
 }
-
-void ConnectionWeight::SetParamIndices(int *param_indices) {
-    for(int i = 0; i < this->n_params; ++i){
-        this->SetParamIndex(param_indices[i], i);
-    }
-}
-
-void ConnectionWeight::SetParamIndex(int value, int idx) {
-    this->param_indices[idx] = value;
-}
-
-double ConnectionWeight::eval() {
-    double product = 1;
-    for(auto e : *this->weight_array) {
-        product *= e;
-    }
-
-    return product;
-}
-

src/NetConnection/ConnectionFunctionGeneral.h

@@ -11,7 +11,7 @@
 #include <functional>
 #include <vector>
 
-class ConnectionWeight {
+class ConnectionFunctionGeneral {
 protected:
     /**
      *
@@ -21,68 +21,40 @@ protected:
     /**
      *
      */
-    int* param_indices = nullptr;
-
-    /**
-     *
-     */
-    int n_params = 0;
+    std::vector<unsigned int> *param_indices = nullptr;
 
 public:
 
     /**
      *
      */
-    ConnectionWeight();
+    ConnectionFunctionGeneral();
 
     /**
      *
      * @param param_count
      * @param f
      */
-    ConnectionWeight(int param_count, std::vector<double>* w_array);
+    ConnectionFunctionGeneral(std::vector<double>* w_array, std::vector<unsigned int> &param_indices, std::string &function_string);
 
     /**
      *
-     * @param value
-     * @param idx
      */
-    void SetParamIndex(int value, int idx);
+    virtual ~ConnectionFunctionGeneral();
 
-    /**
-     *
-     * @param param_ptr
-     */
-    void SetParamIndices(int* param_ptr);
-
-    /**
-     *
-     */
-    virtual ~ConnectionWeight();
 
     /**
      *
      * @return
      */
-    virtual double eval();
+    virtual double eval( );
 
     /**
-     *
-     * @param values
+     * Performs partial derivative of this transfer function according to all parameters. Adds the values multiplied
+     * by alpha to the corresponding gradient vector
      */
+    virtual void eval_partial_derivative(std::vector<double> &weight_gradient, double alpha);
 
-    void set_weights(double *values);
-
-    /**
-     *
-     * @return vector of weights
-     */
-    std::vector<double> get_weights();
-    /**
-     *
-     * @param values
-     */
-    void adjust_weights(double *values);
 
 };
 

src/NetConnection/ConnectionFunctionIdentity.cpp

@@ -5,12 +5,17 @@
  * @date 14.6.18 -
  */
 
-#include "ConnectionWeightIdentity.h"
+#include "ConnectionFunctionIdentity.h"
 
-ConnectionWeightIdentity::ConnectionWeightIdentity(double * val) {
+ConnectionFunctionIdentity::ConnectionFunctionIdentity(double * val, size_t pidx) {
     this->value = val;
+    this->param_idx = pidx;
 }
 
-double ConnectionWeightIdentity::eval() {
+double ConnectionFunctionIdentity::eval() {
     return (*this->value);
+}
+
+void ConnectionFunctionIdentity::eval_partial_derivative(std::vector<double> &weight_gradient, double alpha) {
+    weight_gradient[this->param_idx] += alpha;
 }
\ No newline at end of file

src/NetConnection/ConnectionFunctionIdentity.h

@@ -15,21 +15,29 @@ class ConnectionFunctionGeneral;
 /**
  *
  */
-class ConnectionWeightIdentity:public ConnectionFunctionGeneral {
+class ConnectionFunctionIdentity:public ConnectionFunctionGeneral {
 private:
     double * value = nullptr;
+    size_t param_idx;
 public:
 
     /**
      *
      */
-    ConnectionWeightIdentity(double * value_ptr);
+    ConnectionFunctionIdentity(double * value_ptr, size_t pidx);
 
     /**
      *
      * @return
      */
     double eval() override;
+
+    /**
+     *
+     * @param weight_gradient
+     * @param alpha
+     */
+    void eval_partial_derivative(std::vector<double> &weight_gradient, double alpha) override;
 };
 
 

src/Network/NeuralNetwork.h

 /**
- * DESCRIPTION OF THE FILE
+ * This file contains the header for the NeuralNetwork class representing a function in the form of a directed graph,
+ * in which the vertices are called Neurons (with activation functions) and the edges Connections (with transfer functions)
  *
  * @author Michal Kravčenko
  * @date 13.6.18 -
  */
 
- //TODO pouvazovat o pridani indexu k neuronum, abychom meli urcitou kontrolu nad poradim vstupu a vystupu?
- //TODO rucni nastaveni vstupnich a vystu[nich neuronu
+//TODO preprocess the feed-forward and backward passes for more efficient parallelism
 
 #ifndef INC_4NEURO_NEURALNETWORK_H
 #define INC_4NEURO_NEURALNETWORK_H
 
 #include <vector>
+#include <algorithm>
+#include <utility>
 #include "../Neuron/Neuron.h"
+#include "../NetConnection/ConnectionFunctionGeneral.h"
+#include "../NetConnection/ConnectionFunctionIdentity.h"
 #include "../settings.h"
 
 enum NET_TYPE{GENERAL};
@@ -28,58 +32,134 @@ private:
      */
     NET_TYPE network_type = GENERAL;
 
-     /**
-      *
-      */
-     std::vector<Neuron*> *neurons = nullptr;
+    /**
+     *
+     */
+    std::vector<Neuron*> *neurons = nullptr;
+
+    /**
+     *
+     */
+    std::vector<size_t>* input_neuron_indices = nullptr;
+
+    /**
+     *
+     */
+    std::vector<size_t>* output_neuron_indices = nullptr;
+
+    /**
+     *
+     */
+    std::vector<double>* connection_weights = nullptr;
+
+    /**
+     *
+     */
+    std::vector<double>* neuron_biases = nullptr;
+
+    /**
+     *
+     */
+    std::vector<double>* neuron_potentials = nullptr;
+
+    /**
+     *
+     */
+    std::vector<ConnectionFunctionGeneral*> * connection_list = nullptr;
+
+    /**
+     *
+     */
+    std::vector<std::vector<std::pair<size_t, size_t>>*> * inward_adjacency = nullptr;
+
+    /**
+     *
+     */
+    std::vector<std::vector<std::pair<size_t, size_t>>*> * outward_adjacency = nullptr;
+
+    /**
+     *
+     */
+    std::vector<std::vector<size_t>*> *neuron_layers_feedforward = nullptr;
+
+    /**
+     *
+     */
+    std::vector<std::vector<size_t>*> *neuron_layers_feedbackward = nullptr;
+
+    /**
+     *
+     */
+    size_t n_inputs = 0;
 
-     /**
-      *
-      */
-     std::vector<Neuron*>* input_neurons = nullptr;
+    /**
+     *
+     */
+    size_t n_outputs = 0;
 
-     /**
-      *
-      */
-     std::vector<Neuron*>* output_neurons = nullptr;
+    /**
+     *
+     */
+    size_t last_used_weight_idx = 0;
 
-     std::vector<double>* connection_weights = nullptr;
+    /**
+     *
+     */
+    size_t last_used_bias_idx = 0;
+
+    /**
+     *
+     */
+    size_t n_neurons = 0;
 
-     /**
-      *
-      */
-     size_t n_inputs = 0;
+    /**
+     *
+     */
+    bool in_out_determined = false;
 
-     /**
-      *
-      */
-     size_t n_outputs = 0;
+    /**
+     *
+     */
+    bool layers_analyzed = false;
 
-     /**
-      *
-      */
-     size_t n_weights = 0;
+    /**
+     *
+     */
+    bool delete_weights = true;
 
-     /**
-      *
-      */
-     bool in_out_determined = false;
+    /**
+     *
+     */
+    bool delete_biases = true;
 
-     /**
-      *
-      */
-     bool delete_weights = true;
+    /**
+     * Adds a new connection to the local list of connections
+     * @param con Connection object to be added
+     * @return Returns the index of the added connection among all the connections
+     */
+    size_t add_new_connection_to_list(ConnectionFunctionGeneral* con);
 
-     /**
-      *
-      */
-     std::vector<Neuron*>* active_eval_set = nullptr;
+    /**
+     * Adds a new entry (oriented edge s -> t) to the adjacency list of this network
+     * @param s Index of the source neuron
+     * @param t Index of the target neuron
+     * @param con_idx Index of the connection representing the edge
+     */
+    void add_outward_connection(size_t s, size_t t, size_t con_idx);
 
-     /**
-      *
-      */
-     void determine_inputs_outputs();
+    /**
+     * Adds a new entry (oriented edge s <- t) to the adjacency list of this network
+     * @param s Index of the source neuron
+     * @param t Index of the target neuron
+     * @param con_idx Index of the connection representing the edge
+     */
+    void add_inward_connection(size_t s, size_t t, size_t con_idx);
 
+    /**
+     * Performs one feedforward pass and feedbackward pass during which determines the layers of this neural network
+     * for simpler use during evaluation and learning
+     */
+    void analyze_layer_structure( );
 
 
 public:
@@ -89,11 +169,16 @@ public:
      */
     NeuralNetwork();
 
+    /**
+     *
+     */
+    NeuralNetwork(size_t n_connection_weights, size_t n_neurons);
+
 
     /**
      *
      */
-    ~NeuralNetwork();
+    virtual ~NeuralNetwork();
 
     /**
      * If possible, returns a neural net with 'input_neuron_indices' neurons as inputs and 'output_neuron_indices' as
@@ -105,119 +190,120 @@ public:
      */
     NeuralNetwork* get_subnet(std::vector<size_t> &input_neuron_indices, std::vector<size_t> &output_neuron_indices);
 
+    /**
+     * Replaces the values in @{this->connection_weights} and @{this->neuron_biases} by the provided values
+     * @param parameters
+     */
+    virtual void copy_parameter_space(std::vector<double> *parameters);
+
+    /**
+     * Copies the pointers @{this->connection_weights} and @{this->neuron_biases} from the parental network, sets
+     * flags to not delete the vectors in this object
+     * @param parent_network
+     */
+    virtual void set_parameter_space_pointers( NeuralNetwork &parent_network );
+
     /**
      *
-     * @param[in] input
-     * @param[in,out] output
+     * @param input
+     * @param output
+     * @param custom_weights_and_biases
      */
-    virtual void eval_single(std::vector<double> &input, std::vector<double> &output, double *custom_weights = nullptr);
+    virtual void eval_single(std::vector<double> &input, std::vector<double> &output, std::vector<double> *custom_weights_and_biases = nullptr);
 
 
 
     /**
-     *
+     * Adds a new neuron to the list of neurons. Also assigns a valid bias value to its activation function
      * @param[in] n
      * @return
      */
-    int add_neuron(Neuron* n);
+    size_t add_neuron(Neuron* n, int bias_idx = -1 );
 
     /**
-     *
-     * @param n1_idx
-     * @param n2_idx
+     * Adds a new neuron to this network, does not touch its bias.
+     * @param n
      * @return
      */
-    size_t add_connection_simple(int n1_idx, int n2_idx);
+    size_t add_neuron_no_bias(Neuron *n);
 
     /**
      *
      * @param n1_idx
      * @param n2_idx
-     * @param weight_idx
      * @return
      */
-    size_t add_connection_simple(int n1_idx, int n2_idx, size_t weight_idx);
+    size_t add_connection_simple(size_t n1_idx, size_t n2_idx);
 
     /**