Commit 9965ff6b authored by Michal Kravcenko's avatar Michal Kravcenko

-added a new example solving an ode

parent adf34787
...@@ -14,6 +14,7 @@ ...@@ -14,6 +14,7 @@
#include "../src/NetConnection/ConnectionWeight.h" #include "../src/NetConnection/ConnectionWeight.h"
#include "../src/NetConnection/ConnectionWeightIdentity.h" #include "../src/NetConnection/ConnectionWeightIdentity.h"
#include "../src/Network/NeuralNetwork.h" #include "../src/Network/NeuralNetwork.h"
#include "../src/Network/NeuralNetworkSum.h"
#include "../src/Neuron/Neuron.h" #include "../src/Neuron/Neuron.h"
#include "../src/Neuron/NeuronBinary.h" #include "../src/Neuron/NeuronBinary.h"
#include "../src/Neuron/NeuronLinear.h" #include "../src/Neuron/NeuronLinear.h"
......
...@@ -13,7 +13,7 @@ add_library(4neuro SHARED ...@@ -13,7 +13,7 @@ add_library(4neuro SHARED
NetConnection/ConnectionWeightIdentity.cpp NetConnection/ConnectionWeightIdentity.cpp
LearningMethods/ParticleSwarm.cpp LearningMethods/ParticleSwarm.cpp
DataSet/DataSet.cpp DataSet/DataSet.cpp
ErrorFunction/ErrorFunctions.cpp) ErrorFunction/ErrorFunctions.cpp Network/NeuralNetworkSum.cpp Network/NeuralNetworkSum.h)
target_link_libraries(4neuro boost_serialization) target_link_libraries(4neuro boost_serialization)
...@@ -37,6 +37,9 @@ target_link_libraries(net_test_2 4neuro) ...@@ -37,6 +37,9 @@ target_link_libraries(net_test_2 4neuro)
add_executable(net_test_3 net_test_3.cpp) add_executable(net_test_3 net_test_3.cpp)
target_link_libraries(net_test_3 4neuro) target_link_libraries(net_test_3 4neuro)
add_executable(net_test_ode_1 net_test_ode_1.cpp)
target_link_libraries(net_test_ode_1 4neuro)
############## ##############
# UNIT TESTS # # UNIT TESTS #
############## ##############
......
// //
// Created by martin on 7/13/18. // Created by martin on 7/13/18.
// //
//TODO generovani dat
#ifndef INC_4NEURO_DATASET_H #ifndef INC_4NEURO_DATASET_H
#define INC_4NEURO_DATASET_H #define INC_4NEURO_DATASET_H
......
...@@ -53,6 +53,7 @@ double MSE::eval(double *weights) { ...@@ -53,6 +53,7 @@ double MSE::eval(double *weights) {
MSE_SUM::MSE_SUM() { MSE_SUM::MSE_SUM() {
this->summand = nullptr; this->summand = nullptr;
this->summand_coefficient = nullptr;
this->dimension = 0; this->dimension = 0;
} }
...@@ -60,23 +61,31 @@ MSE_SUM::~MSE_SUM(){ ...@@ -60,23 +61,31 @@ MSE_SUM::~MSE_SUM(){
if( this->summand ){ if( this->summand ){
delete this->summand; delete this->summand;
} }
if( this->summand_coefficient ){
delete this->summand_coefficient;
}
} }
double MSE_SUM::eval(double *weights) { double MSE_SUM::eval(double *weights) {
double output = 0.0; double output = 0.0;
for(ErrorFunction *f: *this->summand){ for( int i = 0; i < this->summand->size(); ++i ){
output += f->eval( weights ); output += this->summand->at( i )->eval( weights ) * this->summand_coefficient->at( i );
} }
return output; return output;
} }
void MSE_SUM::add_error_function(ErrorFunction *F) { void MSE_SUM::add_error_function(ErrorFunction *F, double alpha) {
if(!this->summand){ if(!this->summand){
this->summand = new std::vector<ErrorFunction*>(0); this->summand = new std::vector<ErrorFunction*>(0);
} }
this->summand->push_back(F); this->summand->push_back( F );
if(!this->summand_coefficient){
this->summand_coefficient = new std::vector<double>(0);
}
this->summand_coefficient->push_back( alpha );
if(F->get_dimension() > this->dimension){ if(F->get_dimension() > this->dimension){
this->dimension = F->get_dimension(); this->dimension = F->get_dimension();
......
...@@ -78,7 +78,7 @@ public: ...@@ -78,7 +78,7 @@ public:
* *
* @param F * @param F
*/ */
void add_error_function(ErrorFunction *F); void add_error_function(ErrorFunction *F, double alpha = 1.0);
/** /**
* *
...@@ -88,6 +88,7 @@ public: ...@@ -88,6 +88,7 @@ public:
private: private:
std::vector<ErrorFunction*>* summand; std::vector<ErrorFunction*>* summand;
std::vector<double> *summand_coefficient;
}; };
......
...@@ -46,6 +46,7 @@ Particle::Particle(ErrorFunction* ef, double *domain_bounds) { ...@@ -46,6 +46,7 @@ Particle::Particle(ErrorFunction* ef, double *domain_bounds) {
this->optimal_coordinate[i] = this->coordinate[i]; this->optimal_coordinate[i] = this->coordinate[i];
} }
// printf("coordinate_dim: %d\n", this->coordinate_dim);
this->optimal_value = this->ef->eval(this->coordinate); this->optimal_value = this->ef->eval(this->coordinate);
// this->print_coordinate(); // this->print_coordinate();
......
...@@ -45,7 +45,7 @@ Neuron* Connection::get_neuron_out() { ...@@ -45,7 +45,7 @@ Neuron* Connection::get_neuron_out() {
//} //}
void Connection::pass_signal() { void Connection::pass_signal() {
// printf("passing signal between neurons %d -> %d, value: %f * %f\n", this->neuron_in, this->neuron_out, this->neuron_in->get_state(), this->con->eval()); // printf(" passing signal between neurons %d -> %d, value: %f * %f\n", this->neuron_in, this->neuron_out, this->neuron_in->get_state(), this->con->eval());
this->neuron_out->adjust_potential(this->neuron_in->get_state() * this->con->eval()); this->neuron_out->adjust_potential(this->neuron_in->get_state() * this->con->eval());
} }
......
...@@ -274,21 +274,22 @@ int NeuralNetwork::add_neuron(Neuron *n) { ...@@ -274,21 +274,22 @@ int NeuralNetwork::add_neuron(Neuron *n) {
return n->get_idx(); return n->get_idx();
} }
void NeuralNetwork::add_connection_simple(int n1_idx, int n2_idx) { size_t NeuralNetwork::add_connection_simple(int n1_idx, int n2_idx) {
add_connection_simple(n1_idx, n2_idx, -1); return add_connection_simple(n1_idx, n2_idx, -1);
} }
void NeuralNetwork::add_connection_simple(int n1_idx, int n2_idx, size_t weight_idx) { size_t NeuralNetwork::add_connection_simple(int n1_idx, int n2_idx, size_t weight_idx) {
add_connection_simple(n1_idx, n2_idx, weight_idx, 1); return add_connection_simple(n1_idx, n2_idx, weight_idx, 1);
} }
void NeuralNetwork::add_connection_simple(int n1_idx, int n2_idx, size_t weight_idx, double weight_value) { size_t NeuralNetwork::add_connection_simple(int n1_idx, int n2_idx, size_t weight_idx, double weight_value) {
// TODO generate weight_value automatically from normal distribution // TODO generate weight_value automatically from normal distribution
if(weight_idx < 0 || weight_idx >= this->connection_weights->size()){ if(weight_idx < 0 || weight_idx >= this->connection_weights->size()){
//this weight is a new one, we add it to the system of weights //this weight is a new one, we add it to the system of weights
this->connection_weights->push_back(weight_value); this->connection_weights->push_back(weight_value);
weight_idx = (int)this->connection_weights->size() - 1; weight_idx = (int)this->connection_weights->size() - 1;
this->n_weights++;
} }
Neuron *neuron_out = this->neurons->at(n1_idx); Neuron *neuron_out = this->neurons->at(n1_idx);
Neuron *neuron_in = this->neurons->at(n2_idx); Neuron *neuron_in = this->neurons->at(n2_idx);
...@@ -300,10 +301,42 @@ void NeuralNetwork::add_connection_simple(int n1_idx, int n2_idx, size_t weight_ ...@@ -300,10 +301,42 @@ void NeuralNetwork::add_connection_simple(int n1_idx, int n2_idx, size_t weight_
neuron_out->add_connection_out(u1u2); neuron_out->add_connection_out(u1u2);
neuron_in->add_connection_in(u1u2); neuron_in->add_connection_in(u1u2);
return weight_idx;
}
void NeuralNetwork::add_connection_shared_power1(int n1_idx, int n2_idx, size_t weight_idx) {
std::function<double(double *, size_t*, size_t)> weight_function = [](double * weight_array, size_t * index_array, size_t n_params){
return weight_array[index_array[0]];
};
size_t* weight_indices = new size_t[1];
double* weight_values = new double[1];
weight_indices[0] = weight_idx;
this->add_connection_general( n1_idx, n2_idx, &weight_function, weight_indices, weight_values, 1);
delete [] weight_indices;
delete [] weight_values;
}
void NeuralNetwork::add_connection_shared_power2(int n1_idx, int n2_idx, size_t weight_idx) {
std::function<double(double *, size_t*, size_t)> weight_function = [](double * weight_array, size_t * index_array, size_t n_params){
return weight_array[index_array[0]] * weight_array[index_array[0]];
};
size_t* weight_indices = new size_t[1];
double* weight_values = new double[1];
weight_indices[0] = weight_idx;
this->add_connection_general( n1_idx, n2_idx, &weight_function, weight_indices, weight_values, 1);
delete [] weight_indices;
delete [] weight_values;
} }
void NeuralNetwork::add_connection_general(int n1_idx, int n2_idx, std::function<double(double *, int*, int)> *f, void NeuralNetwork::add_connection_general(int n1_idx, int n2_idx, std::function<double(double *, size_t*, size_t)> *f,
int* weight_indices, double* weight_values, size_t n_weights) { size_t* weight_indices, double* weight_values, size_t n_weights) {
ConnectionWeight *con_weight_u1u2 = new ConnectionWeight(n_weights, this->connection_weights); ConnectionWeight *con_weight_u1u2 = new ConnectionWeight(n_weights, this->connection_weights);
//we analyze weights //we analyze weights
...@@ -328,6 +361,7 @@ void NeuralNetwork::add_connection_general(int n1_idx, int n2_idx, std::function ...@@ -328,6 +361,7 @@ void NeuralNetwork::add_connection_general(int n1_idx, int n2_idx, std::function
neuron_out->add_connection_out(u1u2); neuron_out->add_connection_out(u1u2);
neuron_in->add_connection_in(u1u2); neuron_in->add_connection_in(u1u2);
} }
void NeuralNetwork::determine_inputs_outputs() { void NeuralNetwork::determine_inputs_outputs() {
...@@ -372,6 +406,8 @@ void NeuralNetwork::set_weight_array(std::vector<double> *weight_ptr) { ...@@ -372,6 +406,8 @@ void NeuralNetwork::set_weight_array(std::vector<double> *weight_ptr) {
} }
this->connection_weights = weight_ptr; this->connection_weights = weight_ptr;
this->delete_weights = false; this->delete_weights = false;
this->n_weights = this->connection_weights->size();
} }
void NeuralNetwork::eval_single(std::vector<double> &input, std::vector<double> &output, double * custom_weights) { void NeuralNetwork::eval_single(std::vector<double> &input, std::vector<double> &output, double * custom_weights) {
...@@ -448,8 +484,8 @@ void NeuralNetwork::eval_single(std::vector<double> &input, std::vector<double> ...@@ -448,8 +484,8 @@ void NeuralNetwork::eval_single(std::vector<double> &input, std::vector<double>
//we iterate through the active neurons and propagate the signal //we iterate through the active neurons and propagate the signal
for(i = 0; i < active_set_size[idx1]; ++i){ for(i = 0; i < active_set_size[idx1]; ++i){
active_neuron = this->active_eval_set->at(i + n * idx1); active_neuron = this->active_eval_set->at(i + n * idx1);
active_neuron->activate(); active_neuron->activate();//computes the activation function on its potential
// printf(" active neuron %d, state: %f\n", active_neuron, active_neuron->get_state()); // printf(" active neuron %d, potential: %f, state: %f\n", active_neuron, active_neuron->get_potential(), active_neuron->get_state());
for(Connection* connection: *(active_neuron->get_connections_out())){ for(Connection* connection: *(active_neuron->get_connections_out())){
connection->pass_signal(); connection->pass_signal();
...@@ -463,6 +499,7 @@ void NeuralNetwork::eval_single(std::vector<double> &input, std::vector<double> ...@@ -463,6 +499,7 @@ void NeuralNetwork::eval_single(std::vector<double> &input, std::vector<double>
} }
} }
} }
// printf("-----------\n");
idx1 = idx2; idx1 = idx2;
idx2 = (idx1 + 1) % 2; idx2 = (idx1 + 1) % 2;
...@@ -489,20 +526,25 @@ void NeuralNetwork::copy_weights(double *weights) { ...@@ -489,20 +526,25 @@ void NeuralNetwork::copy_weights(double *weights) {
} }
} }
std::vector<double>* NeuralNetwork::get_weights_ptr(){
return this->connection_weights;
}
void NeuralNetwork::randomize_weights() { void NeuralNetwork::randomize_weights() {
if( this->neurons->size() <= 0 || !this->in_out_determined ){ if( this->n_weights <= 0 ){
return; return;
} }
boost::random::mt19937 gen; boost::random::mt19937 gen;
// Init weight guess ("optimal" for logistic activation functions) // Init weight guess ("optimal" for logistic activation functions)
double r = 4 * sqrt(6./(this->n_inputs + this->n_outputs)); double r = 4 * sqrt(6./(this->n_weights));
boost::random::uniform_real_distribution<> dist(-r, r); boost::random::uniform_real_distribution<> dist(-r, r);
for(unsigned int i = 0; i < this->n_weights; i++) { for(size_t i = 0; i < this->n_weights; i++) {
this->connection_weights->at(i) = dist(gen); this->connection_weights->at(i) = dist(gen);
} }
} }
...@@ -552,3 +594,18 @@ void NeuralNetwork::specify_output_neurons(std::vector<size_t> &output_neurons_i ...@@ -552,3 +594,18 @@ void NeuralNetwork::specify_output_neurons(std::vector<size_t> &output_neurons_i
this->n_outputs = this->output_neurons->size(); this->n_outputs = this->output_neurons->size();
} }
void NeuralNetwork::print_weights() {
printf("Connection weights: ");
if(this->connection_weights){
for( size_t i = 0; i < this->connection_weights->size() - 1; ++i){
printf("%f, ", this->connection_weights->at(i));
}
printf("%f", this->connection_weights->at(this->connection_weights->size() - 1));
}
printf("\n");
}
void NeuralNetwork::print_stats(){
printf("Number of neurons: %d, number of weights: %d\n", this->neurons->size(), this->n_weights);
}
\ No newline at end of file
...@@ -79,11 +79,7 @@ private: ...@@ -79,11 +79,7 @@ private:
*/ */
void determine_inputs_outputs(); void determine_inputs_outputs();
/**
*
* @param weight_ptr
*/
void set_weight_array( std::vector<double>* weight_ptr );
public: public:
...@@ -113,7 +109,7 @@ public: ...@@ -113,7 +109,7 @@ public:
* @param[in] input * @param[in] input
* @param[in,out] output * @param[in,out] output
*/ */
void eval_single(std::vector<double> &input, std::vector<double> &output, double *custom_weights = nullptr); virtual void eval_single(std::vector<double> &input, std::vector<double> &output, double *custom_weights = nullptr);
...@@ -128,25 +124,46 @@ public: ...@@ -128,25 +124,46 @@ public:
* *
* @param n1_idx * @param n1_idx
* @param n2_idx * @param n2_idx
* @return
*/
size_t add_connection_simple(int n1_idx, int n2_idx);
/**
*
* @param n1_idx
* @param n2_idx
* @param weight_idx
* @return
*/
size_t add_connection_simple(int n1_idx, int n2_idx, size_t weight_idx);
/**
*
* @param n1_idx
* @param n2_idx
* @param weight_idx
* @param weight_value
* @return
*/ */
void add_connection_simple(int n1_idx, int n2_idx); size_t add_connection_simple(int n1_idx, int n2_idx, size_t weight_idx, double weight_value);
/** /**
* *
* @param n1_idx * @param n1_idx
* @param n2_idx * @param n2_idx
* @param weight_idx * @param weight_idx
* @param power_degree
*/ */
void add_connection_simple(int n1_idx, int n2_idx, size_t weight_idx); void add_connection_shared_power1(int n1_idx, int n2_idx, size_t weight_idx);
/** /**
* *
* @param[in] n1_idx * @param n1_idx
* @param[in] n2_idx * @param n2_idx
* @param[in] weight_idx * @param weight_idx
* @param[in] weight_value * @param power_degree
*/ */
void add_connection_simple(int n1_idx, int n2_idx, size_t weight_idx, double weight_value); void add_connection_shared_power2(int n1_idx, int n2_idx, size_t weight_idx);
/** /**
* *
...@@ -157,8 +174,14 @@ public: ...@@ -157,8 +174,14 @@ public:
* @param weight_values * @param weight_values
* @param n_weights * @param n_weights
*/ */
void add_connection_general(int n1_idx, int n2_idx, std::function<double(double *, int*, int)> *f, void add_connection_general(int n1_idx, int n2_idx, std::function<double(double *, size_t*, size_t)> *f,
int* weight_indices, double* weight_values, size_t n_weights); size_t* weight_indices, double* weight_values, size_t n_weights);
/**
*
* @param weight_ptr
*/
void set_weight_array( std::vector<double>* weight_ptr );
/** /**
* *
...@@ -166,6 +189,12 @@ public: ...@@ -166,6 +189,12 @@ public:
*/ */
void copy_weights(double *weights); void copy_weights(double *weights);
/**
*
* @return
*/
std::vector<double>* get_weights_ptr();
/** /**
* *
*/ */
...@@ -187,7 +216,7 @@ public: ...@@ -187,7 +216,7 @@ public:
* *
* @return * @return
*/ */
size_t get_n_weights(); virtual size_t get_n_weights();
/** /**
* *
...@@ -201,6 +230,15 @@ public: ...@@ -201,6 +230,15 @@ public:
*/ */
void specify_output_neurons(std::vector<size_t> &output_neurons_indices); void specify_output_neurons(std::vector<size_t> &output_neurons_indices);
/**
*
*/
void print_weights();
/**
*
*/
void print_stats();
}; };
......
/**
* DESCRIPTION OF THE FILE
*
* @author Michal Kravčenko
* @date 18.7.18 -
*/
#include "NeuralNetworkSum.h"
NeuralNetworkSum::NeuralNetworkSum(){
this->summand = nullptr;
this->summand_coefficient = nullptr;
}
NeuralNetworkSum::~NeuralNetworkSum() {
if( this->summand ){
delete this->summand;
}
if( this->summand_coefficient ){
delete this->summand_coefficient;
}
}
void NeuralNetworkSum::add_network(NeuralNetwork *net, double alpha) {
if(!this->summand){
this->summand = new std::vector<NeuralNetwork*>(0);
}
this->summand->push_back( net );
if(!this->summand_coefficient){
this->summand_coefficient = new std::vector<double>(0);
}
this->summand_coefficient->push_back( alpha );
}
void NeuralNetworkSum::eval_single(std::vector<double> &input, std::vector<double> &output, double *custom_weights) {
std::vector<double> mem_output(output.size());
std::fill(output.begin(), output.end(), 0.0);
for(int ni = 0; ni < this->summand->size(); ++ni){
this->summand->at(ni)->eval_single(input, mem_output, custom_weights);
double alpha = this->summand_coefficient->at(ni);
for(int j = 0; j < output.size(); ++j){
output[j] += mem_output[j] * alpha;
}
}
}
size_t NeuralNetworkSum::get_n_weights(){
//TODO stupid solution, assumes the networks share weights
if(this->summand){
return this->summand->at(0)->get_n_weights();
}
return 0;
}
\ No newline at end of file
/**
* DESCRIPTION OF THE FILE
*
* @author Michal Kravčenko
* @date 18.7.18 -
*/
#ifndef INC_4NEURO_NEURALNETWORKSUM_H
#define INC_4NEURO_NEURALNETWORKSUM_H
#include "NeuralNetwork.h"
class NeuralNetworkSum : public NeuralNetwork {
private:
std::vector<NeuralNetwork*> * summand;
std::vector<double> * summand_coefficient;
public:
NeuralNetworkSum();
virtual ~NeuralNetworkSum();
void add_network( NeuralNetwork *net, double alpha = 1.0 );
virtual void eval_single(std::vector<double> &input, std::vector<double> &output, double *custom_weights = nullptr) override;
virtual size_t get_n_weights();
};
#endif //INC_4NEURO_NEURALNETWORKSUM_H
...@@ -28,8 +28,8 @@ void NeuronLogistic::activate( ) { ...@@ -28,8 +28,8 @@ void NeuronLogistic::activate( ) {
double b = this->activation_function_parameters[1]; double b = this->activation_function_parameters[1];
double x = this->potential; double x = this->potential;
double ex = std::pow(E, a - x); double ex = std::pow(E, b - x);
this->state = std::pow(1.0 + ex, -b); this->state = std::pow(1.0 + ex, -a);
} }
...@@ -40,9 +40,9 @@ double NeuronLogistic::activation_function_get_partial_derivative(int param_idx ...@@ -40,9 +40,9 @@ double NeuronLogistic::activation_function_get_partial_derivative(int param_idx
double x = this->potential; double x = this->potential;
if(param_idx == 0){ if(param_idx == 0){
double ex = std::pow(E, a - x); double ex = std::pow(E, b - x);
double exa= -std::pow(ex + 1.0, -b); double exa= -std::pow(ex + 1.0, -a);
return exa * std::log(ex + 1.0); return exa * std::log(ex + 1.0);
} }
...@@ -51,10 +51,10 @@ double NeuronLogistic::activation_function_get_partial_derivative(int param_idx ...@@ -51,10 +51,10 @@ double NeuronLogistic::activation_function_get_partial_derivative(int param_idx
* TODO * TODO
* Could be write as activation_function_get_derivative() * -1 * Could be write as activation_function_get_derivative() * -1
*/ */
double ex = std::pow(E, a - x); double ex = std::pow(E, b - x);
double ex2 = std::pow(ex + 1.0, -b - 1.0); double ex2 = std::pow(ex + 1.0, -a - 1.0);
return -b * ex * ex2; return -a * ex * ex2;
} }
return 0.0; return 0.0;
...@@ -67,9 +67,9 @@ double NeuronLogistic::activation_function_get_derivative( ) { ...@@ -67,9 +67,9 @@ double NeuronLogistic::activation_function_get_derivative( ) {
double b = this->activation_function_parameters[1]; double b = this->activation_function_parameters[1];
double x = this->potential; double x = this->potential;
double ex = std::pow(E, a - x); double ex = std::pow(E, b - x);
double ex2 = std::pow(ex + 1.0, -b - 1.0); double ex2 = std::pow(ex + 1.0, -a - 1.0);
return b * ex * ex2; return a * ex * ex2;
} }
\ No newline at end of file
/**
* Example solving the following ODE:
*
* g(t) = (d^2/d^t)y(t) + 4 (d/dt)y(t) + 4y(t) = 0, for t in [0, 4]
* y(0) = 1
* (d/dt)y(0) = 1
*
* @author Michal Kravčenko
* @date 17.7.18 -
*/