Commit c7ea1acc authored by Michal Kravcenko's avatar Michal Kravcenko

Updated the structure of connections and neurons so they are easier to serialize.

Updated the structure and functionality of NeuralNEtwork class to account for the changes in connections and neurons.
Updated the logic in DESolver to account for the changes in NeuralNetwork
parent df2f6a73
......@@ -5,6 +5,7 @@
* @date 2.7.18 -
*/
#include <iostream>
#include "ParticleSwarm.h"
/**
......@@ -14,41 +15,63 @@
* @param domain_bounds
* @param F
*/
Particle::Particle(ErrorFunction* ef, double *domain_bounds) {
//TODO better generating of random numbers
this->coordinate_dim = ef->get_dimension();
this->coordinate = new std::vector<double>(this->coordinate_dim);
this->velocity = new std::vector<double>(this->coordinate_dim);
this->optimal_coordinate = new std::vector<double>(this->coordinate_dim);
void Particle::randomize_coordinates() {
std::random_device seeder;
std::mt19937 gen(seeder());
std::uniform_real_distribution<double> dist_vel(0.5, 1.0);
std::uniform_real_distribution<double> dist_coord(-1.0, 1.0);
this->domain_bounds = domain_bounds;
for(unsigned int i = 0; i < this->coordinate_dim; ++i){
(*this->velocity)[i] = dist_vel(gen);
(*this->coordinate)[i] = dist_coord(gen);
}
}
void Particle::randomize_parameters() {
std::random_device seeder;
std::mt19937 gen(seeder());
std::uniform_real_distribution<double> dist_vel(0.5, 1.0);
this->r1 = dist_vel(gen);
this->r2 = dist_vel(gen);
this->r3 = dist_vel(gen);
}
void Particle::randomize_velocity() {
std::random_device seeder;
std::mt19937 gen(seeder());
std::uniform_real_distribution<double> dist_vel(0.5, 1.0);
for(unsigned int i = 0; i < this->coordinate_dim; ++i){
(*this->velocity)[i] = dist_vel(gen);
}
}
this->ef = ef;
std::uniform_real_distribution<double> dist_coord(-1.0, 1.0);
Particle::Particle(ErrorFunction* ef, double *domain_bounds) {
//TODO better generating of random numbers
this->domain_bounds = domain_bounds;
this->coordinate_dim = ef->get_dimension();
this->ef = ef;
this->coordinate = new std::vector<double>(this->coordinate_dim);
this->velocity = new std::vector<double>(this->coordinate_dim);
this->optimal_coordinate = new std::vector<double>(this->coordinate_dim);
this->randomize_velocity();
this->randomize_parameters();
this->randomize_coordinates();
for(unsigned int i = 0; i < this->coordinate_dim; ++i){
(*this->coordinate)[i] = dist_coord(gen);
(*this->optimal_coordinate)[i] = (*this->coordinate)[i];
}
// (*this->coordinate) = {-6.35706416, -1.55399893, -1.05305639, 3.07464411};
// printf("coordinate_dim: %d\n", this->coordinate_dim);
this->optimal_value = this->ef->eval(this->coordinate);
// this->print_coordinate();
this->print_coordinate();
}
Particle::~Particle() {
......@@ -94,8 +117,6 @@ double Particle::change_coordinate(double w, double c1, double c2, std::vector<d
double vel_mem;
double output = 0.0;
bool in_domain;
double compensation_coef = 1;
/* Choose random global minima */
std::vector<double> *random_global_best;
......@@ -107,27 +128,31 @@ double Particle::change_coordinate(double w, double c1, double c2, std::vector<d
// TODO use std::sample to choose random vector
//std::sample(global_min_vec.begin(), global_min_vec.end(), std::back_inserter(random_global_best), 1, std::mt19937{std::random_device{}()});
for(unsigned int i = 0; i < this->coordinate_dim; ++i) {
vel_mem = w * (*this->velocity)[i]
+ c1 * this->r1 * ((*this->optimal_coordinate)[i] - (*this->coordinate)[i])
+ c2 * this->r2 * (glob_min_coord[i] - (*this->coordinate)[i]);
// + (c1+c2)/2 * this->r3 * ((*random_global_best)[i] - (*this->coordinate)[i]);
if ((*this->coordinate)[i] + vel_mem > this->domain_bounds[2 * i + 1]) {
this->randomize_velocity();
this->randomize_parameters();
this->randomize_coordinates();
break;
} else if ((*this->coordinate)[i] + vel_mem < this->domain_bounds[2 * i]) {
this->randomize_velocity();
this->randomize_parameters();
this->randomize_coordinates();
break;
}
}
for(unsigned int i = 0; i < this->coordinate_dim; ++i){
vel_mem = w * (*this->velocity)[i]
+ c1 * this->r1 * ((*this->optimal_coordinate)[i] - (*this->coordinate)[i])
+ c2 * this->r2 * (glob_min_coord[i] - (*this->coordinate)[i])
+ (c1+c2)/2 * this->r3 * ((*random_global_best)[i] - (*this->coordinate)[i]);
do{
// printf("%f + %f ?? %f, %f\n", (*this->coordinate)[i], vel_mem, this->domain_bounds[2 * i + 1], this->domain_bounds[2 * i]);
if ((*this->coordinate)[i] + vel_mem > this->domain_bounds[2 * i + 1]) {
in_domain = false;
vel_mem = -penalty_coef * compensation_coef * w * vel_mem;
compensation_coef /= 2;
} else if ((*this->coordinate)[i] + vel_mem < this->domain_bounds[2 * i]) {
in_domain = false;
vel_mem = penalty_coef * compensation_coef * w * vel_mem;
compensation_coef /= 2;
} else {
in_domain = true;
compensation_coef = 1;
}
}while(!in_domain);
(*this->velocity)[i] = vel_mem;
(*this->coordinate)[i] += vel_mem;
......@@ -275,6 +300,7 @@ void ParticleSwarm::optimize( double gamma, double epsilon, double delta) {
for(size_t pi=0; pi < this->n_particles; pi++) {
particle = this->particle_swarm[pi];
tmp_velocity = particle->change_coordinate( this->w, this->c1, this->c2, *this->p_min_glob, global_best_vec);
particle->print_coordinate();
if(tmp_velocity > max_velocity) {
prev_max_velocity = max_velocity;
......@@ -301,6 +327,7 @@ void ParticleSwarm::optimize( double gamma, double epsilon, double delta) {
euclidean_dist /= this->n_particles;
printf("Iteration %d, avg euclidean distance: %f, cluster percent: %f, f-value: %f\n", (int)outer_it, euclidean_dist,
double(cluster.size())/this->n_particles, optimal_value);
std::cout.flush();
// for(unsigned int i=0; i < this->n_particles; i++) {
// printf("Particle %d (%f): \n", i, this->particle_swarm[i]->get_current_value());
......@@ -322,14 +349,14 @@ void ParticleSwarm::optimize( double gamma, double epsilon, double delta) {
if(outer_it < this->iter_max) {
/* Convergence reached */
printf("Found optimum in %d iterations: %10.8f at coordinates: \n", (int)outer_it, optimal_value);
printf("\nFound optimum in %d iterations: %10.8f at coordinates: \n", (int)outer_it, optimal_value);
for (size_t i = 0; i <= this->func_dim - 1; ++i) {
printf("%10.8f \n", (*this->p_min_glob)[i]);
}
} else {
/* Maximal number of iterations reached */
printf("Max number of iterations reached (%d)! Found value %10.8f at coordinates: \n", (int)outer_it, optimal_value);
printf("\nMax number of iterations reached (%d)! Found value %10.8f at coordinates: \n", (int)outer_it, optimal_value);
for (size_t i = 0; i <= this->func_dim - 1; ++i) {
printf("\t%10.8f \n", (*this->p_min_glob)[i]);
}
......
......@@ -43,6 +43,11 @@ private:
double *domain_bounds;
void randomize_coordinates();
void randomize_velocity();
void randomize_parameters();
public:
......
......@@ -13,24 +13,20 @@ ConnectionFunctionGeneral::ConnectionFunctionGeneral() {
}
ConnectionFunctionGeneral::ConnectionFunctionGeneral(std::vector<double>* w_array, std::vector<unsigned int> &param_indices, std::string &function_string) {
this->param_indices = new std::vector<unsigned int>(param_indices);
this->weight_array = w_array;
ConnectionFunctionGeneral::ConnectionFunctionGeneral(std::vector<size_t > &param_indices, std::string &function_string) {
this->param_indices = param_indices;
}
ConnectionFunctionGeneral::~ConnectionFunctionGeneral() {
if(this->param_indices){
delete this->param_indices;
this->param_indices = nullptr;
}
}
double ConnectionFunctionGeneral::eval() {
double ConnectionFunctionGeneral::eval( std::vector<double> &parameter_space ) {
//TODO
return 0.0;
}
void ConnectionFunctionGeneral::eval_partial_derivative(std::vector<double> &weight_gradient, double alpha) {
void ConnectionFunctionGeneral::eval_partial_derivative(std::vector<double> &parameter_space, std::vector<double> &weight_gradient, double alpha) {
//TODO
}
......@@ -13,15 +13,11 @@
class ConnectionFunctionGeneral {
protected:
/**
*
*/
std::vector<double> * weight_array = nullptr;
/**
*
*/
std::vector<unsigned int> *param_indices = nullptr;
std::vector<size_t> param_indices;
public:
......@@ -35,25 +31,25 @@ public:
* @param param_count
* @param f
*/
ConnectionFunctionGeneral(std::vector<double>* w_array, std::vector<unsigned int> &param_indices, std::string &function_string);
ConnectionFunctionGeneral(std::vector<size_t> &param_indices, std::string &function_string);
/**
*
*/
virtual ~ConnectionFunctionGeneral();
virtual ~ConnectionFunctionGeneral( );
/**
*
* @return
*/
virtual double eval( );
virtual double eval( std::vector<double> &parameter_space );
/**
* Performs partial derivative of this transfer function according to all parameters. Adds the values multiplied
* by alpha to the corresponding gradient vector
*/
virtual void eval_partial_derivative(std::vector<double> &weight_gradient, double alpha);
virtual void eval_partial_derivative( std::vector<double> &parameter_space, std::vector<double> &weight_gradient, double alpha );
};
......
......@@ -7,15 +7,29 @@
#include "ConnectionFunctionIdentity.h"
ConnectionFunctionIdentity::ConnectionFunctionIdentity(double * val, size_t pidx) {
this->value = val;
ConnectionFunctionIdentity::ConnectionFunctionIdentity( ) {
this->is_unitary = true;
}
ConnectionFunctionIdentity::ConnectionFunctionIdentity( size_t pidx ) {
this->param_idx = pidx;
this->is_unitary = false;
}
double ConnectionFunctionIdentity::eval() {
return (*this->value);
double ConnectionFunctionIdentity::eval( std::vector<double> &parameter_space ) {
if( this->is_unitary ){
return 1.0;
}
return parameter_space.at(this->param_idx);
}
void ConnectionFunctionIdentity::eval_partial_derivative(std::vector<double> &weight_gradient, double alpha) {
void ConnectionFunctionIdentity::eval_partial_derivative(std::vector<double> &parameter_space, std::vector<double> &weight_gradient, double alpha) {
if( this->is_unitary ){
return;
}
weight_gradient[this->param_idx] += alpha;
}
\ No newline at end of file
......@@ -17,27 +17,35 @@ class ConnectionFunctionGeneral;
*/
class ConnectionFunctionIdentity:public ConnectionFunctionGeneral {
private:
double * value = nullptr;
size_t param_idx;
size_t param_idx = 0;
bool is_unitary = false;
public:
/**
*
*/
ConnectionFunctionIdentity(double * value_ptr, size_t pidx);
ConnectionFunctionIdentity( );
/**
*
*/
ConnectionFunctionIdentity( size_t pidx );
/**
*
* @return
*/
double eval() override;
double eval( std::vector<double> &parameter_space ) override;
/**
*
* @param weight_gradient
* @param alpha
*/
void eval_partial_derivative(std::vector<double> &weight_gradient, double alpha) override;
void eval_partial_derivative(std::vector<double> &parameter_space, std::vector<double> &weight_gradient, double alpha) override;
};
......
This diff is collapsed.
......@@ -21,7 +21,9 @@
enum NET_TYPE{GENERAL};
enum BIAS_TYPE{NEXT, NOCHANGE};
enum BIAS_TYPE{NEXT_BIAS, NO_BIAS, EXISTING_BIAS};
enum SIMPLE_CONNECTION_TYPE{NEXT_WEIGHT, UNITARY_WEIGHT, EXISTING_WEIGHT};
/**
......@@ -59,6 +61,11 @@ private:
*/
std::vector<double>* neuron_biases = nullptr;
/**
*
*/
std::vector<int>* neuron_bias_indices = nullptr;
/**
*
*/
......@@ -89,31 +96,6 @@ private:
*/
std::vector<std::vector<size_t>*> *neuron_layers_feedbackward = nullptr;
/**
*
*/
size_t n_inputs = 0;
/**
*
*/
size_t n_outputs = 0;
/**
*
*/
size_t last_used_weight_idx = 0;
/**
*
*/
size_t last_used_bias_idx = 0;
/**
*
*/
size_t n_neurons = 0;
/**
*
*/
......@@ -171,12 +153,6 @@ public:
*/
NeuralNetwork();
/**
*
*/
NeuralNetwork(size_t n_connection_weights, size_t n_neurons);
/**
*
*/
......@@ -220,22 +196,7 @@ public:
* @param[in] n
* @return
*/
size_t add_neuron(Neuron* n, int bias_idx );
/**
* Adds a new neuron to the list of neurons. Also assigns a valid bias value to its activation function
* @param[in] n
* @return
*/
size_t add_neuron(Neuron* n, BIAS_TYPE bt = NOCHANGE );
/**
* Adds a new neuron to this network, does not touch its bias.
* @param n
* @return
*/
//TODO reformulate to use add_neuron(, BIAS_TYPE)
size_t add_neuron_no_bias(Neuron *n);
size_t add_neuron(Neuron* n, BIAS_TYPE bt = NEXT_BIAS, size_t bias_idx = 0);
/**
*
......@@ -243,26 +204,7 @@ public:
* @param n2_idx
* @return
*/
size_t add_connection_simple(size_t n1_idx, size_t n2_idx);
/**
*
* @param n1_idx
* @param n2_idx
* @param weight_idx
* @return
*/
size_t add_connection_simple(size_t n1_idx, size_t n2_idx, int weight_idx);
/**
*
* @param n1_idx
* @param n2_idx
* @param weight_idx
* @param weight_value
* @return
*/
size_t add_connection_simple(size_t n1_idx, size_t n2_idx, int weight_idx, double weight_value);
size_t add_connection_simple(size_t n1_idx, size_t n2_idx, SIMPLE_CONNECTION_TYPE sct = NEXT_WEIGHT, size_t weight_idx = 0 );
/**
* Take the existing connection with index 'connection_idx' in 'parent_network' and adds it to the structure of this
......@@ -309,6 +251,12 @@ public:
*/
virtual size_t get_n_biases();
/**
*
* @return
*/
virtual size_t get_neuron_bias_index( size_t neuron_idx );
/**
*
* @return
......
......@@ -4,11 +4,6 @@
Neuron::~Neuron() {
}
void Neuron::set_bias(double *b) {
this->bias = b;
}
//template<class Archive>
//void Neuron::serialize(Archive & ar, const unsigned int version) {
// ar << this->potential;
......
......@@ -22,10 +22,6 @@ class Neuron {
friend class boost::serialization::access;
protected:
/**
*
*/
double * bias;
template<class Archive>
void serialize(Archive & ar, const unsigned int version){
......@@ -50,16 +46,7 @@ public:
/**
* Performs the activation function and returns the result
*/
virtual double activate( double x ) = 0;
/**
*
* @param b
*/
virtual void set_bias( double * b = nullptr );
virtual double activate( double x, double b ) = 0;
}; /* end of Neuron class */
......@@ -74,14 +61,14 @@ class IDifferentiable {
* Calculates the derivative with respect to the argument, ie the 'potential'
* @return f'(x), where 'f(x)' is the activation function and 'x' = 'potential'
*/
virtual double activation_function_eval_derivative( double x ) = 0;
virtual double activation_function_eval_derivative( double x, double b ) = 0;
/**
* Calculates the derivative with respect to the bias
* @return d/db f'(x), where 'f(x)' is the activation function, 'x' is the 'potential'
* and 'b' is the bias
*/
virtual double activation_function_eval_derivative_bias( double x ) = 0;
virtual double activation_function_eval_derivative_bias( double x, double b ) = 0;
/**
* Returns a Neuron pointer object with activation function being the partial derivative of
......
......@@ -4,18 +4,11 @@
#include "NeuronBinary.h"
NeuronBinary::NeuronBinary( double * b ) {
this->bias = b;
NeuronBinary::NeuronBinary( ) {
}
double NeuronBinary::activate( double x ) {
double b = 0.0;
if( this->bias ){
b = *this->bias;
}
double NeuronBinary::activate( double x, double b ) {
if(x >= b){
return 1.0;
......
......@@ -32,12 +32,12 @@ public:
* @param[in] threshold Denotes when the neuron is activated
* When neuron potential exceeds 'threshold' value it becomes excited
*/
explicit NeuronBinary( double * b = nullptr );
explicit NeuronBinary( );
/**
* Performs the activation function and stores the result into the 'state' property
*/
double activate( double x ) override;
double activate( double x, double b ) override;
};
......
......@@ -13,15 +13,15 @@ NeuronConstant::NeuronConstant( double c ) {
}
double NeuronConstant::activate( double x ) {
double NeuronConstant::activate( double x, double b ) {
return this->p;
}
double NeuronConstant::activation_function_eval_derivative_bias( double x ) {
double NeuronConstant::activation_function_eval_derivative_bias( double x, double b ) {
return 0.0;
}
double NeuronConstant::activation_function_eval_derivative( double x ) {
double NeuronConstant::activation_function_eval_derivative( double x, double b ) {
return 0.0;
}
......
......@@ -33,7 +33,7 @@ public:
/**
* Evaluates and returns 'c'
*/
double activate( double x ) override;
double activate( double x, double b ) override;
/**
* Calculates the partial derivative of the activation function
......@@ -41,13 +41,13 @@ public:
* @return Partial derivative of the activation function according to the
* 'bias' parameter. Returns 0.0
*/
double activation_function_eval_derivative_bias( double x ) override;
double activation_function_eval_derivative_bias( double x, double b ) override;
/**
* Calculates d/dx of (c) at point x
* @return 0.0
*/
double activation_function_eval_derivative( double x ) override;
double activation_function_eval_derivative( double x, double b ) override;
/**
* Returns a pointer to a Neuron with derivative as its activation function
......
......@@ -6,26 +6,21 @@
NeuronLinear::NeuronLinear( double * b ) {
NeuronLinear::NeuronLinear( ) {
this->bias = b;
}
double NeuronLinear::activate( double x ) {
double b = 0.0;
if( this->bias ){
b = *this->bias;
}
double NeuronLinear::activate( double x, double b ) {
return x + b;
}
double NeuronLinear::activation_function_eval_derivative_bias( double x ) {
double NeuronLinear::activation_function_eval_derivative_bias( double x, double b ) {
return 1.0;
}
double NeuronLinear::activation_function_eval_derivative( double x ) {
double NeuronLinear::activation_function_eval_derivative( double x, double b ) {
return 1.0;
}
......
......@@ -36,12 +36,12 @@ public:
* f(x) = x + b
* @param[in] b Bias
*/
explicit NeuronLinear( double * b = nullptr );
explicit NeuronLinear( );
/**
* Evaluates 'x + b' and stores the result into the 'state' property
*/
double activate( double x ) override;
double activate( double x, double b ) override;
/**
* Calculates the partial derivative of the activation function
......@@ -49,13 +49,13 @@ public:
* @return Partial derivative of the activation function according to the
* 'bias' parameter. Returns 1.0
*/
double activation_function_eval_derivative_bias( double x ) override;
double activation_function_eval_derivative_bias( double x, double b ) override;
/**
* Calculates d/dx of (x + b) at point x
* @return 1.0
*/
double activation_function_eval_derivative( double x ) override;
double activation_function_eval_derivative( double x, double b ) override;
/**
* Returns a pointer to a Neuron with derivative as its activation function
......
......@@ -5,39 +5,34 @@
#include "NeuronLogistic.h"
NeuronLogistic_d2::NeuronLogistic_d2( double * b ) {
this->bias = b;
NeuronLogistic_d2::NeuronLogistic_d2( ) {
}
double NeuronLogistic_d2::activate( double x ) {
double NeuronLogistic_d2::activate( double x, double b ) {
//(e^(b + x) (e^b - e^x))/(e^b + e^x)^3
double b = 0.0;
if( this->bias ){
b = *this->bias;
}
double ex = std::pow(E, x);
double eb = std::pow(E, b);
double denom = (eb + ex);
return (eb*ex*(eb - ex))/((eb + ex)*(eb + ex)*(eb + ex));
return (eb*ex*(eb - ex))/(denom*denom*denom);
}
double NeuronLogistic_d2::activation_function_eval_derivative_bias( double x ) {
double NeuronLogistic_d2::activation_function_eval_derivative_bias( double x, double b ) {
//-(e^(b + x) (-4 e^(b + x) + e^(2 b) + e^(2 x)))/(e^b + e^x)^4
double b = 0.0;
if( this->bias ){
b = *this->bias;