Commit 57f9a4ed authored by Michal Kravcenko's avatar Michal Kravcenko

Updated the first ode example to analyze the correctness of the neural...

Updated the first ode example to analyze the correctness of the neural networks assembled by the DESolver
Corrected some examples to reflect the structural changes in the previous commit
parent c7ea1acc
......@@ -22,7 +22,7 @@ endif (NOT CMAKE_BUILD_TYPE)
# Setting C++ compiler flags #
#--------------------------------#
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall" )
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -mtune=native" )
#--------------------#
# Automatic settings #
......
......@@ -21,9 +21,9 @@ void Particle::randomize_coordinates() {
std::random_device seeder;
std::mt19937 gen(seeder());
std::uniform_real_distribution<double> dist_coord(-1.0, 1.0);
this->domain_bounds = domain_bounds;
for(unsigned int i = 0; i < this->coordinate_dim; ++i){
std::uniform_real_distribution<double> dist_coord(-1, 1);
(*this->coordinate)[i] = dist_coord(gen);
}
}
......@@ -300,7 +300,7 @@ void ParticleSwarm::optimize( double gamma, double epsilon, double delta) {
for(size_t pi=0; pi < this->n_particles; pi++) {
particle = this->particle_swarm[pi];
tmp_velocity = particle->change_coordinate( this->w, this->c1, this->c2, *this->p_min_glob, global_best_vec);
particle->print_coordinate();
// particle->print_coordinate();
if(tmp_velocity > max_velocity) {
prev_max_velocity = max_velocity;
......@@ -325,9 +325,11 @@ void ParticleSwarm::optimize( double gamma, double epsilon, double delta) {
//TODO only in verbose mode
euclidean_dist /= this->n_particles;
printf("Iteration %d, avg euclidean distance: %f, cluster percent: %f, f-value: %f\n", (int)outer_it, euclidean_dist,
double(cluster.size())/this->n_particles, optimal_value);
std::cout.flush();
if(outer_it % 10 == 0){
printf("Iteration %d, avg euclidean distance: %f, cluster percent: %f, f-value: %f\r", (int)outer_it, euclidean_dist,
double(cluster.size())/this->n_particles, optimal_value);
std::cout.flush();
}
// for(unsigned int i=0; i < this->n_particles; i++) {
// printf("Particle %d (%f): \n", i, this->particle_swarm[i]->get_current_value());
......
......@@ -30,7 +30,6 @@ NeuralNetwork::NeuralNetwork() {
this->delete_weights = true;
this->delete_biases = true;
this->layers_analyzed = false;
this->in_out_determined = false;
}
NeuralNetwork::~NeuralNetwork() {
......@@ -369,7 +368,6 @@ size_t NeuralNetwork::add_neuron(Neuron* n, BIAS_TYPE bt, size_t bias_idx) {
this->neurons->push_back(n);
this->in_out_determined = false;
this->layers_analyzed = false;
return this->neurons->size() - 1;
}
......@@ -445,7 +443,7 @@ void NeuralNetwork::set_parameter_space_pointers(NeuralNetwork &parent_network)
}
void NeuralNetwork::eval_single(std::vector<double> &input, std::vector<double> &output, std::vector<double> * custom_weights_and_biases) {
if(!this->in_out_determined || (this->input_neuron_indices->size() * this->output_neuron_indices->size()) <= 0){
if((this->input_neuron_indices->size() * this->output_neuron_indices->size()) <= 0){
std::cerr << "Input and output neurons have not been specified\n" << std::endl;
exit(-1);
}
......@@ -460,6 +458,8 @@ void NeuralNetwork::eval_single(std::vector<double> &input, std::vector<double>
std::cerr << "Error, output size != Network output size\n" << std::endl;
exit(-1);
}
double potential, bias;
int bias_idx;
this->copy_parameter_space( custom_weights_and_biases );
......@@ -475,8 +475,6 @@ void NeuralNetwork::eval_single(std::vector<double> &input, std::vector<double>
}
/* we iterate through all the feed-forward layers and transfer the signals */
double potential, bias;
int bias_idx;
for( auto layer: *this->neuron_layers_feedforward){
/* we iterate through all neurons in this layer and propagate the signal to the neighboring neurons */
......@@ -550,12 +548,12 @@ size_t NeuralNetwork::get_n_biases() {
return this->neuron_biases->size();
}
size_t NeuralNetwork::get_neuron_bias_index(size_t neuron_idx) {
return this->neuron_biases->at( neuron_idx );
int NeuralNetwork::get_neuron_bias_index(size_t neuron_idx) {
return this->neuron_bias_indices->at( neuron_idx );
}
size_t NeuralNetwork::get_n_neurons() {
return this->input_neuron_indices->size();
return this->neurons->size();
}
void NeuralNetwork::specify_input_neurons(std::vector<size_t> &input_neurons_indices) {
......
......@@ -96,12 +96,7 @@ private:
*/
std::vector<std::vector<size_t>*> *neuron_layers_feedbackward = nullptr;
/**
*
*/
bool in_out_determined = false;
/**
/**
*
*/
bool layers_analyzed = false;
......@@ -255,7 +250,7 @@ public:
*
* @return
*/
virtual size_t get_neuron_bias_index( size_t neuron_idx );
virtual int get_neuron_bias_index( size_t neuron_idx );
/**
*
......
......@@ -311,7 +311,7 @@ void DESolver::add_to_differential_equation( size_t equation_idx, MultiIndex &al
void DESolver::set_error_function(size_t equation_idx, ErrorFunctionType F, DataSet *conditions) {
if( equation_idx >= this->n_equations ){
throw std::invalid_argument("The parameter 'equation_idx' is too large! It exceeds the number of differential equations.");
throw std::invalid_argument( "The parameter 'equation_idx' is too large! It exceeds the number of differential equations." );
}
this->errors_functions_types->at( equation_idx ) = F;
......@@ -361,18 +361,49 @@ NeuralNetwork* DESolver::get_solution() {
return this->solution;
}
void DESolver::eval_equation( size_t equation_idx, std::vector<double> *weight_and_biases, std::vector<double> &input ) {
double DESolver::eval_equation( size_t equation_idx, std::vector<double> &weight_and_biases, std::vector<double> &input ) {
std::vector<double> output(1);
this->differential_equations->at( equation_idx )->eval_single( input, output, weight_and_biases );
this->differential_equations->at( equation_idx )->eval_single( input, output, &weight_and_biases );
printf("Input: ");
for( auto e: input ){
printf("%f, ", e);
// printf("Input: ");
// for( auto e: input ){
// printf("%f, ", e);
// }
// printf("\nOutput: ");
// for( auto e: output ){
// printf("%f, ", e);
// }
// printf("\n");
return output[0];
}
double DESolver::eval_total_error(std::vector<double> &weights_and_biases) {
NeuralNetwork *nn;
DataSet *ds;
/* DEFINITION OF THE PARTIAL ERROR FUNCTIONS */
std::vector<ErrorFunction*> error_functions( this->n_equations );
for(size_t i = 0; i < this->n_equations; ++i ){
nn = this->differential_equations->at( i );
ds = this->errors_functions_data_sets->at( i );
if( this->errors_functions_types->at( i ) == ErrorFunctionType::ErrorFuncMSE ){
error_functions[i] = new MSE( nn, ds );
}
else{
//default
error_functions[i] = new MSE( nn, ds );
}
}
printf("\nOutput: ");
for( auto e: output ){
printf("%f, ", e);
/* DEFINITION OF THE GLOBAL ERROR FUNCTION */
ErrorSum total_error;
for(size_t i = 0; i < this->n_equations; ++i ) {
total_error.add_error_function( error_functions[i], 1.0 );
}
printf("\n");
return total_error.eval( &weights_and_biases );
}
\ No newline at end of file
......@@ -165,7 +165,13 @@ public:
/**
* For testing purposes only
*/
void eval_equation( size_t equation_idx, std::vector<double> * weights_and_biases, std::vector<double> &input );
double eval_equation( size_t equation_idx, std::vector<double> &weights_and_biases, std::vector<double> &input );
/**
* For testing purposes only
* @return
*/
double eval_total_error( std::vector<double> &weights_and_biases );
};
......
......@@ -28,7 +28,7 @@ int main() {
DataSet ds(&data_vec);
/* NETWORK DEFINITION */
NeuralNetwork net(4, 0);
NeuralNetwork net;
/* Input neurons */
NeuronLinear *i1 = new NeuronLinear( ); //f(x) = x
......@@ -36,20 +36,23 @@ int main() {
/* Output neuron */
double b = 1.0;//bias
NeuronLinear *o1 = new NeuronLinear( &b ); //f(x) = x + 1
NeuronLinear *o1 = new NeuronLinear( ); //f(x) = x + 1
/* Adding neurons to the net */
int idx1 = net.add_neuron(i1);
int idx2 = net.add_neuron(i2);
int idx3 = net.add_neuron(o1);
size_t idx1 = net.add_neuron(i1, BIAS_TYPE::NO_BIAS);
size_t idx2 = net.add_neuron(i2, BIAS_TYPE::NO_BIAS);
size_t idx3 = net.add_neuron(o1, BIAS_TYPE::NEXT_BIAS);
std::vector<double> *bv = net.get_parameter_ptr_biases();
for(size_t i = 0; i < 1; ++i){
bv->at(i) = 1.0;
}
/* Adding connections */
//net.add_connection_simple(idx1, idx3, -1, 1.0);
//net.add_connection_simple(idx2, idx3, -1, 1.0);
net.add_connection_simple(idx1, idx3);
net.add_connection_simple(idx2, idx3);
net.add_connection_simple(idx1, idx3, SIMPLE_CONNECTION_TYPE::NEXT_WEIGHT);
net.add_connection_simple(idx2, idx3, SIMPLE_CONNECTION_TYPE::NEXT_WEIGHT);
//net.randomize_weights();
......
......@@ -40,27 +40,32 @@ int main() {
NeuronLinear *i2 = new NeuronLinear( ); //f(x) = x
double b = 1;//bias
NeuronLinear *i3 = new NeuronLinear(&b); //f(x) = x + 1
NeuronLinear *i3 = new NeuronLinear( ); //f(x) = x + 1
/* Output neurons */
NeuronLinear *o1 = new NeuronLinear(&b); //f(x) = x + 1
NeuronLinear *o2 = new NeuronLinear(&b); //f(x) = x + 1
NeuronLinear *o1 = new NeuronLinear( ); //f(x) = x + 1
NeuronLinear *o2 = new NeuronLinear( ); //f(x) = x + 1
/* Adding neurons to the nets */
int idx1 = net.add_neuron(i1);
int idx2 = net.add_neuron(i2);
int idx3 = net.add_neuron(o1);
int idx4 = net.add_neuron(i3);
int idx5 = net.add_neuron(o2);
size_t idx1 = net.add_neuron(i1, BIAS_TYPE::NO_BIAS);
size_t idx2 = net.add_neuron(i2, BIAS_TYPE::NO_BIAS);
size_t idx3 = net.add_neuron(o1, BIAS_TYPE::NEXT_BIAS);
size_t idx4 = net.add_neuron(i3, BIAS_TYPE::NEXT_BIAS);
size_t idx5 = net.add_neuron(o2, BIAS_TYPE::NEXT_BIAS);
std::vector<double> *bv = net.get_parameter_ptr_biases();
for(size_t i = 0; i < 3; ++i){
bv->at(i) = 1.0;
}
/* Adding connections */
//net.add_connection_simple(idx1, idx3, -1, 1.0);
//net.add_connection_simple(idx2, idx3, -1, 1.0);
net.add_connection_simple(idx1, idx3); // weight index 0
net.add_connection_simple(idx2, idx3); // weight index 1
net.add_connection_simple(idx4, idx5, 0); // AGAIN weight index 0 - same weight!
net.add_connection_simple(idx1, idx3, SIMPLE_CONNECTION_TYPE::NEXT_WEIGHT); // weight index 0
net.add_connection_simple(idx2, idx3, SIMPLE_CONNECTION_TYPE::NEXT_WEIGHT); // weight index 1
net.add_connection_simple(idx4, idx5, SIMPLE_CONNECTION_TYPE::EXISTING_WEIGHT, 0); // AGAIN weight index 0 - same weight!
net.randomize_weights();
......
......@@ -42,28 +42,30 @@ int main() {
NeuronLinear *i1 = new NeuronLinear(); //f(x) = x
NeuronLinear *i2 = new NeuronLinear(); //f(x) = x
double b = 1;//bias
NeuronLinear *i3 = new NeuronLinear( &b ); //f(x) = x + 1
NeuronLinear *i3 = new NeuronLinear( ); //f(x) = x + 1
/* Output neurons */
NeuronLinear *o1 = new NeuronLinear(&b); //f(x) = x + 1
NeuronLinear *o2 = new NeuronLinear(&b); //f(x) = x + 1
NeuronLinear *o1 = new NeuronLinear( ); //f(x) = x + 1
NeuronLinear *o2 = new NeuronLinear( ); //f(x) = x + 1
/* Adding neurons to the nets */
int idx1 = net.add_neuron(i1);
int idx2 = net.add_neuron(i2);
int idx3 = net.add_neuron(o1);
int idx4 = net.add_neuron(i3);
int idx5 = net.add_neuron(o2);
size_t idx1 = net.add_neuron(i1, BIAS_TYPE::NO_BIAS);
size_t idx2 = net.add_neuron(i2, BIAS_TYPE::NO_BIAS);
size_t idx3 = net.add_neuron(o1, BIAS_TYPE::NEXT_BIAS);
size_t idx4 = net.add_neuron(i3, BIAS_TYPE::NEXT_BIAS);
size_t idx5 = net.add_neuron(o2, BIAS_TYPE::NEXT_BIAS);
std::vector<double> *bv = net.get_parameter_ptr_biases();
for(size_t i = 0; i < 3; ++i){
bv->at(i) = 1.0;
}
/* Adding connections */
//net.add_connection_simple(idx1, idx3, -1, 1.0);
//net.add_connection_simple(idx2, idx3, -1, 1.0);
net.add_connection_simple(idx1, idx3); // weight index 0
net.add_connection_simple(idx2, idx3); // weight index 1
net.add_connection_simple(idx4, idx5, 0); // AGAIN weight index 0 - same weight!
net.add_connection_simple(idx1, idx3, SIMPLE_CONNECTION_TYPE::NEXT_WEIGHT); // weight index 0
net.add_connection_simple(idx2, idx3, SIMPLE_CONNECTION_TYPE::NEXT_WEIGHT); // weight index 1
net.add_connection_simple(idx4, idx5, SIMPLE_CONNECTION_TYPE::EXISTING_WEIGHT, 0); // AGAIN weight index 0 - same weight!
net.randomize_weights();
......
......@@ -566,7 +566,6 @@ void test_odr(double accuracy, size_t n_inner_neurons, size_t train_size, double
/* neumann boundary condition */
solver_01.add_to_differential_equation( 2, alpha_1, 1.0 );
/* SETUP OF THE TRAINING DATA */
std::vector<double> inp, out;
......@@ -574,6 +573,7 @@ void test_odr(double accuracy, size_t n_inner_neurons, size_t train_size, double
/* TRAIN DATA FOR THE GOVERNING DE */
std::vector<std::pair<std::vector<double>, std::vector<double>>> data_vec_g;
std::vector<double> test_points(train_size);
/* ISOTROPIC TRAIN SET */
......@@ -582,6 +582,8 @@ void test_odr(double accuracy, size_t n_inner_neurons, size_t train_size, double
inp = {frac * i};
out = {0.0};
data_vec_g.emplace_back(std::make_pair(inp, out));
test_points[i] = inp[0];
}
/* CHEBYSCHEV TRAIN SET */
......@@ -591,6 +593,8 @@ void test_odr(double accuracy, size_t n_inner_neurons, size_t train_size, double
// inp = {(std::cos(alpha * i) + 1.0) * frac + d1_s};
// out = {0.0};
// data_vec_g.emplace_back(std::make_pair(inp, out));
//
// test_points[i] = inp[0];
// }
DataSet ds_00(&data_vec_g);
......@@ -614,21 +618,68 @@ void test_odr(double accuracy, size_t n_inner_neurons, size_t train_size, double
solver_01.set_error_function( 2, ErrorFunctionType::ErrorFuncMSE, &ds_02 );
std::vector<double> params(3 * n_inner_neurons), params_analytical(3 * n_inner_neurons);
std::random_device seeder;
std::mt19937 gen(seeder());
std::uniform_real_distribution<double> dist(-10.0, 10.0);
std::vector<double> input(1);
for( size_t testi = 0; testi < 50; ++testi ){
double test_error_eq1 = 0.0, total_error = 0.0;
for(size_t i = 0; i < params.size(); ++i){
params[i] = dist(gen);
}
for(size_t i = 0; i < n_inner_neurons; ++i){
params_analytical[3 * i] = params[i];
params_analytical[3 * i + 1] = params[n_inner_neurons + i];
params_analytical[3 * i + 2] = params[2 * n_inner_neurons + i];
}
for(auto d: *ds_00.get_data()){
input = d.first;
double x = input[0];
double analytical_value_f = eval_approx_f(x, n_inner_neurons, params_analytical);
double analytical_value_df = eval_approx_df(x, n_inner_neurons, params_analytical);
double analytical_value_ddf = eval_approx_ddf(x, n_inner_neurons, params_analytical);
double de_solver_value_eq1 = solver_01.eval_equation(0, params, input);
double analytical_value_eq1 = 4 * analytical_value_f + 4 * analytical_value_df + analytical_value_ddf;
test_error_eq1 += (de_solver_value_eq1 - analytical_value_eq1) * (de_solver_value_eq1 - analytical_value_eq1);
}
input[0] = 0.0;
double de_solver_value_eq2 = solver_01.eval_equation(1, params, input);
double analytical_value_eq2 = eval_approx_f(0.0, n_inner_neurons, params_analytical);
double test_error_eq2 = (de_solver_value_eq2 - analytical_value_eq2) * (de_solver_value_eq2 - analytical_value_eq2);
double de_solver_value_eq3 = solver_01.eval_equation(2, params, input);
double analytical_value_eq3 = eval_approx_df(0.0, n_inner_neurons, params_analytical);
double test_error_eq3 = (de_solver_value_eq3 - analytical_value_eq3) * (de_solver_value_eq3 - analytical_value_eq3);
double total_error_de_solver = solver_01.eval_total_error(params);
double total_error_analytical = eval_error_function(params_analytical, n_inner_neurons, test_points);
printf("\tRepresentation test %6d, error of eq1: %10.8f, error of eq2: %10.8f, error of eq3: %10.8f, total error: %10.8f\n", (int)testi, std::sqrt(test_error_eq1), std::sqrt(test_error_eq2), std::sqrt(test_error_eq3), (total_error_analytical - total_error_de_solver) * (total_error_analytical - total_error_de_solver));
}
return;
/* PARTICLE SWARM TRAINING METHOD SETUP */
//must encapsulate each of the partial error functions
double *domain_bounds = new double[ 6 * n_inner_neurons ];
for(unsigned int i = 0; i < 3 * n_inner_neurons; ++i){
domain_bounds[2 * i] = -800.0;
domain_bounds[2 * i + 1] = 800.0;
domain_bounds[2 * i] = -10.0;
domain_bounds[2 * i + 1] = 10.0;
}
double c1 = 0.05, c2 = 0.0, w = 0.3;
double c1 = 0.5, c2 = 0.8, w = 0.7;
double gamma = 0.5, epsilon = 0.0000000000002, delta = 1.1;
double gamma = 0.5, epsilon = 0.02, delta = 0.9;
solver_01.solve_via_particle_swarm( domain_bounds, c1, c2, w, n_particles, max_iters, gamma, epsilon, delta );
......@@ -642,7 +693,7 @@ void test_odr(double accuracy, size_t n_inner_neurons, size_t train_size, double
parameters[3 * i + 2] = biases_params->at(i);
}
std::vector<double> input(1), output(1);
std::vector<double> output(1);
double x;
for(unsigned int i = 0; i < n_test_points; ++i){
......@@ -661,7 +712,7 @@ void test_odr(double accuracy, size_t n_inner_neurons, size_t train_size, double
int main() {
unsigned int n_inner_neurons = 2;
unsigned int train_size = 200;
unsigned int train_size = 150;
double accuracy = 1e-4;
double ds = 0.0;
double de = 4.0;
......@@ -682,9 +733,8 @@ int main() {
//
// std::random_device seeder;
// std::mt19937 gen(seeder());
// std::uniform_real_distribution<double> dist(-1.0, 1.0);
// for(unsigned int i = 0; i < 3 * n_inner_neurons; ++i){
// init_guess[i] = dist(gen);
// std::uniform_real_distribution<double> dist(-10.0, 10.0);
// for(unsigned int i = 0; i < init_guess.size(); ++i){
// init_guess[i] = dist(gen);
// }
// if(!optimize_biases){
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment