Skip to content
Snippets Groups Projects
Commit abc9193b authored by Michal Kravcenko's avatar Michal Kravcenko
Browse files

MOD: update a few examples

parent 664bfecf
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
src/examples/net_test_1.cpp
+ 1
− 1
View file @ abc9193b
......@@ -60,7 +60,7 @@ void optimize_via_particle_swarm( l4n::NeuralNetwork &net, l4n::ErrorFunction &e
void optimize_via_gradient_descent( l4n::NeuralNetwork &net, l4n::ErrorFunction &ef ){
std::cout << "***********************************************************************************************************************" <<std::endl;
l4n::GradientDescent gd( 1e-6, 1000 );
l4n::GradientDescentBB gd( 1e-6, 1000 );
gd.optimize( ef );
......
src/examples/net_test_2.cpp
+ 1
− 1
View file @ abc9193b
......@@ -57,7 +57,7 @@ void optimize_via_particle_swarm( l4n::NeuralNetwork &net, l4n::ErrorFunction &e
void optimize_via_gradient_descent( l4n::NeuralNetwork &net, l4n::ErrorFunction &ef ){
l4n::GradientDescent gd( 1e-6, 1000 );
l4n::GradientDescentBB gd( 1e-6, 1000 );
gd.optimize( ef );
......
src/examples/net_test_3.cpp
+ 78
− 153
View file @ abc9193b
/**
* Example of a set of neural networks sharing some edge weights
* The system of equations associated with the net in this example is not regular
* minimizes the function: [(2y+0.5)^2 + (2x+y+0.25)^2] / 2 + [(4.5x + 0.37)^2] / 1
* minimum [0.010024714] at (x, y) = (-333/4370, -9593/43700) = (-0.076201373, -0.219519451)
* Example testing the correctness of back-propagation implementation
* */
//
// Created by martin on 7/16/18.
//
#include <iostream>
#include <cstdio>
#include <fstream>
#include <vector>
#include <chrono>
#include "4neuro.h"
void optimize_via_particle_swarm( l4n::NeuralNetwork &net, l4n::ErrorFunction &ef ){
/* TRAINING METHOD SETUP */
std::vector<double> domain_bounds(2 * (net.get_n_weights() + net.get_n_biases()));
for(size_t i = 0; i < domain_bounds.size() / 2; ++i){
domain_bounds[2 * i] = -10;
domain_bounds[2 * i + 1] = 10;
}
double c1 = 1.7;
double c2 = 1.7;
double w = 0.7;
size_t n_particles = 50;
size_t iter_max = 1000;
/* if the maximal velocity from the previous step is less than 'gamma' times the current maximal velocity, then one
* terminating criterion is met */
double gamma = 0.5;
/* if 'delta' times 'n' particles are in the centroid neighborhood given by the radius 'epsilon', then the second
* terminating criterion is met ('n' is the total number of particles) */
double epsilon = 0.02;
double delta = 0.7;
l4n::ParticleSwarm swarm_01(
&domain_bounds,
c1,
c2,
w,
gamma,
epsilon,
delta,
n_particles,
iter_max
);
swarm_01.optimize( ef );
std::vector<double> *parameters = swarm_01.get_parameters();
net.copy_parameter_space(parameters);
std::cout << "Run finished! Error of the network[Particle swarm]: " << ef.eval( nullptr ) << std::endl;
std::cout << "***********************************************************************************************************************" <<std::endl;
}
void optimize_via_gradient_descent( l4n::NeuralNetwork &net, l4n::ErrorFunction &ef ){
#include <utility>
#include <algorithm>
#include <assert.h>
l4n::GradientDescent gd( 1e-6, 1000 );
#include <4neuro.h>
gd.optimize( ef );
#include <boost/random/mersenne_twister.hpp>
#include <boost/random/uniform_int_distribution.hpp>
#include <boost/random/uniform_real_distribution.hpp>
std::vector<double> *parameters = gd.get_parameters();
net.copy_parameter_space(parameters);
/* ERROR CALCULATION */
std::cout << "Run finished! Error of the network[Gradient descent]: " << ef.eval( nullptr )<< std::endl;
std::cout << "***********************************************************************************************************************" <<std::endl;
}
double get_difference(std::vector<double> &a, std::vector<double> &b){
int main() {
std::cout << "Running lib4neuro example 3: Use of the particle swarm method to train a set of networks sharing some edge weights" << std::endl;
std::cout << "********************************************************************************************************************" <<std::endl;
double out = 0.0, m;
/* TRAIN DATA DEFINITION */
std::vector<std::pair<std::vector<double>, std::vector<double>>> data_vec_01, data_vec_02;
std::vector<double> inp, out;
for( size_t i = 0; i < a.size(); ++i ){
inp = {0, 1};
out = {0.5};
data_vec_01.emplace_back(std::make_pair(inp, out));
std::cout << a[i] << " - " << b[i] << std::endl;
inp = {1, 0.5};
out = {0.75};
data_vec_01.emplace_back(std::make_pair(inp, out));
m = a[i]-b[i];
out += m * m;
}
l4n::DataSet ds_01(&data_vec_01);
return std::sqrt(out);
}
inp = {1.25};
out = {0.63};
data_vec_02.emplace_back(std::make_pair(inp, out));
l4n::DataSet ds_02(&data_vec_02);
/* NETWORK DEFINITION */
l4n::NeuralNetwork net;
void calculate_gradient_analytical(std::vector<double> &input, std::vector<double> &parameter_biases, std::vector<double> &parameter_weights, size_t n_hidden_neurons, std::vector<double> &gradient_analytical ){
/* Input neurons */
l4n::NeuronLinear *i1 = new l4n::NeuronLinear(); //f(x) = x
l4n::NeuronLinear *i2 = new l4n::NeuronLinear(); //f(x) = x
double a, b, y, x = input[0];
for( size_t i = 0; i < n_hidden_neurons; ++i ){
a = parameter_weights[i];
b = parameter_biases[i];
y = parameter_weights[n_hidden_neurons + i];
l4n::NeuronLinear *i3 = new l4n::NeuronLinear( ); //f(x) = x
gradient_analytical[i] += y * x * std::exp(b - a * x) / ((1+std::exp(b - a * x))*(1+std::exp(b - a * x)));
gradient_analytical[n_hidden_neurons + i] += 1.0 / ((1+std::exp(b - a * x)));
gradient_analytical[2*n_hidden_neurons + i] -= y * std::exp(b - a * x) / ((1+std::exp(b - a * x))*(1+std::exp(b - a * x)));
}
/* Output neurons */
l4n::NeuronLinear *o1 = new l4n::NeuronLinear( ); //f(x) = x
l4n::NeuronLinear *o2 = new l4n::NeuronLinear( ); //f(x) = x
}
int main(int argc, char** argv) {
int n_tests = 1;
int n_hidden_neurons = 2;
try {
/* Numbers of neurons in layers (including input and output layers) */
std::vector<unsigned int> neuron_numbers_in_layers(3);
neuron_numbers_in_layers[0] = neuron_numbers_in_layers[2] = 1;
neuron_numbers_in_layers[1] = n_hidden_neurons;
/* Adding neurons to the nets */
size_t idx1 = net.add_neuron(i1, l4n::BIAS_TYPE::NO_BIAS);
size_t idx2 = net.add_neuron(i2, l4n::BIAS_TYPE::NO_BIAS);
size_t idx3 = net.add_neuron(o1, l4n::BIAS_TYPE::NEXT_BIAS);
size_t idx4 = net.add_neuron(i3, l4n::BIAS_TYPE::NEXT_BIAS);
size_t idx5 = net.add_neuron(o2, l4n::BIAS_TYPE::NEXT_BIAS);
/* Fully connected feed-forward network with linear activation functions for input and output */
/* layers and the specified activation fns for the hidden ones (each entry = layer)*/
std::vector<l4n::NEURON_TYPE> hidden_type_v = { l4n::NEURON_TYPE::LOGISTIC, l4n::NEURON_TYPE::LOGISTIC, l4n::NEURON_TYPE::LOGISTIC, l4n::NEURON_TYPE::LOGISTIC, l4n::NEURON_TYPE::LOGISTIC }; // hidden_type_v = {l4n::NEURON_TYPE::LOGISTIC, l4n::NEURON_TYPE::LINEAR}
l4n::FullyConnectedFFN nn1(&neuron_numbers_in_layers, &hidden_type_v);
nn1.randomize_parameters();
/* Adding connections */
net.add_connection_simple(idx1, idx3, l4n::SIMPLE_CONNECTION_TYPE::NEXT_WEIGHT); // weight index 0
net.add_connection_simple(idx2, idx3, l4n::SIMPLE_CONNECTION_TYPE::NEXT_WEIGHT); // weight index 1
net.add_connection_simple(idx4, idx5, l4n::SIMPLE_CONNECTION_TYPE::EXISTING_WEIGHT, 0); // AGAIN weight index 0 - same weight!
boost::random::mt19937 gen(std::time(0));
boost::random::uniform_real_distribution<> dist(-1, 1);
net.randomize_weights();
size_t n_parameters = nn1.get_n_weights() + nn1.get_n_biases();
std::vector<double> gradient_backprogation(n_parameters);
std::vector<double> gradient_analytical(n_parameters);
std::vector<double> *parameter_biases = nn1.get_parameter_ptr_biases();
std::vector<double> *parameter_weights = nn1.get_parameter_ptr_weights();
std::vector<double> error_derivative = {1};
/* specification of the input/output neurons */
std::vector<size_t> net_input_neurons_indices(3);
std::vector<size_t> net_output_neurons_indices(2);
net_input_neurons_indices[0] = idx1;
net_input_neurons_indices[1] = idx2;
net_input_neurons_indices[2] = idx4;
size_t n_good = 0, n_bad = 0;
net_output_neurons_indices[0] = idx3;
net_output_neurons_indices[1] = idx5;
nn1.write_weights();
nn1.write_biases();
for(int i = 0; i < n_tests; ++i){
net.specify_input_neurons(net_input_neurons_indices);
net.specify_output_neurons(net_output_neurons_indices);
std::vector<double> input(1);
std::vector<double> output(1);
input[0] = dist(gen);
output[0] = 0;
/* CONSTRUCTION OF SUBNETWORKS */
//TODO subnetworks retain the number of weights, could be optimized to include only the used weights
//TODO this example is not working properly, subnet method is not implemented
std::vector<size_t> subnet_01_input_neurons, subnet_01_output_neurons;
std::vector<size_t> subnet_02_input_neurons, subnet_02_output_neurons;
subnet_01_input_neurons.push_back(idx1);
subnet_01_input_neurons.push_back(idx2);
subnet_01_output_neurons.push_back(idx3);
l4n::NeuralNetwork *subnet_01 = net.get_subnet( subnet_01_input_neurons, subnet_01_output_neurons );
std::fill(gradient_backprogation.begin(), gradient_backprogation.end(), 0);
std::fill(gradient_analytical.begin(), gradient_analytical.end(), 0);
subnet_02_input_neurons.push_back(idx4);
subnet_02_output_neurons.push_back(idx5);
l4n::NeuralNetwork *subnet_02 = net.get_subnet( subnet_02_input_neurons, subnet_02_output_neurons );
nn1.eval_single_debug(input, output);
if(subnet_01 && subnet_02){
/* COMPLEX ERROR FUNCTION SPECIFICATION */
l4n::MSE mse_01(subnet_01, &ds_01);
l4n::MSE mse_02(subnet_02, &ds_02);
calculate_gradient_analytical(input, *parameter_biases, *parameter_weights, n_hidden_neurons, gradient_analytical );
nn1.add_to_gradient_single_debug(input, error_derivative, 1, gradient_backprogation);
l4n::ErrorSum mse_sum;
mse_sum.add_error_function( &mse_01 );
mse_sum.add_error_function( &mse_02 );
double diff = get_difference(gradient_backprogation, gradient_analytical);
/* PARTICLE SWARM LEARNING */
// net.randomize_weights();
// optimize_via_particle_swarm( net, mse_sum );
if ( diff < 1e-32 ){
n_good++;
}
else{
n_bad++;
}
}
/* GRADIENT DESCENT LEARNING */
auto start = std::chrono::system_clock::now();
std::cout << "Good gradients: " << n_good << ", Bad gradients: " << n_bad << std::endl;
net.randomize_weights();
optimize_via_gradient_descent( net, mse_sum );
auto end = std::chrono::system_clock::now();
std::chrono::duration<double> elapsed_seconds = end - start;
std::cout << "elapsed time: " << elapsed_seconds.count() << std::endl;
}
else{
std::cout << "We apologize, this example is unfinished as we are in the process of developing methods for efficient subnetwork definition" << std::endl;
}
return 0;
if(subnet_01){
delete subnet_01;
subnet_01 = nullptr;
}
if(subnet_02){
delete subnet_02;
subnet_02 = nullptr;
catch (const std::exception& e) {
std::cerr << e.what() << std::endl;
exit(EXIT_FAILURE);
}
return 0;
}
\ No newline at end of file
src/examples/simulator.cpp
+ 51
− 60
View file @ abc9193b
......@@ -12,11 +12,17 @@
#include <assert.h>
#include "4neuro.h"
#include "../CrossValidator/CrossValidator.h"
int main(int argc, char** argv) {
bool normalize_data = true;
double prec = 1e-9;
int restart_interval = 500;
int max_n_iters_gradient = 10000;
int max_n_iters_swarm = 50;
int n_particles_swarm = 100;
int batch_size = 0;
int max_number_of_cycles = 10;
try {
/* PHASE 1 - TRAINING DATA LOADING, NETWORK ASSEMBLY AND PARTICLE SWARM OPTIMIZATION */
l4n::CSVReader reader1("/home/fluffymoo/Dropbox/data_BACK_RH_1.csv", ";", true); // File, separator, skip 1st line
......@@ -25,17 +31,19 @@ int main(int argc, char** argv) {
/* PHASE 1 - NEURAL NETWORK SPECIFICATION */
/* Create data set for both the first training of the neural network */
/* Specify which columns are inputs or outputs */
std::vector<unsigned int> inputs1 = { 0 }; // Possible multiple inputs, e.g. {0,3}, column indices starting from 0
std::vector<unsigned int> outputs1 = { 1 }; // Possible multiple outputs, e.g. {1,2}
l4n::DataSet ds1 = reader1.get_data_set(&inputs1, &outputs1); // Creation of data-set for NN
ds1.normalize(); // Normalization of data to prevent numerical problems
std::vector<unsigned int> inputs = { 0 }; // Possible multiple inputs, e.g. {0,3}, column indices starting from 0
std::vector<unsigned int> outputs = { 2 }; // Possible multiple outputs, e.g. {1,2}
l4n::DataSet ds1 = reader1.get_data_set(&inputs, &outputs); // Creation of data-set for NN
if(normalize_data){
ds1.normalize(); // Normalization of data to prevent numerical problems
}
/* Numbers of neurons in layers (including input and output layers) */
std::vector<unsigned int> neuron_numbers_in_layers = { 1, 2, 1 };
/* Fully connected feed-forward network with linear activation functions for input and output */
/* layers and the specified activation fns for the hidden ones (each entry = layer)*/
std::vector<l4n::NEURON_TYPE> hidden_type_v = { l4n::NEURON_TYPE::LOGISTIC}; // hidden_type_v = {l4n::NEURON_TYPE::LOGISTIC, l4n::NEURON_TYPE::LINEAR}
std::vector<l4n::NEURON_TYPE> hidden_type_v = { l4n::NEURON_TYPE::LOGISTIC, l4n::NEURON_TYPE::LOGISTIC, l4n::NEURON_TYPE::LOGISTIC, l4n::NEURON_TYPE::LOGISTIC, l4n::NEURON_TYPE::LOGISTIC }; // hidden_type_v = {l4n::NEURON_TYPE::LOGISTIC, l4n::NEURON_TYPE::LINEAR}
l4n::FullyConnectedFFN nn1(&neuron_numbers_in_layers, &hidden_type_v);
/* Error function */
......@@ -44,8 +52,8 @@ int main(int argc, char** argv) {
/* Particle Swarm method domain*/
std::vector<double> domain_bounds(2 * (nn1.get_n_weights() + nn1.get_n_biases()));
for (size_t i = 0; i < domain_bounds.size() / 2; ++i) {
domain_bounds[2 * i] = -10;
domain_bounds[2 * i + 1] = 10;
domain_bounds[2 * i] = -0.1;
domain_bounds[2 * i + 1] = 0.1;
}
// Parameters of the Particle Swarm
......@@ -65,56 +73,39 @@ int main(int argc, char** argv) {
0.5,
0.3,
0.7,
150,
200);
/* Weight and bias randomization in the network accordingly to the uniform distribution */
nn1.randomize_parameters();
/* Particle Swarm optimization */
ps.optimize(mse1);
/* Save Neural network parameters to file */
nn1.save_text("test_net_Particle_Swarm.4n");
/* PHASE 3 - LOADING NN FROM FILE AND TRAINING NO 2 - GRADIENT DESCENT */
l4n::NeuralNetwork nn2("test_net_Particle_Swarm.4n");
/* Training data loading for the second phase */
l4n::CSVReader reader2("/home/fluffymoo/Dropbox/data_BACK_RH_1.csv", ";", true); // File, separator, skip 1st line
reader2.read(); // Read from the file
/* Create data set for both the first training of the neural network */
/* Specify which columns are inputs or outputs */
std::vector<unsigned int> inputs2 = { 0 }; // Possible multiple inputs, e.g. {0,3}, column indices starting from 0
std::vector<unsigned int> outputs2 = { 1 }; // Possible multiple outputs, e.g. {1,2}
l4n::DataSet ds2 = reader2.get_data_set(&inputs2, &outputs2); // Creation of data-set for NN
ds2.normalize(); // Normalization of data to prevent numerical problems
/* Error function */
l4n::MSE mse2(&nn2, &ds2); // First parameter - neural network, second parameter - data-set
n_particles_swarm,
max_n_iters_swarm);
// Parameters of the gradient descent
// 1) Threshold for the successful ending of the optimization - deviation from minima
// 2) Number of iterations to reset step size to tolerance/10.0
// 3) Maximal number of iterations - optimization will stop after that, even if not converged
l4n::GradientDescent gs(1e-6, 150, 50000);
l4n::GradientDescentBB gs(prec, restart_interval, max_n_iters_gradient, batch_size);
l4n::GradientDescentSingleItem gs_si(prec, 0, 5000);
l4n::LearningSequence learning_sequence( 1e-6, max_number_of_cycles );
learning_sequence.add_learning_method( &ps );
learning_sequence.add_learning_method( &gs );
learning_sequence.add_learning_method( &gs_si );
learning_sequence.add_learning_method( &gs );
/* Weight and bias randomization in the network accordingly to the uniform distribution */
nn1.randomize_parameters();
/* Gradient Descent Optimization */
gs.optimize(mse2); // Network training
/* Complex Optimization */
learning_sequence.optimize(mse1); // Network training
/* Save Neural network parameters to file */
nn2.save_text("test_net_Gradient_Descent.4n");
nn1.save_text("test_net_Gradient_Descent.4n");
/* PHASE 4 - TESTING DATA */
/* Output file specification */
// /* Output file specification */
std::string filename = "simulator_output.txt";
std::ofstream output_file(filename);
if (!output_file.is_open()) {
throw std::runtime_error("File '" + filename + "' can't be opened!");
}
/* Neural network loading */
//
// /* Neural network loading */
l4n::NeuralNetwork nn3("test_net_Gradient_Descent.4n");
/* Check of the saved network - write to the file */
......@@ -122,25 +113,25 @@ int main(int argc, char** argv) {
nn3.write_stats(&output_file);
nn3.write_weights(&output_file);
nn3.write_biases(&output_file);
/* Evaluate network on an arbitrary data-set and save results into the file */
//
// /* Evaluate network on an arbitrary data-set and save results into the file */
l4n::CSVReader reader3("/home/fluffymoo/Dropbox/data_BACK_RH_1.csv", ";", true); // File, separator, skip 1st line
reader3.read(); // Read from the file
/* Create data set for both the testing of the neural network */
/* Specify which columns are inputs or outputs */
std::vector<unsigned int> inputs3 = { 0 }; // Possible multiple inputs, e.g. {0,3}, column indices starting from 0
std::vector<unsigned int> outputs3 = { 1 }; // Possible multiple outputs, e.g. {1,2}
l4n::DataSet ds3 = reader3.get_data_set(&inputs3, &outputs3); // Creation of data-set for NN
ds3.normalize(); // Normalization of data to prevent numerical problems
output_file << std::endl << "Evaluating network on the dataset: " << std::endl;
ds3.store_data_text(&output_file);
//
// /* Create data set for both the testing of the neural network */
// /* Specify which columns are inputs or outputs */
//
l4n::DataSet ds3 = reader3.get_data_set(&inputs, &outputs); // Creation of data-set for NN
if(normalize_data){
ds3.normalize(); // Normalization of data to prevent numerical problems
}
//
// output_file << std::endl << "Evaluating network on the dataset: " << std::endl;
// ds3.store_data_text(&output_file);
//
output_file << "Output and the error:" << std::endl;
/* Error function */
//
// /* Error function */
l4n::MSE mse3(&nn3, &ds3); // First parameter - neural network, second parameter - data-set
mse3.eval_on_data_set(&ds3, &output_file);
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment