net_test_3.cpp

/**
 * Example of a set of neural networks sharing some edge weights
 * The system of equations associated with the net in this example is not regular
 * minimizes the function: [(2y+0.5)^2 + (2x+y+0.25)^2] / 2 + [(4.5x + 0.37)^2] / 1
 * minimum [0.010024714] at (x, y) = (-333/4370, -9593/43700) = (-0.076201373, -0.219519451)
 * */

//
// Created by martin on 7/16/18.
//

#include <vector>

#include "4neuro.h"

int main() {
    std::cout << "Running lib4neuro example   3: Use of the particle swarm method to train a set of networks sharing some edge weights" << std::endl;
    std::cout << "********************************************************************************************************************" <<std::endl;

    /* TRAIN DATA DEFINITION */
    std::vector<std::pair<std::vector<double>, std::vector<double>>> data_vec_01, data_vec_02;
    std::vector<double> inp, out;

    inp = {0, 1};
    out = {0.5};
    data_vec_01.emplace_back(std::make_pair(inp, out));

    inp = {1, 0.5};
    out = {0.75};
    data_vec_01.emplace_back(std::make_pair(inp, out));

    l4n::DataSet ds_01(&data_vec_01);


    inp = {1.25};
    out = {0.63};
    data_vec_02.emplace_back(std::make_pair(inp, out));
    l4n::DataSet ds_02(&data_vec_02);

    /* NETWORK DEFINITION */
    l4n::NeuralNetwork net;

    /* Input neurons */
    l4n::NeuronLinear *i1 = new l4n::NeuronLinear();  //f(x) = x
    l4n::NeuronLinear *i2 = new l4n::NeuronLinear();  //f(x) = x

    l4n::NeuronLinear *i3 = new l4n::NeuronLinear( ); //f(x) = x

    /* Output neurons */
    l4n::NeuronLinear *o1 = new l4n::NeuronLinear( );  //f(x) = x
    l4n::NeuronLinear *o2 = new l4n::NeuronLinear( );  //f(x) = x



    /* Adding neurons to the nets */
    size_t idx1 = net.add_neuron(i1, l4n::BIAS_TYPE::NO_BIAS);
    size_t idx2 = net.add_neuron(i2, l4n::BIAS_TYPE::NO_BIAS);
    size_t idx3 = net.add_neuron(o1, l4n::BIAS_TYPE::NEXT_BIAS);
    size_t idx4 = net.add_neuron(i3, l4n::BIAS_TYPE::NEXT_BIAS);
    size_t idx5 = net.add_neuron(o2, l4n::BIAS_TYPE::NEXT_BIAS);

    /* Adding connections */
    net.add_connection_simple(idx1, idx3, l4n::SIMPLE_CONNECTION_TYPE::NEXT_WEIGHT); // weight index 0
    net.add_connection_simple(idx2, idx3, l4n::SIMPLE_CONNECTION_TYPE::NEXT_WEIGHT); // weight index 1
    net.add_connection_simple(idx4, idx5, l4n::SIMPLE_CONNECTION_TYPE::EXISTING_WEIGHT, 0); // AGAIN weight index 0 - same weight!

    net.randomize_weights();

    /* specification of the input/output neurons */
    std::vector<size_t> net_input_neurons_indices(3);
    std::vector<size_t> net_output_neurons_indices(2);
    net_input_neurons_indices[0] = idx1;
    net_input_neurons_indices[1] = idx2;
    net_input_neurons_indices[2] = idx4;

    net_output_neurons_indices[0] = idx3;
    net_output_neurons_indices[1] = idx5;

    net.specify_input_neurons(net_input_neurons_indices);
    net.specify_output_neurons(net_output_neurons_indices);


    /* CONSTRUCTION OF SUBNETWORKS */
    //TODO subnetworks retain the number of weights, could be optimized to include only the used weights
    std::vector<size_t> subnet_01_input_neurons, subnet_01_output_neurons;
    std::vector<size_t> subnet_02_input_neurons, subnet_02_output_neurons;

    subnet_01_input_neurons.push_back(idx1);
    subnet_01_input_neurons.push_back(idx2);
    subnet_01_output_neurons.push_back(idx3);
    l4n::NeuralNetwork *subnet_01 = net.get_subnet( subnet_01_input_neurons, subnet_01_output_neurons );

    subnet_02_input_neurons.push_back(idx4);
    subnet_02_output_neurons.push_back(idx5);
    l4n::NeuralNetwork *subnet_02 = net.get_subnet( subnet_02_input_neurons, subnet_02_output_neurons );

    if(subnet_01 && subnet_02){
        /* COMPLEX ERROR FUNCTION SPECIFICATION */
        l4n::MSE mse_01(subnet_01, &ds_01);
        l4n::MSE mse_02(subnet_02, &ds_02);

        l4n::ErrorSum mse_sum;
        mse_sum.add_error_function( &mse_01 );
        mse_sum.add_error_function( &mse_02 );

        /* TRAINING METHOD SETUP */
        std::vector<double> domain_bounds = {-10.0, 10.0, -10.0, 10.0,-10.0, 10.0, -10.0, 10.0,-10.0, 10.0, -10.0, 10.0,-10.0, 10.0, -10.0, 10.0,-10.0, 10.0, -10.0, 10.0,-10.0, 10.0, -10.0, 10.0,-10.0, 10.0, -10.0, 10.0};
        ParticleSwarm swarm_01(&mse_sum, &domain_bounds);

        /* if the maximal velocity from the previous step is less than 'gamma' times the current maximal velocity, then one
         * terminating criterion is met */
        double gamma = 0.5;

        /* if 'delta' times 'n' particles are in the centroid neighborhood given by the radius 'epsilon', then the second
         * terminating criterion is met ('n' is the total number of particles) */
        double epsilon = 0.02;
        double delta = 0.9;
        swarm_01.optimize(gamma, epsilon, delta);


    }
    else{
        std::cout << "We apologize, this example is unfinished as we are in the process of developing methods for efficient subnetwork definition" << std::endl;
    }

    if(subnet_01){
        delete subnet_01;
        subnet_01 = nullptr;
    }

    if(subnet_02){
        delete subnet_02;
        subnet_02 = nullptr;
    }

    return 0;
}