diff --git a/CMakeLists.txt b/CMakeLists.txt
index 38f6ea91bd4b67a9aee3ff6dc79634484fc64328..c8eb260605d135155c57eb7e931d264f26329621 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -22,7 +22,7 @@ endif (NOT CMAKE_BUILD_TYPE)
# Setting C++ compiler flags #
#--------------------------------#
set(CMAKE_CXX_STANDARD 17)
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall" )
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -mtune=native" )
#--------------------#
# Automatic settings #
diff --git a/src/LearningMethods/ParticleSwarm.cpp b/src/LearningMethods/ParticleSwarm.cpp
index 5370623884c606fee265e4b955d5aeb93e7df019..3f1caefc95336e6e71a8f283a94dcafdc7022ceb 100644
--- a/src/LearningMethods/ParticleSwarm.cpp
+++ b/src/LearningMethods/ParticleSwarm.cpp
@@ -21,9 +21,9 @@ void Particle::randomize_coordinates() {
std::random_device seeder;
std::mt19937 gen(seeder());
- std::uniform_real_distribution<double> dist_coord(-1.0, 1.0);
this->domain_bounds = domain_bounds;
for(unsigned int i = 0; i < this->coordinate_dim; ++i){
+ std::uniform_real_distribution<double> dist_coord(-1, 1);
(*this->coordinate)[i] = dist_coord(gen);
}
}
@@ -300,7 +300,7 @@ void ParticleSwarm::optimize( double gamma, double epsilon, double delta) {
for(size_t pi=0; pi < this->n_particles; pi++) {
particle = this->particle_swarm[pi];
tmp_velocity = particle->change_coordinate( this->w, this->c1, this->c2, *this->p_min_glob, global_best_vec);
- particle->print_coordinate();
+// particle->print_coordinate();
if(tmp_velocity > max_velocity) {
prev_max_velocity = max_velocity;
@@ -325,9 +325,11 @@ void ParticleSwarm::optimize( double gamma, double epsilon, double delta) {
//TODO only in verbose mode
euclidean_dist /= this->n_particles;
- printf("Iteration %d, avg euclidean distance: %f, cluster percent: %f, f-value: %f\n", (int)outer_it, euclidean_dist,
- double(cluster.size())/this->n_particles, optimal_value);
- std::cout.flush();
+ if(outer_it % 10 == 0){
+ printf("Iteration %d, avg euclidean distance: %f, cluster percent: %f, f-value: %f\r", (int)outer_it, euclidean_dist,
+ double(cluster.size())/this->n_particles, optimal_value);
+ std::cout.flush();
+ }
// for(unsigned int i=0; i < this->n_particles; i++) {
// printf("Particle %d (%f): \n", i, this->particle_swarm[i]->get_current_value());
diff --git a/src/Network/NeuralNetwork.cpp b/src/Network/NeuralNetwork.cpp
index 99eb85fa96fae079fe05a4c63c1e500d363f0814..f204caee8eaa1333fbb069ae0abf4aca5b5e648c 100644
--- a/src/Network/NeuralNetwork.cpp
+++ b/src/Network/NeuralNetwork.cpp
@@ -30,7 +30,6 @@ NeuralNetwork::NeuralNetwork() {
this->delete_weights = true;
this->delete_biases = true;
this->layers_analyzed = false;
- this->in_out_determined = false;
}
NeuralNetwork::~NeuralNetwork() {
@@ -369,7 +368,6 @@ size_t NeuralNetwork::add_neuron(Neuron* n, BIAS_TYPE bt, size_t bias_idx) {
this->neurons->push_back(n);
- this->in_out_determined = false;
this->layers_analyzed = false;
return this->neurons->size() - 1;
}
@@ -445,7 +443,7 @@ void NeuralNetwork::set_parameter_space_pointers(NeuralNetwork &parent_network)
}
void NeuralNetwork::eval_single(std::vector<double> &input, std::vector<double> &output, std::vector<double> * custom_weights_and_biases) {
- if(!this->in_out_determined || (this->input_neuron_indices->size() * this->output_neuron_indices->size()) <= 0){
+ if((this->input_neuron_indices->size() * this->output_neuron_indices->size()) <= 0){
std::cerr << "Input and output neurons have not been specified\n" << std::endl;
exit(-1);
}
@@ -460,6 +458,8 @@ void NeuralNetwork::eval_single(std::vector<double> &input, std::vector<double>
std::cerr << "Error, output size != Network output size\n" << std::endl;
exit(-1);
}
+ double potential, bias;
+ int bias_idx;
this->copy_parameter_space( custom_weights_and_biases );
@@ -475,8 +475,6 @@ void NeuralNetwork::eval_single(std::vector<double> &input, std::vector<double>
}
/* we iterate through all the feed-forward layers and transfer the signals */
- double potential, bias;
- int bias_idx;
for( auto layer: *this->neuron_layers_feedforward){
/* we iterate through all neurons in this layer and propagate the signal to the neighboring neurons */
@@ -550,12 +548,12 @@ size_t NeuralNetwork::get_n_biases() {
return this->neuron_biases->size();
}
-size_t NeuralNetwork::get_neuron_bias_index(size_t neuron_idx) {
- return this->neuron_biases->at( neuron_idx );
+int NeuralNetwork::get_neuron_bias_index(size_t neuron_idx) {
+ return this->neuron_bias_indices->at( neuron_idx );
}
size_t NeuralNetwork::get_n_neurons() {
- return this->input_neuron_indices->size();
+ return this->neurons->size();
}
void NeuralNetwork::specify_input_neurons(std::vector<size_t> &input_neurons_indices) {
diff --git a/src/Network/NeuralNetwork.h b/src/Network/NeuralNetwork.h
index fb9918f0836907856d608affda2cf50d0d7f9ca3..589815aa1da3f641f5547efadb1beb815d2298d1 100644
--- a/src/Network/NeuralNetwork.h
+++ b/src/Network/NeuralNetwork.h
@@ -96,12 +96,7 @@ private:
*/
std::vector<std::vector<size_t>*> *neuron_layers_feedbackward = nullptr;
- /**
- *
- */
- bool in_out_determined = false;
-
- /**
+ /**
*
*/
bool layers_analyzed = false;
@@ -255,7 +250,7 @@ public:
*
* @return
*/
- virtual size_t get_neuron_bias_index( size_t neuron_idx );
+ virtual int get_neuron_bias_index( size_t neuron_idx );
/**
*
diff --git a/src/Solvers/DESolver.cpp b/src/Solvers/DESolver.cpp
index e4a2d75fa971583ef0a1797577290779cd07f5fb..dfc9b2aa38e1b86092bd96b932db89ff7a2c1e9c 100644
--- a/src/Solvers/DESolver.cpp
+++ b/src/Solvers/DESolver.cpp
@@ -311,7 +311,7 @@ void DESolver::add_to_differential_equation( size_t equation_idx, MultiIndex &al
void DESolver::set_error_function(size_t equation_idx, ErrorFunctionType F, DataSet *conditions) {
if( equation_idx >= this->n_equations ){
- throw std::invalid_argument("The parameter 'equation_idx' is too large! It exceeds the number of differential equations.");
+ throw std::invalid_argument( "The parameter 'equation_idx' is too large! It exceeds the number of differential equations." );
}
this->errors_functions_types->at( equation_idx ) = F;
@@ -361,18 +361,49 @@ NeuralNetwork* DESolver::get_solution() {
return this->solution;
}
-void DESolver::eval_equation( size_t equation_idx, std::vector<double> *weight_and_biases, std::vector<double> &input ) {
+double DESolver::eval_equation( size_t equation_idx, std::vector<double> &weight_and_biases, std::vector<double> &input ) {
std::vector<double> output(1);
- this->differential_equations->at( equation_idx )->eval_single( input, output, weight_and_biases );
+ this->differential_equations->at( equation_idx )->eval_single( input, output, &weight_and_biases );
- printf("Input: ");
- for( auto e: input ){
- printf("%f, ", e);
+// printf("Input: ");
+// for( auto e: input ){
+// printf("%f, ", e);
+// }
+// printf("\nOutput: ");
+// for( auto e: output ){
+// printf("%f, ", e);
+// }
+// printf("\n");
+
+ return output[0];
+}
+
+double DESolver::eval_total_error(std::vector<double> &weights_and_biases) {
+
+ NeuralNetwork *nn;
+ DataSet *ds;
+
+ /* DEFINITION OF THE PARTIAL ERROR FUNCTIONS */
+ std::vector<ErrorFunction*> error_functions( this->n_equations );
+ for(size_t i = 0; i < this->n_equations; ++i ){
+ nn = this->differential_equations->at( i );
+ ds = this->errors_functions_data_sets->at( i );
+
+ if( this->errors_functions_types->at( i ) == ErrorFunctionType::ErrorFuncMSE ){
+ error_functions[i] = new MSE( nn, ds );
+ }
+ else{
+ //default
+ error_functions[i] = new MSE( nn, ds );
+ }
}
- printf("\nOutput: ");
- for( auto e: output ){
- printf("%f, ", e);
+
+ /* DEFINITION OF THE GLOBAL ERROR FUNCTION */
+ ErrorSum total_error;
+ for(size_t i = 0; i < this->n_equations; ++i ) {
+ total_error.add_error_function( error_functions[i], 1.0 );
}
- printf("\n");
+
+ return total_error.eval( &weights_and_biases );
}
\ No newline at end of file
diff --git a/src/Solvers/DESolver.h b/src/Solvers/DESolver.h
index dd3f48a74d34eb0f342913f858cd9607c2906063..e9e953605f642bff8b89c4411bad5734ed45df05 100644
--- a/src/Solvers/DESolver.h
+++ b/src/Solvers/DESolver.h
@@ -165,7 +165,13 @@ public:
/**
* For testing purposes only
*/
- void eval_equation( size_t equation_idx, std::vector<double> * weights_and_biases, std::vector<double> &input );
+ double eval_equation( size_t equation_idx, std::vector<double> &weights_and_biases, std::vector<double> &input );
+
+ /**
+ * For testing purposes only
+ * @return
+ */
+ double eval_total_error( std::vector<double> &weights_and_biases );
};
diff --git a/src/examples/net_test_1.cpp b/src/examples/net_test_1.cpp
index 1c46189a2d55ce8ed50df43cdcd50af8fa17e87b..b232a3ccc9f8e22e79ba4953b0d4bb80b8eb0aa7 100644
--- a/src/examples/net_test_1.cpp
+++ b/src/examples/net_test_1.cpp
@@ -28,7 +28,7 @@ int main() {
DataSet ds(&data_vec);
/* NETWORK DEFINITION */
- NeuralNetwork net(4, 0);
+ NeuralNetwork net;
/* Input neurons */
NeuronLinear *i1 = new NeuronLinear( ); //f(x) = x
@@ -36,20 +36,23 @@ int main() {
/* Output neuron */
double b = 1.0;//bias
- NeuronLinear *o1 = new NeuronLinear( &b ); //f(x) = x + 1
+ NeuronLinear *o1 = new NeuronLinear( ); //f(x) = x + 1
/* Adding neurons to the net */
- int idx1 = net.add_neuron(i1);
- int idx2 = net.add_neuron(i2);
- int idx3 = net.add_neuron(o1);
+ size_t idx1 = net.add_neuron(i1, BIAS_TYPE::NO_BIAS);
+ size_t idx2 = net.add_neuron(i2, BIAS_TYPE::NO_BIAS);
+ size_t idx3 = net.add_neuron(o1, BIAS_TYPE::NEXT_BIAS);
+
+ std::vector<double> *bv = net.get_parameter_ptr_biases();
+ for(size_t i = 0; i < 1; ++i){
+ bv->at(i) = 1.0;
+ }
/* Adding connections */
- //net.add_connection_simple(idx1, idx3, -1, 1.0);
- //net.add_connection_simple(idx2, idx3, -1, 1.0);
- net.add_connection_simple(idx1, idx3);
- net.add_connection_simple(idx2, idx3);
+ net.add_connection_simple(idx1, idx3, SIMPLE_CONNECTION_TYPE::NEXT_WEIGHT);
+ net.add_connection_simple(idx2, idx3, SIMPLE_CONNECTION_TYPE::NEXT_WEIGHT);
//net.randomize_weights();
diff --git a/src/examples/net_test_2.cpp b/src/examples/net_test_2.cpp
index 27d8bf45501e629a8b0e17432584778c60644dd4..04aae6d7c1a996f4e1784d3b68beed8f827e453e 100644
--- a/src/examples/net_test_2.cpp
+++ b/src/examples/net_test_2.cpp
@@ -40,27 +40,32 @@ int main() {
NeuronLinear *i2 = new NeuronLinear( ); //f(x) = x
double b = 1;//bias
- NeuronLinear *i3 = new NeuronLinear(&b); //f(x) = x + 1
+ NeuronLinear *i3 = new NeuronLinear( ); //f(x) = x + 1
/* Output neurons */
- NeuronLinear *o1 = new NeuronLinear(&b); //f(x) = x + 1
- NeuronLinear *o2 = new NeuronLinear(&b); //f(x) = x + 1
+ NeuronLinear *o1 = new NeuronLinear( ); //f(x) = x + 1
+ NeuronLinear *o2 = new NeuronLinear( ); //f(x) = x + 1
/* Adding neurons to the nets */
- int idx1 = net.add_neuron(i1);
- int idx2 = net.add_neuron(i2);
- int idx3 = net.add_neuron(o1);
- int idx4 = net.add_neuron(i3);
- int idx5 = net.add_neuron(o2);
+ size_t idx1 = net.add_neuron(i1, BIAS_TYPE::NO_BIAS);
+ size_t idx2 = net.add_neuron(i2, BIAS_TYPE::NO_BIAS);
+ size_t idx3 = net.add_neuron(o1, BIAS_TYPE::NEXT_BIAS);
+ size_t idx4 = net.add_neuron(i3, BIAS_TYPE::NEXT_BIAS);
+ size_t idx5 = net.add_neuron(o2, BIAS_TYPE::NEXT_BIAS);
+
+ std::vector<double> *bv = net.get_parameter_ptr_biases();
+ for(size_t i = 0; i < 3; ++i){
+ bv->at(i) = 1.0;
+ }
/* Adding connections */
//net.add_connection_simple(idx1, idx3, -1, 1.0);
//net.add_connection_simple(idx2, idx3, -1, 1.0);
- net.add_connection_simple(idx1, idx3); // weight index 0
- net.add_connection_simple(idx2, idx3); // weight index 1
- net.add_connection_simple(idx4, idx5, 0); // AGAIN weight index 0 - same weight!
+ net.add_connection_simple(idx1, idx3, SIMPLE_CONNECTION_TYPE::NEXT_WEIGHT); // weight index 0
+ net.add_connection_simple(idx2, idx3, SIMPLE_CONNECTION_TYPE::NEXT_WEIGHT); // weight index 1
+ net.add_connection_simple(idx4, idx5, SIMPLE_CONNECTION_TYPE::EXISTING_WEIGHT, 0); // AGAIN weight index 0 - same weight!
net.randomize_weights();
diff --git a/src/examples/net_test_3.cpp b/src/examples/net_test_3.cpp
index 187a74a469e64e30ff9cac5cdce47569827cda29..fa469485d7951a644f4ced341f4f662a61507746 100644
--- a/src/examples/net_test_3.cpp
+++ b/src/examples/net_test_3.cpp
@@ -42,28 +42,30 @@ int main() {
NeuronLinear *i1 = new NeuronLinear(); //f(x) = x
NeuronLinear *i2 = new NeuronLinear(); //f(x) = x
- double b = 1;//bias
- NeuronLinear *i3 = new NeuronLinear( &b ); //f(x) = x + 1
+ NeuronLinear *i3 = new NeuronLinear( ); //f(x) = x + 1
/* Output neurons */
- NeuronLinear *o1 = new NeuronLinear(&b); //f(x) = x + 1
- NeuronLinear *o2 = new NeuronLinear(&b); //f(x) = x + 1
+ NeuronLinear *o1 = new NeuronLinear( ); //f(x) = x + 1
+ NeuronLinear *o2 = new NeuronLinear( ); //f(x) = x + 1
/* Adding neurons to the nets */
- int idx1 = net.add_neuron(i1);
- int idx2 = net.add_neuron(i2);
- int idx3 = net.add_neuron(o1);
- int idx4 = net.add_neuron(i3);
- int idx5 = net.add_neuron(o2);
+ size_t idx1 = net.add_neuron(i1, BIAS_TYPE::NO_BIAS);
+ size_t idx2 = net.add_neuron(i2, BIAS_TYPE::NO_BIAS);
+ size_t idx3 = net.add_neuron(o1, BIAS_TYPE::NEXT_BIAS);
+ size_t idx4 = net.add_neuron(i3, BIAS_TYPE::NEXT_BIAS);
+ size_t idx5 = net.add_neuron(o2, BIAS_TYPE::NEXT_BIAS);
+
+ std::vector<double> *bv = net.get_parameter_ptr_biases();
+ for(size_t i = 0; i < 3; ++i){
+ bv->at(i) = 1.0;
+ }
/* Adding connections */
- //net.add_connection_simple(idx1, idx3, -1, 1.0);
- //net.add_connection_simple(idx2, idx3, -1, 1.0);
- net.add_connection_simple(idx1, idx3); // weight index 0
- net.add_connection_simple(idx2, idx3); // weight index 1
- net.add_connection_simple(idx4, idx5, 0); // AGAIN weight index 0 - same weight!
+ net.add_connection_simple(idx1, idx3, SIMPLE_CONNECTION_TYPE::NEXT_WEIGHT); // weight index 0
+ net.add_connection_simple(idx2, idx3, SIMPLE_CONNECTION_TYPE::NEXT_WEIGHT); // weight index 1
+ net.add_connection_simple(idx4, idx5, SIMPLE_CONNECTION_TYPE::EXISTING_WEIGHT, 0); // AGAIN weight index 0 - same weight!
net.randomize_weights();
diff --git a/src/examples/net_test_ode_1.cpp b/src/examples/net_test_ode_1.cpp
index f458e54eb8323e3fd7f0298f43d74ee43003e0d3..106ee9d1a9912c40d1f69ec0624197d6ab811c79 100644
--- a/src/examples/net_test_ode_1.cpp
+++ b/src/examples/net_test_ode_1.cpp
@@ -566,7 +566,6 @@ void test_odr(double accuracy, size_t n_inner_neurons, size_t train_size, double
/* neumann boundary condition */
solver_01.add_to_differential_equation( 2, alpha_1, 1.0 );
-
/* SETUP OF THE TRAINING DATA */
std::vector<double> inp, out;
@@ -574,6 +573,7 @@ void test_odr(double accuracy, size_t n_inner_neurons, size_t train_size, double
/* TRAIN DATA FOR THE GOVERNING DE */
std::vector<std::pair<std::vector<double>, std::vector<double>>> data_vec_g;
+ std::vector<double> test_points(train_size);
/* ISOTROPIC TRAIN SET */
@@ -582,6 +582,8 @@ void test_odr(double accuracy, size_t n_inner_neurons, size_t train_size, double
inp = {frac * i};
out = {0.0};
data_vec_g.emplace_back(std::make_pair(inp, out));
+
+ test_points[i] = inp[0];
}
/* CHEBYSCHEV TRAIN SET */
@@ -591,6 +593,8 @@ void test_odr(double accuracy, size_t n_inner_neurons, size_t train_size, double
// inp = {(std::cos(alpha * i) + 1.0) * frac + d1_s};
// out = {0.0};
// data_vec_g.emplace_back(std::make_pair(inp, out));
+//
+// test_points[i] = inp[0];
// }
DataSet ds_00(&data_vec_g);
@@ -614,21 +618,68 @@ void test_odr(double accuracy, size_t n_inner_neurons, size_t train_size, double
solver_01.set_error_function( 2, ErrorFunctionType::ErrorFuncMSE, &ds_02 );
+ std::vector<double> params(3 * n_inner_neurons), params_analytical(3 * n_inner_neurons);
+ std::random_device seeder;
+ std::mt19937 gen(seeder());
+ std::uniform_real_distribution<double> dist(-10.0, 10.0);
+
+ std::vector<double> input(1);
+ for( size_t testi = 0; testi < 50; ++testi ){
+ double test_error_eq1 = 0.0, total_error = 0.0;
+ for(size_t i = 0; i < params.size(); ++i){
+ params[i] = dist(gen);
+ }
+ for(size_t i = 0; i < n_inner_neurons; ++i){
+ params_analytical[3 * i] = params[i];
+ params_analytical[3 * i + 1] = params[n_inner_neurons + i];
+ params_analytical[3 * i + 2] = params[2 * n_inner_neurons + i];
+ }
+
+ for(auto d: *ds_00.get_data()){
+ input = d.first;
+ double x = input[0];
+
+ double analytical_value_f = eval_approx_f(x, n_inner_neurons, params_analytical);
+ double analytical_value_df = eval_approx_df(x, n_inner_neurons, params_analytical);
+ double analytical_value_ddf = eval_approx_ddf(x, n_inner_neurons, params_analytical);
+
+ double de_solver_value_eq1 = solver_01.eval_equation(0, params, input);
+ double analytical_value_eq1 = 4 * analytical_value_f + 4 * analytical_value_df + analytical_value_ddf;
+ test_error_eq1 += (de_solver_value_eq1 - analytical_value_eq1) * (de_solver_value_eq1 - analytical_value_eq1);
+
+ }
+ input[0] = 0.0;
+ double de_solver_value_eq2 = solver_01.eval_equation(1, params, input);
+ double analytical_value_eq2 = eval_approx_f(0.0, n_inner_neurons, params_analytical);
+ double test_error_eq2 = (de_solver_value_eq2 - analytical_value_eq2) * (de_solver_value_eq2 - analytical_value_eq2);
+
+ double de_solver_value_eq3 = solver_01.eval_equation(2, params, input);
+ double analytical_value_eq3 = eval_approx_df(0.0, n_inner_neurons, params_analytical);
+ double test_error_eq3 = (de_solver_value_eq3 - analytical_value_eq3) * (de_solver_value_eq3 - analytical_value_eq3);
+
+ double total_error_de_solver = solver_01.eval_total_error(params);
+
+ double total_error_analytical = eval_error_function(params_analytical, n_inner_neurons, test_points);
+
+ printf("\tRepresentation test %6d, error of eq1: %10.8f, error of eq2: %10.8f, error of eq3: %10.8f, total error: %10.8f\n", (int)testi, std::sqrt(test_error_eq1), std::sqrt(test_error_eq2), std::sqrt(test_error_eq3), (total_error_analytical - total_error_de_solver) * (total_error_analytical - total_error_de_solver));
+ }
+
+return;
/* PARTICLE SWARM TRAINING METHOD SETUP */
//must encapsulate each of the partial error functions
double *domain_bounds = new double[ 6 * n_inner_neurons ];
for(unsigned int i = 0; i < 3 * n_inner_neurons; ++i){
- domain_bounds[2 * i] = -800.0;
- domain_bounds[2 * i + 1] = 800.0;
+ domain_bounds[2 * i] = -10.0;
+ domain_bounds[2 * i + 1] = 10.0;
}
- double c1 = 0.05, c2 = 0.0, w = 0.3;
+ double c1 = 0.5, c2 = 0.8, w = 0.7;
- double gamma = 0.5, epsilon = 0.0000000000002, delta = 1.1;
+ double gamma = 0.5, epsilon = 0.02, delta = 0.9;
solver_01.solve_via_particle_swarm( domain_bounds, c1, c2, w, n_particles, max_iters, gamma, epsilon, delta );
@@ -642,7 +693,7 @@ void test_odr(double accuracy, size_t n_inner_neurons, size_t train_size, double
parameters[3 * i + 2] = biases_params->at(i);
}
- std::vector<double> input(1), output(1);
+ std::vector<double> output(1);
double x;
for(unsigned int i = 0; i < n_test_points; ++i){
@@ -661,7 +712,7 @@ void test_odr(double accuracy, size_t n_inner_neurons, size_t train_size, double
int main() {
unsigned int n_inner_neurons = 2;
- unsigned int train_size = 200;
+ unsigned int train_size = 150;
double accuracy = 1e-4;
double ds = 0.0;
double de = 4.0;
@@ -682,9 +733,8 @@ int main() {
//
// std::random_device seeder;
// std::mt19937 gen(seeder());
-// std::uniform_real_distribution<double> dist(-1.0, 1.0);
-// for(unsigned int i = 0; i < 3 * n_inner_neurons; ++i){
-// init_guess[i] = dist(gen);
+// std::uniform_real_distribution<double> dist(-10.0, 10.0);
+// for(unsigned int i = 0; i < init_guess.size(); ++i){
// init_guess[i] = dist(gen);
// }
// if(!optimize_biases){