diff --git a/CMakeLists.txt b/CMakeLists.txt
index 1ea507921a4155151d0b757eaddd975562368767..0d83c68044266fd10dc2131c4af68dc86fa74511 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -22,7 +22,7 @@ if (NOT CMAKE_BUILD_TYPE)
"Choose the type of build, options are: None Debug Release."
FORCE)
elseif(CMAKE_BUILD_TYPE STREQUAL "Debug")
- #TODO rewrite to use add_compile_definitions
+ add_compile_definitions("L4N_DEBUG")
endif (NOT CMAKE_BUILD_TYPE)
#------------------------#
diff --git a/build_scripts/linux/clean_dependencies.sh b/build_scripts/linux/clean_dependencies.sh
index 3b4ead63bb41002d099ab633e7c62349c5ee8cb7..1bbae3c794a4ee4ddee92fab1fcbcce8b7385abe 100755
--- a/build_scripts/linux/clean_dependencies.sh
+++ b/build_scripts/linux/clean_dependencies.sh
@@ -1,4 +1,5 @@
#!/bin/sh
-rm ../../external_dependencies/boost/*
-rm ../../external_dependencies/exprtk/*
+rm -rf ../../external_dependencies/boost/*
+rm -rf ../../external_dependencies/exprtk/*
+rm -rf ../../external_dependencies/turtle/*
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 5d018c5c95c68ee42a757c69ec576f21e0c966f0..0b8c2abc63f633246f87d1ed999fc5bd5025a11e 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -49,6 +49,7 @@ if ("${BUILD_LIB}" STREQUAL "yes")
Exception/Exceptions.cpp
CSVReader/CSVReader.cpp
CrossValidator/CrossValidator.cpp
+ NormalizationStrategy/NormalizationStrategy.cpp
)
target_link_libraries(
diff --git a/src/CrossValidator/CrossValidator.cpp b/src/CrossValidator/CrossValidator.cpp
index 373ec7aa8ba87e9858af4e524f1f1ead58ef6cf8..6be29fca98a78cbc9549216ed673b1b59ce3a0e5 100644
--- a/src/CrossValidator/CrossValidator.cpp
+++ b/src/CrossValidator/CrossValidator.cpp
@@ -12,14 +12,13 @@ namespace lib4neuro {
}
LIB4NEURO_API void CrossValidator::run_k_fold_test(unsigned int k, unsigned int tests_number) {
-
NeuralNetwork *net = this->ef->get_network_instance();
for(unsigned int i = 0; i < tests_number; i++) {
- std::cout << "Cross-validation run " << i+1 << std::endl;
+ COUT_DEBUG(<< "Cross-validation run " << i+1 << std::endl);
this->ef->divide_data_train_test(1.0/k);
- std::cout << "Cross-validation run " << i+1 << ", number of train data points: " << this->ef->get_dataset()->get_n_elements() << std::endl;
+ COUT_DEBUG(<< "number of train data points: " << this->ef->get_dataset()->get_n_elements() << std::endl);
net->print_weights();
net->randomize_parameters();
diff --git a/src/DataSet/DataSet.cpp b/src/DataSet/DataSet.cpp
index 83403f8c524aece678cc9aec6f71f4e940313426..35ddee5c5ad4d95092d32cabddc69e01487ba470 100644
--- a/src/DataSet/DataSet.cpp
+++ b/src/DataSet/DataSet.cpp
@@ -2,12 +2,17 @@
// Created by martin on 7/13/18.
//
+#include <algorithm>
#include "DataSetSerialization.h"
namespace lib4neuro {
- DataSet::DataSet() {}
+ DataSet::DataSet() {
+ this->n_elements = 0;
+ this->input_dim = 0;
+ this->output_dim = 0;
+ }
DataSet::DataSet(std::string file_path) {
std::ifstream ifs(file_path);
@@ -47,7 +52,6 @@ namespace lib4neuro {
this->add_isotropic_data(bounds, no_elems_in_one_dim, output_func);
}
-
void DataSet::add_data_pair(std::vector<double> &inputs, std::vector<double> &outputs) {
if(this->n_elements == 0 && this->input_dim == 0 && this->output_dim == 0) {
this->input_dim = inputs.size();
@@ -184,4 +188,44 @@ namespace lib4neuro {
return v_combined;
}
+ void DataSet::normalize() {
+ /* Find maximum and minimum values */
+ this->max_inp_val = this->min_inp_val = this->data[0].first.at(0);
+ double tmp, tmp2;
+ for(auto pair : this->data) {
+ /* Finding maximum */
+ //TODO make more efficiently
+ tmp = *std::max_element(pair.first.begin(), pair.first.end());
+ tmp2 = *std::max_element(pair.second.begin(), pair.second.end());
+
+ tmp = std::max(tmp, tmp2);
+
+ if (tmp > this->max_inp_val) {
+ this->max_inp_val = tmp;
+ }
+
+ /* Finding minimum */
+ tmp = *std::min_element(pair.first.begin(), pair.first.end());
+ tmp2 = *std::min_element(pair.second.begin(), pair.second.end());
+
+ tmp = std::min(tmp, tmp2);
+
+ if (tmp < this->min_inp_val) {
+ this->min_inp_val = tmp;
+ }
+ }
+
+ /* Normalize every number in the data set */
+ for(auto& pair : this->data) {
+ for(auto& v : pair.first) {
+ v = normalization_strategy->normalize(v, this->max_inp_val, this->min_inp_val);
+ }
+
+ for(auto& v : pair.second) {
+ v = normalization_strategy->normalize(v, this->max_inp_val, this->min_inp_val);
+ }
+ }
+
+ }
+
}
\ No newline at end of file
diff --git a/src/DataSet/DataSet.h b/src/DataSet/DataSet.h
index d31762c992d57ae91b2521c9da1e3b351222dd22..b12e4f5634e4096943a35a07924290868d1c0963 100644
--- a/src/DataSet/DataSet.h
+++ b/src/DataSet/DataSet.h
@@ -14,6 +14,7 @@
#include "../settings.h"
#include "../Exception/Exceptions.h"
+#include "../NormalizationStrategy/NormalizationStrategy.h"
namespace lib4neuro {
/**
@@ -39,6 +40,16 @@ namespace lib4neuro {
*/
size_t output_dim = 0;
+ /**
+ * Maximum input value
+ */
+ double max_inp_val;
+
+ /**
+ * Minimum input value
+ */
+ double min_inp_val;
+
/**
* Stored data in the format of pairs of corresponding
* input and output vectors
@@ -48,6 +59,13 @@ namespace lib4neuro {
template<class T>
std::vector<std::vector<T>> cartesian_product(const std::vector<std::vector<T>> *v);
+ /**
+ *
+ */
+ //TODO let user choose in the constructor!
+ NormalizationStrategy* normalization_strategy = new DoubleUnitStrategy;
+
+
public:
/**
@@ -171,6 +189,14 @@ namespace lib4neuro {
* Stores the DataSet object to the binary file
*/
LIB4NEURO_API void store_text(std::string &file_path);
+
+ /**
+ * Normalizes the data set into [-1,1] interval
+ *
+ * Former maximum and minimum values are stored into the
+ * private variables max_inp_val and min_inp_val
+ */
+ LIB4NEURO_API void normalize();
};
}
#endif //INC_4NEURO_DATASET_H
diff --git a/src/ErrorFunction/ErrorFunctions.cpp b/src/ErrorFunction/ErrorFunctions.cpp
index 164b375b8f3141a7aa08d29c581aa106cb3b14b8..394ba231c8c4d9543857b92ae4b5297d61ba431a 100644
--- a/src/ErrorFunction/ErrorFunctions.cpp
+++ b/src/ErrorFunction/ErrorFunctions.cpp
@@ -30,7 +30,6 @@ namespace lib4neuro {
boost::random::uniform_int_distribution<> dist(0, ds_size - 1);
size_t test_set_size = ceil(ds_size * percent_test);
- printf("new train set size: %d\n", test_set_size);
std::vector<unsigned int> test_indices;
test_indices.reserve(test_set_size);
@@ -115,6 +114,9 @@ namespace lib4neuro {
for (auto el: *data) { // Iterate through every element in the test set
+ /* Input normalization to [0,1] interval */
+ //TODO
+
this->net->eval_single(el.first, error_derivative,
¶ms); // Compute the net output and store it into 'output' variable
diff --git a/src/Network/NeuralNetwork.cpp b/src/Network/NeuralNetwork.cpp
index 94c8d4889ce3ad05b1dfb145a09fc58519038746..06606068bb47b6b0dc0f8a3dc5a38ab6579db879 100644
--- a/src/Network/NeuralNetwork.cpp
+++ b/src/Network/NeuralNetwork.cpp
@@ -12,8 +12,6 @@
#include "NeuralNetwork.h"
#include "NeuralNetworkSerialization.h"
-/* Seed for the random number generator */
-boost::random::mt19937 gen;
namespace lib4neuro {
NeuralNetwork::NeuralNetwork() {
@@ -595,7 +593,7 @@ namespace lib4neuro {
void NeuralNetwork::randomize_weights() {
-// boost::random::mt19937 gen;
+ boost::random::mt19937 gen(std::time(0));
// Init weight guess ("optimal" for logistic activation functions)
double r = 4 * sqrt(6. / (this->connection_weights->size()));
@@ -609,7 +607,7 @@ namespace lib4neuro {
void NeuralNetwork::randomize_biases() {
-// boost::random::mt19937 gen;
+ boost::random::mt19937 gen(std::time(0));
// Init weight guess ("optimal" for logistic activation functions)
boost::random::uniform_real_distribution<> dist(-1, 1);
@@ -911,8 +909,9 @@ namespace lib4neuro {
unsigned int inp_dim = neuron_numbers->at(0); //!< Network input dimension
unsigned int out_dim = neuron_numbers->back(); //!< Network output dimension
- printf("# of inputs: %d\n", inp_dim);
- printf("# of outputs: %d\n", out_dim);
+ COUT_DEBUG(<< "Fully connected feed-forward network:" << std::endl)
+ COUT_DEBUG(<< "# of inputs: " << inp_dim << std::endl);
+ COUT_DEBUG(<< "# of outputs: " << out_dim << std::endl);
std::vector<size_t> input_layer_neuron_indices;
std::vector<size_t> previous_layer_neuron_indices;
diff --git a/src/Network/NeuralNetwork.h b/src/Network/NeuralNetwork.h
index 223aa26ce435d78eb97f88d148649ac80b820273..02c5b3408377d73b6d5d895e881fd5c55485e193 100644
--- a/src/Network/NeuralNetwork.h
+++ b/src/Network/NeuralNetwork.h
@@ -205,8 +205,6 @@ namespace lib4neuro {
LIB4NEURO_API virtual void eval_single(std::vector<double> &input, std::vector<double> &output,
std::vector<double> *custom_weights_and_biases = nullptr);
-
-
/**
*
* @param error_derivative
diff --git a/src/Neuron/Neuron.h b/src/Neuron/Neuron.h
index fd81dde8aef8eb6be179f34e133fc338d8c99ac8..fb8142d4223c7c7953b36ad5ad9fd443488c1334 100644
--- a/src/Neuron/Neuron.h
+++ b/src/Neuron/Neuron.h
@@ -15,7 +15,7 @@
namespace lib4neuro {
- enum NeuronType {
+ enum NEURON_TYPE {
BINARY,
CONSTANT,
LINEAR,
diff --git a/src/Neuron/NeuronLogistic.cpp b/src/Neuron/NeuronLogistic.cpp
index 1af9fde526e7821d78305582b712f0fe691cb61e..b9dcca793f7d9c6acdde5cf0b0ffaf6ab6df3793 100644
--- a/src/Neuron/NeuronLogistic.cpp
+++ b/src/Neuron/NeuronLogistic.cpp
@@ -97,9 +97,21 @@ namespace lib4neuro {
double NeuronLogistic::activation_function_eval_derivative_bias(double x, double b) {
//-e^(b - x)/(e^(b - x) + 1)^2
double ex = std::pow(E, b - x);
+// if(std::isinf(ex)) {
+// ex = 10e35;
+// }
+
double denom = (ex + 1);
- return -ex / (denom * denom);
+ double res = -ex / (denom * denom);
+
+// if(std::isnan(res)) {
+// throw std::runtime_error("Derivative results in NeuronLogistic is NaN!");
+// } else if (std::abs(res) < 10e-8) {
+// res = 0;
+// }
+
+ return res;
}
diff --git a/src/NormalizationStrategy/NormalizationStrategy.cpp b/src/NormalizationStrategy/NormalizationStrategy.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..7e83d15ff683bfeea17bb820d2863a2854557b7d
--- /dev/null
+++ b/src/NormalizationStrategy/NormalizationStrategy.cpp
@@ -0,0 +1,11 @@
+//
+// Created by martin on 21.11.18.
+//
+
+#include "NormalizationStrategy.h"
+
+DoubleUnitStrategy::DoubleUnitStrategy() {}
+
+double DoubleUnitStrategy::normalize(double n, double max, double min) {
+ return 2*(n - min)/(max - min) - 1;
+}
\ No newline at end of file
diff --git a/src/NormalizationStrategy/NormalizationStrategy.h b/src/NormalizationStrategy/NormalizationStrategy.h
new file mode 100644
index 0000000000000000000000000000000000000000..ec7548cad5166517844df07960b16175830af8d4
--- /dev/null
+++ b/src/NormalizationStrategy/NormalizationStrategy.h
@@ -0,0 +1,28 @@
+//
+// Created by martin on 21.11.18.
+//
+
+#ifndef LIB4NEURO_NORMALIZATIONSTRATEGY_H
+#define LIB4NEURO_NORMALIZATIONSTRATEGY_H
+
+class NormalizationStrategy {
+public:
+ virtual double normalize(double n, double max, double min) = 0;
+};
+
+class DoubleUnitStrategy : public NormalizationStrategy {
+public:
+ DoubleUnitStrategy();
+
+ /**
+ * Normalizes the input value to the interval [-1,1]
+ *
+ * @param n
+ * @param max
+ * @param min
+ * @return
+ */
+ double normalize(double n, double max, double min) override;
+};
+
+#endif //LIB4NEURO_NORMALIZATIONSTRATEGY_H
diff --git a/src/examples/main.cpp b/src/examples/main.cpp
index 5c766604a55e579be9cee8a94e103291c63ce6d0..b8d4580f84f48697389504d0a0575956de56a728 100644
--- a/src/examples/main.cpp
+++ b/src/examples/main.cpp
@@ -17,18 +17,21 @@
int main(int argc, char** argv){
+ /* Read data from the file */
l4n::CSVReader reader("/home/martin/Desktop/ANN_DATA_1_SET.txt", "\t", true);
reader.read();
-// std::vector<std::vector<std::string>>* data = reader.get_data();
-
+ /* Create data set for both the training and testing of the neural network */
std::vector<unsigned int> inputs = {2,3,4,5,6,7,8,26,27,28};
std::vector<unsigned int> outputs = {17,18,19,20,21,22,23,24,25};
l4n::DataSet ds = reader.get_data_set(&inputs, &outputs);
+ /* Normalize data in the set for easier training of the network */
+ ds.normalize();
+
/* Neural network construction */
- std::vector<unsigned int> neuron_numbers_in_layers = {10,9};
- l4n::FullyConnectedFFN nn(&neuron_numbers_in_layers, l4n::NeuronType::LOGISTIC);
+ std::vector<unsigned int> neuron_numbers_in_layers = {10,10,10,9};
+ l4n::FullyConnectedFFN nn(&neuron_numbers_in_layers, l4n::NEURON_TYPE::LOGISTIC);
/* Error function */
l4n::MSE mse(&nn, &ds);
@@ -36,17 +39,38 @@ int main(int argc, char** argv){
/* Domain */
std::vector<double> domain_bounds(2 * (nn.get_n_weights() + nn.get_n_biases()));
- for(size_t i = 0; i < domain_bounds.size() / 2; ++i){
- domain_bounds[2 * i] = -20;
- domain_bounds[2 * i + 1] = 20;
- }
/* Training method */
-// l4n::ParticleSwarm ps(&domain_bounds);
- l4n::GradientDescent gs;
+// for(size_t i = 0; i < domain_bounds.size() / 2; ++i){
+// domain_bounds[2 * i] = -10;
+// domain_bounds[2 * i + 1] = 10;
+// }
+// l4n::ParticleSwarm ps(&domain_bounds,
+// 1.711897,
+// 1.711897,
+// 0.711897,
+// 0.5,
+// 20,
+// 0.7,
+// 600,
+// 1000);
+ l4n::GradientDescent gs(1e-3, 1);
+
+ nn.randomize_weights();
/* Cross - validation */
l4n::CrossValidator cv(&gs, &mse);
cv.run_k_fold_test(10, 3);
+ /* Save network to the file */
+ nn.save_text("test_net.4n");
+
+ /* Check of the saved network */
+ std::cout << "The original network info:" << std::endl;
+ nn.print_stats();
+
+ l4n::NeuralNetwork nn_loaded("test_net.4n");
+ std::cout << "The loaded network info:" << std::endl;
+ nn_loaded.print_stats();
+
return 0;
}
diff --git a/src/examples/seminar.cpp b/src/examples/seminar.cpp
index 45117e492854bf2c83a1f46a6af9a4b31dd56065..a52a1425b388166a3876bc67bd3093678c6923ee 100644
--- a/src/examples/seminar.cpp
+++ b/src/examples/seminar.cpp
@@ -10,7 +10,6 @@
#include <fstream>
#include "4neuro.h"
-#include "../Solvers/DESolver.h"
int main() {
@@ -18,14 +17,14 @@ int main() {
std::cout << "********************************************************************************************************************************************" <<std::endl;
- NeuralNetwork XOR;
- unsigned int i1 = XOR.add_neuron( new NeuronLinear( ), BIAS_TYPE::NO_BIAS );
- unsigned int i2 = XOR.add_neuron( new NeuronLinear( ), BIAS_TYPE::NO_BIAS );
+ l4n::NeuralNetwork XOR;
+ unsigned int i1 = XOR.add_neuron( new l4n::NeuronLinear( ), l4n::BIAS_TYPE::NO_BIAS );
+ unsigned int i2 = XOR.add_neuron( new l4n::NeuronLinear( ), l4n::BIAS_TYPE::NO_BIAS );
- unsigned int h1 = XOR.add_neuron( new NeuronLogistic( ) );
- unsigned int h2 = XOR.add_neuron( new NeuronLogistic( ) );
+ unsigned int h1 = XOR.add_neuron( new l4n::NeuronLogistic( ) );
+ unsigned int h2 = XOR.add_neuron( new l4n::NeuronLogistic( ) );
- unsigned int o1 = XOR.add_neuron( new NeuronLinear( ), BIAS_TYPE::NO_BIAS );
+ unsigned int o1 = XOR.add_neuron( new l4n::NeuronLinear( ), l4n::BIAS_TYPE::NO_BIAS );
XOR.add_connection_simple( i1, h1 );
XOR.add_connection_simple( i2, h1 );
@@ -56,7 +55,7 @@ int main() {
out = {0};
data_vec.emplace_back(std::make_pair(inp, out));
- DataSet ds(&data_vec);
+ l4n::DataSet ds(&data_vec);
/* specification of the input/output neurons */
std::vector<size_t> net_input_neurons_indices(2);
@@ -71,7 +70,7 @@ int main() {
/* ERROR FUNCTION SPECIFICATION */
- MSE mse(&XOR, &ds);
+ l4n::MSE mse(&XOR, &ds);
@@ -98,17 +97,15 @@ int main() {
double epsilon = 0.02;
double delta = 0.7;
- ParticleSwarm swarm_01(
- &domain_bounds,
- c1,
- c2,
- w,
- gamma,
- epsilon,
- delta,
- n_particles,
- iter_max
- );
+ l4n::ParticleSwarm swarm_01(&domain_bounds,
+ c1,
+ c2,
+ w,
+ gamma,
+ epsilon,
+ delta,
+ n_particles,
+ iter_max);
swarm_01.optimize( mse );
std::vector<double> *parameters = swarm_01.get_parameters( );
diff --git a/src/message.h b/src/message.h
index b04fccb24f4abdbdb320d66f0fc735e53e0bff0c..a0f0a05de6e41061e3a43884cf006178c873ee28 100644
--- a/src/message.h
+++ b/src/message.h
@@ -7,7 +7,12 @@
#include <cassert>
-#define MSG_INFO(str) std::cout << "INFO: " << str << std::endl;
-#define MSG_DEBUG(str) assert(std::cerr << "DEBUG:" << str << std::endl);
+#define COUT_INFO(inp) std::cout << "INFO: " inp;
+
+#ifdef L4N_DEBUG
+#define COUT_DEBUG(inp) assert(std::cerr << "DEBUG: " inp);
+#else
+#define COUT_DEBUG(inp)
+#endif // L4N_DEBUG
#endif //PROJECT_MESSAGE_H