From 04d751a222725304dc26a72a4be0b0d251605f27 Mon Sep 17 00:00:00 2001
From: mrovma <martin.mrovec@vsb.cz>
Date: Thu, 9 May 2019 16:59:12 +0200
Subject: [PATCH] ENH: Added new neurons - Absolute, LinearSaturated
---
include/4neuro.h | 2 +
src/CMakeLists.txt | 2 +
src/DataSet/DataSet.cpp | 12 ++++
src/DataSet/DataSet.h | 6 ++
src/Neuron/NeuronAbsolute.cpp | 39 +++++++++++
src/Neuron/NeuronAbsolute.h | 70 +++++++++++++++++++
src/Neuron/NeuronAbsoluteSerialization.h | 48 +++++++++++++
src/Neuron/NeuronLinearSaturated.cpp | 64 +++++++++++++++++
src/Neuron/NeuronLinearSaturated.h | 36 ++++++++++
.../NeuronLinearSaturatedSerialization.h | 49 +++++++++++++
10 files changed, 328 insertions(+)
create mode 100644 src/Neuron/NeuronAbsolute.cpp
create mode 100644 src/Neuron/NeuronAbsolute.h
create mode 100644 src/Neuron/NeuronAbsoluteSerialization.h
create mode 100644 src/Neuron/NeuronLinearSaturated.cpp
create mode 100644 src/Neuron/NeuronLinearSaturated.h
create mode 100644 src/Neuron/NeuronLinearSaturatedSerialization.h
diff --git a/include/4neuro.h b/include/4neuro.h
index 6f076b20..731b1bb0 100644
--- a/include/4neuro.h
+++ b/include/4neuro.h
@@ -12,9 +12,11 @@
#include "../src/Neuron/NeuronLinear.h"
#include "../src/Neuron/NeuronLogistic.h"
#include "../src/Neuron/NeuronBiased.h"
+#include "../src/Neuron/NeuronAbsolute.h"
#include "../src/Neuron/NeuronBinaryBiased.h"
#include "../src/Neuron/NeuronFilter.h"
#include "../src/Neuron/NeuronRectifier.h"
+#include "../src/Neuron/NeuronLinearSaturated.h"
#include "../src/Solvers/DESolver.h"
#include "../src/ErrorFunction/ErrorFunctions.h"
#include "../src/LearningMethods/ParticleSwarm.h"
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index d9d2f75c..f8f6b9b6 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -87,10 +87,12 @@ IF("${BUILD_LIB}" STREQUAL "yes")
Neuron/NeuronBinary.cpp
Neuron/NeuronConstant.cpp
Neuron/NeuronLinear.cpp
+ Neuron/NeuronAbsolute.cpp
Neuron/NeuronLogistic.cpp
Neuron/NeuronRectifier.cpp
Neuron/NeuronFilter.cpp
Neuron/NeuronBinaryBiased.cpp
+ Neuron/NeuronLinearSaturated.cpp
Network/NeuralNetwork.cpp
Network/NeuralNetworkSum.cpp
NetConnection/ConnectionFunctionGeneral.cpp
diff --git a/src/DataSet/DataSet.cpp b/src/DataSet/DataSet.cpp
index a87d9ab4..425ccb10 100644
--- a/src/DataSet/DataSet.cpp
+++ b/src/DataSet/DataSet.cpp
@@ -487,4 +487,16 @@ namespace lib4neuro {
return newData;
}
}
+
+ void DataSet::add_zero_output_columns(size_t n_columns)
+ {
+ for (size_t i = 0; i < this->n_elements; i++)
+ {
+ for (size_t j = 0; j < n_columns; j++)
+ {
+ this->data.at(i).second.push_back(0);
+ }
+ }
+ this->output_dim += n_columns;
+ }
}
diff --git a/src/DataSet/DataSet.h b/src/DataSet/DataSet.h
index aa850d8d..f6be02b0 100644
--- a/src/DataSet/DataSet.h
+++ b/src/DataSet/DataSet.h
@@ -308,6 +308,12 @@ namespace lib4neuro {
*/
LIB4NEURO_API std::vector<std::pair<std::vector<double>, std::vector<double>>>
get_random_data_batch(size_t max);
+
+ /**
+ * Adds a new output column filled with zeros
+ * @param n_columns Number of columns to be inserted
+ */
+ LIB4NEURO_API void add_zero_output_columns(size_t n_columns);
};
}
#endif //INC_4NEURO_DATASET_H
diff --git a/src/Neuron/NeuronAbsolute.cpp b/src/Neuron/NeuronAbsolute.cpp
new file mode 100644
index 00000000..2e54269f
--- /dev/null
+++ b/src/Neuron/NeuronAbsolute.cpp
@@ -0,0 +1,39 @@
+
+#include <boost/serialization/export.hpp>
+
+#include "NeuronAbsolute.h"
+#include "NeuronConstant.h"
+#include "NeuronSerialization.h"
+#include "NeuronAbsoluteSerialization.h"
+#include "exceptions.h"
+
+BOOST_CLASS_EXPORT_IMPLEMENT(lib4neuro::NeuronAbsolute);
+
+namespace lib4neuro {
+ NeuronAbsolute::NeuronAbsolute() {}
+
+ double NeuronAbsolute::activate(double x,
+ double b) {
+ //this->activation_val = abs(x + b);
+ this->activation_val = pow((x + b),2);
+ return this->activation_val;
+ }
+
+ double NeuronAbsolute::activation_function_eval_derivative_bias(double x,
+ double b) {
+ //return (x + b > 0)? 1.0 : -1.0;
+ return 2 * (x + b);
+ }
+
+ double NeuronAbsolute::activation_function_eval_derivative(double x,
+ double b) {
+ //return (x + b > 0) ? 1.0 : -1.0;
+ return 2 * (x + b);
+ }
+
+
+ Neuron* NeuronAbsolute::get_derivative() {//TODO: implement this function correctly
+ THROW_NOT_IMPLEMENTED_ERROR("");
+ }
+
+}
\ No newline at end of file
diff --git a/src/Neuron/NeuronAbsolute.h b/src/Neuron/NeuronAbsolute.h
new file mode 100644
index 00000000..47665409
--- /dev/null
+++ b/src/Neuron/NeuronAbsolute.h
@@ -0,0 +1,70 @@
+ /**
+* DESCRIPTION OF THE CLASS
+*
+* @author Martin Beseda
+* @author Martin Mrovec
+* @author Michal KravÄŤenko
+* @date 2017 - 2018
+*/
+
+#ifndef INC_4NEURO_NEURONABSOLUTE_H
+#define INC_4NEURO_NEURONABSOLUTE_H
+
+#include "Neuron.h"
+
+namespace lib4neuro {
+
+ /**
+ * Linear neuron class - uses activation function in the form f(x)=a*x + b,
+ * 'x' being the neuron's potential
+ */
+ class NeuronAbsolute : public NeuronDifferentiable {
+
+ public:
+
+ /**
+ * Struct used to access private properties from
+ * the serialization function
+ */
+ struct access;
+
+ /**
+ * Constructs the object of the Linear neuron with activation function
+ * f(x) = x + b
+ * @param[in] b Bias
+ */
+ LIB4NEURO_API explicit NeuronAbsolute();
+
+ /**
+ * Evaluates 'x + b' and stores the result into the 'state' property
+ */
+ LIB4NEURO_API double activate(double x,
+ double b) override;
+
+ /**
+ * Calculates the partial derivative of the activation function
+ * f(x) = x + b at point x
+ * @return Partial derivative of the activation function according to the
+ * 'bias' parameter. Returns 1.0
+ */
+ LIB4NEURO_API double activation_function_eval_derivative_bias(double x,
+ double b) override;
+
+ /**
+ * Calculates d/dx of (x + b) at point x
+ * @return 1.0
+ */
+ LIB4NEURO_API double activation_function_eval_derivative(double x,
+ double b) override;
+
+ /**
+ * Returns a pointer to a Neuron with derivative as its activation function
+ * @return
+ */
+ LIB4NEURO_API Neuron* get_derivative() override;
+
+ };
+
+}
+
+#endif //INC_4NEURO_NeuronAbsolute_H
diff --git a/src/Neuron/NeuronAbsoluteSerialization.h b/src/Neuron/NeuronAbsoluteSerialization.h
new file mode 100644
index 00000000..74db6d8a
--- /dev/null
+++ b/src/Neuron/NeuronAbsoluteSerialization.h
@@ -0,0 +1,48 @@
+
+#ifndef LIB4NEURO_NEURONABSOLUTESERIALIZATION_H
+#define LIB4NEURO_NEURONABSOLUTESERIALIZATION_H
+
+#include <boost/serialization/base_object.hpp>
+#include <boost/archive/text_oarchive.hpp>
+#include <boost/archive/text_iarchive.hpp>
+#include <boost/serialization/export.hpp>
+
+#include "NeuronAbsolute.h"
+#include "NeuronSerialization.h"
+
+BOOST_CLASS_EXPORT_KEY(lib4neuro::NeuronAbsolute);
+
+namespace lib4neuro {
+ struct NeuronAbsolute::access {
+ template<class Archive>
+ static void serialize(Archive& ar,
+ NeuronAbsolute& n,
+ const unsigned int version) {
+ ar & boost::serialization::base_object<Neuron>(n);
+ }
+ };
+}
+
+namespace boost {
+ namespace serialization {
+
+ /**
+ * Serialization function
+ * @tparam Archive Boost library template
+ * @param ar Boost parameter - filled automatically during serialization!
+ * @param n NeuronAbsolute instance
+ * @param version Boost parameter - filled automatically during serialization!
+ */
+ template<class Archive>
+ void serialize(Archive& ar,
+ lib4neuro::NeuronAbsolute& n,
+ const unsigned int version) {
+ lib4neuro::NeuronAbsolute::access::serialize(ar,
+ n,
+ version);
+ }
+
+ } // namespace serialization
+} // namespace boost
+
+#endif //LIB4NEURO_NeuronAbsoluteSERIALIZATION_H
diff --git a/src/Neuron/NeuronLinearSaturated.cpp b/src/Neuron/NeuronLinearSaturated.cpp
new file mode 100644
index 00000000..4904b682
--- /dev/null
+++ b/src/Neuron/NeuronLinearSaturated.cpp
@@ -0,0 +1,64 @@
+
+#include <boost/serialization/export.hpp>
+#include <iostream>
+#include "NeuronLinearSaturated.h"
+#include "NeuronBinary.h"
+#include "NeuronSerialization.h"
+#include "NeuronLinearSaturatedSerialization.h"
+#include "exceptions.h"
+
+BOOST_CLASS_EXPORT_IMPLEMENT(lib4neuro::NeuronLinearSaturated);
+
+namespace lib4neuro {
+ NeuronLinearSaturated::NeuronLinearSaturated(double saturation_point) {
+ this->saturation_point = saturation_point;
+ }
+
+ double NeuronLinearSaturated::activate(double x,
+ double b) {
+ if (x + b < 0) {
+ this->activation_val = 0.0;
+ }
+ else if (x + b > this->saturation_point) {
+ this->activation_val = this->saturation_point;
+ }
+ else {
+ this->activation_val = x + b;
+ }
+ return this->activation_val;
+ }
+
+ double NeuronLinearSaturated::activation_function_eval_derivative_bias(double x,
+ double b) {
+ // f'(0) = 0 for the purposes of training
+ if (x + b < 0) {
+ return 0.0;
+ }
+ else if (x + b > this->saturation_point) {
+ return 0.0;
+ }
+ else {
+ return 1.0;
+ }
+ }
+
+ double NeuronLinearSaturated::activation_function_eval_derivative(double x,
+ double b) {
+ // f'(0) = 0 for the purposes of training
+ if (x + b < 0) {
+ return 0.0;
+ }
+ else if (x + b > this->saturation_point) {
+ return 0.0;
+ }
+ else {
+ return 1.0;
+ }
+ }
+
+ Neuron* NeuronLinearSaturated::get_derivative() {
+ THROW_NOT_IMPLEMENTED_ERROR("");
+ }
+
+
+}
\ No newline at end of file
diff --git a/src/Neuron/NeuronLinearSaturated.h b/src/Neuron/NeuronLinearSaturated.h
new file mode 100644
index 00000000..def4d910
--- /dev/null
+++ b/src/Neuron/NeuronLinearSaturated.h
@@ -0,0 +1,36 @@
+
+#include "Neuron.h"
+
+#ifndef INC_4NEURO_NEURONLINEARSATURATED_H
+#define INC_4NEURO_NEURONLINEARSATURATED_H
+
+namespace lib4neuro {
+
+ /**
+ * LinearSaturated linear unit neuron class
+ *
+ */
+ class NeuronLinearSaturated : public NeuronDifferentiable {
+
+ protected:
+ double saturation_point;
+
+ public:
+ struct access;
+
+ LIB4NEURO_API explicit NeuronLinearSaturated(double saturation_point = 0.0);
+
+ LIB4NEURO_API double activate(double x,
+ double b) override;
+
+ LIB4NEURO_API double activation_function_eval_derivative_bias(double x,
+ double b) override;
+
+ LIB4NEURO_API double activation_function_eval_derivative(double x,
+ double b) override;
+
+ LIB4NEURO_API Neuron* get_derivative() override;
+ };
+}
+
+#endif //INC_4NEURO_NEURONLINEARSATURATED_H
diff --git a/src/Neuron/NeuronLinearSaturatedSerialization.h b/src/Neuron/NeuronLinearSaturatedSerialization.h
new file mode 100644
index 00000000..22e19c19
--- /dev/null
+++ b/src/Neuron/NeuronLinearSaturatedSerialization.h
@@ -0,0 +1,49 @@
+
+#ifndef LIB4NEURO_NEURONLINEARSATURATEDSERIALIZATION_H
+#define LIB4NEURO_NEURONLINEARSATURATEDSERIALIZATION_H
+
+#include <boost/serialization/base_object.hpp>
+#include <boost/archive/text_oarchive.hpp>
+#include <boost/archive/text_iarchive.hpp>
+#include <boost/serialization/export.hpp>
+
+#include "NeuronLinearSaturated.h"
+#include "NeuronSerialization.h"
+
+BOOST_CLASS_EXPORT_KEY(lib4neuro::NeuronLinearSaturated);
+
+namespace lib4neuro {
+ struct NeuronLinearSaturated::access {
+ template<class Archive>
+ static void serialize(Archive& ar,
+ NeuronLinearSaturated& n,
+ const unsigned int version) {
+ ar & boost::serialization::base_object<Neuron>(n);
+ ar & n.saturation_point;
+ }
+ };
+}
+
+namespace boost {
+ namespace serialization {
+
+ /**
+ * Serialization function
+ * @tparam Archive Boost library template
+ * @param ar Boost parameter - filled automatically during serialization!
+ * @param n NeuronLinearSaturated instance
+ * @param version Boost parameter - filled automatically during serialization!
+ */
+ template<class Archive>
+ void serialize(Archive& ar,
+ lib4neuro::NeuronLinearSaturated& n,
+ const unsigned int version) {
+ lib4neuro::NeuronLinearSaturated::access::serialize(ar,
+ n,
+ version);
+ }
+
+ } // namespace serialization
+} // namespace boost
+
+#endif //LIB4NEURO_NEURONLINEARSATURATEDSERIALIZATION_H
--
GitLab