ENH: Added new neurons - Absolute, LinearSaturated

include/4neuro.h

@@ -12,9 +12,11 @@
 #include "../src/Neuron/NeuronLinear.h"
 #include "../src/Neuron/NeuronLogistic.h"
 #include "../src/Neuron/NeuronBiased.h"
+#include "../src/Neuron/NeuronAbsolute.h"
 #include "../src/Neuron/NeuronBinaryBiased.h"
 #include "../src/Neuron/NeuronFilter.h"
 #include "../src/Neuron/NeuronRectifier.h"
+#include "../src/Neuron/NeuronLinearSaturated.h"
 #include "../src/Solvers/DESolver.h"
 #include "../src/ErrorFunction/ErrorFunctions.h"
 #include "../src/LearningMethods/ParticleSwarm.h"

src/CMakeLists.txt

@@ -87,10 +87,12 @@ IF("${BUILD_LIB}" STREQUAL "yes")
         Neuron/NeuronBinary.cpp
         Neuron/NeuronConstant.cpp
         Neuron/NeuronLinear.cpp
+        Neuron/NeuronAbsolute.cpp
         Neuron/NeuronLogistic.cpp
         Neuron/NeuronRectifier.cpp
         Neuron/NeuronFilter.cpp
         Neuron/NeuronBinaryBiased.cpp
+        Neuron/NeuronLinearSaturated.cpp
         Network/NeuralNetwork.cpp
         Network/NeuralNetworkSum.cpp
         NetConnection/ConnectionFunctionGeneral.cpp

src/DataSet/DataSet.cpp

@@ -487,4 +487,16 @@ namespace lib4neuro {
             return newData;
         }
     }
+	
+	void DataSet::add_zero_output_columns(size_t n_columns)
+	{
+		for (size_t i = 0; i < this->n_elements; i++)
+		{
+			for (size_t j = 0; j < n_columns; j++)
+			{
+				this->data.at(i).second.push_back(0);
+			}
+		}
+		this->output_dim += n_columns;
+	}
 }

src/DataSet/DataSet.h

@@ -308,6 +308,12 @@ namespace lib4neuro {
          */
         LIB4NEURO_API std::vector<std::pair<std::vector<double>, std::vector<double>>>
         get_random_data_batch(size_t max);
+
+		/**
+		 * Adds a new output column filled with zeros 
+		 * @param n_columns Number of columns to be inserted
+		 */
+		LIB4NEURO_API void add_zero_output_columns(size_t n_columns);
     };
 }
 #endif //INC_4NEURO_DATASET_H

src/Neuron/NeuronAbsolute.cpp

+
+#include <boost/serialization/export.hpp>
+
+#include "NeuronAbsolute.h"
+#include "NeuronConstant.h"
+#include "NeuronSerialization.h"
+#include "NeuronAbsoluteSerialization.h"
+#include "exceptions.h"
+
+BOOST_CLASS_EXPORT_IMPLEMENT(lib4neuro::NeuronAbsolute);
+
+namespace lib4neuro {
+	NeuronAbsolute::NeuronAbsolute() {}
+
+	double NeuronAbsolute::activate(double x,
+		double b) {
+		//this->activation_val = abs(x + b);
+		this->activation_val = pow((x + b),2);
+		return this->activation_val;
+	}
+
+	double NeuronAbsolute::activation_function_eval_derivative_bias(double x,
+		double b) {
+		//return (x + b > 0)? 1.0 : -1.0;
+		return 2 * (x + b);
+	}
+
+	double NeuronAbsolute::activation_function_eval_derivative(double x,
+		double b) {
+		//return (x + b > 0) ? 1.0 : -1.0;
+		return 2 * (x + b);
+	}
+
+
+	Neuron* NeuronAbsolute::get_derivative() {//TODO: implement this function correctly
+		THROW_NOT_IMPLEMENTED_ERROR("");
+	}
+
+}
\ No newline at end of file

src/Neuron/NeuronAbsolute.h

+                                         /**
+* DESCRIPTION OF THE CLASS
+*
+* @author Martin Beseda
+* @author Martin Mrovec
+* @author Michal Kravčenko
+* @date 2017 - 2018
+*/
+
+#ifndef INC_4NEURO_NEURONABSOLUTE_H
+#define INC_4NEURO_NEURONABSOLUTE_H
+
+#include "Neuron.h"
+
+namespace lib4neuro {
+
+	/**
+	* Linear neuron class - uses activation function in the form f(x)=a*x + b,
+	* 'x' being the neuron's potential
+	*/
+	class NeuronAbsolute : public NeuronDifferentiable {
+
+	public:
+
+		/**
+		* Struct used to access private properties from
+		* the serialization function
+		*/
+		struct access;
+
+		/**
+		* Constructs the object of the Linear neuron with activation function
+		* f(x) = x + b
+		* @param[in] b Bias
+		*/
+		LIB4NEURO_API explicit NeuronAbsolute();
+
+		/**
+		* Evaluates 'x + b' and stores the result into the 'state' property
+		*/
+		LIB4NEURO_API double activate(double x,
+			double b) override;
+
+		/**
+		* Calculates the partial derivative of the activation function
+		* f(x) = x + b at point x
+		* @return Partial derivative of the activation function according to the
+		* 'bias' parameter. Returns 1.0
+		*/
+		LIB4NEURO_API double activation_function_eval_derivative_bias(double x,
+			double b) override;
+
+		/**
+		* Calculates d/dx of (x + b) at point x
+		* @return 1.0
+		*/
+		LIB4NEURO_API double activation_function_eval_derivative(double x,
+			double b) override;
+
+		/**
+		* Returns a pointer to a Neuron with derivative as its activation function
+		* @return
+		*/
+		LIB4NEURO_API Neuron* get_derivative() override;
+
+	};
+
+}
+
+#endif //INC_4NEURO_NeuronAbsolute_H

src/Neuron/NeuronAbsoluteSerialization.h

+
+#ifndef LIB4NEURO_NEURONABSOLUTESERIALIZATION_H
+#define LIB4NEURO_NEURONABSOLUTESERIALIZATION_H
+
+#include <boost/serialization/base_object.hpp>
+#include <boost/archive/text_oarchive.hpp>
+#include <boost/archive/text_iarchive.hpp>
+#include <boost/serialization/export.hpp>
+
+#include "NeuronAbsolute.h"
+#include "NeuronSerialization.h"
+
+BOOST_CLASS_EXPORT_KEY(lib4neuro::NeuronAbsolute);
+
+namespace lib4neuro {
+    struct NeuronAbsolute::access {
+        template<class Archive>
+        static void serialize(Archive& ar,
+                              NeuronAbsolute& n,
+                              const unsigned int version) {
+            ar & boost::serialization::base_object<Neuron>(n);
+        }
+    };
+}
+
+namespace boost {
+    namespace serialization {
+
+        /**
+         * Serialization function
+         * @tparam Archive Boost library template
+         * @param ar Boost parameter - filled automatically during serialization!
+         * @param n NeuronAbsolute instance
+         * @param version Boost parameter - filled automatically during serialization!
+         */
+        template<class Archive>
+        void serialize(Archive& ar,
+                       lib4neuro::NeuronAbsolute& n,
+                       const unsigned int version) {
+            lib4neuro::NeuronAbsolute::access::serialize(ar,
+                                                       n,
+                                                       version);
+        }
+
+    } // namespace serialization
+} // namespace boost
+
+#endif //LIB4NEURO_NeuronAbsoluteSERIALIZATION_H

src/Neuron/NeuronLinearSaturated.cpp

+
+#include <boost/serialization/export.hpp>
+#include <iostream>
+#include "NeuronLinearSaturated.h"
+#include "NeuronBinary.h"
+#include "NeuronSerialization.h"
+#include "NeuronLinearSaturatedSerialization.h"
+#include "exceptions.h"
+
+BOOST_CLASS_EXPORT_IMPLEMENT(lib4neuro::NeuronLinearSaturated);
+
+namespace lib4neuro {
+    NeuronLinearSaturated::NeuronLinearSaturated(double saturation_point) {
+		this->saturation_point = saturation_point;
+	}
+
+    double NeuronLinearSaturated::activate(double x,
+                                     double b) {
+		if (x + b < 0) {
+			this->activation_val = 0.0;
+		}
+		else if (x + b > this->saturation_point) {
+			this->activation_val = this->saturation_point;
+		}
+		else {
+			this->activation_val = x + b;
+		}
+        return this->activation_val;
+    }
+
+    double NeuronLinearSaturated::activation_function_eval_derivative_bias(double x,
+                                                                		 double b) {
+        // f'(0) = 0 for the purposes of training
+		if (x + b < 0) {
+			return 0.0;
+		}
+		else if (x + b > this->saturation_point) {
+			return 0.0;
+		}
+		else {
+			return 1.0;
+		}
+    }
+
+    double NeuronLinearSaturated::activation_function_eval_derivative(double x,
+                                                                double b) {
+        // f'(0) = 0 for the purposes of training
+		if (x + b < 0) {
+			return 0.0;
+		}
+		else if (x + b > this->saturation_point) {
+			return 0.0;
+		}
+		else {
+			return 1.0;
+		}
+    }
+
+    Neuron* NeuronLinearSaturated::get_derivative() {
+        THROW_NOT_IMPLEMENTED_ERROR("");
+    }
+
+
+}
\ No newline at end of file

src/Neuron/NeuronLinearSaturated.h

+
+#include "Neuron.h"
+
+#ifndef INC_4NEURO_NEURONLINEARSATURATED_H
+#define INC_4NEURO_NEURONLINEARSATURATED_H
+
+namespace lib4neuro {
+
+    /**
+     * LinearSaturated linear unit neuron class 
+     * 
+     */
+    class NeuronLinearSaturated : public NeuronDifferentiable {
+	
+	protected:
+		double saturation_point;
+	
+	public:
+        struct access;
+
+        LIB4NEURO_API explicit NeuronLinearSaturated(double saturation_point = 0.0);
+
+        LIB4NEURO_API double activate(double x,
+                                      double b) override;
+
+        LIB4NEURO_API double activation_function_eval_derivative_bias(double x,
+                                                                      double b) override;
+
+        LIB4NEURO_API double activation_function_eval_derivative(double x,
+                                                                 double b) override;
+
+        LIB4NEURO_API Neuron* get_derivative() override;
+    };
+}
+
+#endif //INC_4NEURO_NEURONLINEARSATURATED_H

src/Neuron/NeuronLinearSaturatedSerialization.h

+
+#ifndef LIB4NEURO_NEURONLINEARSATURATEDSERIALIZATION_H
+#define LIB4NEURO_NEURONLINEARSATURATEDSERIALIZATION_H
+
+#include <boost/serialization/base_object.hpp>
+#include <boost/archive/text_oarchive.hpp>
+#include <boost/archive/text_iarchive.hpp>
+#include <boost/serialization/export.hpp>
+
+#include "NeuronLinearSaturated.h"
+#include "NeuronSerialization.h"
+
+BOOST_CLASS_EXPORT_KEY(lib4neuro::NeuronLinearSaturated);
+
+namespace lib4neuro {
+    struct NeuronLinearSaturated::access {
+        template<class Archive>
+        static void serialize(Archive& ar,
+                              NeuronLinearSaturated& n,
+                              const unsigned int version) {
+            ar & boost::serialization::base_object<Neuron>(n);
+			ar & n.saturation_point;
+        }
+    };
+}
+
+namespace boost {
+    namespace serialization {
+
+        /**
+         * Serialization function
+         * @tparam Archive Boost library template
+         * @param ar Boost parameter - filled automatically during serialization!
+         * @param n NeuronLinearSaturated instance
+         * @param version Boost parameter - filled automatically during serialization!
+         */
+        template<class Archive>
+        void serialize(Archive& ar,
+                       lib4neuro::NeuronLinearSaturated& n,
+                       const unsigned int version) {
+            lib4neuro::NeuronLinearSaturated::access::serialize(ar,
+                                                          n,
+                                                          version);
+        }
+
+    } // namespace serialization
+} // namespace boost
+
+#endif //LIB4NEURO_NEURONLINEARSATURATEDSERIALIZATION_H