ErrorFunctions.h 23.1 KB
Newer Older
1
2
3
#ifndef INC_4NEURO_ERRORFUNCTION_H
#define INC_4NEURO_ERRORFUNCTION_H

4
5
#include "../settings.h"

Martin Beseda's avatar
Martin Beseda committed
6
7
#include "../Network/NeuralNetwork.h"
#include "../DataSet/DataSet.h"
8

9
10
//TODO HEAVY refactoring needed!

11
12
13
14
15
16
17
18
namespace arma {
    template<typename eT>
    class Mat;

    template<typename eT>
    class Col;
}

19
20
namespace lib4neuro {

21
    //TODO write smarter using ErrorFunction abstract class?
22
    enum ErrorFunctionType {
23
24
25
        ErrorFuncMSE,
        ErrorFuncRMSE,
        ErrorFuncSMAPE
26
27
28
29
30
31
32
33
34
35
    };

    class ErrorFunction {
    public:

        /**
         *
         * @param weights
         * @return
         */
Martin Beseda's avatar
Martin Beseda committed
36
37
        virtual double eval(std::vector<double>* weights = nullptr,
                            bool denormalize_data = false,
38
                            bool verbose = false) = 0;
39

40
41
42
43
44
45
46
47
        /**
         *
         * @param input
         * @param output
         * @param weights
         * @return
         */
        virtual double eval_on_single_input(std::vector<double>* input,
48
49
                                            std::vector<double>* output,
                                            std::vector<double>* weights = nullptr) = 0;
50

51
52
53
54
55
56
        /**
         *
         * @return
         */
        LIB4NEURO_API virtual size_t get_dimension();

Martin Beseda's avatar
Martin Beseda committed
57
58
59
60
        /**
         *
         * @param params
         * @param grad
61
62
         * @param alpha
         * @param batch
Martin Beseda's avatar
Martin Beseda committed
63
64
         */
        virtual void
65
66
        calculate_error_gradient(std::vector<double>& params,
                                 std::vector<double>& grad,
67
68
                                 double alpha = 1.0,
                                 size_t batch = 0) = 0;
Martin Beseda's avatar
Martin Beseda committed
69

70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
		/**
         *
         * @param params
         * @param grad
         * @param alpha
         * @param batch
         */
        virtual void
        calculate_error_gradient_normalized(
			std::vector<double>& params,
            std::vector<double>& grad,
            double alpha = 1.0,
            size_t batch = 0
		) = 0;

85
86
87
88
89
90
91
92
93
        /**
         *
         * @param params
         * @param grad
         * @param alpha
         * @param batch
         */
        virtual void
        analyze_error_gradient(std::vector<double>& params,
Martin Beseda's avatar
Martin Beseda committed
94
95
96
                               std::vector<double>& grad,
                               double alpha = 1.0,
                               size_t batch = 0) = 0;
97

Martin Beseda's avatar
Martin Beseda committed
98
99
100
101
        /**
         *
         * @return
         */
102
        virtual std::vector<double> get_parameters() = 0;
Martin Beseda's avatar
Martin Beseda committed
103
104

        /**
105
         *
Martin Beseda's avatar
Martin Beseda committed
106
107
         * @return
         */
108
        virtual size_t get_n_data_set() = 0;
109
110
111
112
113

        /**
         *
         * @return
         */
114
        virtual size_t get_n_test_data_set() = 0;
115
116
117
118
119

        /**
         *
         * @return
         */
120
121
122
123
124
125
        virtual size_t get_n_outputs() = 0;

        /**
         *
         * @param params
         */
126
        virtual void set_parameters(std::vector<double>& params) = 0;
127
128
129
130
131
132

        /**
         *
         * @param percent_train
         * @return
         */
kra568's avatar
kra568 committed
133
134
135
136
137
138
139
140
141
        LIB4NEURO_API virtual void divide_data_train_test(double percent_test);

        /**
         *
         * @param percent_train
         * @return
         */
        LIB4NEURO_API virtual size_t divide_data_worst_subset(
            std::vector<size_t> &subset_indices,
142
143
144
145
            std::vector<int> &active_subset,
            std::vector<float> &entry_errors,
			size_t expansion_len,
			float tolerance
kra568's avatar
kra568 committed
146
        );
147
148
149
150

        /**
         *
         */
kra568's avatar
kra568 committed
151
        LIB4NEURO_API virtual void return_full_data_set_for_training();
152
153
154

        /**
         *
155
156
         * @param jacobian
         * @param rhs
157
         */
158
159
        virtual void get_jacobian_and_rhs(std::vector<std::vector<double>>& jacobian,
                                          std::vector<double>& rhs) = 0;
160

161
162
163
164
        virtual void get_jacobian_and_rhs(std::vector<std::vector<double>>& jacobian,
                                          std::vector<double>& rhs,
                                          std::vector<std::pair<std::vector<double>, std::vector<double>>>& data) = 0;

165
166
167
        /**
         *
         */
Martin Beseda's avatar
Martin Beseda committed
168
169
        virtual double eval_on_test_data(std::vector<double>* weights = nullptr,
                                         bool verbose = false) = 0;
Martin Beseda's avatar
Martin Beseda committed
170

171
172
173
174
175
176
        /**
         *
         * @param results_file_path
         * @param weights
         * @return
         */
Martin Beseda's avatar
Martin Beseda committed
177
178
179
        virtual double eval_on_test_data(std::string results_file_path,
                                         std::vector<double>* weights = nullptr,
                                         bool verbose = false) = 0;
180
181
182
183
184
185
186

        /**
         *
         * @param results_file_path
         * @param weights
         * @return
         */
Martin Beseda's avatar
Martin Beseda committed
187
188
189
        virtual double eval_on_test_data(std::ofstream* results_file_path,
                                         std::vector<double>* weights = nullptr,
                                         bool verbose = false) = 0;
190
191
192
193
194
195
196

        /**
         *
         * @param data_set
         * @param weights
         * @return
         */
Martin Beseda's avatar
Martin Beseda committed
197
198
199
        virtual double eval_on_data_set(DataSet* data_set,
                                        std::vector<double>* weights = nullptr,
                                        bool verbose = false) = 0;
200
201
202
203
204
205
206
207
208

        /**
         *
         * @param data_set
         * @param weights
         * @param results_file_path
         * @return
         */
        virtual double
Martin Beseda's avatar
Martin Beseda committed
209
210
211
212
        eval_on_data_set(DataSet* data_set,
                         std::string results_file_path,
                         std::vector<double>* weights = nullptr,
                         bool verbose = false) = 0;
213
214
215
216
217
218
219
220

        /**
         *
         * @param data_set
         * @param results_file_path
         * @param weights
         * @return
         */
221
222
        virtual double eval_on_data_set(DataSet* data_set,
                                        std::ofstream* results_file_path = nullptr,
223
                                        std::vector<double>* weights = nullptr,
224
                                        bool verbose = false) = 0;
225

226
227
228
229
230
231
232
        /**
         *
         * @param i
         * @param parameter_vector
         * @param error_vector
         * @return
         */
Martin Beseda's avatar
Martin Beseda committed
233
234
235
        virtual double eval_single_item_by_idx(size_t i,
                                               std::vector<double>* parameter_vector,
                                               std::vector<double>& error_vector) = 0;
236
237
238
239
240
241

        /**
         *
         * @param error_vector
         * @param gradient_vector
         */
Martin Beseda's avatar
Martin Beseda committed
242
243
        virtual void calculate_error_gradient_single(std::vector<double>& error_vector,
                                                     std::vector<double>& gradient_vector) = 0;
244

Michal Kravcenko's avatar
Michal Kravcenko committed
245
246
247
248
249
250
251
        /**
         *
         * @param input
         * @param output
         * @param gradient
         * @param h
         */
252
        virtual void
253
254
255
        calculate_residual_gradient(std::vector<double>* input,
                                    std::vector<double>* output,
                                    std::vector<double>* gradient,
256
257
                                    double h = 1e-3) = 0;

Michal Kravcenko's avatar
Michal Kravcenko committed
258
259
260
261
262
263
264
        /**
         *
         * @param input
         * @param output
         * @param parameters
         * @return
         */
265
        virtual double
266
267
268
        calculate_single_residual(std::vector<double>* input,
                                  std::vector<double>* output,
                                  std::vector<double>* parameters = nullptr) = 0;
269

270
271
272
273
274
275
        /**
         *
         * @param scaling
         */
        virtual void randomize_parameters(double scaling) = 0;

kra568's avatar
kra568 committed
276
277
278
279
280
        [[nodiscard]] std::vector<NeuralNetwork*>& get_nets();

        [[nodiscard]] DataSet* get_dataset() const;

        void set_dataset(DataSet* ds);
281
282
283
284
285
286
		
		virtual void add_to_hessian_and_rhs_single(
			arma::Mat<double>& H,
			arma::Col<double>& rhs,
			size_t entry_idx
		);
kra568's avatar
kra568 committed
287

288
289
290
291
292
293
    protected:

        /**
         *
         */
        size_t dimension = 0;
Martin Beseda's avatar
Martin Beseda committed
294
295
296
297

        /**
         *
         */
298
        std::vector<NeuralNetwork*> nets;
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313

        /**
         *
         */
        DataSet* ds = nullptr;

        /**
         *
         */
        DataSet* ds_full = nullptr;

        /**
         *
         */
        DataSet* ds_test = nullptr;
314
315
    };

316

Martin Beseda's avatar
Martin Beseda committed
317
    class MSE : public ErrorFunction {
kra568's avatar
kra568 committed
318
319
    private:
        bool rescale_error;
320
321
322
323
324
325

    public:
        /**
         * Constructor for single neural network
         * @param net
         * @param ds
kra568's avatar
kra568 committed
326
         * @param rescale_error if true, the means square error is used, if false, the absolute value of the error is used
327
         */
Martin Beseda's avatar
Martin Beseda committed
328
        LIB4NEURO_API MSE(NeuralNetwork* net,
kra568's avatar
kra568 committed
329
330
331
                          DataSet* ds,
                          bool rescale_error = true
                          );
332

333
334
335
336
337
338

        /**
         *
         * @param jacobian
         * @param rhs
         */
339
340
        LIB4NEURO_API virtual void get_jacobian_and_rhs(std::vector<std::vector<double>>& jacobian,
                                                        std::vector<double>& rhs) override;
341
342
343
344

        LIB4NEURO_API virtual void get_jacobian_and_rhs(std::vector<std::vector<double>>& jacobian,
                                                        std::vector<double>& rhs,
                                                        std::vector<std::pair<std::vector<double>, std::vector<double>>>& data) override;
345
346
347
348
349
        /**
         *
         * @param weights
         * @return
         */
350
        LIB4NEURO_API double eval(std::vector<double>* weights = nullptr,
351
352
                                  bool denormalize_data = false,
                                  bool verbose = false) override;
Martin Beseda's avatar
Martin Beseda committed
353

Martin Beseda's avatar
Martin Beseda committed
354
355
356
357
        /**
         *
         * @param params
         * @param grad
358
359
         * @param alpha
         * @param batch
Martin Beseda's avatar
Martin Beseda committed
360
361
         */
        LIB4NEURO_API void
362
363
        calculate_error_gradient(std::vector<double>& params,
                                 std::vector<double>& grad,
364
365
                                 double alpha = 1.0,
                                 size_t batch = 0) override;
366

367
368
369
370
371
372
373
		/**
         *
         * @param params
         * @param grad
         * @param alpha
         * @param batch
         */
374
        void
375
376
377
378
379
380
381
        calculate_error_gradient_normalized(
			std::vector<double>& params,
            std::vector<double>& grad,
            double alpha = 1.0,
            size_t batch = 0
		) override;
		
382
383
384
385
386
387
388
389
390
        /**
         *
         * @param params
         * @param grad
         * @param alpha
         * @param batch
         */
        LIB4NEURO_API void
        analyze_error_gradient(std::vector<double>& params,
Martin Beseda's avatar
Martin Beseda committed
391
392
393
                               std::vector<double>& grad,
                               double alpha = 1.0,
                               size_t batch = 0) override;
394

395
        /**
396
397
398
399
400
401
         * Evaluates the function f(x) = 0 - MSE(x) for a
         * specified input x
         *
         * @param input
         * @return
         */
Michal Kravcenko's avatar
Michal Kravcenko committed
402
        LIB4NEURO_API
Martin Beseda's avatar
Martin Beseda committed
403
404
405
        double calculate_single_residual(std::vector<double>* input,
                                         std::vector<double>* output,
                                         std::vector<double>* parameters) override;
406
407
408

        /**
         * Compute gradient of the residual function f(x) = 0 - MSE(x) for a specific input x.
409
410
411
412
413
414
415
         * The method uses the central difference method.
         *
         * @param[in] input Vector being a single input
         * @param[out] gradient Resulting gradient
         * @param[in] h Step used in the central difference
         */
        LIB4NEURO_API void
416
417
418
        calculate_residual_gradient(std::vector<double>* input,
                                    std::vector<double>* output,
                                    std::vector<double>* gradient,
Martin Beseda's avatar
Martin Beseda committed
419
                                    double h = 1e-3) override;
420
421
422
423
424
425

        /**
         *
         * @param input
         * @return
         */
426
427
        LIB4NEURO_API double eval_on_single_input(std::vector<double>* input,
                                                  std::vector<double>* output,
428
                                                  std::vector<double>* weights = nullptr) override;
429

Martin Beseda's avatar
Martin Beseda committed
430
431
        /**
         *
432
         * @param weights
Martin Beseda's avatar
Martin Beseda committed
433
434
         * @return
         */
Martin Beseda's avatar
Martin Beseda committed
435
436
        LIB4NEURO_API double eval_on_test_data(std::vector<double>* weights = nullptr,
                                               bool verbose = false) override;
Martin Beseda's avatar
Martin Beseda committed
437

438
439
440
441
442
443
        /**
         *
         * @param results_file_path
         * @param weights
         * @return
         */
444
        LIB4NEURO_API double eval_on_test_data(std::string results_file_path = nullptr,
445
                                               std::vector<double>* weights = nullptr,
446
                                               bool verbose = false) override;
447

448
449
450
451
452
453
        /**
         *
         * @param results_file_path
         * @param weights
         * @return
         */
454
        LIB4NEURO_API double eval_on_test_data(std::ofstream* results_file_path,
455
                                               std::vector<double>* weights = nullptr,
456
                                               bool verbose = false) override;
457

458
459
460
461
462
463
464
465
466
        /**
         *
         * @param data_set
         * @param results_file_path
         * @param weights
         * @return
         */
        LIB4NEURO_API double eval_on_data_set(DataSet* data_set,
                                              std::ofstream* results_file_path,
467
                                              std::vector<double>* weights = nullptr,
468
                                              bool verbose = false) override;
469
470
471
472
473
474
475

        /**
         *
         * @param data_set
         * @param weights
         * @return
         */
476
        LIB4NEURO_API double eval_on_data_set(DataSet* data_set,
477
                                              std::vector<double>* weights = nullptr,
478
                                              bool verbose = false) override;
479
480
481
482
483
484
485
486

        /**
         *
         * @param data_set
         * @param results_file_path
         * @param weights
         * @return
         */
487
488
        LIB4NEURO_API double eval_on_data_set(DataSet* data_set,
                                              std::string results_file_path,
489
                                              std::vector<double>* weights = nullptr,
490
                                              bool verbose = false) override;
491

492
493
494
495
496
497
498
        /**
         *
         * @param i
         * @param parameter_vector
         * @param error_vector
         * @return
         */
Martin Beseda's avatar
Martin Beseda committed
499
500
501
        LIB4NEURO_API double eval_single_item_by_idx(size_t i,
                                                     std::vector<double>* parameter_vector,
                                                     std::vector<double>& error_vector) override;
502
503
504
505
506
507

        /**
         *
         * @param error_vector
         * @param gradient_vector
         */
Martin Beseda's avatar
Martin Beseda committed
508
509
        LIB4NEURO_API void calculate_error_gradient_single(std::vector<double>& error_vector,
                                                           std::vector<double>& gradient_vector) override;
510

511
512
513
514
515
516
517
518
519
520
        /**
         *
         * @return
         */
        LIB4NEURO_API virtual std::vector<double> get_parameters() override;

        /**
         *
         * @param params
         */
521
        LIB4NEURO_API virtual void set_parameters(std::vector<double>& params) override;
522

523
524
525
526
        /**
         *
         * @return
         */
527
        LIB4NEURO_API virtual size_t get_n_data_set() override;
528
529
530
531
532

        /**
         *
         * @return
         */
533
        LIB4NEURO_API virtual size_t get_n_test_data_set() override;
534
535
536
537
538
539
540
541
542
543
544
545

        /**
         *
         * @return
         */
        LIB4NEURO_API virtual size_t get_n_outputs() override;

        /**
         *
         * @param scaling
         */
        LIB4NEURO_API virtual void randomize_parameters(double scaling) override;
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
    };

    class ErrorSum : public ErrorFunction {
    public:
        /**
         *
         */
        LIB4NEURO_API ErrorSum();

        /**
         *
         */
        LIB4NEURO_API ~ErrorSum();

        /**
         *
         * @param weights
         * @return
         */
565
        LIB4NEURO_API double eval(std::vector<double>* weights = nullptr,
566
                                  bool denormalize_data = false,
567
                                  bool verbose = false);
568

569
570
571
572
        LIB4NEURO_API double eval_on_single_input(std::vector<double>* input,
                                                  std::vector<double>* output,
                                                  std::vector<double>* weights = nullptr) override;

573
574
575
576
577
        /**
         *
         * @param weights
         * @return
         */
Martin Beseda's avatar
Martin Beseda committed
578
579
        LIB4NEURO_API double eval_on_test_data(std::vector<double>* weights = nullptr,
                                               bool verbose = false) override;
580
581
582
583
584
585
586

        /**
         *
         * @param results_file_path
         * @param weights
         * @return
         */
587
        LIB4NEURO_API double eval_on_test_data(std::string results_file_path,
588
                                               std::vector<double>* weights = nullptr,
589
                                               bool verbose = false) override;
590

591
592
        /**
         *
593
         * @param results_file_path
594
595
596
         * @param weights
         * @return
         */
597
        LIB4NEURO_API double eval_on_test_data(std::ofstream* results_file_path,
598
                                               std::vector<double>* weights = nullptr,
599
                                               bool verbose = false) override;
600
601
602
603
604
605
606

        /**
         *
         * @param data_set
         * @param weights
         * @return
         */
607
        LIB4NEURO_API double eval_on_data_set(DataSet* data_set,
608
                                              std::vector<double>* weights = nullptr,
609
                                              bool verbose = false) override;
610
611
612
613
614
615
616
617

        /**
         *
         * @param data_set
         * @param results_file_path
         * @param weights
         * @return
         */
618
619
        LIB4NEURO_API double eval_on_data_set(DataSet* data_set,
                                              std::string results_file_path,
620
                                              std::vector<double>* weights = nullptr,
621
                                              bool verbose = false) override;
622
623
624
625
626
627
628
629
630
631

        /**
         *
         * @param data_set
         * @param results_file_path
         * @param weights
         * @return
         */
        LIB4NEURO_API double eval_on_data_set(DataSet* data_set,
                                              std::ofstream* results_file_path,
632
                                              std::vector<double>* weights = nullptr,
633
                                              bool verbose = false) override;
634

635
636
637
638
639
640
641
        /**
         *
         * @param i
         * @param parameter_vector
         * @param error_vector
         * @return
         */
Martin Beseda's avatar
Martin Beseda committed
642
643
644
        LIB4NEURO_API virtual double eval_single_item_by_idx(size_t i,
                                                             std::vector<double>* parameter_vector,
                                                             std::vector<double>& error_vector) override;
645
646
647
648
649
650

        /**
         *
         * @param error_vector
         * @param gradient_vector
         */
Martin Beseda's avatar
Martin Beseda committed
651
652
        LIB4NEURO_API virtual void calculate_error_gradient_single(std::vector<double>& error_vector,
                                                                   std::vector<double>& gradient_vector) override;
653

654
655
656
657
        /**
         *
         * @param F
         */
Martin Beseda's avatar
Martin Beseda committed
658
659
        LIB4NEURO_API void add_error_function(ErrorFunction* F,
                                              double alpha = 1.0);
660
661
662
663
664
665
666

        /**
         *
         * @return
         */
        LIB4NEURO_API size_t get_dimension() override;

Martin Beseda's avatar
Martin Beseda committed
667
668
669
670
        /**
         *
         * @param params
         * @param grad
671
672
         * @param alpha
         * @param batch
Martin Beseda's avatar
Martin Beseda committed
673
674
         */
        LIB4NEURO_API void
675
676
        calculate_error_gradient(std::vector<double>& params,
                                 std::vector<double>& grad,
677
678
                                 double alpha = 1.0,
                                 size_t batch = 0) override;
679
680
681
682
683
684
685
686
687
688
689
690
691
692
		/**
         *
         * @param params
         * @param grad
         * @param alpha
         * @param batch
         */
        virtual void
        calculate_error_gradient_normalized(
			std::vector<double>& params,
            std::vector<double>& grad,
            double alpha = 1.0,
            size_t batch = 0
		) override;								 
693
694
695
696
697
698
699
700
701
        /**
         *
         * @param params
         * @param grad
         * @param alpha
         * @param batch
         */
        LIB4NEURO_API void
        analyze_error_gradient(std::vector<double>& params,
Martin Beseda's avatar
Martin Beseda committed
702
703
704
                               std::vector<double>& grad,
                               double alpha = 1.0,
                               size_t batch = 0) override;
705
706

        LIB4NEURO_API void
707
708
709
        calculate_residual_gradient(std::vector<double>* input,
                                    std::vector<double>* output,
                                    std::vector<double>* gradient,
710
711
712
                                    double h = 1e-3) override;

        LIB4NEURO_API double
713
714
715
        calculate_single_residual(std::vector<double>* input,
                                  std::vector<double>* output,
                                  std::vector<double>* parameters = nullptr) override;
716
717


Martin Beseda's avatar
Martin Beseda committed
718
719
720
721
        /**
         *
         * @return
         */
722
723
724
725
726
727
        LIB4NEURO_API virtual std::vector<double> get_parameters() override;

        /**
         *
         * @param params
         */
728
        LIB4NEURO_API virtual void set_parameters(std::vector<double>& params) override;
729
730
731
732
733
734

        /**
         *
         * @return
         */
        LIB4NEURO_API virtual size_t get_n_data_set() override;
Martin Beseda's avatar
Martin Beseda committed
735

736
737
738
739
        /**
         *
         * @return
         */
740
        LIB4NEURO_API virtual size_t get_n_test_data_set() override;
Martin Beseda's avatar
Martin Beseda committed
741

Michal Kravcenko's avatar
Michal Kravcenko committed
742
743
744
745
        /**
         *
         * @return
         */
746
747
748
749
        LIB4NEURO_API virtual size_t get_n_outputs() override;

        /**
         *
kra568's avatar
kra568 committed
750
751
752
753
754
755
756
757
758
         * @param percent_train
         * @return
         */
        LIB4NEURO_API virtual void divide_data_train_test(double percent_test) override;

        /**
         *
         * @param percent_train
         * @return
759
         */
kra568's avatar
kra568 committed
760
761
        LIB4NEURO_API virtual size_t divide_data_worst_subset(
            std::vector<size_t> &subset_indices,
762
763
764
765
            std::vector<int> &active_subset,
            std::vector<float> &entry_errors,
			size_t expansion_len,
			float tolerance
kra568's avatar
kra568 committed
766
        ) override;
767
768
769
770
771
772
773
774
775
776
777
778

        /**
         *
         */
        LIB4NEURO_API virtual void return_full_data_set_for_training() override;

        /**
         *
         * @param jacobian
         * @param rhs
         */
        LIB4NEURO_API virtual void get_jacobian_and_rhs(
779
780
            std::vector<std::vector<double>>& jacobian,
            std::vector<double>& rhs) override;
781

782
783
784
785
        LIB4NEURO_API virtual void get_jacobian_and_rhs(std::vector<std::vector<double>>& jacobian,
                                                        std::vector<double>& rhs,
                                                        std::vector<std::pair<std::vector<double>, std::vector<double>>>& data) override;

786
787
788
789
790
        /**
         *
         * @param scaling
         */
        LIB4NEURO_API virtual void randomize_parameters(double scaling) override;
Michal Kravcenko's avatar
Michal Kravcenko committed
791

792
    protected:
793
        std::vector<ErrorFunction*>* summand;
794
        std::vector<double>        summand_coefficient;
795
796
    };
}
797
798

#endif //INC_4NEURO_ERRORFUNCTION_H