Network.hpp:

//------------------------------------------------
//  NETWORK.HPP
//------------------------------------------------

#ifndef NETWORK_HPP
#define NETWORK_HPP


// Libraries
#include <vector>

// My Libraries
#include "neuron.hpp"
#include "rne.hpp"
#include "defs.hpp"

// Typedefs and Structures


// Network : Controls the dynamic of the neural network

class network
{
    private:
        std::vector<unsigned int> _network_structure;

        std::vector<std::vector<neuron>> _neurons;
 
        rne _random_engine;

        const double _learnrate, _layerfact;

        void _initialize_random_rates();

        void _reset_netoutputs();
        void _calculate_netoutputs(std::vector<double> input);
        void _train_network(std::vector<double> errors);

    public:
        network(std::vector<unsigned int> neurons_per_layer, rne &random_engine, double learning_rate = 0.1);
        
        double train_epoch(testcases tests);    //testcases from defs.hpp
        std::vector<double> solve(std::vector<double> input);
};

#endif

Network.cpp:

//------------------------------------------------
//  NETWORK.CPP
//------------------------------------------------

#include "network.hpp"

// Libraries
#include "config.hpp"
#include "errfct.hpp"

// CODE

using namespace std;

network::network(vector<unsigned int> neurons_per_layer, rne &random_engine, double learning_rate) :
    _network_structure(neurons_per_layer), _neurons(neurons_per_layer.size() ),
    _random_engine(random_engine), _learnrate(learning_rate), _layerfact(1.2)
{
    for (unsigned int i = 0; i < neurons_per_layer.size(); ++i)
    {
        _neurons[i].resize(neurons_per_layer[i], neuron(essential::fact_linear, essential::dfact_linear) );
    }

#ifdef BIAS_NEURON
    _neurons[0].resize(neurons_per_layer[0] + 1);
#endif //BIAS_NEURON
    
    _initialize_random_rates();
}


// Methods

void network::_initialize_random_rates()
{
    for (vector<vector<neuron>>::iterator layer = ++_neurons.begin();
            layer != _neurons.end(); ++layer)
    {
#ifdef FEED_FORWARD
        vector<vector<neuron>>::iterator prev_layer = layer - 1;
        for (auto& successors: *layer)
        {
            static double rate;
            for (auto& predecessors: *prev_layer)
            {
                rate = _random_engine.splitrange(0.05,0.5);
                link_neurons(predecessors, successors, rate);
            }
        }
#endif //FEED_FORWARD

#ifdef FULL_FORWARD
        for (vector<vector<neuron>>::iterator prev_layers = _neurons.begin();
                prev_layers != layer; ++prev_layers)
        {
            for (auto& successors: *layer)
            {
                static double rate;
                for (auto& predecessors: *prev_layers)
                {
                    rate  = _random_engine.splitrange(0.05,0.5);
                    link_neurons(predecessors, successors, rate);
                }
            }
        }
#endif //FULL_FORWARD
    }
}

void network::_reset_netoutputs()
{
    for (auto& layer: _neurons)
    {
        for (auto& neuron: layer)
        {
            neuron.out = 0;
        }
    }
}

void network::_calculate_netoutputs(vector<double> input)
{
    unsigned input_size = input.size();
    if (input_size != _network_structure.front() )
    {
        //Throw an error --- will be implemented
    }

    for (unsigned i = 0; i < input_size; ++i)
    {
        _neurons.front()[i].out = input[i];
    }

#ifdef BIAS_NEURON
    _neurons.front().back().out = 1; // The Bias neuron is the last neuron of the input layer, and always
    // sends a constant signal
#endif

    for (vector<vector<neuron>>::iterator layer = ++_neurons.begin(); layer != _neurons.end(); ++layer)
    {
        for (auto& neuron: *layer)
        {
            neuron.activate();
        }
    }
}

void network::_train_network(vector<double> errors)
{
    double lrate = _learnrate;

    // Train the output neurons
    for (unsigned i = 0; i < _network_structure.back(); ++i) 
    {
        _neurons.back()[i].calculate_delta(errors[i]);
        _neurons.back()[i].train_connection(lrate);
    }

    // Train all, but the input/output neurons
    for (vector<vector<neuron>>::reverse_iterator rlayer = ++_neurons.rbegin();
            rlayer != --_neurons.rend(); ++rlayer)
    {
        for (auto& neuron: *rlayer)
        {
            neuron.calculate_delta();
            neuron.train_connection(lrate);
        }
        lrate *= _layerfact;
    }
}


double network::train_epoch(testcases tests)
{
    vector<vector<double>> errors;
    for (auto& test: tests)
    {
        if (test[0].size() != _network_structure.front() || test[1].size() != _network_structure.back() )
        {
            //Throw an error
        }

        _reset_netoutputs();
        _calculate_netoutputs(test.front() );

        vector<double> tmp_error(_network_structure.back() );
        for (unsigned j = 0; j < tmp_error.size(); ++j)
        {
            tmp_error[j] = test.back()[j] - _neurons.back()[j].out;
        }
        errors.push_back(tmp_error);

#ifdef LEARN_ONLINE
        _train_network(errors.back() );
#endif //LEARN_ONLINE
    }
#ifdef LEARN_OFFLINE
    vector<double> avg_error(_network_structure.back(), 0);
    for (unsigned i = 0; i < _network_structure.back(); ++i)
    {
        for (auto const& error: errors)
        {
            avg_error[i] += error[i];
        }
        avg_error[i] /= (double)errors.size();
    }
    _train_network(avg_error);
#endif //LEARN_OFFLINE

    vector<double> specific_errors;
    for (auto const& error: errors)
    {
        specific_errors.push_back(_errfct(error) );
    }

    double totalerror = 0;
    for (auto& specific_error: specific_errors)
    {
        totalerror += specific_error;
    }

    return totalerror;
}

vector<double> network::solve(vector<double> input)
{
    _reset_netoutputs();
    _calculate_netoutputs(input);
    
    vector<double> solution;
    for (auto const& output_neuron: _neurons.back() )
    {
        solution.push_back(output_neuron.out);
    }
    return solution;
}

These are the two interesting classes, yet there is more code around them. I hesitate to post it all here, to keep everything compact, so I will only provide links right now --- if needed I will paste edit my question.

• fact.hpp

• fact.cpp

• errfct.hpp

• errfct.cpp

• rne.hpp

• rne.cpp

• defs.hpp

• config.hpp

• main.cpp

• What general advice do you have for a novice programmer?

• Where do I deviate from best practice, where is my code hard to read/ unlogical?

• Is my overall design/ ideas good? How do I better plan such projects?

• Can I avoid a bunch of nested conditional compilation steps and still have control over the features of my code at compile time?

Network.hpp (updated; old network.hpp):

//------------------------------------------------
//  NETWORK.HPP
//------------------------------------------------

#ifndef NETWORK_HPP
#define NETWORK_HPP


// Libraries
#include <vector>

// My Libraries
#include "neuron.hpp"
#include "rne.hpp"
#include "defs.hpp"

// Typedefs and Structures

enum brain_param    { FEED_FORWARD, FULL_FORWARD };

enum time_param     { OFFLINE, ONLINE };
enum method_param   { BACKPROP, RPROP };

struct learning_param
{
    time_param timing;
    method_param method;
    learning_param(time_param t_in, method_param m_in) : timing(t_in), method(m_in) {}
};

// Network : Controls the dynamic of the neural network

class network
{
    private:
        std::vector<unsigned int> _network_structure;

        std::vector<std::vector<neuron>> _neurons;
        neuron _bias;

        rne _random_engine;

        void _initialize_random_rates();

        void _reset_netoutputs();
        void _calculate_netoutputs(std::vector<double> input);
        void _train_network(std::vector<double> errors, double learning_rate, double layer_fact);

    public:
        network(std::vector<unsigned int> neurons_per_layer, rne &random_engine);
        
        double train_epoch(testcases tests, double learning_rate = 0.01, double layer_fact = 1.2);
            //testcases from defs.hpp
        std::vector<double> solve(std::vector<double> input);

        brain_param brain_structure;
        learning_param learning_strategy;
        bool BIAS_NEURON;
};

#endif

Network.cpp (updated, old network.cpp):

//------------------------------------------------
//  NETWORK.CPP
//------------------------------------------------

#include "network.hpp"

// Libraries
#include <iostream>
#include <algorithm> //std::shuffle

#include "config.hpp"
#include "errfct.hpp"

// CODE

using namespace std;

// Constructors

network::network(vector<unsigned int> neurons_per_layer, rne &random_engine) :
    _network_structure(neurons_per_layer), _neurons(neurons_per_layer.size() ),
    _random_engine(random_engine), brain_structure(FEED_FORWARD), 
    learning_strategy(OFFLINE, BACKPROP), BIAS_NEURON(true)
{
    for (unsigned int i = 0; i < neurons_per_layer.size() - 1; ++i)
    {
        _neurons[i].resize(neurons_per_layer[i], neuron(essential::fact_tanh, essential::dfact_tanh) );
    }
    _neurons.back().resize(neurons_per_layer.back(), neuron(essential::fact_linear, essential::dfact_linear) );

    _initialize_random_rates();
}


// Methods

void network::_initialize_random_rates()
{
    double rate;

    for (vector<vector<neuron>>::iterator layer = _neurons.begin() + 1;
            layer != _neurons.end(); ++layer)
    {
        switch (brain_structure)
        {
            case FEED_FORWARD:
            {
                vector<vector<neuron>>::iterator prev_layer = layer - 1;
                for (auto& successors: *layer)
                {
                    for (auto& predecessors: *prev_layer)
                    {
                        rate = _random_engine.splitrange(0.05,0.5);
                        link_neurons(predecessors, successors, rate);
                    }
                
                    // Link every neuron against the bias neuron. If BIAS_NEUROM == false
                    // the output of _bias is set to 0, so that the link becomes unimportant.
                    // This is used that the _bias neuron can be switched on and off.
                    rate = _random_engine.splitrange(0.05,0.5);
                    link_neurons(_bias, successors, rate);
                }
                break;
            }   // END FEED_FORWARD

            case FULL_FORWARD:
            {
                for (vector<vector<neuron>>::iterator prev_layers = _neurons.begin();
                        prev_layers != layer; ++prev_layers)
                {
                    for (auto& successors: *layer)
                    {
                        for (auto& predecessors: *prev_layers)
                        {
                            rate  = _random_engine.splitrange(0.05,0.5);
                            link_neurons(predecessors, successors, rate);
                        }
                        
                        // Link every neuron against the bias neuron. If BIAS_NEUROM == false
                        // the output of _bias is set to 0, so that the link becomes unimportant.
                        // This is used that the _bias neuron can be switched on and off.
                        rate = _random_engine.splitrange(0.05,0.5);
                        link_neurons(_bias, successors, rate);
                    }
                }
                break;
            }   // END FULL_FORWARD

            default:
            {
            }
        }   // END switch
    }
}

void network::_reset_netoutputs()
{
    for (auto& layer: _neurons)
    {
        for (auto& neuron: layer)
        {
            neuron.out = 0;
        }
    }
}

void network::_calculate_netoutputs(vector<double> input)
{
    unsigned input_size = input.size();
    if (input_size != _network_structure.front() )
    {
        //Throw an error --- will be implemented
    }

    for (unsigned i = 0; i < input_size; ++i)
    {
        _neurons.front()[i].out = input[i];
    }

    if (BIAS_NEURON)
    {
        _bias.out = 1;
            // The Bias neuron is an additional neuron of the input layer, and always sends a constant signal
    }
    else
    {
        _bias.out = 0;
    }

    for (vector<vector<neuron>>::iterator layer = ++_neurons.begin(); layer != _neurons.end(); ++layer)
    {
        for (auto& neuron: *layer)
        {
            neuron.activate();
        }
    }
}

void network::_train_network(vector<double> errors, double learning_rate, double layer_fact)
{
    // Train the output neurons
    for (unsigned i = 0; i < _network_structure.back(); ++i) 
    {
        _neurons.back()[i].calculate_delta(errors[i]);
        _neurons.back()[i].train_connection(learning_rate);
    }

    // Train all, but the input/output neurons
    for (vector<vector<neuron>>::reverse_iterator rlayer = ++_neurons.rbegin();
            rlayer != --_neurons.rend(); ++rlayer)
    {
        learning_rate *= layer_fact;
        for (auto& neuron: *rlayer)
        {
            neuron.calculate_delta();
            neuron.train_connection(learning_rate);
        }
    }
}


double network::train_epoch(testcases tests, double learning_rate, double layer_fact)
{
    vector<vector<double>> errors;
 
    switch (learning_strategy.timing)
    {
        case OFFLINE:
        {
            for (auto& test: tests)
            {
                if (test[0].size() != _network_structure.front() || test[1].size() != _network_structure.back() )
                {
                    //Throw an error
                }

                _reset_netoutputs();
                _calculate_netoutputs(test.front() );

                vector<double> tmp_error(_network_structure.back() );
                for (unsigned j = 0; j < tmp_error.size(); ++j)
                {
                    tmp_error[j] = test.back()[j] - _neurons.back()[j].out;
                }
                errors.push_back(tmp_error);
            }

            vector<double> avg_error(_network_structure.back(), 0);
            for (unsigned i = 0; i < _network_structure.back(); ++i)
            {
                for (auto const& error: errors)
                {
                    avg_error[i] += error[i];
                }
                avg_error[i] /= (double)errors.size();
            }
            _train_network(avg_error, learning_rate, layer_fact);
            break;
        }   // END OFFLINE

        case ONLINE:
        {
            std::shuffle(tests.begin(), tests.end(), _random_engine.get_engine() );
            for (auto& test: tests)
            {
                if (test[0].size() != _network_structure.front() || test[1].size() != _network_structure.back() )
                {
                    //Throw an error
                }

                _reset_netoutputs();
                _calculate_netoutputs(test.front() );

                vector<double> tmp_error(_network_structure.back() );
                for (unsigned j = 0; j < tmp_error.size(); ++j)
                {
                    tmp_error[j] = test.back()[j] - _neurons.back()[j].out;
                }
                errors.push_back(tmp_error);
                _train_network(errors.back(), learning_rate, layer_fact);
            }
            break;
        }   // END ONLINE

        default:
        {
        }
    }

    vector<double> specific_errors;
    for (auto const& error: errors)
    {
        specific_errors.push_back(_errfct(error) );
    }

    double totalerror = 0;
    for (auto& specific_error: specific_errors)
    {
        totalerror += specific_error;
    }

    return totalerror;
}

vector<double> network::solve(vector<double> input)
{
    _reset_netoutputs();
    _calculate_netoutputs(input);
    
    vector<double> solution;
    for (auto const& output_neuron: _neurons.back() )
    {
        solution.push_back(output_neuron.out);
    }
    return solution;
}

These are the two interesting classes, yet there is more code around them. I hesitate to post it all here, to keep everything compact, so I will only provide links right now --- if needed I will edit my question.

• network repository (The source I posted here is from commit d37db4b / 09.11)

EDIT the old source files

• fact.hpp (old)

• fact.cpp (old)

• errfct.hpp (old)

• errfct.cpp (old)

• rne.hpp (old)

• rne.cpp (old)

• defs.hpp (old)

• config.hpp (old)

• main.cpp (old)

EDIT: I moved a lot of the compile time options into enum's. I am still pretty sure, this isn't the optimal solution, yet I think it is better than before.

• What general advice do you have for a novice programmer?

• Where do I deviate from best practice, where is my code hard to read/ unlogical?

• Is my overall design/ ideas good? How do I better plan such projects?

• (old question, see edit) Can I avoid a bunch of nested conditional compilation steps and still have control over the features of my code at compile time?

• I currently rely on enum's to choose at run time the features of code that I want. This works better for me than the previous approach, but how can I further improve on this? It still feels a little clunky.

EDIT

The greatest thing I changed are the conditional compilation flags: I removed them. It just felt more and more bad. I now use enum's and switch statements to decide at run time which part of code I want. Maybe I come back later to some compile time features. Additionally, I changed the learning_rate and layer_fact (the factor by with the learning rate grows by each layer) from public variables to function arguments. To keep the code compatible (and because I like default arguments) I assigned a default value to them. Quick questions: are numerical default arguments magic numbers?