model.hpp

Go to the documentation of this file.

#ifndef BPLYFMIFNNWSGMLLEKBJMAJDBRSPHHRAYMOHTWSTCMNMFSLLYNQTTCCAQXKXSLMSLKESHRASL
#define BPLYFMIFNNWSGMLLEKBJMAJDBRSPHHRAYMOHTWSTCMNMFSLLYNQTTCCAQXKXSLMSLKESHRASL
 
#include "./includes.hpp"
#include "./operation.hpp"
#include "./place_holder.hpp"
#include "./tensor.hpp"
#include "./utils/better_assert.hpp"
#include "./utils/context_cast.hpp"
#include "./utils/tqdm.hpp"
 
namespace ceras
{
    template< Expression Ex >
    void make_trainable( Ex& ex , bool t )
    {
        if constexpr (is_variable_v<Ex>)
        {
            ex.trainable( t );
        }
        else if constexpr (is_binary_operator_v<Ex>)
        {
            make_trainable( ex.lhs_op_, t );
            make_trainable( ex.rhs_op_, t );
        }
        else if constexpr (is_unary_operator_v<Ex>)
        {
            make_trainable( ex.op_, t );
        }
    }
 
    template< Expression Ex, Place_Holder Ph, Expression Ey >
    auto replace_placeholder_with_expression( Ex const& ex, Ph const& old_place_holder, Ey const& new_expression )
    {
        if constexpr (is_value_v<Ex> || is_constant_v<Ex> || is_variable_v<Ex>)
        {
            return ex;
        }
        else if constexpr (is_place_holder_v<Ex>)
        {
            return new_expression; // assuming only one place holder in the model
            //return (ex == old_place_holder) ? new_expression : ex;
        }
        else if constexpr (is_unary_operator_v<Ex>)
        {
            return make_unary_operator( ex.forward_action_, ex.backward_action_, ex.name_ )( replace_placeholder_with_expression( ex.op_, old_place_holder, new_expression ) );
        }
        else if constexpr (is_binary_operator_v<Ex>)
        {
            return make_binary_operator( ex.forward_action_, ex.backward_action_, ex.name_ )
                                       ( replace_placeholder_with_expression( ex.lhs_op_, old_place_holder, new_expression ),
                                         replace_placeholder_with_expression( ex.rhs_op_, old_place_holder, new_expression ) );
        }
        else
        {
            better_assert( false, "replace::Should never reach here!" );
        }
    }
 
    template< typename Model, typename Optimizer, typename Loss >
    struct compiled_model
    {
        typedef typename Model::input_layer_type io_layer_type;
        typedef decltype(std::declval<Optimizer>()(std::declval<Loss&>())) optimizer_type; // defined because the compiled optimizer takes a reference to an expression as its parameter
 
        Model model_;
        io_layer_type input_place_holder_;
        io_layer_type ground_truth_place_holder_;
        Loss loss_; // MeanSquaredError()( model_.output() )( find a input );
        Optimizer optimizer_; // Adam( ... )
        optimizer_type compiled_optimizer_;
 
        compiled_model( Model const& m, io_layer_type const& input_place_holder, io_layer_type const& ground_truth_place_holder, Loss const& loss, Optimizer const& optimizer ):
            model_{m}, input_place_holder_{input_place_holder}, ground_truth_place_holder_{ground_truth_place_holder}, loss_{loss}, optimizer_{optimizer}, compiled_optimizer_{ optimizer_(loss_) }  { }
 
        template< Tensor Tsor >
        auto evaluate( Tsor const& inputs, Tsor const& outputs, unsigned long batch_size=32 )
        {
            // extract size of samples
            unsigned long const samples = *(inputs.shape().begin());
            unsigned long const loops = samples / batch_size;
 
            // prepare tensor for inputs
            std::vector<unsigned long> batch_input_shape = inputs.shape();
            batch_input_shape[0] = batch_size;
            Tsor input_samples{ batch_input_shape };
            unsigned long const input_size_per_batch = input_samples.size();
 
            // prepare tensor for outputs
            std::vector<unsigned long> batch_output_shape = outputs.shape();
            batch_output_shape[0] = batch_size;
            Tsor output_samples{ batch_output_shape };
            unsigned long const output_size_per_batch = output_samples.size();
 
            // bind tensors to place holders
            //session<Tsor> s;
            auto& s = get_default_session<Tsor>();
            s.bind( input_place_holder_, input_samples );
            s.bind( ground_truth_place_holder_, output_samples );
 
            typedef typename Tsor::value_type value_type;
            value_type validation_error = 0;
 
            learning_phase = 0; // for different behaviours in normalization and drop-out layers
 
            for ( auto l : tq::trange( loops ) )
            {
                // feed data
                std::copy_n( inputs.data() + l * input_size_per_batch, input_size_per_batch, input_samples.data() );
                std::copy_n( outputs.data() + l * output_size_per_batch, output_size_per_batch, output_samples.data() );
                // forward pass
                auto error = s.run( loss_ ).as_scalar();
                // in case of training split, do backpropagation
                validation_error += error;
            }
 
            learning_phase = 1; // for different behaviours in normalization and drop-out layers
 
            return validation_error / loops;
        }
 
        template< Tensor Tsor >
        auto fit( Tsor const& inputs, Tsor const& outputs, unsigned long batch_size, unsigned long epoch=1, int verbose=0, double validation_split=0.0 )
        {
            // extract size of samples
            unsigned long const samples = *(inputs.shape().begin());
            unsigned long const loops_per_epoch = samples / batch_size;
            unsigned long const training_loops = ( 1.0 - validation_split ) * loops_per_epoch;
            unsigned long const validation_loops = loops_per_epoch - training_loops;
 
            // prepare tensor for inputs
            std::vector<unsigned long> batch_input_shape = inputs.shape();
            batch_input_shape[0] = batch_size;
            Tsor input_samples{ batch_input_shape };
            unsigned long const input_size_per_batch = input_samples.size();
 
            // prepare tensor for outputs
            std::vector<unsigned long> batch_output_shape = outputs.shape();
            batch_output_shape[0] = batch_size;
            Tsor output_samples{ batch_output_shape };
            unsigned long const output_size_per_batch = output_samples.size();
 
            // bind tensors to place holders
            //session<Tsor> s;
            auto& s = get_default_session<Tsor>();//.get();
            s.bind( input_place_holder_, input_samples );
            s.bind( ground_truth_place_holder_, output_samples );
 
            // collect training errors
            typedef typename Tsor::value_type value_type;
            std::vector<value_type> training_errors;
            std::vector<value_type> validation_errors;
 
            learning_phase = 1; // for different behaviours in normalization and drop-out layers
 
            for ( auto e : range( epoch ) )
            {
                value_type training_error = 0;
                value_type validation_error = 0;
                for ( auto l : tq::trange( loops_per_epoch ) )
                {
                    // feed data
                    std::copy_n( inputs.data() + l * input_size_per_batch, input_size_per_batch, input_samples.data() );
                    std::copy_n( outputs.data() + l * output_size_per_batch, output_size_per_batch, output_samples.data() );
                    // forward pass
                    auto error = s.run( loss_ ).as_scalar();
                    // in case of training split, do backpropagation
                    if ( l <= training_loops )
                    {
                        training_error += error;
                        s.run( compiled_optimizer_ );
                    }
                    else // in case of validation split, just collect errors
                    {
                        validation_error += error;
                    }
                }
                training_errors.push_back( training_error / training_loops );
                validation_errors.push_back( validation_error / validation_loops );
                if ( verbose )
                    std::cout << "\nTraining error: " << training_error / training_loops << " and validation error: " << validation_error / validation_loops << " at epoch: " << e+1 << "/" << epoch;
                std::cout << std::endl;
            }
            return std::make_tuple( training_errors, validation_errors );
        }
 
        template< Tensor Tsor >
        auto train_on_batch( Tsor const& input, Tsor const& output )
        {
            learning_phase = 1; // for different behaviours in normalization and drop-out layers
            auto& s = get_default_session<Tsor>();//.get();
            s.bind( input_place_holder_, input );
            s.bind( ground_truth_place_holder_, output );
            //debug_log( "Training on batch forward pass." );
            auto error = s.run( loss_ );
            //debug_log( "Training on batch backward pass." );
            s.run( compiled_optimizer_ );
            return error.as_scalar();
        }
 
        template< Tensor Tsor>
        auto predict( Tsor const& input_tensor )
        {
            auto m = model_;
            return m.predict( input_tensor );
        }
 
        template< Expression Exp >
        auto operator()( Exp const& ex ) const noexcept
        {
            return model_( ex );
        }
 
        void trainable( bool t )
        {
            model_.trainable( t );
        }
    };
 
    template< typename Model, typename Optimizer, typename Loss >
    inline auto make_compiled_model( Model const& m, Loss const& l, Optimizer const& o )
    {
        auto input_place_holder = m.input();
        auto ground_truth_place_holder = typename Model::input_layer_type{};
        auto loss = l( m.output() )( ground_truth_place_holder );
        auto optimizer = o( loss );
        return compiled_model{ m, input_place_holder, ground_truth_place_holder, loss, o};
    }
 
    template< Expression Ex, Place_Holder Ph >
    struct model
    {
        typedef Ph       input_layer_type;
        typedef Ex       output_layer_type;
 
        output_layer_type expression_;   
        input_layer_type place_holder_;//< input layer of the model.
 
 
        input_layer_type input() const noexcept { return place_holder_; }
 
        output_layer_type output() const noexcept { return expression_; }
 
        model( input_layer_type const& place_holder, output_layer_type const& expression ) :  expression_{expression}, place_holder_{place_holder} {}
 
        template< Tensor Tsor>
        auto predict( Tsor const& input_tensor )
        {
            learning_phase = 0; // for different behaviours in normalization and drop-out layers
 
            //session<Tsor> s;
            auto& s = get_default_session<Tsor>();//.get();
            s.bind( place_holder_, input_tensor );
 
            auto ans = s.run( expression_ );
 
            learning_phase = 1; // restore learning phase
 
            return ans;
        }
 
        template< Expression Exp >
        auto operator()( Exp const& ex ) const noexcept
        {
            return replace_placeholder_with_expression( expression_, place_holder_, ex );
        }
 
        template< typename Loss, typename Optimizer >
        auto compile( Loss const& l, Optimizer const& o )
        {
            return make_compiled_model( *this, l, o );
        }
 
        void trainable( bool t )
        {
            make_trainable( expression_, t );
        }
 
 
        void save_weights( std::string const& file )
        {
            auto& s = get_default_session<tensor<float>>();
            s.serialize( file );
        }
 
        void load_weights( std::string const& file )
        {
            auto& s = get_default_session<tensor<float>>();
            s.deserialize( file );
        }
 
 
        void summary(std::string const& file_name=std::string{}) const noexcept
        {
            auto g = computation_graph( expression_ );
 
            if ( file_name.empty() )
            {
                std::cout << g << std::endl;
                return;
            }
 
            std::ofstream ofs{ file_name };
            ofs << g;
        }
 
 
    };
 
}//namespace ceras
 
#endif//BPLYFMIFNNWSGMLLEKBJMAJDBRSPHHRAYMOHTWSTCMNMFSLLYNQTTCCAQXKXSLMSLKESHRASLCalculate the loss for the model in test model