layers_abstract.h 12.6 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
// Copyright (C) 2015  Davis E. King (davis@dlib.net)
// License: Boost Software License   See LICENSE.txt for the full license.
#undef DLIB_DNn_LAYERS_ABSTRACT_H_
#ifdef DLIB_DNn_LAYERS_ABSTRACT_H_

#include "tensor_abstract.h"
#include "core_abstract.h"


namespace dlib
{

// ----------------------------------------------------------------------------------------

Davis King's avatar
Davis King committed
15
    class SUBNET 
16
17
18
    {
        /*!
            WHAT THIS OBJECT REPRESENTS
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
                This object represents a deep neural network.  In particular, it is
                the simplified interface through which layer objects interact with their
                subnetworks.  A layer's two important tasks are to (1) take outputs from its
                subnetwork and forward propagate them though itself and (2) to backwards
                propagate an error gradient through itself and onto its subnetwork.
                The idea of a subnetwork is illustrated in the following diagram:

                  +---------------------------------------------------------+
                  | loss <-- layer1 <-- layer2 <-- ... <-- layern <-- input |
                  +---------------------------------------------------------+
                                      ^                            ^
                                      \__ subnetwork for layer1 __/

                Therefore, by "subnetwork" we mean the part of the network closer to the
                input.  
34
35
36
        !*/

    public:
37
        // You aren't allowed to copy subnetworks from inside a layer.
Davis King's avatar
Davis King committed
38
39
        SUBNET(const SUBNET&) = delete;
        SUBNET& operator=(const SUBNET&) = delete;
40
41
42

        const tensor& get_output(
        ) const;
43
44
45
46
47
48
        /*!
            ensures
                - returns the output of this subnetwork.  This is the data that the next
                  layer in the network will take as input.
                - have_same_dimensions(#get_gradient_input(), get_output()) == true
        !*/
49
50
51

        tensor& get_gradient_input(
        );
52
53
54
55
56
57
58
59
60
61
62
        /*!
            ensures
                - returns the error gradient for this subnetwork.  That is, this is the
                  error gradient that this network will use to update itself.  Therefore,
                  when performing back propagation, layers that sit on top of this
                  subnetwork write their back propagated error gradients into
                  get_gradient_input().  Or to put it another way, during back propagation,
                  layers take the contents of their get_gradient_input() and back propagate
                  it through themselves and store the results into their subnetwork's
                  get_gradient_input().
        !*/
63

Davis King's avatar
Davis King committed
64
        const NEXT_SUBNET& subnet(
65
        ) const;
66
67
68
        /*!
            ensures
                - returns the subnetwork of *this network.  With respect to the diagram
Davis King's avatar
Davis King committed
69
                  above, if *this was layer1 then subnet() would return the network that
70
71
                  begins with layer2.
        !*/
72

Davis King's avatar
Davis King committed
73
        NEXT_SUBNET& subnet(
74
        );
75
76
77
        /*!
            ensures
                - returns the subnetwork of *this network.  With respect to the diagram
Davis King's avatar
Davis King committed
78
                  above, if *this was layer1 then subnet() would return the network that
79
80
                  begins with layer2.
        !*/
81
82
83
84
85
86
87
88
89
90
91
92
    };

// ----------------------------------------------------------------------------------------

    class EXAMPLE_LAYER_
    {
        /*!
            WHAT THIS OBJECT REPRESENTS
                Each layer in a deep neural network can be thought of as a function,
                f(data,parameters), that takes in a data tensor, some parameters, and
                produces an output tensor.  You create an entire deep network by composing
                these functions.  Importantly, you are able to use a wide range of
93
                different functions to accommodate the task you are trying to accomplish.
94
                Dlib includes a number of common layer types but if you want to define your
95
96
                own then you simply implement a class with the same interface as
                EXAMPLE_LAYER_.
97

98
99
                Note that there is no dlib::EXAMPLE_LAYER_ type.  It is shown here purely
                to document the interface that a layer object must implement.
100
101
102
103
104
105
106
107
108
        !*/

    public:

        EXAMPLE_LAYER_(
        );
        /*!
            ensures
                - Default constructs this object.  This function is not required to do
109
110
                  anything in particular but it must exist, that is, it is required that
                  layer objects be default constructable. 
111
112
        !*/

113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
        EXAMPLE_LAYER_(
            const some_other_layer_type& item
        );
        /*!
            ensures
                - Constructs this object from item.  This form of constructor is optional
                  but it allows you to provide a conversion from one layer type to another.
                  For example, the following code is valid only if my_layer2 can be
                  constructed from my_layer1:
                    relu<fc<my_layer1<fc<input<matrix<float>>>>>> my_dnn1;
                    relu<fc<my_layer2<fc<input<matrix<float>>>>>> my_dnn2(my_dnn1);
                  This kind of pattern is useful if you want to use one type of layer
                  during training but a different type of layer during testing since it
                  allows you to easily convert between related deep neural network types.  
        !*/

Davis King's avatar
Davis King committed
129
        template <typename SUBNET>
130
        void setup (
Davis King's avatar
Davis King committed
131
            const SUBNET& sub
132
133
134
        );
        /*!
            requires
Davis King's avatar
Davis King committed
135
                - SUBNET implements the SUBNET interface defined at the top of this file.
136
137
138
            ensures
                - performs any necessary initial memory allocations and/or sets parameters
                  to their initial values prior to learning.  Therefore, calling setup
Davis King's avatar
Davis King committed
139
140
141
                  destroys any previously learned parameters.  Also, typically setup()
                  would look at the dimensions of the outputs of sub and configure the
                  number of parameters in *this accordingly.
142
143
        !*/

Davis King's avatar
Davis King committed
144
        template <typename SUBNET>
145
        void forward(
Davis King's avatar
Davis King committed
146
            const SUBNET& sub, 
147
148
149
150
            resizable_tensor& output
        );
        /*!
            requires
Davis King's avatar
Davis King committed
151
                - SUBNET implements the SUBNET interface defined at the top of this file.
152
153
                - setup() has been called.
            ensures
154
                - Runs the output of the subnetwork through this layer and stores the
155
                  output into #output.  In particular, forward() can use any of the outputs
Davis King's avatar
Davis King committed
156
                  in sub (e.g. sub.get_output(), sub.subnet().get_output(), etc.) to
157
158
159
160
                  compute whatever it wants.
                - #output.num_samples() == sub.get_output().num_samples()
        !*/

Davis King's avatar
Davis King committed
161
        template <typename SUBNET>
162
163
        void backward(
            const tensor& gradient_input, 
Davis King's avatar
Davis King committed
164
            SUBNET& sub, 
165
166
167
168
            tensor& params_grad
        );
        /*!
            requires
Davis King's avatar
Davis King committed
169
                - SUBNET implements the SUBNET interface defined at the top of this file.
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
                - setup() has been called.
                - gradient_input has the same dimensions as the output of forward(sub,output).
                - have_same_dimensions(sub.get_gradient_input(), sub.get_output()) == true
                - have_same_dimensions(params_grad, get_layer_params()) == true
            ensures
                - This function outputs the gradients of this layer with respect to the
                  input data from sub and also with respect to this layer's parameters.
                  These gradients are stored into #sub and #params_grad, respectively. To be
                  precise, the gradients are taken of a function f(sub,get_layer_params())
                  which is defined thusly:   
                    - let OUT be the output of forward(sub,OUT).
                    - let f(sub,get_layer_params()) == dot(OUT, gradient_input)
                  Then we define the following gradient vectors: 
                    - PARAMETER_GRADIENT == gradient of f(sub,get_layer_params()) with
                      respect to get_layer_params(). 
                    - for all valid I:
                        - DATA_GRADIENT_I == gradient of f(sub,get_layer_params()) with
                          respect to layer<I>(sub).get_output() (recall that forward() can
Davis King's avatar
Davis King committed
188
                          draw inputs from the immediate sub layer, sub.subnet(), or
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
                          any earlier layer.  So you must consider the gradients with
                          respect to all inputs drawn from sub)
                  Finally, backward() adds these gradients into the output by performing:
                    - params_grad += PARAMETER_GRADIENT
                    - for all valid I:
                        - layer<I>(sub).get_gradient_input() += DATA_GRADIENT_I
        !*/

        const tensor& get_layer_params(
        ) const; 
        /*!
            ensures
                - returns the parameters that define the behavior of forward().
        !*/

        tensor& get_layer_params(
        ); 
        /*!
            ensures
                - returns the parameters that define the behavior of forward().
        !*/

    };

    // For each layer you define, always define an add_layer template so that layers can be
    // easily composed.  Moreover, the convention is that the layer class ends with an _
    // while the add_layer template has the same name but without the trailing _.
Davis King's avatar
Davis King committed
216
217
    template <typename SUBNET>
    using EXAMPLE_LAYER = add_layer<EXAMPLE_LAYER_, SUBNET>;
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258

// ----------------------------------------------------------------------------------------
// ----------------------------------------------------------------------------------------
// ----------------------------------------------------------------------------------------

    class fc_
    {
        /*!
            WHAT THIS OBJECT REPRESENTS
                This is an implementation of the EXAMPLE_LAYER_ interface defined above.
                In particular, it defines a fully connected layer that takes an input
                tensor and multiplies it by a weight matrix and outputs the results.
        !*/

    public:
        fc_(
        );
        /*!
            ensures
                - #get_num_outputs() == 1
        !*/

        explicit fc_(
            unsigned long num_outputs
        );
        /*!
            ensures
                - #get_num_outputs() == num_outputs
        !*/

        unsigned long get_num_outputs (
        ) const; 
        /*!
            ensures
                - This layer outputs column vectors that contain get_num_outputs()
                  elements. That is, the output tensor T from forward() will be such that:
                    - T.num_samples() == however many samples were given to forward().
                    - T.nr() == get_num_outputs()
                    - The rest of the dimensions of T will be 1.
        !*/

Davis King's avatar
Davis King committed
259
260
261
        template <typename SUBNET> void setup (const SUBNET& sub);
        template <typename SUBNET> void forward(const SUBNET& sub, resizable_tensor& output);
        template <typename SUBNET> void backward(const tensor& gradient_input, SUBNET& sub, tensor& params_grad);
262
263
264
265
266
267
268
269
        const tensor& get_layer_params() const; 
        tensor& get_layer_params(); 
        /*!
            These functions are implemented as described in the EXAMPLE_LAYER_ interface.
        !*/
    };


Davis King's avatar
Davis King committed
270
271
    template <typename SUBNET>
    using fc = add_layer<fc_, SUBNET>;
272
273
274
275
276

// ----------------------------------------------------------------------------------------

    class relu_
    {
Davis King's avatar
Davis King committed
277
278
279
280
281
282
283
284
285
        /*!
            WHAT THIS OBJECT REPRESENTS
                This is an implementation of the EXAMPLE_LAYER_ interface defined above.
                In particular, it defines a rectified linear layer.  Therefore, it passes
                its inputs though the function f(x)=max(x,0) where f() is applied pointwise
                across the input tensor.
                
        !*/

286
287
288
289
290
    public:

        relu_(
        );

Davis King's avatar
Davis King committed
291
292
293
        template <typename SUBNET> void setup (const SUBNET& sub);
        template <typename SUBNET> void forward(const SUBNET& sub, resizable_tensor& output);
        template <typename SUBNET> void backward(const tensor& gradient_input, SUBNET& sub, tensor& params_grad);
294
295
296
297
298
299
300
301
        const tensor& get_layer_params() const; 
        tensor& get_layer_params(); 
        /*!
            These functions are implemented as described in the EXAMPLE_LAYER_ interface.
        !*/
    };


Davis King's avatar
Davis King committed
302
303
    template <typename SUBNET>
    using relu = add_layer<relu_, SUBNET>;
304
305
306
307
308

// ----------------------------------------------------------------------------------------

}

309
#endif // DLIB_DNn_LAYERS_ABSTRACT_H_
310