Added outline for the CUDA portion of the DNN toolkit.

dlib/dnn/cublas.h

+// Copyright (C) 2015  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_DNN_CuBLAS_H_
+#define DLIB_DNN_CuBLAS_H_
+
+#ifdef DLIB_USE_CUDA
+
+#include "tensor.h"
+
+namespace dlib
+{
+    namespace cuda 
+    {
+
+    // -----------------------------------------------------------------------------------
+
+        struct cublas_error : public error
+        {
+            cublas_error(const std::string& message): error(message) {}
+        };
+
+    // -----------------------------------------------------------------------------------
+
+        class cublas_context
+        {
+        public:
+            cublas_context(const cublas_context&) = delete;
+            cublas_context& operator=(const cublas_context&) = delete;
+
+            cublas_context()
+            {
+                // TODO
+            }
+
+            ~cublas_context()
+            {
+                // TODO
+            }
+        private:
+
+            void* handle;
+        };
+
+    // -----------------------------------------------------------------------------------
+
+        void gemm (
+            cublas_context& context,
+            float beta,
+            tensor& dest,
+            float alpha,
+            const tensor& lhs 
+            bool trans_lhs,
+            const tensor& rhs,
+            bool trans_rhs
+        );
+
+    // ------------------------------------------------------------------------------------
+
+    }  
+}
+
+#endif // DLIB_USE_CUDA
+
+#endif // DLIB_DNN_CuBLAS_H_
+
+

dlib/dnn/cuda.h

+// Copyright (C) 2015  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_DNN_CuDA_H_
+#define DLIB_DNN_CuDA_H_
+
+#ifdef DLIB_USE_CUDA
+
+#include "tensor.h"
+
+namespace dlib
+{
+    namespace cuda 
+    {
+
+    // -----------------------------------------------------------------------------------
+
+        void affine_transform(
+            resizable_tensor& dest,
+            const tensor& src,
+            const float A,
+            const float B
+        );
+        /*!
+            ensures
+                - have_same_dimensions(#dest,src) == true
+                - #dest == A*src + B
+        !*/
+
+    // -----------------------------------------------------------------------------------
+
+        void affine_transform(
+            resizable_tensor& dest,
+            const tensor& src,
+            const tensor& A,
+            const tensor& B
+        );
+        /*!
+            requires
+                - A.num_samples() == 1
+                - B.num_samples() == 1
+                - A.nr() == B.nr() == src.nr()
+                - A.nc() == B.nc() == src.nc()
+                - A.k()  == B.k()  == src.k()
+            ensures
+                - have_same_dimensions(#dest,src) == true
+                - #dest == A*src + B
+                  (done for each sample in src)
+        !*/
+
+    // -----------------------------------------------------------------------------------
+
+    // TODO, add versions of batch_normalize() that output the gradients.
+        void batch_normalize (
+            resizable_tensor& dest,
+            resizable_tensor& means,
+            resizable_tensor& vars,
+            const tensor& src,
+            const tensor& gamma, 
+            const tensor& beta 
+        );
+        /*!
+            requires
+                - gamma.num_samples() == 1
+                - beta.num_samples() == 1
+                - gamma.nr() == beta.nr() == src.nr()
+                - gamma.nc() == beta.nc() == src.nc()
+                - gamma.k()  == beta.k()  == src.k()
+            ensures
+                - have_same_dimensions(#dest, src) == true
+                - #means.num_samples() == 1
+                - #vars.num_samples() == 1
+                - means.nr() == vars.nr() == src.nr()
+                - means.nc() == vars.nc() == src.nc()
+                - means.k()  == vars.k()  == src.k()
+                - #src == the batch normalized version of src.
+                - #means == the mean values of the contents of src.
+                - #vars == the variance values of the contents of src.
+        !*/
+
+        void batch_normalize_conv (
+            resizable_tensor& dest,
+            resizable_tensor& means,
+            resizable_tensor& vars,
+            const tensor& src,
+            const tensor& gamma, 
+            const tensor& beta 
+        );
+        /*!
+            requires
+                - gamma.num_samples()==gamma.nr()==gamma.nc() == 1
+                - beta.num_samples() ==beta.nr() ==gamma.nc() == 1
+                - gamma.k()  == beta.k()  == src.k()
+            ensures
+                - have_same_dimensions(#dest, src) == true
+                - #means.num_samples()==means.nr()==means.nc() == 1
+                - #vars.num_samples() ==vars.nr() ==vars.nc() == 1
+                - means.k()  == vars.k()  == src.k()
+                - #src == the batch normalized version of src.
+                - #means == the mean values of the contents of src.
+                - #vars == the variance values of the contents of src.
+        !*/
+
+    // -----------------------------------------------------------------------------------
+
+        class dropout
+        {
+            /*!
+            !*/
+        public:
+
+            dropout(const dropout&) = delete;
+            dropout& operator=(const dropout&) = delete;
+
+            dropout(float drop_rate = 0.5);
+            dropout(float drop_rate, int seed);
+
+            void operator() (
+                resizable_tensor& dest,
+                resizable_tensor& random_mask,
+                const tensor& src
+            );
+            /*!
+                ensures
+                    - have_same_dimensions(src, #dest) == true
+                    - have_same_dimensions(src, #random_mask) == true
+            !*/
+
+            void get_gradient(
+                const tensor& gradient_input, 
+                const tensor& random_mask,
+                tensor& grad 
+            );
+            /*!
+                requires
+                    - have_same_dimensions(gradient_input, random_mask) == true
+                    - have_same_dimensions(gradient_input, grad) == true
+                ensures
+                    - let OUT and MASK be the output of (*this)(OUT,MASK,src)
+                    - let f(src) == dot(gradient_input,OUT)
+                    - Then this function computes the gradient of f() with respect to src
+                      and adds it to grad.
+            !*/
+        };
+
+    // -----------------------------------------------------------------------------------
+
+    } 
+}
+
+#endif // DLIB_USE_CUDA
+
+#endif // DLIB_DNN_CuDA_H_
+

dlib/dnn/cudnn.h

+// Copyright (C) 2015  Davis E. King (davis@dlib.net)
+// License: Boost Software License   See LICENSE.txt for the full license.
+#ifndef DLIB_DNN_CuDNN_H_
+#define DLIB_DNN_CuDNN_H_
+
+#ifdef DLIB_USE_CUDA
+
+#include "tensor.h"
+
+namespace dlib
+{
+    namespace cuda 
+    {
+
+    // -----------------------------------------------------------------------------------
+
+        struct cudnn_error : public error
+        {
+            cudnn_error(const std::string& message): error(message) {}
+        };
+
+    // ------------------------------------------------------------------------------------
+
+        class cudnn_context
+        {
+        public:
+            cudnn_context(const cudnn_context&) = delete;
+            cudnn_context& operator=(const cudnn_context&) = delete;
+
+            cudnn_context()
+            {
+                // TODO
+                // cudnnCreate()
+            }
+
+            ~cudnn_context()
+            {
+                // TODO
+                // cudnnDestroy()
+            }
+        private:
+
+            void* handle;
+        };
+
+    // ------------------------------------------------------------------------------------
+
+        class tensor_descriptor
+        {
+            /*!
+                Each tensor object will carry a tensor_descriptor in it when compiled with
+                CUDA.
+            !*/
+
+        public:
+            tensor_descriptor(const tensor_descriptor&) = delete;
+            tensor_descriptor& operator=(const tensor_descriptor&) = delete;
+
+            tensor_descriptor()
+            {
+                //  cudnnCreateTensorDescriptor()
+
+                // Also call cudnnSetTensor4dDescriptor() somewhere.  Probably in a set()
+                // method.
+            }
+
+            ~tensor_descriptor()
+            {
+                // cudnnDestroyTensorDescriptor()
+            }
+
+        private:
+
+            void* handle;
+        };
+
+    // ------------------------------------------------------------------------------------
+
+        void add(
+            cudnn_context& context, 
+            float beta,
+            tensor& dest,
+            float alpha,
+            const tensor& src
+        );
+        /*!
+            requires
+                - dest.num_samples()==src.num_samples() || src.num_samples()==1
+                - dest.nr()==src.nr() || src.nr()==1
+                - dest.nc()==src.nc() || src.nc()==1
+                - dest.k()==src.k()   || src.k()==1
+            ensures
+                - performs: dest = beta*dest + alpha*src
+                   TODO, clarify:
+
+                   calls cudnnAddTensor_v3()
+
+                   This function adds the scaled values of one bias tensor to another
+                   tensor. Each dimension of the bias tensor must match the coresponding
+                   dimension of the srcDest tensor or must be equal to 1. In the latter
+                   case, the same value from the bias tensor for thoses dimensions will be
+                   used to blend into the srcDest tensor.
+        !*/
+
+        void set_tensor (
+            cudnn_context& context,
+            tensor& t,
+            float value
+        );
+        /*!
+            ensures
+                - sets all elements in t equal to value.
+                  Uses cudnnSetTensor().
+        !*/
+
+        void scale_tensor (
+            cudnn_context& context,
+            tensor& t,
+            float value
+        );
+        /*!
+            ensures
+                - scales all elements of t by the given value.  I.e. for all elements E in
+                  t, this function performs:
+                    - E = E*value
+
+                uses cudnnScaleTensor()
+        !*/
+
+    // ------------------------------------------------------------------------------------
+
+        class conv
+        {
+        public:
+            conv(const conv&) = delete;
+            conv& operator=(const conv&) = delete;
+
+            conv(
+                cudnn_context& context,
+                const tensor& data,
+                const tensor& filters,
+                int stride_y,
+                int stride_x
+            );
+
+            void operator() (
+                resizable_tensor& output,
+                const tensor& data,
+                const tensor& filters
+            );
+            /*!
+                requires
+                    - the dimensions of data and filters are the same as the ones given 
+                      to the constructor.
+                ensures
+                    - convolves filters over data.  
+                    - filters contains filters.num_samples() filters. 
+                    - #output.num_samples() == data.num_samples()
+                    - #output.k() == filters.num_samples()
+                    - #output.nr() == 1+(data.nr()-1)/stride_y
+                    - #output.nc() == 1+(data.nc()-1)/stride_x
+            !*/
+
+            // get gradient of data: 4.49. cudnnConvolutionBackwardData_v3
+            void get_gradient_for_data (
+                const tensor& gradient_input, 
+                const tensor& filters,
+                tensor& data_gradient
+            );
+            /*!
+                requires
+                    - filters has the same dimensions as the filters object give to the
+                      constructor.
+                    - data_gradient has the same dimensions as the data object give to the
+                      constructor.
+                    - gradient_input has the same dimensions as the output of operator().
+                ensures
+                    - let OUT be the output of (*this)(OUT,data,filters).
+                    - let f(data,filters) == dot(OUT, gradient_input)
+                    - This function finds the gradient of f() with respect to data
+                      and adds this gradient to data_gradient.
+            !*/
+
+            // get gradient of filters: 4.44. cudnnConvolutionBackwardFilter_v3
+            void get_gradient_for_filters (
+                const tensor& gradient_input, 
+                const tensor& data,
+                tensor& filters_gradient
+            );
+            /*!
+                requires
+                    - filters_gradient has the same dimensions as the filters object give
+                      to the constructor.
+                    - data has the same dimensions as the data object give to the constructor.
+                    - gradient_input has the same dimensions as the output of operator().
+                ensures
+                    - let OUT be the output of (*this)(OUT,data,filters).
+                    - let f(data,filters) == dot(OUT, gradient_input)
+                    - This function finds the gradient of f() with respect to filters 
+                      and adds this gradient to filters_gradient.
+            !*/
+
+        };
+
+    // ------------------------------------------------------------------------------------
+
+        void soft_max (
+            cudnn_context& context,
+            resizable_tensor& dest,
+            const tensor& src
+        );
+        /*!
+            probably uses CUDNN_SOFTMAX_MODE_CHANNEL 
+        !*/
+
+        void soft_max_gradient (
+            cudnn_context& context,
+            tensor& grad,
+            const tensor& src,
+            const tensor& gradient_input
+        );
+        /*!
+            - let OUT be the output of soft_max(context,OUT,src)
+            - let f(src) == dot(gradient_input,OUT)
+            - Then this function computes the gradient of f() with respect to src
+              and adds it to grad.
+        !*/
+
+    // ------------------------------------------------------------------------------------
+
+        class max_pool
+        {
+            /*!
+                CUDNN_POOLING_MAX
+            !*/
+        public:
+
+            max_pool(const max_pool&) = delete;
+            max_pool& operator=(const max_pool&) = delete;
+
+            // cudnnCreatePoolingDescriptor(), cudnnSetPooling2dDescriptor()
+            max_pool (
+                cudnn_context& context,
+                int window_height,
+                int window_width,
+                int stride_y,
+                int stride_x
+            );
+
+            // cudnnDestroyPoolingDescriptor ()
+            ~max_pool(
+            );
+
+            // cudnnGetPooling2dForwardOutputDim(), cudnnPoolingForward()
+            void operator() (
+                resizable_tensor& dest,
+                const tensor& src
+            );
+            /*!
+            !*/
+
+            // cudnnPoolingBackward()
+            void get_gradient(
+                const tensor& gradient_input, 
+                const tensor& src,
+                tensor& grad 
+            );
+            /*!
+                - let OUT be the output of (*this)(OUT,src)
+                - let f(src) == dot(gradient_input,OUT)
+                - Then this function computes the gradient of f() with respect to src and
+                  adds it to grad.
+            !*/
+        };
+
+        // TODO, make the order of parameters of all these functions consistent.
+
+    // ------------------------------------------------------------------------------------
+
+        // cudnnActivationForward(), CUDNN_ACTIVATION_SIGMOID
+        void sigmoid (
+            cudnn_context& context,
+            resizable_tensor& dest,
+            const tensor& src
+        );
+        /*!
+            ensures
+                - have_same_dimensions(#dest, src) == true
+                - for all valid i:
+                    - #dest.host()[i] == 1/(1+std::exp(-src.host()[i])) 
+        !*/
+
+        // cudnnActivationBackward()
+        void sigmoid_gradient (
+            cudnn_context& context,
+            tensor& grad,
+            const tensor& src,
+            const tensor& gradient_input
+        );
+        /*!
+            requires
+                - have_same_dimensions(src,gradient_input) == true 
+                - have_same_dimensions(src,grad) == true 
+            ensures
+                - let OUT be the output of sigmoid(context,OUT,src)
+                - let f(src) == dot(gradient_input,OUT)
+                - Then this function computes the gradient of f() with respect to src and
+                  adds it to grad.
+        !*/
+
+    // ------------------------------------------------------------------------------------
+
+        // cudnnActivationForward(), CUDNN_ACTIVATION_RELU
+        void relu (
+            cudnn_context& context,
+            resizable_tensor& dest,
+            const tensor& src
+        );
+        /*!
+            ensures
+                - have_same_dimensions(#dest, src) == true
+                - for all valid i:
+                    - #dest.host()[i] == std::max(0,src.host()[i]) 
+        !*/
+
+        // cudnnActivationBackward()
+        void relu_gradient (
+            cudnn_context& context,
+            tensor& grad,
+            const tensor& src,
+            const tensor& gradient_input
+        );
+        /*!
+            requires
+                - have_same_dimensions(src,gradient_input) == true 
+                - have_same_dimensions(src,grad) == true 
+            ensures
+                - let OUT be the output of relu(context,OUT,src)
+                - let f(src) == dot(gradient_input,OUT)
+                - Then this function computes the gradient of f() with respect to src and
+                  adds it to grad.
+        !*/
+
+    // ------------------------------------------------------------------------------------
+
+        // cudnnActivationForward(), CUDNN_ACTIVATION_TANH
+        void tanh (
+            cudnn_context& context,
+            resizable_tensor& dest,
+            const tensor& src
+        );
+        /*!
+            ensures
+                - have_same_dimensions(#dest, src) == true
+                - for all valid i:
+                    - #dest.host()[i] == std::tanh(src.host()[i]) 
+        !*/
+
+        // cudnnActivationBackward()
+        void tanh_gradient (
+            cudnn_context& context,
+            tensor& grad,
+            const tensor& src,
+            const tensor& gradient_input
+        );
+        /*!
+            requires
+                - have_same_dimensions(src,gradient_input) == true 
+                - have_same_dimensions(src,grad) == true 
+            ensures
+                - let OUT be the output of tanh(context,OUT,src)
+                - let f(src) == dot(gradient_input,OUT)
+                - Then this function computes the gradient of f() with respect to src and
+                  adds it to grad.
+        !*/
+
+    // ------------------------------------------------------------------------------------
+
+    } 
+}
+
+#endif // DLIB_USE_CUDA
+
+#endif // DLIB_DNN_CuDNN_H_
+