driver.cpp 33.2 KB
Newer Older
Chao Liu's avatar
Chao Liu committed
1
#include <iostream>
Chao Liu's avatar
Chao Liu committed
2
3
#include <numeric>
#include <initializer_list>
Chao Liu's avatar
Chao Liu committed
4
#include <cstdlib>
Chao Liu's avatar
Chao Liu committed
5
#include <stdlib.h>
Chao Liu's avatar
Chao Liu committed
6
7
8
#include "config.hpp"
#include "ConstantTensorDescriptor.hpp"
#include "device.hpp"
Chao Liu's avatar
Chao Liu committed
9
#include "conv_common.hpp"
Chao Liu's avatar
Chao Liu committed
10
#include "device_convolution_direct_v2_nchw_kcyx_nkhw.hpp"
Chao Liu's avatar
Chao Liu committed
11
#include "device_convolution_implicit_gemm_v1_chwn_cyxk_khwn.hpp"
Chao Liu's avatar
Chao Liu committed
12
#include "device_convolution_implicit_gemm_v1_nchw_cyxk_nkhw.hpp"
Chao Liu's avatar
Chao Liu committed
13
#include "device_convolution_implicit_gemm_v2_chwn_cyxk_khwn.hpp"
Chao Liu's avatar
Chao Liu committed
14
#include "device_convolution_implicit_gemm_v3_nchw_cyxk_nkhw.hpp"
15
#include "device_convolution_implicit_gemm_v4_nchw_kcyx_nkhw.hpp"
Chao Liu's avatar
Chao Liu committed
16

Chao Liu's avatar
Chao Liu committed
17
struct GeneratorTensor_1
Chao Liu's avatar
Chao Liu committed
18
19
{
    template <class... Is>
Chao Liu's avatar
Chao Liu committed
20
    double operator()(Is... is)
Chao Liu's avatar
Chao Liu committed
21
    {
Chao Liu's avatar
Chao Liu committed
22
        return 1;
Chao Liu's avatar
Chao Liu committed
23
24
25
    }
};

Chao Liu's avatar
Chao Liu committed
26
27
28
29
30
31
32
33
34
35
36
37
struct GeneratorTensor_2
{
    int min_value = 0;
    int max_value = 1;

    template <class... Is>
    double operator()(Is...)
    {
        return (std::rand() % (max_value - min_value)) + min_value;
    }
};

38
39
40
41
42
43
44
struct GeneratorTensor_3
{
    template <class... Is>
    double operator()(Is... is)
    {
        std::array<index_t, sizeof...(Is)> dims = {{static_cast<index_t>(is)...}};

45
        auto f_acc = [](auto a, auto b) { return 100 * a + b; };
46

47
        return std::accumulate(dims.begin(), dims.end(), index_t(0), f_acc);
48
49
50
    }
};

Chao Liu's avatar
Chao Liu committed
51
52
53
54
55
struct GeneratorTensor_Checkboard
{
    template <class... Ts>
    double operator()(Ts... Xs) const
    {
Chao Liu's avatar
Chao Liu committed
56
        std::array<index_t, sizeof...(Ts)> dims = {{Xs...}};
Chao Liu's avatar
Chao Liu committed
57
58
59
        return std::accumulate(dims.begin(),
                               dims.end(),
                               true,
Chao Liu's avatar
Chao Liu committed
60
                               [](bool init, index_t x) -> int { return init != (x % 2); })
Chao Liu's avatar
Chao Liu committed
61
62
63
64
65
                   ? 1
                   : -1;
    }
};

66
67
68
69
70
71
72
template <class TIn,
          class TWei,
          class TOut,
          class ConvStrides,
          class ConvDilations,
          class LowerPads,
          class UpperPads>
73
74
75
void host_direct_convolution(const Tensor<TIn>& in_nchw,
                             const Tensor<TWei>& wei_kcyx,
                             Tensor<TOut>& out_nkhw,
76
77
                             ConvStrides,
                             ConvDilations,
78
79
                             LowerPads,
                             UpperPads)
Chao Liu's avatar
Chao Liu committed
80
{
Chao Liu's avatar
Chao Liu committed
81
82
    index_t h_pad_low = LowerPads{}.Get(Number<0>{});
    index_t w_pad_low = LowerPads{}.Get(Number<1>{});
83

Chao Liu's avatar
Chao Liu committed
84
85
    index_t h_pad_up = UpperPads{}.Get(Number<0>{});
    index_t w_pad_up = UpperPads{}.Get(Number<1>{});
86

Chao Liu's avatar
Chao Liu committed
87
88
    auto f = [&](auto n, auto k, auto ho, auto wo) {
        double v = 0;
Chao Liu's avatar
Chao Liu committed
89
        for(int c = 0; c < wei_kcyx.mDesc.GetLengths()[1]; ++c)
Chao Liu's avatar
Chao Liu committed
90
        {
Chao Liu's avatar
Chao Liu committed
91
            for(int y = 0; y < wei_kcyx.mDesc.GetLengths()[2]; ++y)
Chao Liu's avatar
Chao Liu committed
92
            {
93
                int hi = ho * ConvStrides{}[0] + y * ConvDilations{}[0] - h_pad_low;
Chao Liu's avatar
Chao Liu committed
94
                for(int x = 0; x < wei_kcyx.mDesc.GetLengths()[3]; ++x)
Chao Liu's avatar
Chao Liu committed
95
                {
96
                    int wi = wo * ConvStrides{}[1] + x * ConvDilations{}[1] - w_pad_low;
97
98
99
                    if(hi >= 0 && hi < in_nchw.mDesc.GetLengths()[2] && wi >= 0 &&
                       wi < in_nchw.mDesc.GetLengths()[3])
                    {
100
                        v += double(in_nchw(n, c, hi, wi)) * double(wei_kcyx(k, c, y, x));
101
                    }
Chao Liu's avatar
Chao Liu committed
102
103
104
                }
            }
        }
105
        out_nkhw(n, k, ho, wo) = v;
Chao Liu's avatar
Chao Liu committed
106
107
108
    };

    auto f_par = make_ParallelTensorFunctor(f,
109
110
111
112
                                            out_nkhw.mDesc.GetLengths()[0],
                                            out_nkhw.mDesc.GetLengths()[1],
                                            out_nkhw.mDesc.GetLengths()[2],
                                            out_nkhw.mDesc.GetLengths()[3]);
Chao Liu's avatar
Chao Liu committed
113

Chao Liu's avatar
Chao Liu committed
114
    f_par(std::thread::hardware_concurrency());
Chao Liu's avatar
Chao Liu committed
115
116
}

117
118
119
120
121
122
template <class TIn, class TWei, class TOut, class LowerPads, class UpperPads>
void host_winograd_3x3_convolution(const Tensor<TIn>& in_nchw,
                                   const Tensor<TWei>& wei_kcyx,
                                   Tensor<TOut>& out_nkhw,
                                   LowerPads,
                                   UpperPads)
Chao Liu's avatar
Chao Liu committed
123
{
Chao Liu's avatar
Chao Liu committed
124
125
    constexpr std::size_t HoPerTile = 2;
    constexpr std::size_t WoPerTile = 2;
Chao Liu's avatar
Chao Liu committed
126

Chao Liu's avatar
Chao Liu committed
127
128
129
130
    std::size_t N  = in_nchw.mDesc.GetLengths()[0];
    std::size_t C  = in_nchw.mDesc.GetLengths()[1];
    std::size_t HI = in_nchw.mDesc.GetLengths()[2];
    std::size_t WI = in_nchw.mDesc.GetLengths()[3];
Chao Liu's avatar
Chao Liu committed
131

Chao Liu's avatar
Chao Liu committed
132
133
134
    std::size_t K = wei_kcyx.mDesc.GetLengths()[0];
    std::size_t Y = wei_kcyx.mDesc.GetLengths()[2];
    std::size_t X = wei_kcyx.mDesc.GetLengths()[3];
Chao Liu's avatar
Chao Liu committed
135

136
137
    std::size_t HO = out_nkhw.mDesc.GetLengths()[2];
    std::size_t WO = out_nkhw.mDesc.GetLengths()[3];
Chao Liu's avatar
Chao Liu committed
138

Chao Liu's avatar
Chao Liu committed
139
140
    index_t h_pad_low = LowerPads{}.Get(Number<0>{});
    index_t w_pad_low = LowerPads{}.Get(Number<1>{});
141

Chao Liu's avatar
Chao Liu committed
142
143
    index_t h_pad_up = UpperPads{}.Get(Number<0>{});
    index_t w_pad_up = UpperPads{}.Get(Number<1>{});
144

Chao Liu's avatar
Chao Liu committed
145
146
    std::size_t HiPerTile = HoPerTile + Y - 1;
    std::size_t WiPerTile = WoPerTile + X - 1;
Chao Liu's avatar
Chao Liu committed
147

Chao Liu's avatar
Chao Liu committed
148
149
    std::size_t HTile = (HO + HoPerTile - 1) / HoPerTile;
    std::size_t WTile = (WO + WoPerTile - 1) / WoPerTile;
Chao Liu's avatar
Chao Liu committed
150

151
152
153
154
155
    Tensor<double> in_hold({N, C, HTile, WTile, HiPerTile, WiPerTile});
    Tensor<double> in_transform({N, C, HTile, WTile, HiPerTile, WiPerTile});
    Tensor<double> wei_transform({K, C, HiPerTile, WiPerTile});
    Tensor<double> out_transform({N, K, HTile, WTile, HiPerTile, HiPerTile});
    Tensor<double> out_hold({N, K, HTile, WTile, HoPerTile, WoPerTile});
Chao Liu's avatar
Chao Liu committed
156

Chao Liu's avatar
Chao Liu committed
157
158
    auto f_in_hold = [&](auto n, auto c, auto htile, auto wtile) {
        for(int j = 0; j < HiPerTile; ++j)
Chao Liu's avatar
Chao Liu committed
159
        {
Chao Liu's avatar
Chao Liu committed
160
161
            int hi = HoPerTile * htile + j - h_pad_low;
            for(int i = 0; i < WiPerTile; ++i)
Chao Liu's avatar
Chao Liu committed
162
            {
Chao Liu's avatar
Chao Liu committed
163
                int wi = WoPerTile * wtile + i - w_pad_low;
164
165
166
167

                if(hi >= 0 && hi < in_nchw.mDesc.GetLengths()[2] && wi >= 0 &&
                   wi < in_nchw.mDesc.GetLengths()[3])
                {
Chao Liu's avatar
Chao Liu committed
168
                    in_hold(n, c, htile, wtile, j, i) = in_nchw(n, c, hi, wi);
169
170
171
                }
                else
                {
172
                    in_hold(n, c, htile, wtile, j, i) = TIn(0);
173
                }
Chao Liu's avatar
Chao Liu committed
174
175
176
177
            }
        }
    };

Chao Liu's avatar
Chao Liu committed
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
    auto f_in_transform = [&](auto n, auto c, auto htile, auto wtile) {
        in_transform(n, c, htile, wtile, 0, 0) =
            in_hold(n, c, htile, wtile, 0, 0) - in_hold(n, c, htile, wtile, 0, 2) -
            in_hold(n, c, htile, wtile, 2, 0) + in_hold(n, c, htile, wtile, 2, 2);
        in_transform(n, c, htile, wtile, 0, 1) =
            in_hold(n, c, htile, wtile, 0, 1) + in_hold(n, c, htile, wtile, 0, 2) -
            in_hold(n, c, htile, wtile, 2, 1) - in_hold(n, c, htile, wtile, 2, 2);
        in_transform(n, c, htile, wtile, 0, 2) =
            -in_hold(n, c, htile, wtile, 0, 1) + in_hold(n, c, htile, wtile, 0, 2) +
            in_hold(n, c, htile, wtile, 2, 1) - in_hold(n, c, htile, wtile, 2, 2);
        in_transform(n, c, htile, wtile, 0, 3) =
            in_hold(n, c, htile, wtile, 0, 1) - in_hold(n, c, htile, wtile, 0, 3) -
            in_hold(n, c, htile, wtile, 2, 1) + in_hold(n, c, htile, wtile, 2, 3);

        in_transform(n, c, htile, wtile, 1, 0) =
            in_hold(n, c, htile, wtile, 1, 0) - in_hold(n, c, htile, wtile, 1, 2) +
            in_hold(n, c, htile, wtile, 2, 0) - in_hold(n, c, htile, wtile, 2, 2);
        in_transform(n, c, htile, wtile, 1, 1) =
            in_hold(n, c, htile, wtile, 1, 1) + in_hold(n, c, htile, wtile, 1, 2) +
            in_hold(n, c, htile, wtile, 2, 1) + in_hold(n, c, htile, wtile, 2, 2);
        in_transform(n, c, htile, wtile, 1, 2) =
            -in_hold(n, c, htile, wtile, 1, 1) + in_hold(n, c, htile, wtile, 1, 2) -
            in_hold(n, c, htile, wtile, 2, 1) + in_hold(n, c, htile, wtile, 2, 2);
        in_transform(n, c, htile, wtile, 1, 3) =
            in_hold(n, c, htile, wtile, 1, 1) - in_hold(n, c, htile, wtile, 1, 3) +
            in_hold(n, c, htile, wtile, 2, 1) - in_hold(n, c, htile, wtile, 2, 3);

        in_transform(n, c, htile, wtile, 2, 0) =
            -in_hold(n, c, htile, wtile, 1, 0) + in_hold(n, c, htile, wtile, 1, 2) +
            in_hold(n, c, htile, wtile, 2, 0) - in_hold(n, c, htile, wtile, 2, 2);
        in_transform(n, c, htile, wtile, 2, 1) =
            -in_hold(n, c, htile, wtile, 1, 1) - in_hold(n, c, htile, wtile, 1, 2) +
            in_hold(n, c, htile, wtile, 2, 1) + in_hold(n, c, htile, wtile, 2, 2);
        in_transform(n, c, htile, wtile, 2, 2) =
            in_hold(n, c, htile, wtile, 1, 1) - in_hold(n, c, htile, wtile, 1, 2) -
            in_hold(n, c, htile, wtile, 2, 1) + in_hold(n, c, htile, wtile, 2, 2);
        in_transform(n, c, htile, wtile, 2, 3) =
            -in_hold(n, c, htile, wtile, 1, 1) + in_hold(n, c, htile, wtile, 1, 3) +
            in_hold(n, c, htile, wtile, 2, 1) - in_hold(n, c, htile, wtile, 2, 3);

        in_transform(n, c, htile, wtile, 3, 0) =
            in_hold(n, c, htile, wtile, 1, 0) - in_hold(n, c, htile, wtile, 1, 2) -
            in_hold(n, c, htile, wtile, 3, 0) + in_hold(n, c, htile, wtile, 3, 2);
        in_transform(n, c, htile, wtile, 3, 1) =
            in_hold(n, c, htile, wtile, 1, 1) + in_hold(n, c, htile, wtile, 1, 2) -
            in_hold(n, c, htile, wtile, 3, 1) - in_hold(n, c, htile, wtile, 3, 2);
        in_transform(n, c, htile, wtile, 3, 2) =
            -in_hold(n, c, htile, wtile, 1, 1) + in_hold(n, c, htile, wtile, 1, 2) +
            in_hold(n, c, htile, wtile, 3, 1) - in_hold(n, c, htile, wtile, 3, 2);
        in_transform(n, c, htile, wtile, 3, 3) =
            in_hold(n, c, htile, wtile, 1, 1) - in_hold(n, c, htile, wtile, 1, 3) -
            in_hold(n, c, htile, wtile, 3, 1) + in_hold(n, c, htile, wtile, 3, 3);
Chao Liu's avatar
Chao Liu committed
230
231
232
    };

    auto f_wei_transform = [&](auto k, auto c) {
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
        wei_transform(k, c, 0, 0) = double(wei_kcyx(k, c, 0, 0));
        wei_transform(k, c, 0, 1) = 0.5 * double(wei_kcyx(k, c, 0, 0)) +
                                    0.5 * double(wei_kcyx(k, c, 0, 1)) +
                                    0.5 * double(wei_kcyx(k, c, 0, 2));
        wei_transform(k, c, 0, 2) = 0.5 * double(wei_kcyx(k, c, 0, 0)) -
                                    0.5 * double(wei_kcyx(k, c, 0, 1)) +
                                    0.5 * double(wei_kcyx(k, c, 0, 2));
        wei_transform(k, c, 0, 3) = double(wei_kcyx(k, c, 0, 2));

        wei_transform(k, c, 1, 0) = 0.5 * double(wei_kcyx(k, c, 0, 0)) +
                                    0.5 * double(wei_kcyx(k, c, 1, 0)) +
                                    0.5 * double(wei_kcyx(k, c, 2, 0));
        wei_transform(k, c, 1, 1) =
            0.25 * double(wei_kcyx(k, c, 0, 0)) + 0.25 * double(wei_kcyx(k, c, 0, 1)) +
            0.25 * double(wei_kcyx(k, c, 0, 2)) + 0.25 * double(wei_kcyx(k, c, 1, 0)) +
            0.25 * double(wei_kcyx(k, c, 1, 1)) + 0.25 * double(wei_kcyx(k, c, 1, 2)) +
            0.25 * double(wei_kcyx(k, c, 2, 0)) + 0.25 * double(wei_kcyx(k, c, 2, 1)) +
            0.25 * double(wei_kcyx(k, c, 2, 2));
        wei_transform(k, c, 1, 2) =
            0.25 * double(wei_kcyx(k, c, 0, 0)) - 0.25 * double(wei_kcyx(k, c, 0, 1)) +
            0.25 * double(wei_kcyx(k, c, 0, 2)) + 0.25 * double(wei_kcyx(k, c, 1, 0)) -
            0.25 * double(wei_kcyx(k, c, 1, 1)) + 0.25 * double(wei_kcyx(k, c, 1, 2)) +
            0.25 * double(wei_kcyx(k, c, 2, 0)) - 0.25 * double(wei_kcyx(k, c, 2, 1)) +
            0.25 * double(wei_kcyx(k, c, 2, 2));
        wei_transform(k, c, 1, 3) = 0.5 * double(wei_kcyx(k, c, 0, 2)) +
                                    0.5 * double(wei_kcyx(k, c, 1, 2)) +
                                    0.5 * double(wei_kcyx(k, c, 2, 2));

        wei_transform(k, c, 2, 0) = 0.5 * double(wei_kcyx(k, c, 0, 0)) -
                                    0.5 * double(wei_kcyx(k, c, 1, 0)) +
                                    0.5 * double(wei_kcyx(k, c, 2, 0));
        wei_transform(k, c, 2, 1) =
            0.25 * double(wei_kcyx(k, c, 0, 0)) + 0.25 * double(wei_kcyx(k, c, 0, 1)) +
            0.25 * double(wei_kcyx(k, c, 0, 2)) - 0.25 * double(wei_kcyx(k, c, 1, 0)) -
            0.25 * double(wei_kcyx(k, c, 1, 1)) - 0.25 * double(wei_kcyx(k, c, 1, 2)) +
            0.25 * double(wei_kcyx(k, c, 2, 0)) + 0.25 * double(wei_kcyx(k, c, 2, 1)) +
            0.25 * double(wei_kcyx(k, c, 2, 2));
        wei_transform(k, c, 2, 2) =
            0.25 * double(wei_kcyx(k, c, 0, 0)) - 0.25 * double(wei_kcyx(k, c, 0, 1)) +
            0.25 * double(wei_kcyx(k, c, 0, 2)) - 0.25 * double(wei_kcyx(k, c, 1, 0)) +
            0.25 * double(wei_kcyx(k, c, 1, 1)) - 0.25 * double(wei_kcyx(k, c, 1, 2)) +
            0.25 * double(wei_kcyx(k, c, 2, 0)) - 0.25 * double(wei_kcyx(k, c, 2, 1)) +
            0.25 * double(wei_kcyx(k, c, 2, 2));
        wei_transform(k, c, 2, 3) = 0.5 * double(wei_kcyx(k, c, 0, 2)) -
                                    0.5 * double(wei_kcyx(k, c, 1, 2)) +
                                    0.5 * double(wei_kcyx(k, c, 2, 2));

        wei_transform(k, c, 3, 0) = double(wei_kcyx(k, c, 2, 0));
        wei_transform(k, c, 3, 1) = 0.5 * double(wei_kcyx(k, c, 2, 0)) +
                                    0.5 * double(wei_kcyx(k, c, 2, 1)) +
                                    0.5 * double(wei_kcyx(k, c, 2, 2));
        wei_transform(k, c, 3, 2) = 0.5 * double(wei_kcyx(k, c, 2, 0)) -
                                    0.5 * double(wei_kcyx(k, c, 2, 1)) +
                                    0.5 * double(wei_kcyx(k, c, 2, 2));
        wei_transform(k, c, 3, 3) = double(wei_kcyx(k, c, 2, 2));
Chao Liu's avatar
Chao Liu committed
288
289
    };

Chao Liu's avatar
Chao Liu committed
290
291
    auto f_out_transform = [&](auto n, auto k, auto htile, auto wtile) {
        for(int j = 0; j < HiPerTile; ++j)
Chao Liu's avatar
Chao Liu committed
292
        {
Chao Liu's avatar
Chao Liu committed
293
            for(int i = 0; i < WiPerTile; ++i)
Chao Liu's avatar
Chao Liu committed
294
295
296
297
            {
                double v = 0;
                for(int c = 0; c < C; ++c)
                {
Chao Liu's avatar
Chao Liu committed
298
                    v += in_transform(n, c, htile, wtile, j, i) * wei_transform(k, c, j, i);
Chao Liu's avatar
Chao Liu committed
299
300
                }

Chao Liu's avatar
Chao Liu committed
301
                out_transform(n, k, htile, wtile, j, i) = v;
Chao Liu's avatar
Chao Liu committed
302
303
304
305
            }
        }
    };

Chao Liu's avatar
Chao Liu committed
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
    auto f_out_hold = [&](auto n, auto k, auto htile, auto wtile) {
        out_hold(n, k, htile, wtile, 0, 0) =
            out_transform(n, k, htile, wtile, 0, 0) + out_transform(n, k, htile, wtile, 0, 1) +
            out_transform(n, k, htile, wtile, 0, 2) + out_transform(n, k, htile, wtile, 1, 0) +
            out_transform(n, k, htile, wtile, 1, 1) + out_transform(n, k, htile, wtile, 1, 2) +
            out_transform(n, k, htile, wtile, 2, 0) + out_transform(n, k, htile, wtile, 2, 1) +
            out_transform(n, k, htile, wtile, 2, 2);
        out_hold(n, k, htile, wtile, 0, 1) =
            out_transform(n, k, htile, wtile, 0, 1) - out_transform(n, k, htile, wtile, 0, 2) -
            out_transform(n, k, htile, wtile, 0, 3) + out_transform(n, k, htile, wtile, 1, 1) -
            out_transform(n, k, htile, wtile, 1, 2) - out_transform(n, k, htile, wtile, 1, 3) +
            out_transform(n, k, htile, wtile, 2, 1) - out_transform(n, k, htile, wtile, 2, 2) -
            out_transform(n, k, htile, wtile, 2, 3);
        out_hold(n, k, htile, wtile, 1, 0) =
            out_transform(n, k, htile, wtile, 1, 0) + out_transform(n, k, htile, wtile, 1, 1) +
            out_transform(n, k, htile, wtile, 1, 2) - out_transform(n, k, htile, wtile, 2, 0) -
            out_transform(n, k, htile, wtile, 2, 1) - out_transform(n, k, htile, wtile, 2, 2) -
            out_transform(n, k, htile, wtile, 3, 0) - out_transform(n, k, htile, wtile, 3, 1) -
            out_transform(n, k, htile, wtile, 3, 2);
        out_hold(n, k, htile, wtile, 1, 1) =
            out_transform(n, k, htile, wtile, 1, 1) - out_transform(n, k, htile, wtile, 1, 2) -
            out_transform(n, k, htile, wtile, 1, 3) - out_transform(n, k, htile, wtile, 2, 1) +
            out_transform(n, k, htile, wtile, 2, 2) + out_transform(n, k, htile, wtile, 2, 3) -
            out_transform(n, k, htile, wtile, 3, 1) + out_transform(n, k, htile, wtile, 3, 2) +
            out_transform(n, k, htile, wtile, 3, 3);
Chao Liu's avatar
Chao Liu committed
331
332
    };

Chao Liu's avatar
Chao Liu committed
333
334
    auto f_out = [&](auto n, auto k, auto htile, auto wtile) {
        for(int j = 0; j < HoPerTile; ++j)
Chao Liu's avatar
Chao Liu committed
335
        {
Chao Liu's avatar
Chao Liu committed
336
337
            std::size_t ho = HoPerTile * htile + j;
            for(int i = 0; i < WoPerTile; ++i)
Chao Liu's avatar
Chao Liu committed
338
            {
339
                std::size_t wo = WoPerTile * wtile + i;
340
                out_nkhw(n, k, ho, wo) = out_hold(n, k, htile, wtile, j, i);
Chao Liu's avatar
Chao Liu committed
341
342
343
344
345
346
            }
        }
    };

    std::size_t num_thread = std::thread::hardware_concurrency();

Chao Liu's avatar
Chao Liu committed
347
348
    make_ParallelTensorFunctor(f_in_hold, N, C, HTile, WTile)(num_thread);
    make_ParallelTensorFunctor(f_in_transform, N, C, HTile, WTile)(num_thread);
Chao Liu's avatar
Chao Liu committed
349
    make_ParallelTensorFunctor(f_wei_transform, K, C)(num_thread);
Chao Liu's avatar
Chao Liu committed
350
351
352
    make_ParallelTensorFunctor(f_out_transform, N, K, HTile, WTile)(num_thread);
    make_ParallelTensorFunctor(f_out_hold, N, K, HTile, WTile)(num_thread);
    make_ParallelTensorFunctor(f_out, N, K, HTile, WTile)(num_thread);
Chao Liu's avatar
Chao Liu committed
353
354
355
356
357
358
}

template <class T>
void check_error(const Tensor<T>& ref, const Tensor<T>& result)
{
    float error     = 0;
Chao Liu's avatar
Chao Liu committed
359
    float max_diff  = -1;
Chao Liu's avatar
Chao Liu committed
360
361
362
    float ref_value = 0, result_value = 0;
    for(int i = 0; i < ref.mData.size(); ++i)
    {
363
364
        error += std::abs(double(ref.mData[i]) - double(result.mData[i]));
        float diff = std::abs(double(ref.mData[i]) - double(result.mData[i]));
Chao Liu's avatar
Chao Liu committed
365
366
367
368
369
370
371
372
373
374
375
376
        if(max_diff < diff)
        {
            max_diff     = diff;
            ref_value    = ref.mData[i];
            result_value = result.mData[i];
        }
    }

    std::cout << "error: " << error << std::endl;
    std::cout << "max_diff: " << max_diff << ", " << ref_value << ", " << result_value << std::endl;
}

Chao Liu's avatar
Chao Liu committed
377
int main(int argc, char* argv[])
Chao Liu's avatar
Chao Liu committed
378
{
Chao Liu's avatar
Chao Liu committed
379
380
    using namespace ck;

Chao Liu's avatar
Chao Liu committed
381
382
#if 0
    constexpr index_t N  = 8;
Chao Liu's avatar
Chao Liu committed
383
    constexpr index_t C  = 16;
Chao Liu's avatar
Chao Liu committed
384
385
386
387
388
389
390
391
    constexpr index_t HI = 3;
    constexpr index_t WI = 18;
    constexpr index_t K  = 128;
    constexpr index_t Y  = 3;
    constexpr index_t X  = 3;

    constexpr index_t HPad = 0;
    constexpr index_t WPad = 0;
392
#elif 0
393
    // 3x3, 34x34
394
    constexpr index_t N  = 128;
395
    constexpr index_t C  = 256;
396
397
    constexpr index_t HI = 34;
    constexpr index_t WI = 34;
398
399
400
    constexpr index_t K  = 128;
    constexpr index_t Y  = 3;
    constexpr index_t X  = 3;
Chao Liu's avatar
Chao Liu committed
401

402
403
404
    using ConvStrides   = Sequence<2, 2>;
    using ConvDilations = Sequence<1, 1>;

Chao Liu's avatar
Chao Liu committed
405
406
    constexpr index_t HPad = 0;
    constexpr index_t WPad = 0;
Chao Liu's avatar
Chao Liu committed
407
#elif 0
408
    // 3x3, 56x56
Chao Liu's avatar
Chao Liu committed
409
410
    constexpr index_t N  = 64;
    constexpr index_t C  = 64;
411
412
    constexpr index_t HI = 56;
    constexpr index_t WI = 56;
Chao Liu's avatar
Chao Liu committed
413
414
415
    constexpr index_t K  = 128;
    constexpr index_t Y  = 3;
    constexpr index_t X  = 3;
Chao Liu's avatar
Chao Liu committed
416
417
418

    constexpr index_t HPad = 0;
    constexpr index_t WPad = 0;
Chao Liu's avatar
Chao Liu committed
419
#elif 0
Chao Liu's avatar
Chao Liu committed
420
421
422
423
424
    // 3x3 filter, 28x28 image
    constexpr index_t N  = 128;
    constexpr index_t C  = 256;
    constexpr index_t HI = 28;
    constexpr index_t WI = 28;
425
    constexpr index_t K  = 128;
Chao Liu's avatar
Chao Liu committed
426
427
428
    constexpr index_t Y  = 3;
    constexpr index_t X  = 3;

Chao Liu's avatar
Chao Liu committed
429
    using ConvStrides   = Sequence<1, 1>;
430
431
    using ConvDilations = Sequence<1, 1>;

Chao Liu's avatar
Chao Liu committed
432
433
    constexpr index_t HPad = 0;
    constexpr index_t WPad = 0;
Chao Liu's avatar
Chao Liu committed
434
#elif 0
Chao Liu's avatar
Chao Liu committed
435
    // 1x1 filter, 28x28 image
436
437
    constexpr index_t N  = 128;
    constexpr index_t C  = 512;
Chao Liu's avatar
Chao Liu committed
438
439
440
441
442
443
    constexpr index_t HI = 28;
    constexpr index_t WI = 28;
    constexpr index_t K  = 512;
    constexpr index_t Y  = 1;
    constexpr index_t X  = 1;

Chao Liu's avatar
Chao Liu committed
444
445
446
    using ConvStrides   = Sequence<1, 1>;
    using ConvDilations = Sequence<1, 1>;

Chao Liu's avatar
Chao Liu committed
447
448
    constexpr index_t HPad = 0;
    constexpr index_t WPad = 0;
449
450
#elif 0
    // 3x3 filter, 20x84 image, 1x1 padding
Chao Liu's avatar
Chao Liu committed
451
452
453
454
455
456
457
458
459
460
    constexpr index_t N  = 16;
    constexpr index_t C  = 256;
    constexpr index_t HI = 20;
    constexpr index_t WI = 84;
    constexpr index_t K  = 256;
    constexpr index_t Y  = 3;
    constexpr index_t X  = 3;

    constexpr index_t HPad = 1;
    constexpr index_t WPad = 1;
Chao Liu's avatar
Chao Liu committed
461
462
#elif 0
    // 3x3 filter, 112x112 image, 1x1 padding
Chao Liu's avatar
Chao Liu committed
463
464
465
466
467
468
469
470
471
472
    constexpr index_t N  = 16;
    constexpr index_t C  = 64;
    constexpr index_t HI = 112;
    constexpr index_t WI = 112;
    constexpr index_t K  = 128;
    constexpr index_t Y  = 3;
    constexpr index_t X  = 3;

    constexpr index_t HPad = 1;
    constexpr index_t WPad = 1;
473
#elif 0
474
475
476
477
478
479
480
481
482
483
484
    // 5x5 filter, 20x86 image
    constexpr index_t N  = 16;
    constexpr index_t C  = 256;
    constexpr index_t HI = 20;
    constexpr index_t WI = 86;
    constexpr index_t K  = 512;
    constexpr index_t Y  = 5;
    constexpr index_t X  = 5;

    constexpr index_t HPad = 0;
    constexpr index_t WPad = 0;
Chao Liu's avatar
Chao Liu committed
485
486
#elif 0
    // 5x5 filter, 20x86 image, 1x1 padding
Chao Liu's avatar
Chao Liu committed
487
488
489
490
491
492
493
494
495
496
    constexpr index_t N  = 16;
    constexpr index_t C  = 256;
    constexpr index_t HI = 20;
    constexpr index_t WI = 86;
    constexpr index_t K  = 512;
    constexpr index_t Y  = 5;
    constexpr index_t X  = 5;

    constexpr index_t HPad = 1;
    constexpr index_t WPad = 1;
Chao Liu's avatar
Chao Liu committed
497
498
#elif 0
    // 5x5 filter, 28x28 image, 2x2 padding
Chao Liu's avatar
Chao Liu committed
499
500
501
502
503
504
505
506
507
508
    constexpr index_t N  = 16;
    constexpr index_t C  = 192;
    constexpr index_t HI = 28;
    constexpr index_t WI = 28;
    constexpr index_t K  = 32;
    constexpr index_t Y  = 5;
    constexpr index_t X  = 5;

    constexpr index_t HPad = 2;
    constexpr index_t WPad = 2;
Chao Liu's avatar
Chao Liu committed
509
#elif 0
510
    // 3x3 filter, 14x14 image
Chao Liu's avatar
Chao Liu committed
511
    constexpr index_t N  = 128;
512
    constexpr index_t C  = 256;
Chao Liu's avatar
Chao Liu committed
513
514
    constexpr index_t HI = 14;
    constexpr index_t WI = 14;
515
516
517
    constexpr index_t K  = 128;
    constexpr index_t Y  = 3;
    constexpr index_t X  = 3;
Chao Liu's avatar
Chao Liu committed
518
519
520

    constexpr index_t HPad = 0;
    constexpr index_t WPad = 0;
Chao Liu's avatar
Chao Liu committed
521
#elif 0
522
    // 1x1 filter, 14x14 image
Chao Liu's avatar
Chao Liu committed
523
524
525
526
527
528
529
530
    constexpr index_t N  = 128;
    constexpr index_t C  = 512;
    constexpr index_t HI = 14;
    constexpr index_t WI = 14;
    constexpr index_t K  = 512;
    constexpr index_t Y  = 1;
    constexpr index_t X  = 1;

531
532
533
    using ConvStrides   = Sequence<1, 1>;
    using ConvDilations = Sequence<1, 1>;

Chao Liu's avatar
Chao Liu committed
534
535
536
537
538
539
540
541
542
543
544
545
    constexpr index_t HPad = 0;
    constexpr index_t WPad = 0;
#elif 0
    // 1x1 filter, 7x7 image
    constexpr index_t N  = 128;
    constexpr index_t C  = 512;
    constexpr index_t HI = 7;
    constexpr index_t WI = 7;
    constexpr index_t K  = 2048;
    constexpr index_t Y  = 1;
    constexpr index_t X  = 1;

546
547
    constexpr index_t HPad = 0;
    constexpr index_t WPad = 0;
Chao Liu's avatar
Chao Liu committed
548
#elif 0
549
550
    // 1x1 filter, 73x73 image
    constexpr index_t N  = 128;
Chao Liu's avatar
Chao Liu committed
551
    constexpr index_t C  = 512;
552
553
554
555
556
557
    constexpr index_t HI = 73;
    constexpr index_t WI = 73;
    constexpr index_t K  = 128;
    constexpr index_t Y  = 1;
    constexpr index_t X  = 1;

Chao Liu's avatar
Chao Liu committed
558
559
    constexpr index_t HPad = 0;
    constexpr index_t WPad = 0;
Chao Liu's avatar
Chao Liu committed
560
#elif 0
Chao Liu's avatar
Chao Liu committed
561
    // 1x1 filter, 8x8 image
Chao Liu's avatar
Chao Liu committed
562
    // cudnn@V100 68%, ck@V100 72%, ck@P100 52%, ck@VII 42%
Chao Liu's avatar
Chao Liu committed
563
564
565
566
567
568
569
570
571
572
573
574
575
    constexpr index_t N  = 64;
    constexpr index_t C  = 1536;
    constexpr index_t HI = 8;
    constexpr index_t WI = 8;
    constexpr index_t K  = 256;
    constexpr index_t Y  = 1;
    constexpr index_t X  = 1;

    using ConvStrides   = Sequence<1, 1>;
    using ConvDilations = Sequence<1, 1>;

    constexpr index_t HPad = 0;
    constexpr index_t WPad = 0;
Chao Liu's avatar
Chao Liu committed
576
#elif 1
Chao Liu's avatar
Chao Liu committed
577
    // 1x1 filter, 8x8 image
Chao Liu's avatar
Chao Liu committed
578
    // cudnn@V100 77%, ck@V100 76%, ck@P100 79%, ck@VII 51%
Chao Liu's avatar
Chao Liu committed
579
580
581
582
583
584
585
586
587
588
589
590
591
    constexpr index_t N  = 128;
    constexpr index_t C  = 2048;
    constexpr index_t HI = 8;
    constexpr index_t WI = 8;
    constexpr index_t K  = 384;
    constexpr index_t Y  = 1;
    constexpr index_t X  = 1;

    using ConvStrides   = Sequence<1, 1>;
    using ConvDilations = Sequence<1, 1>;

    constexpr index_t HPad = 0;
    constexpr index_t WPad = 0;
Chao Liu's avatar
Chao Liu committed
592
#elif 0
Chao Liu's avatar
Chao Liu committed
593
    // 1x1 filter, 7x7 image
Chao Liu's avatar
Chao Liu committed
594
    // cudnn@V100 82%, ck@V100 76%, ck@P100 67%, ck@VII 64%
Chao Liu's avatar
Chao Liu committed
595
596
597
598
599
600
601
602
603
604
605
606
607
    constexpr index_t N  = 128;
    constexpr index_t C  = 832;
    constexpr index_t HI = 7;
    constexpr index_t WI = 7;
    constexpr index_t K  = 384;
    constexpr index_t Y  = 1;
    constexpr index_t X  = 1;

    using ConvStrides   = Sequence<1, 1>;
    using ConvDilations = Sequence<1, 1>;

    constexpr index_t HPad = 0;
    constexpr index_t WPad = 0;
Chao Liu's avatar
Chao Liu committed
608
#elif 0
Chao Liu's avatar
Chao Liu committed
609
    // 1x1 filter, 8x8 image
Chao Liu's avatar
Chao Liu committed
610
    // cudnn@V100 83%, ck@V100 75%, ck@P100 78%, ck@VII 65%
Chao Liu's avatar
Chao Liu committed
611
612
613
614
615
616
617
618
619
620
621
622
623
    constexpr index_t N  = 128;
    constexpr index_t C  = 1280;
    constexpr index_t HI = 8;
    constexpr index_t WI = 8;
    constexpr index_t K  = 384;
    constexpr index_t Y  = 1;
    constexpr index_t X  = 1;

    using ConvStrides   = Sequence<1, 1>;
    using ConvDilations = Sequence<1, 1>;

    constexpr index_t HPad = 0;
    constexpr index_t WPad = 0;
Chao Liu's avatar
Chao Liu committed
624
#elif 0
Chao Liu's avatar
Chao Liu committed
625
    // 1x1 filter, 14x14 image
Chao Liu's avatar
Chao Liu committed
626
    // cudnn@V100 62%, ck@V100 68%, ck@P100 70%, ck@VII 50%
Chao Liu's avatar
Chao Liu committed
627
628
629
630
631
632
633
634
635
636
637
638
639
    constexpr index_t N  = 128;
    constexpr index_t C  = 512;
    constexpr index_t HI = 14;
    constexpr index_t WI = 14;
    constexpr index_t K  = 128;
    constexpr index_t Y  = 1;
    constexpr index_t X  = 1;

    using ConvStrides   = Sequence<1, 1>;
    using ConvDilations = Sequence<1, 1>;

    constexpr index_t HPad = 0;
    constexpr index_t WPad = 0;
Chao Liu's avatar
Chao Liu committed
640
#elif 0
Chao Liu's avatar
Chao Liu committed
641
    // 1x1 filter, 8x8 image
Chao Liu's avatar
Chao Liu committed
642
    // cudnn@V100 74%, ck@V100 57%, ck@P100 78%, ck@VII 61%
Chao Liu's avatar
Chao Liu committed
643
644
645
646
647
648
649
650
651
652
653
654
655
    constexpr index_t N  = 64;
    constexpr index_t C  = 1536;
    constexpr index_t HI = 8;
    constexpr index_t WI = 8;
    constexpr index_t K  = 384;
    constexpr index_t Y  = 1;
    constexpr index_t X  = 1;

    using ConvStrides   = Sequence<1, 1>;
    using ConvDilations = Sequence<1, 1>;

    constexpr index_t HPad = 0;
    constexpr index_t WPad = 0;
Chao Liu's avatar
Chao Liu committed
656
#elif 0
Chao Liu's avatar
Chao Liu committed
657
    // 1x1 filter, 28x28 image
Chao Liu's avatar
Chao Liu committed
658
    // cudnn@V100 86%, ck@V100 84%, ck@P100 80%, ck@VII 69%
Chao Liu's avatar
Chao Liu committed
659
660
661
662
663
664
665
666
667
668
669
670
671
    constexpr index_t N  = 128;
    constexpr index_t C  = 256;
    constexpr index_t HI = 28;
    constexpr index_t WI = 28;
    constexpr index_t K  = 128;
    constexpr index_t Y  = 1;
    constexpr index_t X  = 1;

    using ConvStrides   = Sequence<1, 1>;
    using ConvDilations = Sequence<1, 1>;

    constexpr index_t HPad = 0;
    constexpr index_t WPad = 0;
Chao Liu's avatar
Chao Liu committed
672
#elif 0
Chao Liu's avatar
Chao Liu committed
673
    // 1x1 filter, 7x7 image
Chao Liu's avatar
Chao Liu committed
674
    // cudnn@V100 71%, ck@V100 55%, ck@P100 70%, ck@VII 62%
Chao Liu's avatar
Chao Liu committed
675
676
677
678
679
680
681
682
683
684
685
686
687
    constexpr index_t N  = 128;
    constexpr index_t C  = 832;
    constexpr index_t HI = 7;
    constexpr index_t WI = 7;
    constexpr index_t K  = 256;
    constexpr index_t Y  = 1;
    constexpr index_t X  = 1;

    using ConvStrides   = Sequence<1, 1>;
    using ConvDilations = Sequence<1, 1>;

    constexpr index_t HPad = 0;
    constexpr index_t WPad = 0;
Chao Liu's avatar
Chao Liu committed
688
#elif 0
Chao Liu's avatar
Chao Liu committed
689
    // 3x3 filter, 2x2 stride, 35x35 input, 17x17 output
Chao Liu's avatar
Chao Liu committed
690
    // cudnn@V100 90%, ck@V100 93%, ck@P100 83%, ck@VII 81%
Chao Liu's avatar
Chao Liu committed
691
692
693
694
695
696
697
698
699
700
701
702
703
    constexpr index_t N  = 128;
    constexpr index_t C  = 288;
    constexpr index_t HI = 35;
    constexpr index_t WI = 35;
    constexpr index_t K  = 384;
    constexpr index_t Y  = 3;
    constexpr index_t X  = 3;

    using ConvStrides   = Sequence<2, 2>;
    using ConvDilations = Sequence<1, 1>;

    constexpr index_t HPad = 0;
    constexpr index_t WPad = 0;
Chao Liu's avatar
Chao Liu committed
704
#elif 0
Chao Liu's avatar
Chao Liu committed
705
    // 1x1 filter, 17x17 input
Chao Liu's avatar
Chao Liu committed
706
    // cudnn@V100 81%, ck@V100 76%, ck@P100 70%, ck@VII 76%
Chao Liu's avatar
Chao Liu committed
707
708
709
710
711
712
713
714
715
716
717
718
719
    constexpr index_t N  = 128;
    constexpr index_t C  = 768;
    constexpr index_t HI = 17;
    constexpr index_t WI = 17;
    constexpr index_t K  = 128;
    constexpr index_t Y  = 1;
    constexpr index_t X  = 1;

    using ConvStrides   = Sequence<1, 1>;
    using ConvDilations = Sequence<1, 1>;

    constexpr index_t HPad = 0;
    constexpr index_t WPad = 0;
Chao Liu's avatar
Chao Liu committed
720
#elif 0
Chao Liu's avatar
Chao Liu committed
721
    // 1x1 filter, 14x14 image
Chao Liu's avatar
Chao Liu committed
722
    // cudnn@V100 73%, ck@V100 71%, ck@P100 70%, ck@VII 64%
Chao Liu's avatar
Chao Liu committed
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
    constexpr index_t N  = 128;
    constexpr index_t C  = 528;
    constexpr index_t HI = 14;
    constexpr index_t WI = 14;
    constexpr index_t K  = 128;
    constexpr index_t Y  = 1;
    constexpr index_t X  = 1;

    using ConvStrides   = Sequence<1, 1>;
    using ConvDilations = Sequence<1, 1>;

    constexpr index_t HPad = 0;
    constexpr index_t WPad = 0;
#elif 0
    // 1x1 filter, 14x14 image
Chao Liu's avatar
Chao Liu committed
738
    // cudnn@V100 73%, ck@V100 72%, ck@P100 79%, ck@VII 75%
Chao Liu's avatar
Chao Liu committed
739
740
741
742
743
744
745
746
747
748
749
750
751
    constexpr index_t N  = 128;
    constexpr index_t C  = 528;
    constexpr index_t HI = 14;
    constexpr index_t WI = 14;
    constexpr index_t K  = 256;
    constexpr index_t Y  = 1;
    constexpr index_t X  = 1;

    using ConvStrides   = Sequence<1, 1>;
    using ConvDilations = Sequence<1, 1>;

    constexpr index_t HPad = 0;
    constexpr index_t WPad = 0;
Chao Liu's avatar
Chao Liu committed
752
#elif 0
Chao Liu's avatar
Chao Liu committed
753
    // 1x1 filter, 7x7 image
Chao Liu's avatar
Chao Liu committed
754
    // cudnn@V100 49%, ck@V100 50%, ck@P100 61%, ck@VII 52%
Chao Liu's avatar
Chao Liu committed
755
756
757
758
    constexpr index_t N  = 128;
    constexpr index_t C  = 832;
    constexpr index_t HI = 7;
    constexpr index_t WI = 7;
Chao Liu's avatar
Chao Liu committed
759
760
761
    constexpr index_t K  = 128;
    constexpr index_t Y  = 1;
    constexpr index_t X  = 1;
Chao Liu's avatar
Chao Liu committed
762
763
764
765

    using ConvStrides   = Sequence<1, 1>;
    using ConvDilations = Sequence<1, 1>;

Chao Liu's avatar
Chao Liu committed
766
767
    constexpr index_t HPad = 0;
    constexpr index_t WPad = 0;
Chao Liu's avatar
Chao Liu committed
768
#endif
Chao Liu's avatar
Chao Liu committed
769

770
771
772
    auto lower_pads = Sequence<HPad, WPad>{};
    auto upper_pads = Sequence<HPad, WPad>{};

Chao Liu's avatar
Chao Liu committed
773
774
    auto in_nchw_desc  = make_ConstantTensorDescriptor_packed(Sequence<N, C, HI, WI>{});
    auto wei_kcyx_desc = make_ConstantTensorDescriptor_packed(Sequence<K, C, Y, X>{});
775
    auto out_nkhw_desc = get_convolution_with_padding_output_default_4d_tensor_descriptor(
776
        in_nchw_desc, wei_kcyx_desc, ConvStrides{}, ConvDilations{}, lower_pads, upper_pads);
Chao Liu's avatar
Chao Liu committed
777

Chao Liu's avatar
Chao Liu committed
778
    ostream_ConstantTensorDescriptor(in_nchw_desc, std::cout << "in_nchw_desc: ");
Chao Liu's avatar
Chao Liu committed
779
    ostream_ConstantTensorDescriptor(wei_kcyx_desc, std::cout << "wei_kcyx_desc: ");
Chao Liu's avatar
Chao Liu committed
780
    ostream_ConstantTensorDescriptor(out_nkhw_desc, std::cout << "out_nkhw_desc: ");
Chao Liu's avatar
Chao Liu committed
781

Chao Liu's avatar
Chao Liu committed
782
783
    using in_data_t  = float;
    using out_data_t = float;
784
785
786
787
    Tensor<in_data_t> in_nchw(make_TensorDescriptor(in_nchw_desc));
    Tensor<in_data_t> wei_kcyx(make_TensorDescriptor(wei_kcyx_desc));
    Tensor<out_data_t> out_nkhw_host(make_TensorDescriptor(out_nkhw_desc));
    Tensor<out_data_t> out_nkhw_device(make_TensorDescriptor(out_nkhw_desc));
Chao Liu's avatar
Chao Liu committed
788

Chao Liu's avatar
Chao Liu committed
789
    std::size_t num_thread = std::thread::hardware_concurrency();
Chao Liu's avatar
Chao Liu committed
790

Chao Liu's avatar
Chao Liu committed
791
792
793
794
795
796
797
    if(argc != 3)
    {
        printf("arg1: do_verification, arg2: nrepeat\n");
        exit(1);
    }

    bool do_verification = atoi(argv[1]);
Chao Liu's avatar
Chao Liu committed
798
    index_t nrepeat      = atoi(argv[2]);
799
800
801

    if(do_verification)
    {
Chao Liu's avatar
Chao Liu committed
802
#if 0
803
        in_nchw.GenerateTensorValue(GeneratorTensor_1{}, num_thread);
Chao Liu's avatar
Chao Liu committed
804
        wei_kcyx.GenerateTensorValue(GeneratorTensor_1{}, num_thread);
Chao Liu's avatar
Chao Liu committed
805
806
807
#elif 0
        in_nchw.GenerateTensorValue(GeneratorTensor_1{}, num_thread);
        wei_kcyx.GenerateTensorValue(GeneratorTensor_2{-5, 5}, num_thread);
808
809
810
#elif 0
        in_nchw.GenerateTensorValue(GeneratorTensor_3{}, num_thread);
        wei_kcyx.GenerateTensorValue(GeneratorTensor_1{}, num_thread);
Chao Liu's avatar
Chao Liu committed
811
#elif 1
812
        in_nchw.GenerateTensorValue(GeneratorTensor_2{-5, 5}, num_thread);
Chao Liu's avatar
Chao Liu committed
813
        wei_kcyx.GenerateTensorValue(GeneratorTensor_2{-5, 5}, num_thread);
Chao Liu's avatar
Chao Liu committed
814
#elif 0
815
816
817
818
819
820
        in_nchw.GenerateTensorValue(GeneratorTensor_2{1, 5}, num_thread);

        auto gen_wei = [](auto... is) {
            return GeneratorTensor_2{1, 5}(is...) * GeneratorTensor_Checkboard{}(is...);
        };
        wei_kcyx.GenerateTensorValue(gen_wei, num_thread);
Chao Liu's avatar
Chao Liu committed
821
#endif
822
    }
Chao Liu's avatar
Chao Liu committed
823

Chao Liu's avatar
Chao Liu committed
824
#if 0
Chao Liu's avatar
Chao Liu committed
825
    device_convolution_direct_v2_nchw_kcyx_nkhw
Chao Liu's avatar
Chao Liu committed
826
        in_nchw_desc, in_nchw, wei_kcyx_desc, wei_kcyx, out_nkhw_desc, out_nkhw_device, nrepeat);
Chao Liu's avatar
Chao Liu committed
827
#elif 0
828
    device_convolution_implicit_gemm_v1_chwn_cyxk_khwn
Chao Liu's avatar
Chao Liu committed
829
        in_nchw_desc, in_nchw, wei_kcyx_desc, wei_kcyx, out_nkhw_desc, out_nkhw_device, nrepeat);
Chao Liu's avatar
Chao Liu committed
830
#elif 0
Chao Liu's avatar
Chao Liu committed
831
    device_convolution_implicit_gemm_v1_nchw_cyxk_nkhw
Chao Liu's avatar
Chao Liu committed
832
        in_nchw_desc, in_nchw, wei_kcyx_desc, wei_kcyx, out_nkhw_desc, out_nkhw_device, nrepeat);
833
#elif 0
Chao Liu's avatar
Chao Liu committed
834
    device_convolution_implicit_gemm_v2_chwn_cyxk_khwn
Chao Liu's avatar
Chao Liu committed
835
        in_nchw_desc, in_nchw, wei_kcyx_desc, wei_kcyx, out_nkhw_desc, out_nkhw_device, nrepeat);
Chao Liu's avatar
Chao Liu committed
836
#elif 0
Chao Liu's avatar
Chao Liu committed
837
838
    device_convolution_implicit_gemm_v3_nchw_cyxk_nkhw(
        in_nchw_desc, in_nchw, wei_kcyx_desc, wei_kcyx, out_nkhw_desc, out_nkhw_device, nrepeat);
Chao Liu's avatar
Chao Liu committed
839
#elif 1
Chao Liu's avatar
Chao Liu committed
840
841
842
843
844
845
846
847
848
    device_convolution_implicit_gemm_v4_nchw_kcyx_nkhw(in_nchw_desc,
                                                       in_nchw,
                                                       wei_kcyx_desc,
                                                       wei_kcyx,
                                                       out_nkhw_desc,
                                                       out_nkhw_device,
                                                       ConvStrides{},
                                                       ConvDilations{},
                                                       nrepeat);
849
#elif 0
Chao Liu's avatar
Chao Liu committed
850
    device_implicit_gemm_convolution_1_chwn_cyxk_khwn_padded(in_nchw_desc,
Chao Liu's avatar
Chao Liu committed
851
                                                             in_nchw,
Chao Liu's avatar
Chao Liu committed
852
853
                                                             wei_kcyx_desc,
                                                             wei_kcyx,
Chao Liu's avatar
Chao Liu committed
854
855
856
857
858
                                                             out_nkhw_desc,
                                                             out_nkhw_device,
                                                             lower_pads,
                                                             upper_pads,
                                                             nrepeat);
859
#endif
Chao Liu's avatar
Chao Liu committed
860

861
    if(do_verification)
862
    {
Chao Liu's avatar
Chao Liu committed
863
#if 1
864
865
        if(Y == 3 && X == 3 && ConvStrides{}[0] == 1 && ConvStrides{}[1] == 1 &&
           ConvDilations{}[0] == 1 && ConvDilations{}[1] == 1)
866
        {
Chao Liu's avatar
Chao Liu committed
867
            host_winograd_3x3_convolution(in_nchw, wei_kcyx, out_nkhw_host, lower_pads, upper_pads);
868
869
        }
        else
Chao Liu's avatar
Chao Liu committed
870
#endif
871
        {
872
873
874
875
876
877
878
            host_direct_convolution(in_nchw,
                                    wei_kcyx,
                                    out_nkhw_host,
                                    ConvStrides{},
                                    ConvDilations{},
                                    lower_pads,
                                    upper_pads);
879
880
        }
        check_error(out_nkhw_host, out_nkhw_device);
Chao Liu's avatar
Chao Liu committed
881

Chao Liu's avatar
Chao Liu committed
882
#if 0
883
        LogRange(std::cout << "in_nchw : ", in_nchw.mData, ",") << std::endl;
Chao Liu's avatar
Chao Liu committed
884
        LogRange(std::cout << "wei_kcyx: ", wei_kcyx.mData, ",") << std::endl;
885
886
        LogRange(std::cout << "out_nkhw_host  : ", out_nkhw_host.mData, ",") << std::endl;
        LogRange(std::cout << "out_nkhw_device: ", out_nkhw_device.mData, ",") << std::endl;
Chao Liu's avatar
Chao Liu committed
887
#endif
888
    }
889
}