driver.hip.cpp 26 KB
Newer Older
Chao Liu's avatar
Chao Liu committed
1
#include <iostream>
Chao Liu's avatar
Chao Liu committed
2
3
#include <numeric>
#include <initializer_list>
Chao Liu's avatar
Chao Liu committed
4
#include <cstdlib>
Chao Liu's avatar
Chao Liu committed
5
#include <stdlib.h>
Chao Liu's avatar
Chao Liu committed
6
#include "config.h"
Chao Liu's avatar
Chao Liu committed
7
#include "tensor.hpp"
8
9
#include "ConstantTensorDescriptor.hip.hpp"
#include "conv_common.hip.hpp"
Chao Liu's avatar
Chao Liu committed
10
//#include "device_direct_convolution_1.hpp"
Chao Liu's avatar
Chao Liu committed
11
#include "device_direct_convolution_2_nchw_kcyx_nkhw.hpp"
12
#include "device_direct_convolution_2_vectorized_nchw_kcyx_nkhw.hpp"
Chao Liu's avatar
Chao Liu committed
13
14
15
//#include "device_implicit_gemm_convolution_1_chwn_cyxk_khwn.hpp"
//#include "device_implicit_gemm_convolution_1_chwn_cyxk_khwn_padded.hpp"
//#include "device_implicit_gemm_convolution_2_chwn_cyxk_khwn.hpp"
Chao Liu's avatar
Chao Liu committed
16

Chao Liu's avatar
Chao Liu committed
17
struct GeneratorTensor_1
Chao Liu's avatar
Chao Liu committed
18
19
{
    template <class... Is>
Chao Liu's avatar
Chao Liu committed
20
    double operator()(Is... is)
Chao Liu's avatar
Chao Liu committed
21
    {
Chao Liu's avatar
Chao Liu committed
22
        return 1;
Chao Liu's avatar
Chao Liu committed
23
24
25
    }
};

Chao Liu's avatar
Chao Liu committed
26
27
28
29
30
31
32
33
34
35
36
37
struct GeneratorTensor_2
{
    int min_value = 0;
    int max_value = 1;

    template <class... Is>
    double operator()(Is...)
    {
        return (std::rand() % (max_value - min_value)) + min_value;
    }
};

Chao Liu's avatar
Chao Liu committed
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
struct GeneratorTensor_Checkboard
{
    template <class... Ts>
    double operator()(Ts... Xs) const
    {
        std::array<unsigned long, sizeof...(Ts)> dims = {{Xs...}};
        return std::accumulate(dims.begin(),
                               dims.end(),
                               true,
                               [](bool init, unsigned long x) -> int { return init != (x % 2); })
                   ? 1
                   : -1;
    }
};

Chao Liu's avatar
Chao Liu committed
53
54
55
56
57
58
// this is ugly, only for 4d
template <class TConstTensorDesc>
void ostream_ConstantTensorDescriptor(TConstTensorDesc, std::ostream& os = std::cout)
{
    static_assert(TConstTensorDesc::nDim == 4, "nDim is not 4");

Chao Liu's avatar
Chao Liu committed
59
60
61
62
    constexpr auto I0   = Number<0>{};
    constexpr auto I1   = Number<1>{};
    constexpr auto I2   = Number<2>{};
    constexpr auto I3   = Number<3>{};
Chao Liu's avatar
Chao Liu committed
63
64
65
66
67
68
69
70
71
72
73
74
75
76
    constexpr auto desc = TConstTensorDesc{};

    os << "Lengths: {" << desc.GetLength(I0) << ", " << desc.GetLength(I1) << ", "
       << desc.GetLength(I2) << ", " << desc.GetLength(I3) << "}, "
       << "Strides: {" << desc.GetStride(I0) << ", " << desc.GetStride(I1) << ", "
       << desc.GetStride(I2) << ", " << desc.GetStride(I3) << "}" << std::endl;
}

// this is ugly, only for 4d
template <class TConstTensorDesc>
auto make_TensorDescriptor(TConstTensorDesc)
{
    static_assert(TConstTensorDesc::nDim == 4, "nDim is not 4");

Chao Liu's avatar
Chao Liu committed
77
78
79
80
    constexpr auto I0   = Number<0>{};
    constexpr auto I1   = Number<1>{};
    constexpr auto I2   = Number<2>{};
    constexpr auto I3   = Number<3>{};
Chao Liu's avatar
Chao Liu committed
81
82
83
84
85
86
87
88
89
90
    constexpr auto desc = TConstTensorDesc{};

    std::initializer_list<unsigned> lengths = {
        desc.GetLength(I0), desc.GetLength(I1), desc.GetLength(I2), desc.GetLength(I3)};
    std::initializer_list<unsigned> strides = {
        desc.GetStride(I0), desc.GetStride(I1), desc.GetStride(I2), desc.GetStride(I3)};

    return TensorDescriptor(lengths, strides);
}

91
92
template <class T, class LowerPads, class UpperPads>
void host_direct_convolution(
Chao Liu's avatar
Chao Liu committed
93
    const Tensor<T>& in_nchw, const Tensor<T>& wei_kcyx, Tensor<T>& out, LowerPads, UpperPads)
Chao Liu's avatar
Chao Liu committed
94
{
95
96
97
98
99
100
    unsigned h_pad_low = LowerPads{}.Get(Number<0>{});
    unsigned w_pad_low = LowerPads{}.Get(Number<1>{});

    unsigned h_pad_up = UpperPads{}.Get(Number<0>{});
    unsigned w_pad_up = UpperPads{}.Get(Number<1>{});

Chao Liu's avatar
Chao Liu committed
101
102
    auto f = [&](auto n, auto k, auto ho, auto wo) {
        double v = 0;
Chao Liu's avatar
Chao Liu committed
103
        for(int c = 0; c < wei_kcyx.mDesc.GetLengths()[1]; ++c)
Chao Liu's avatar
Chao Liu committed
104
        {
Chao Liu's avatar
Chao Liu committed
105
            for(int y = 0; y < wei_kcyx.mDesc.GetLengths()[2]; ++y)
Chao Liu's avatar
Chao Liu committed
106
            {
107
                int hi = ho + y - h_pad_low;
Chao Liu's avatar
Chao Liu committed
108
                for(int x = 0; x < wei_kcyx.mDesc.GetLengths()[3]; ++x)
Chao Liu's avatar
Chao Liu committed
109
                {
110
111
112
113
                    int wi = wo + x - w_pad_low;
                    if(hi >= 0 && hi < in_nchw.mDesc.GetLengths()[2] && wi >= 0 &&
                       wi < in_nchw.mDesc.GetLengths()[3])
                    {
114
                        v += double(in_nchw(n, c, hi, wi)) * double(wei_kcyx(k, c, y, x));
115
                    }
Chao Liu's avatar
Chao Liu committed
116
117
118
119
120
121
122
123
124
125
126
127
                }
            }
        }
        out(n, k, ho, wo) = v;
    };

    auto f_par = make_ParallelTensorFunctor(f,
                                            out.mDesc.GetLengths()[0],
                                            out.mDesc.GetLengths()[1],
                                            out.mDesc.GetLengths()[2],
                                            out.mDesc.GetLengths()[3]);

Chao Liu's avatar
Chao Liu committed
128
    f_par(std::thread::hardware_concurrency());
Chao Liu's avatar
Chao Liu committed
129
130
}

131
132
template <class T, class LowerPads, class UpperPads>
void host_winograd_3x3_convolution(
Chao Liu's avatar
Chao Liu committed
133
    const Tensor<T>& in_nchw, const Tensor<T>& wei_kcyx, Tensor<T>& out, LowerPads, UpperPads)
Chao Liu's avatar
Chao Liu committed
134
{
Chao Liu's avatar
Chao Liu committed
135
136
    constexpr std::size_t HoPerTile = 2;
    constexpr std::size_t WoPerTile = 2;
Chao Liu's avatar
Chao Liu committed
137

Chao Liu's avatar
Chao Liu committed
138
139
140
141
    std::size_t N  = in_nchw.mDesc.GetLengths()[0];
    std::size_t C  = in_nchw.mDesc.GetLengths()[1];
    std::size_t HI = in_nchw.mDesc.GetLengths()[2];
    std::size_t WI = in_nchw.mDesc.GetLengths()[3];
Chao Liu's avatar
Chao Liu committed
142

Chao Liu's avatar
Chao Liu committed
143
144
145
    std::size_t K = wei_kcyx.mDesc.GetLengths()[0];
    std::size_t Y = wei_kcyx.mDesc.GetLengths()[2];
    std::size_t X = wei_kcyx.mDesc.GetLengths()[3];
Chao Liu's avatar
Chao Liu committed
146
147
148
149

    std::size_t HO = out.mDesc.GetLengths()[2];
    std::size_t WO = out.mDesc.GetLengths()[3];

150
151
152
153
154
155
    unsigned h_pad_low = LowerPads{}.Get(Number<0>{});
    unsigned w_pad_low = LowerPads{}.Get(Number<1>{});

    unsigned h_pad_up = UpperPads{}.Get(Number<0>{});
    unsigned w_pad_up = UpperPads{}.Get(Number<1>{});

Chao Liu's avatar
Chao Liu committed
156
157
    std::size_t HiPerTile = HoPerTile + Y - 1;
    std::size_t WiPerTile = WoPerTile + X - 1;
Chao Liu's avatar
Chao Liu committed
158

Chao Liu's avatar
Chao Liu committed
159
160
    std::size_t HTile = (HO + HoPerTile - 1) / HoPerTile;
    std::size_t WTile = (WO + WoPerTile - 1) / WoPerTile;
Chao Liu's avatar
Chao Liu committed
161

162
163
164
165
166
    Tensor<double> in_hold({N, C, HTile, WTile, HiPerTile, WiPerTile});
    Tensor<double> in_transform({N, C, HTile, WTile, HiPerTile, WiPerTile});
    Tensor<double> wei_transform({K, C, HiPerTile, WiPerTile});
    Tensor<double> out_transform({N, K, HTile, WTile, HiPerTile, HiPerTile});
    Tensor<double> out_hold({N, K, HTile, WTile, HoPerTile, WoPerTile});
Chao Liu's avatar
Chao Liu committed
167

Chao Liu's avatar
Chao Liu committed
168
169
    auto f_in_hold = [&](auto n, auto c, auto htile, auto wtile) {
        for(int j = 0; j < HiPerTile; ++j)
Chao Liu's avatar
Chao Liu committed
170
        {
Chao Liu's avatar
Chao Liu committed
171
172
            int hi = HoPerTile * htile + j - h_pad_low;
            for(int i = 0; i < WiPerTile; ++i)
Chao Liu's avatar
Chao Liu committed
173
            {
Chao Liu's avatar
Chao Liu committed
174
                int wi = WoPerTile * wtile + i - w_pad_low;
175
176
177
178

                if(hi >= 0 && hi < in_nchw.mDesc.GetLengths()[2] && wi >= 0 &&
                   wi < in_nchw.mDesc.GetLengths()[3])
                {
Chao Liu's avatar
Chao Liu committed
179
                    in_hold(n, c, htile, wtile, j, i) = in_nchw(n, c, hi, wi);
180
181
182
                }
                else
                {
Chao Liu's avatar
Chao Liu committed
183
                    in_hold(n, c, htile, wtile, j, i) = T(0);
184
                }
Chao Liu's avatar
Chao Liu committed
185
186
187
188
            }
        }
    };

Chao Liu's avatar
Chao Liu committed
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
    auto f_in_transform = [&](auto n, auto c, auto htile, auto wtile) {
        in_transform(n, c, htile, wtile, 0, 0) =
            in_hold(n, c, htile, wtile, 0, 0) - in_hold(n, c, htile, wtile, 0, 2) -
            in_hold(n, c, htile, wtile, 2, 0) + in_hold(n, c, htile, wtile, 2, 2);
        in_transform(n, c, htile, wtile, 0, 1) =
            in_hold(n, c, htile, wtile, 0, 1) + in_hold(n, c, htile, wtile, 0, 2) -
            in_hold(n, c, htile, wtile, 2, 1) - in_hold(n, c, htile, wtile, 2, 2);
        in_transform(n, c, htile, wtile, 0, 2) =
            -in_hold(n, c, htile, wtile, 0, 1) + in_hold(n, c, htile, wtile, 0, 2) +
            in_hold(n, c, htile, wtile, 2, 1) - in_hold(n, c, htile, wtile, 2, 2);
        in_transform(n, c, htile, wtile, 0, 3) =
            in_hold(n, c, htile, wtile, 0, 1) - in_hold(n, c, htile, wtile, 0, 3) -
            in_hold(n, c, htile, wtile, 2, 1) + in_hold(n, c, htile, wtile, 2, 3);

        in_transform(n, c, htile, wtile, 1, 0) =
            in_hold(n, c, htile, wtile, 1, 0) - in_hold(n, c, htile, wtile, 1, 2) +
            in_hold(n, c, htile, wtile, 2, 0) - in_hold(n, c, htile, wtile, 2, 2);
        in_transform(n, c, htile, wtile, 1, 1) =
            in_hold(n, c, htile, wtile, 1, 1) + in_hold(n, c, htile, wtile, 1, 2) +
            in_hold(n, c, htile, wtile, 2, 1) + in_hold(n, c, htile, wtile, 2, 2);
        in_transform(n, c, htile, wtile, 1, 2) =
            -in_hold(n, c, htile, wtile, 1, 1) + in_hold(n, c, htile, wtile, 1, 2) -
            in_hold(n, c, htile, wtile, 2, 1) + in_hold(n, c, htile, wtile, 2, 2);
        in_transform(n, c, htile, wtile, 1, 3) =
            in_hold(n, c, htile, wtile, 1, 1) - in_hold(n, c, htile, wtile, 1, 3) +
            in_hold(n, c, htile, wtile, 2, 1) - in_hold(n, c, htile, wtile, 2, 3);

        in_transform(n, c, htile, wtile, 2, 0) =
            -in_hold(n, c, htile, wtile, 1, 0) + in_hold(n, c, htile, wtile, 1, 2) +
            in_hold(n, c, htile, wtile, 2, 0) - in_hold(n, c, htile, wtile, 2, 2);
        in_transform(n, c, htile, wtile, 2, 1) =
            -in_hold(n, c, htile, wtile, 1, 1) - in_hold(n, c, htile, wtile, 1, 2) +
            in_hold(n, c, htile, wtile, 2, 1) + in_hold(n, c, htile, wtile, 2, 2);
        in_transform(n, c, htile, wtile, 2, 2) =
            in_hold(n, c, htile, wtile, 1, 1) - in_hold(n, c, htile, wtile, 1, 2) -
            in_hold(n, c, htile, wtile, 2, 1) + in_hold(n, c, htile, wtile, 2, 2);
        in_transform(n, c, htile, wtile, 2, 3) =
            -in_hold(n, c, htile, wtile, 1, 1) + in_hold(n, c, htile, wtile, 1, 3) +
            in_hold(n, c, htile, wtile, 2, 1) - in_hold(n, c, htile, wtile, 2, 3);

        in_transform(n, c, htile, wtile, 3, 0) =
            in_hold(n, c, htile, wtile, 1, 0) - in_hold(n, c, htile, wtile, 1, 2) -
            in_hold(n, c, htile, wtile, 3, 0) + in_hold(n, c, htile, wtile, 3, 2);
        in_transform(n, c, htile, wtile, 3, 1) =
            in_hold(n, c, htile, wtile, 1, 1) + in_hold(n, c, htile, wtile, 1, 2) -
            in_hold(n, c, htile, wtile, 3, 1) - in_hold(n, c, htile, wtile, 3, 2);
        in_transform(n, c, htile, wtile, 3, 2) =
            -in_hold(n, c, htile, wtile, 1, 1) + in_hold(n, c, htile, wtile, 1, 2) +
            in_hold(n, c, htile, wtile, 3, 1) - in_hold(n, c, htile, wtile, 3, 2);
        in_transform(n, c, htile, wtile, 3, 3) =
            in_hold(n, c, htile, wtile, 1, 1) - in_hold(n, c, htile, wtile, 1, 3) -
            in_hold(n, c, htile, wtile, 3, 1) + in_hold(n, c, htile, wtile, 3, 3);
Chao Liu's avatar
Chao Liu committed
241
242
243
    };

    auto f_wei_transform = [&](auto k, auto c) {
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
        wei_transform(k, c, 0, 0) = double(wei_kcyx(k, c, 0, 0));
        wei_transform(k, c, 0, 1) = 0.5 * double(wei_kcyx(k, c, 0, 0)) +
                                    0.5 * double(wei_kcyx(k, c, 0, 1)) +
                                    0.5 * double(wei_kcyx(k, c, 0, 2));
        wei_transform(k, c, 0, 2) = 0.5 * double(wei_kcyx(k, c, 0, 0)) -
                                    0.5 * double(wei_kcyx(k, c, 0, 1)) +
                                    0.5 * double(wei_kcyx(k, c, 0, 2));
        wei_transform(k, c, 0, 3) = double(wei_kcyx(k, c, 0, 2));

        wei_transform(k, c, 1, 0) = 0.5 * double(wei_kcyx(k, c, 0, 0)) +
                                    0.5 * double(wei_kcyx(k, c, 1, 0)) +
                                    0.5 * double(wei_kcyx(k, c, 2, 0));
        wei_transform(k, c, 1, 1) =
            0.25 * double(wei_kcyx(k, c, 0, 0)) + 0.25 * double(wei_kcyx(k, c, 0, 1)) +
            0.25 * double(wei_kcyx(k, c, 0, 2)) + 0.25 * double(wei_kcyx(k, c, 1, 0)) +
            0.25 * double(wei_kcyx(k, c, 1, 1)) + 0.25 * double(wei_kcyx(k, c, 1, 2)) +
            0.25 * double(wei_kcyx(k, c, 2, 0)) + 0.25 * double(wei_kcyx(k, c, 2, 1)) +
            0.25 * double(wei_kcyx(k, c, 2, 2));
        wei_transform(k, c, 1, 2) =
            0.25 * double(wei_kcyx(k, c, 0, 0)) - 0.25 * double(wei_kcyx(k, c, 0, 1)) +
            0.25 * double(wei_kcyx(k, c, 0, 2)) + 0.25 * double(wei_kcyx(k, c, 1, 0)) -
            0.25 * double(wei_kcyx(k, c, 1, 1)) + 0.25 * double(wei_kcyx(k, c, 1, 2)) +
            0.25 * double(wei_kcyx(k, c, 2, 0)) - 0.25 * double(wei_kcyx(k, c, 2, 1)) +
            0.25 * double(wei_kcyx(k, c, 2, 2));
        wei_transform(k, c, 1, 3) = 0.5 * double(wei_kcyx(k, c, 0, 2)) +
                                    0.5 * double(wei_kcyx(k, c, 1, 2)) +
                                    0.5 * double(wei_kcyx(k, c, 2, 2));

        wei_transform(k, c, 2, 0) = 0.5 * double(wei_kcyx(k, c, 0, 0)) -
                                    0.5 * double(wei_kcyx(k, c, 1, 0)) +
                                    0.5 * double(wei_kcyx(k, c, 2, 0));
        wei_transform(k, c, 2, 1) =
            0.25 * double(wei_kcyx(k, c, 0, 0)) + 0.25 * double(wei_kcyx(k, c, 0, 1)) +
            0.25 * double(wei_kcyx(k, c, 0, 2)) - 0.25 * double(wei_kcyx(k, c, 1, 0)) -
            0.25 * double(wei_kcyx(k, c, 1, 1)) - 0.25 * double(wei_kcyx(k, c, 1, 2)) +
            0.25 * double(wei_kcyx(k, c, 2, 0)) + 0.25 * double(wei_kcyx(k, c, 2, 1)) +
            0.25 * double(wei_kcyx(k, c, 2, 2));
        wei_transform(k, c, 2, 2) =
            0.25 * double(wei_kcyx(k, c, 0, 0)) - 0.25 * double(wei_kcyx(k, c, 0, 1)) +
            0.25 * double(wei_kcyx(k, c, 0, 2)) - 0.25 * double(wei_kcyx(k, c, 1, 0)) +
            0.25 * double(wei_kcyx(k, c, 1, 1)) - 0.25 * double(wei_kcyx(k, c, 1, 2)) +
            0.25 * double(wei_kcyx(k, c, 2, 0)) - 0.25 * double(wei_kcyx(k, c, 2, 1)) +
            0.25 * double(wei_kcyx(k, c, 2, 2));
        wei_transform(k, c, 2, 3) = 0.5 * double(wei_kcyx(k, c, 0, 2)) -
                                    0.5 * double(wei_kcyx(k, c, 1, 2)) +
                                    0.5 * double(wei_kcyx(k, c, 2, 2));

        wei_transform(k, c, 3, 0) = double(wei_kcyx(k, c, 2, 0));
        wei_transform(k, c, 3, 1) = 0.5 * double(wei_kcyx(k, c, 2, 0)) +
                                    0.5 * double(wei_kcyx(k, c, 2, 1)) +
                                    0.5 * double(wei_kcyx(k, c, 2, 2));
        wei_transform(k, c, 3, 2) = 0.5 * double(wei_kcyx(k, c, 2, 0)) -
                                    0.5 * double(wei_kcyx(k, c, 2, 1)) +
                                    0.5 * double(wei_kcyx(k, c, 2, 2));
        wei_transform(k, c, 3, 3) = double(wei_kcyx(k, c, 2, 2));
Chao Liu's avatar
Chao Liu committed
299
300
    };

Chao Liu's avatar
Chao Liu committed
301
302
    auto f_out_transform = [&](auto n, auto k, auto htile, auto wtile) {
        for(int j = 0; j < HiPerTile; ++j)
Chao Liu's avatar
Chao Liu committed
303
        {
Chao Liu's avatar
Chao Liu committed
304
            for(int i = 0; i < WiPerTile; ++i)
Chao Liu's avatar
Chao Liu committed
305
306
307
308
            {
                double v = 0;
                for(int c = 0; c < C; ++c)
                {
Chao Liu's avatar
Chao Liu committed
309
                    v += in_transform(n, c, htile, wtile, j, i) * wei_transform(k, c, j, i);
Chao Liu's avatar
Chao Liu committed
310
311
                }

Chao Liu's avatar
Chao Liu committed
312
                out_transform(n, k, htile, wtile, j, i) = v;
Chao Liu's avatar
Chao Liu committed
313
314
315
316
            }
        }
    };

Chao Liu's avatar
Chao Liu committed
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
    auto f_out_hold = [&](auto n, auto k, auto htile, auto wtile) {
        out_hold(n, k, htile, wtile, 0, 0) =
            out_transform(n, k, htile, wtile, 0, 0) + out_transform(n, k, htile, wtile, 0, 1) +
            out_transform(n, k, htile, wtile, 0, 2) + out_transform(n, k, htile, wtile, 1, 0) +
            out_transform(n, k, htile, wtile, 1, 1) + out_transform(n, k, htile, wtile, 1, 2) +
            out_transform(n, k, htile, wtile, 2, 0) + out_transform(n, k, htile, wtile, 2, 1) +
            out_transform(n, k, htile, wtile, 2, 2);
        out_hold(n, k, htile, wtile, 0, 1) =
            out_transform(n, k, htile, wtile, 0, 1) - out_transform(n, k, htile, wtile, 0, 2) -
            out_transform(n, k, htile, wtile, 0, 3) + out_transform(n, k, htile, wtile, 1, 1) -
            out_transform(n, k, htile, wtile, 1, 2) - out_transform(n, k, htile, wtile, 1, 3) +
            out_transform(n, k, htile, wtile, 2, 1) - out_transform(n, k, htile, wtile, 2, 2) -
            out_transform(n, k, htile, wtile, 2, 3);
        out_hold(n, k, htile, wtile, 1, 0) =
            out_transform(n, k, htile, wtile, 1, 0) + out_transform(n, k, htile, wtile, 1, 1) +
            out_transform(n, k, htile, wtile, 1, 2) - out_transform(n, k, htile, wtile, 2, 0) -
            out_transform(n, k, htile, wtile, 2, 1) - out_transform(n, k, htile, wtile, 2, 2) -
            out_transform(n, k, htile, wtile, 3, 0) - out_transform(n, k, htile, wtile, 3, 1) -
            out_transform(n, k, htile, wtile, 3, 2);
        out_hold(n, k, htile, wtile, 1, 1) =
            out_transform(n, k, htile, wtile, 1, 1) - out_transform(n, k, htile, wtile, 1, 2) -
            out_transform(n, k, htile, wtile, 1, 3) - out_transform(n, k, htile, wtile, 2, 1) +
            out_transform(n, k, htile, wtile, 2, 2) + out_transform(n, k, htile, wtile, 2, 3) -
            out_transform(n, k, htile, wtile, 3, 1) + out_transform(n, k, htile, wtile, 3, 2) +
            out_transform(n, k, htile, wtile, 3, 3);
Chao Liu's avatar
Chao Liu committed
342
343
    };

Chao Liu's avatar
Chao Liu committed
344
345
    auto f_out = [&](auto n, auto k, auto htile, auto wtile) {
        for(int j = 0; j < HoPerTile; ++j)
Chao Liu's avatar
Chao Liu committed
346
        {
Chao Liu's avatar
Chao Liu committed
347
348
            std::size_t ho = HoPerTile * htile + j;
            for(int i = 0; i < WoPerTile; ++i)
Chao Liu's avatar
Chao Liu committed
349
            {
Chao Liu's avatar
Chao Liu committed
350
351
                std::size_t wo    = WoPerTile * wtile + i;
                out(n, k, ho, wo) = out_hold(n, k, htile, wtile, j, i);
Chao Liu's avatar
Chao Liu committed
352
353
354
355
356
357
            }
        }
    };

    std::size_t num_thread = std::thread::hardware_concurrency();

Chao Liu's avatar
Chao Liu committed
358
359
    make_ParallelTensorFunctor(f_in_hold, N, C, HTile, WTile)(num_thread);
    make_ParallelTensorFunctor(f_in_transform, N, C, HTile, WTile)(num_thread);
Chao Liu's avatar
Chao Liu committed
360
    make_ParallelTensorFunctor(f_wei_transform, K, C)(num_thread);
Chao Liu's avatar
Chao Liu committed
361
362
363
    make_ParallelTensorFunctor(f_out_transform, N, K, HTile, WTile)(num_thread);
    make_ParallelTensorFunctor(f_out_hold, N, K, HTile, WTile)(num_thread);
    make_ParallelTensorFunctor(f_out, N, K, HTile, WTile)(num_thread);
Chao Liu's avatar
Chao Liu committed
364
365
366
367
368
}

template <class T>
void check_error(const Tensor<T>& ref, const Tensor<T>& result)
{
369
370
    // printf("\n");

Chao Liu's avatar
Chao Liu committed
371
    float error     = 0;
Chao Liu's avatar
Chao Liu committed
372
    float max_diff  = -1;
Chao Liu's avatar
Chao Liu committed
373
374
375
    float ref_value = 0, result_value = 0;
    for(int i = 0; i < ref.mData.size(); ++i)
    {
376
377
        error += std::abs(double(ref.mData[i]) - double(result.mData[i]));
        float diff = std::abs(double(ref.mData[i]) - double(result.mData[i]));
Chao Liu's avatar
Chao Liu committed
378
379
380
381
382
383
        if(max_diff < diff)
        {
            max_diff     = diff;
            ref_value    = ref.mData[i];
            result_value = result.mData[i];
        }
384
385

        // printf("{%f, %f}", double(ref.mData[i]), double(result.mData[i]));
Chao Liu's avatar
Chao Liu committed
386
    }
387
    // printf("\n");
Chao Liu's avatar
Chao Liu committed
388
389
390
391
392

    std::cout << "error: " << error << std::endl;
    std::cout << "max_diff: " << max_diff << ", " << ref_value << ", " << result_value << std::endl;
}

Chao Liu's avatar
Chao Liu committed
393
int main(int argc, char* argv[])
Chao Liu's avatar
Chao Liu committed
394
{
Chao Liu's avatar
Chao Liu committed
395
#if 0
Chao Liu's avatar
Chao Liu committed
396
    constexpr unsigned N  = 1;
Chao Liu's avatar
Chao Liu committed
397
    constexpr unsigned C  = 1;
Chao Liu's avatar
Chao Liu committed
398
399
    constexpr unsigned HI = 28;
    constexpr unsigned WI = 28;
Chao Liu's avatar
tune  
Chao Liu committed
400
    constexpr unsigned K  = 1;
Chao Liu's avatar
Chao Liu committed
401
402
    constexpr unsigned Y  = 3;
    constexpr unsigned X  = 3;
Chao Liu's avatar
Chao Liu committed
403

Chao Liu's avatar
Chao Liu committed
404
405
    constexpr unsigned HPad = 0;
    constexpr unsigned WPad = 0;
Chao Liu's avatar
Chao Liu committed
406
#elif 1
407
    // 3x3, 34x34
408
409
    constexpr unsigned N  = 64;
    constexpr unsigned C  = 256;
Chao Liu's avatar
Chao Liu committed
410
411
    constexpr unsigned HI = 34;
    constexpr unsigned WI = 34;
412
413
414
    constexpr unsigned K  = 64;
    constexpr unsigned Y  = 3;
    constexpr unsigned X  = 3;
415
416
417

    constexpr unsigned HPad = 0;
    constexpr unsigned WPad = 0;
Chao Liu's avatar
Chao Liu committed
418
#elif 0
419
    // 3x3, 56x56
Chao Liu's avatar
Chao Liu committed
420
421
    constexpr unsigned N  = 64;
    constexpr unsigned C  = 64;
Chao Liu's avatar
Chao Liu committed
422
423
    constexpr unsigned HI = 56;
    constexpr unsigned WI = 56;
Chao Liu's avatar
Chao Liu committed
424
    constexpr unsigned K  = 64;
Chao Liu's avatar
Chao Liu committed
425
426
    constexpr unsigned Y  = 3;
    constexpr unsigned X  = 3;
Chao Liu's avatar
Chao Liu committed
427
#elif 0
428
429
430
431
432
433
    // 3x3, 58x58
    constexpr unsigned N  = 64;
    constexpr unsigned C  = 64;
    constexpr unsigned HI = 58;
    constexpr unsigned WI = 58;
    constexpr unsigned K  = 64;
Chao Liu's avatar
Chao Liu committed
434
435
    constexpr unsigned Y  = 3;
    constexpr unsigned X  = 3;
436
437
#elif 0
    // 5x5, 36x36
Chao Liu's avatar
Chao Liu committed
438
    constexpr unsigned N  = 64;
Chao Liu's avatar
Chao Liu committed
439
440
441
    constexpr unsigned C  = 256;
    constexpr unsigned HI = 36;
    constexpr unsigned WI = 36;
Chao Liu's avatar
Chao Liu committed
442
    constexpr unsigned K  = 64;
Chao Liu's avatar
Chao Liu committed
443
444
    constexpr unsigned Y  = 5;
    constexpr unsigned X  = 5;
Chao Liu's avatar
Chao Liu committed
445
446
447

    constexpr unsigned HPad = 0;
    constexpr unsigned WPad = 0;
448
449
450
451
452
453
454
#elif 0
    // 7x7, 38x38
    constexpr unsigned N  = 64;
    constexpr unsigned C  = 256;
    constexpr unsigned HI = 38;
    constexpr unsigned WI = 38;
    constexpr unsigned K  = 64;
Chao Liu's avatar
Chao Liu committed
455
456
    constexpr unsigned Y  = 7;
    constexpr unsigned X  = 7;
Chao Liu's avatar
Chao Liu committed
457
458
459

    constexpr unsigned HPad = 0;
    constexpr unsigned WPad = 0;
460
#elif 0
461
462
463
464
465
466
    // 3x3, 58x58
    constexpr unsigned N  = 16;
    constexpr unsigned C  = 128;
    constexpr unsigned HI = 58;
    constexpr unsigned WI = 58;
    constexpr unsigned K  = 256;
Chao Liu's avatar
Chao Liu committed
467
468
    constexpr unsigned Y  = 3;
    constexpr unsigned X  = 3;
469
470
471
472
473
474
475
#elif 0
    // 3x3 filter, 58x58 image, 0x0 padding
    constexpr unsigned N  = 16;
    constexpr unsigned C  = 128;
    constexpr unsigned HI = 58;
    constexpr unsigned WI = 58;
    constexpr unsigned K  = 256;
Chao Liu's avatar
Chao Liu committed
476
477
    constexpr unsigned Y  = 3;
    constexpr unsigned X  = 3;
478
479
480

    constexpr unsigned HPad = 0;
    constexpr unsigned WPad = 0;
Chao Liu's avatar
Chao Liu committed
481
#elif 0
482
483
484
485
486
487
    // 3x3 filter, 56x56 image, 1x1 padding
    constexpr unsigned N  = 16;
    constexpr unsigned C  = 128;
    constexpr unsigned HI = 56;
    constexpr unsigned WI = 56;
    constexpr unsigned K  = 256;
Chao Liu's avatar
Chao Liu committed
488
489
    constexpr unsigned Y  = 3;
    constexpr unsigned X  = 3;
490
491
492
493
494
495
496
497
498
499

    constexpr unsigned HPad = 1;
    constexpr unsigned WPad = 1;
#elif 0
    // 3x3 filter, 28x28 image, 1x1 padding
    constexpr unsigned N  = 16;
    constexpr unsigned C  = 256;
    constexpr unsigned HI = 28;
    constexpr unsigned WI = 28;
    constexpr unsigned K  = 512;
Chao Liu's avatar
Chao Liu committed
500
501
    constexpr unsigned Y  = 3;
    constexpr unsigned X  = 3;
502
503
504

    constexpr unsigned HPad = 1;
    constexpr unsigned WPad = 1;
Chao Liu's avatar
Chao Liu committed
505
#elif 1
Chao Liu's avatar
Chao Liu committed
506
507
508
509
510
511
    // 1x1 filter, 28x28 image
    constexpr unsigned N  = 16;
    constexpr unsigned C  = 256;
    constexpr unsigned HI = 28;
    constexpr unsigned WI = 28;
    constexpr unsigned K  = 512;
Chao Liu's avatar
Chao Liu committed
512
513
    constexpr unsigned Y  = 1;
    constexpr unsigned X  = 1;
Chao Liu's avatar
Chao Liu committed
514
515
516

    constexpr unsigned HPad = 0;
    constexpr unsigned WPad = 0;
517
518
519
520
521
522
523
#elif 0
    // 3x3 filter, 20x84 image, 1x1 padding
    constexpr unsigned N  = 16;
    constexpr unsigned C  = 256;
    constexpr unsigned HI = 20;
    constexpr unsigned WI = 84;
    constexpr unsigned K  = 256;
Chao Liu's avatar
Chao Liu committed
524
525
    constexpr unsigned Y  = 3;
    constexpr unsigned X  = 3;
526
527
528

    constexpr unsigned HPad = 1;
    constexpr unsigned WPad = 1;
Chao Liu's avatar
Chao Liu committed
529
530
531
532
533
534
535
#elif 0
    // 3x3 filter, 112x112 image, 1x1 padding
    constexpr unsigned N  = 16;
    constexpr unsigned C  = 64;
    constexpr unsigned HI = 112;
    constexpr unsigned WI = 112;
    constexpr unsigned K  = 128;
Chao Liu's avatar
Chao Liu committed
536
537
    constexpr unsigned Y  = 3;
    constexpr unsigned X  = 3;
Chao Liu's avatar
Chao Liu committed
538
539
540
541
542
543
544
545
546
547

    constexpr unsigned HPad = 1;
    constexpr unsigned WPad = 1;
#elif 0
    // 5x5 filter, 20x86 image, 1x1 padding
    constexpr unsigned N  = 16;
    constexpr unsigned C  = 256;
    constexpr unsigned HI = 20;
    constexpr unsigned WI = 86;
    constexpr unsigned K  = 512;
Chao Liu's avatar
Chao Liu committed
548
549
    constexpr unsigned Y  = 5;
    constexpr unsigned X  = 5;
Chao Liu's avatar
Chao Liu committed
550
551
552
553
554
555
556
557
558
559

    constexpr unsigned HPad = 1;
    constexpr unsigned WPad = 1;
#elif 0
    // 5x5 filter, 28x28 image, 2x2 padding
    constexpr unsigned N  = 16;
    constexpr unsigned C  = 192;
    constexpr unsigned HI = 28;
    constexpr unsigned WI = 28;
    constexpr unsigned K  = 32;
Chao Liu's avatar
Chao Liu committed
560
561
    constexpr unsigned Y  = 5;
    constexpr unsigned X  = 5;
Chao Liu's avatar
Chao Liu committed
562
563
564

    constexpr unsigned HPad = 2;
    constexpr unsigned WPad = 2;
Chao Liu's avatar
Chao Liu committed
565
#endif
Chao Liu's avatar
Chao Liu committed
566

567
568
569
    auto lower_pads = Sequence<HPad, WPad>{};
    auto upper_pads = Sequence<HPad, WPad>{};

Chao Liu's avatar
Chao Liu committed
570
    auto in_nchw_desc  = make_ConstantTensorDescriptor(Sequence<N, C, HI, WI>{});
Chao Liu's avatar
Chao Liu committed
571
    auto wei_kcyx_desc = make_ConstantTensorDescriptor(Sequence<K, C, Y, X>{});
572
    auto out_nkhw_desc = get_convolution_with_padding_output_default_4d_tensor_descriptor(
Chao Liu's avatar
Chao Liu committed
573
        in_nchw_desc, wei_kcyx_desc, lower_pads, upper_pads);
Chao Liu's avatar
Chao Liu committed
574

Chao Liu's avatar
Chao Liu committed
575
    ostream_ConstantTensorDescriptor(in_nchw_desc, std::cout << "in_nchw_desc: ");
Chao Liu's avatar
Chao Liu committed
576
    ostream_ConstantTensorDescriptor(wei_kcyx_desc, std::cout << "wei_kcyx_desc: ");
Chao Liu's avatar
Chao Liu committed
577
    ostream_ConstantTensorDescriptor(out_nkhw_desc, std::cout << "out_nkhw_desc: ");
Chao Liu's avatar
Chao Liu committed
578

Chao Liu's avatar
Chao Liu committed
579
580
581
582
583
    using Float = float;
    Tensor<Float> in_nchw(make_TensorDescriptor(in_nchw_desc));
    Tensor<Float> wei_kcyx(make_TensorDescriptor(wei_kcyx_desc));
    Tensor<Float> out_nkhw_host(make_TensorDescriptor(out_nkhw_desc));
    Tensor<Float> out_nkhw_device(make_TensorDescriptor(out_nkhw_desc));
Chao Liu's avatar
Chao Liu committed
584

Chao Liu's avatar
Chao Liu committed
585
    std::size_t num_thread = std::thread::hardware_concurrency();
Chao Liu's avatar
Chao Liu committed
586

Chao Liu's avatar
Chao Liu committed
587
588
589
590
591
592
593
594
    if(argc != 3)
    {
        printf("arg1: do_verification, arg2: nrepeat\n");
        exit(1);
    }

    bool do_verification = atoi(argv[1]);
    unsigned nrepeat     = atoi(argv[2]);
595
596
597

    if(do_verification)
    {
Chao Liu's avatar
Chao Liu committed
598
#if 0
599
        in_nchw.GenerateTensorValue(GeneratorTensor_1{}, num_thread);
Chao Liu's avatar
Chao Liu committed
600
        wei_kcyx.GenerateTensorValue(GeneratorTensor_1{}, num_thread);
Chao Liu's avatar
Chao Liu committed
601
#elif 1
602
        in_nchw.GenerateTensorValue(GeneratorTensor_2{-5, 5}, num_thread);
Chao Liu's avatar
Chao Liu committed
603
        wei_kcyx.GenerateTensorValue(GeneratorTensor_2{-5, 5}, num_thread);
Chao Liu's avatar
Chao Liu committed
604
#elif 1
605
606
607
608
609
610
        in_nchw.GenerateTensorValue(GeneratorTensor_2{1, 5}, num_thread);

        auto gen_wei = [](auto... is) {
            return GeneratorTensor_2{1, 5}(is...) * GeneratorTensor_Checkboard{}(is...);
        };
        wei_kcyx.GenerateTensorValue(gen_wei, num_thread);
Chao Liu's avatar
Chao Liu committed
611
#endif
612
    }
Chao Liu's avatar
Chao Liu committed
613

Chao Liu's avatar
Chao Liu committed
614
#if 1
Chao Liu's avatar
Chao Liu committed
615
#if 0
Chao Liu's avatar
Chao Liu committed
616
    device_direct_convolution_1
617
#elif 0
Chao Liu's avatar
Chao Liu committed
618
    device_direct_convolution_2_nchw_kcyx_nkhw
619
620
#elif 1
    device_direct_convolution_2_vectorized_nchw_kcyx_nkhw
Chao Liu's avatar
Chao Liu committed
621
#elif 0
Chao Liu's avatar
Chao Liu committed
622
    device_implicit_gemm_convolution_1_chwn_cyxk_khwn
Chao Liu's avatar
Chao Liu committed
623
#elif 0
Chao Liu's avatar
Chao Liu committed
624
    device_implicit_gemm_convolution_2_chwn_cyxk_khwn
625
#endif
Chao Liu's avatar
Chao Liu committed
626
    (in_nchw_desc, in_nchw, wei_kcyx_desc, wei_kcyx, out_nkhw_desc, out_nkhw_device, nrepeat);
627

Chao Liu's avatar
Chao Liu committed
628
#elif 1
Chao Liu's avatar
Chao Liu committed
629
    device_implicit_gemm_convolution_1_chwn_cyxk_khwn_padded(in_nchw_desc,
Chao Liu's avatar
Chao Liu committed
630
                                                             in_nchw,
Chao Liu's avatar
Chao Liu committed
631
632
                                                             wei_kcyx_desc,
                                                             wei_kcyx,
Chao Liu's avatar
Chao Liu committed
633
634
635
636
637
                                                             out_nkhw_desc,
                                                             out_nkhw_device,
                                                             lower_pads,
                                                             upper_pads,
                                                             nrepeat);
638
#endif
Chao Liu's avatar
Chao Liu committed
639

640
    if(do_verification)
641
    {
Chao Liu's avatar
Chao Liu committed
642
        if(Y == 3 && X == 3)
643
        {
Chao Liu's avatar
Chao Liu committed
644
            host_winograd_3x3_convolution(in_nchw, wei_kcyx, out_nkhw_host, lower_pads, upper_pads);
645
646
647
        }
        else
        {
Chao Liu's avatar
Chao Liu committed
648
            host_direct_convolution(in_nchw, wei_kcyx, out_nkhw_host, lower_pads, upper_pads);
649
650
        }
        check_error(out_nkhw_host, out_nkhw_device);
Chao Liu's avatar
Chao Liu committed
651

Chao Liu's avatar
Chao Liu committed
652
#if 0
653
        LogRange(std::cout << "in_nchw : ", in_nchw.mData, ",") << std::endl;
Chao Liu's avatar
Chao Liu committed
654
        LogRange(std::cout << "wei_kcyx: ", wei_kcyx.mData, ",") << std::endl;
655
656
        LogRange(std::cout << "out_nkhw_host  : ", out_nkhw_host.mData, ",") << std::endl;
        LogRange(std::cout << "out_nkhw_device: ", out_nkhw_device.mData, ",") << std::endl;
Chao Liu's avatar
Chao Liu committed
657
#endif
658
    }
659
}