driver.hip.cpp 27.3 KB
Newer Older
Chao Liu's avatar
Chao Liu committed
1
#include <iostream>
Chao Liu's avatar
Chao Liu committed
2
3
#include <numeric>
#include <initializer_list>
Chao Liu's avatar
Chao Liu committed
4
#include <cstdlib>
Chao Liu's avatar
Chao Liu committed
5
#include <stdlib.h>
Chao Liu's avatar
Chao Liu committed
6
#include "config.h"
Chao Liu's avatar
Chao Liu committed
7
#include "tensor.hpp"
8
9
#include "ConstantTensorDescriptor.hip.hpp"
#include "conv_common.hip.hpp"
Chao Liu's avatar
Chao Liu committed
10
//#include "device_direct_convolution_1.hpp"
Chao Liu's avatar
Chao Liu committed
11
#include "device_direct_convolution_2_nchw_kcyx_nkhw.hpp"
Chao Liu's avatar
Chao Liu committed
12
//#include "device_direct_convolution_2_vectorized_nchw_kcyx_nkhw.hpp"
Chao Liu's avatar
Chao Liu committed
13
#include "device_convolution_implicit_gemm_v1_chwn_cyxk_khwn.hpp"
14
#include "device_convolution_implicit_gemm_v1_nchw_cyxk_khwn.hpp"
Chao Liu's avatar
Chao Liu committed
15
//#include "device_implicit_gemm_convolution_1_chwn_cyxk_khwn_padded.hpp"
Chao Liu's avatar
Chao Liu committed
16
#include "device_convolution_implicit_gemm_v2_chwn_cyxk_khwn.hpp"
Chao Liu's avatar
Chao Liu committed
17

Chao Liu's avatar
Chao Liu committed
18
struct GeneratorTensor_1
Chao Liu's avatar
Chao Liu committed
19
20
{
    template <class... Is>
Chao Liu's avatar
Chao Liu committed
21
    double operator()(Is... is)
Chao Liu's avatar
Chao Liu committed
22
    {
Chao Liu's avatar
Chao Liu committed
23
        return 1;
Chao Liu's avatar
Chao Liu committed
24
25
26
    }
};

Chao Liu's avatar
Chao Liu committed
27
28
29
30
31
32
33
34
35
36
37
38
struct GeneratorTensor_2
{
    int min_value = 0;
    int max_value = 1;

    template <class... Is>
    double operator()(Is...)
    {
        return (std::rand() % (max_value - min_value)) + min_value;
    }
};

39
40
41
42
43
44
45
46
47
48
49
50
51
struct GeneratorTensor_3
{
    int min_value = 0;
    int max_value = 9;

    template <class... Is>
    double operator()(Is... is)
    {
        std::array<index_t, sizeof...(Is)> dims = {{static_cast<index_t>(is)...}};

#if 0
        auto f_acc = std::plus<index_t>{};
#else
52
        auto f_acc = [](auto a, auto b) { return 10 * a + b; };
53
54
#endif

55
        return std::accumulate(dims.begin(), dims.end(), index_t(0), f_acc);
56
57
58
    }
};

Chao Liu's avatar
Chao Liu committed
59
60
61
62
63
struct GeneratorTensor_Checkboard
{
    template <class... Ts>
    double operator()(Ts... Xs) const
    {
Chao Liu's avatar
Chao Liu committed
64
        std::array<index_t, sizeof...(Ts)> dims = {{Xs...}};
Chao Liu's avatar
Chao Liu committed
65
66
67
        return std::accumulate(dims.begin(),
                               dims.end(),
                               true,
Chao Liu's avatar
Chao Liu committed
68
                               [](bool init, index_t x) -> int { return init != (x % 2); })
Chao Liu's avatar
Chao Liu committed
69
70
71
72
73
                   ? 1
                   : -1;
    }
};

Chao Liu's avatar
Chao Liu committed
74
75
76
77
78
79
// this is ugly, only for 4d
template <class TConstTensorDesc>
void ostream_ConstantTensorDescriptor(TConstTensorDesc, std::ostream& os = std::cout)
{
    static_assert(TConstTensorDesc::nDim == 4, "nDim is not 4");

Chao Liu's avatar
Chao Liu committed
80
81
82
83
    constexpr auto I0   = Number<0>{};
    constexpr auto I1   = Number<1>{};
    constexpr auto I2   = Number<2>{};
    constexpr auto I3   = Number<3>{};
Chao Liu's avatar
Chao Liu committed
84
85
86
87
88
89
90
91
92
93
94
95
96
97
    constexpr auto desc = TConstTensorDesc{};

    os << "Lengths: {" << desc.GetLength(I0) << ", " << desc.GetLength(I1) << ", "
       << desc.GetLength(I2) << ", " << desc.GetLength(I3) << "}, "
       << "Strides: {" << desc.GetStride(I0) << ", " << desc.GetStride(I1) << ", "
       << desc.GetStride(I2) << ", " << desc.GetStride(I3) << "}" << std::endl;
}

// this is ugly, only for 4d
template <class TConstTensorDesc>
auto make_TensorDescriptor(TConstTensorDesc)
{
    static_assert(TConstTensorDesc::nDim == 4, "nDim is not 4");

Chao Liu's avatar
Chao Liu committed
98
99
100
101
    constexpr auto I0   = Number<0>{};
    constexpr auto I1   = Number<1>{};
    constexpr auto I2   = Number<2>{};
    constexpr auto I3   = Number<3>{};
Chao Liu's avatar
Chao Liu committed
102
103
    constexpr auto desc = TConstTensorDesc{};

Chao Liu's avatar
Chao Liu committed
104
    std::initializer_list<index_t> lengths = {
Chao Liu's avatar
Chao Liu committed
105
        desc.GetLength(I0), desc.GetLength(I1), desc.GetLength(I2), desc.GetLength(I3)};
Chao Liu's avatar
Chao Liu committed
106
    std::initializer_list<index_t> strides = {
Chao Liu's avatar
Chao Liu committed
107
108
109
110
111
        desc.GetStride(I0), desc.GetStride(I1), desc.GetStride(I2), desc.GetStride(I3)};

    return TensorDescriptor(lengths, strides);
}

112
113
114
115
116
117
template <class TIn, class TWei, class TOut, class LowerPads, class UpperPads>
void host_direct_convolution(const Tensor<TIn>& in_nchw,
                             const Tensor<TWei>& wei_kcyx,
                             Tensor<TOut>& out_nkhw,
                             LowerPads,
                             UpperPads)
Chao Liu's avatar
Chao Liu committed
118
{
Chao Liu's avatar
Chao Liu committed
119
120
    index_t h_pad_low = LowerPads{}.Get(Number<0>{});
    index_t w_pad_low = LowerPads{}.Get(Number<1>{});
121

Chao Liu's avatar
Chao Liu committed
122
123
    index_t h_pad_up = UpperPads{}.Get(Number<0>{});
    index_t w_pad_up = UpperPads{}.Get(Number<1>{});
124

Chao Liu's avatar
Chao Liu committed
125
126
    auto f = [&](auto n, auto k, auto ho, auto wo) {
        double v = 0;
Chao Liu's avatar
Chao Liu committed
127
        for(int c = 0; c < wei_kcyx.mDesc.GetLengths()[1]; ++c)
Chao Liu's avatar
Chao Liu committed
128
        {
Chao Liu's avatar
Chao Liu committed
129
            for(int y = 0; y < wei_kcyx.mDesc.GetLengths()[2]; ++y)
Chao Liu's avatar
Chao Liu committed
130
            {
131
                int hi = ho + y - h_pad_low;
Chao Liu's avatar
Chao Liu committed
132
                for(int x = 0; x < wei_kcyx.mDesc.GetLengths()[3]; ++x)
Chao Liu's avatar
Chao Liu committed
133
                {
134
135
136
137
                    int wi = wo + x - w_pad_low;
                    if(hi >= 0 && hi < in_nchw.mDesc.GetLengths()[2] && wi >= 0 &&
                       wi < in_nchw.mDesc.GetLengths()[3])
                    {
138
                        v += double(in_nchw(n, c, hi, wi)) * double(wei_kcyx(k, c, y, x));
139
                    }
Chao Liu's avatar
Chao Liu committed
140
141
142
                }
            }
        }
143
        out_nkhw(n, k, ho, wo) = v;
Chao Liu's avatar
Chao Liu committed
144
145
146
    };

    auto f_par = make_ParallelTensorFunctor(f,
147
148
149
150
                                            out_nkhw.mDesc.GetLengths()[0],
                                            out_nkhw.mDesc.GetLengths()[1],
                                            out_nkhw.mDesc.GetLengths()[2],
                                            out_nkhw.mDesc.GetLengths()[3]);
Chao Liu's avatar
Chao Liu committed
151

Chao Liu's avatar
Chao Liu committed
152
    f_par(std::thread::hardware_concurrency());
Chao Liu's avatar
Chao Liu committed
153
154
}

155
156
157
158
159
160
template <class TIn, class TWei, class TOut, class LowerPads, class UpperPads>
void host_winograd_3x3_convolution(const Tensor<TIn>& in_nchw,
                                   const Tensor<TWei>& wei_kcyx,
                                   Tensor<TOut>& out_nkhw,
                                   LowerPads,
                                   UpperPads)
Chao Liu's avatar
Chao Liu committed
161
{
Chao Liu's avatar
Chao Liu committed
162
163
    constexpr std::size_t HoPerTile = 2;
    constexpr std::size_t WoPerTile = 2;
Chao Liu's avatar
Chao Liu committed
164

Chao Liu's avatar
Chao Liu committed
165
166
167
168
    std::size_t N  = in_nchw.mDesc.GetLengths()[0];
    std::size_t C  = in_nchw.mDesc.GetLengths()[1];
    std::size_t HI = in_nchw.mDesc.GetLengths()[2];
    std::size_t WI = in_nchw.mDesc.GetLengths()[3];
Chao Liu's avatar
Chao Liu committed
169

Chao Liu's avatar
Chao Liu committed
170
171
172
    std::size_t K = wei_kcyx.mDesc.GetLengths()[0];
    std::size_t Y = wei_kcyx.mDesc.GetLengths()[2];
    std::size_t X = wei_kcyx.mDesc.GetLengths()[3];
Chao Liu's avatar
Chao Liu committed
173

174
175
    std::size_t HO = out_nkhw.mDesc.GetLengths()[2];
    std::size_t WO = out_nkhw.mDesc.GetLengths()[3];
Chao Liu's avatar
Chao Liu committed
176

Chao Liu's avatar
Chao Liu committed
177
178
    index_t h_pad_low = LowerPads{}.Get(Number<0>{});
    index_t w_pad_low = LowerPads{}.Get(Number<1>{});
179

Chao Liu's avatar
Chao Liu committed
180
181
    index_t h_pad_up = UpperPads{}.Get(Number<0>{});
    index_t w_pad_up = UpperPads{}.Get(Number<1>{});
182

Chao Liu's avatar
Chao Liu committed
183
184
    std::size_t HiPerTile = HoPerTile + Y - 1;
    std::size_t WiPerTile = WoPerTile + X - 1;
Chao Liu's avatar
Chao Liu committed
185

Chao Liu's avatar
Chao Liu committed
186
187
    std::size_t HTile = (HO + HoPerTile - 1) / HoPerTile;
    std::size_t WTile = (WO + WoPerTile - 1) / WoPerTile;
Chao Liu's avatar
Chao Liu committed
188

189
190
191
192
193
    Tensor<double> in_hold({N, C, HTile, WTile, HiPerTile, WiPerTile});
    Tensor<double> in_transform({N, C, HTile, WTile, HiPerTile, WiPerTile});
    Tensor<double> wei_transform({K, C, HiPerTile, WiPerTile});
    Tensor<double> out_transform({N, K, HTile, WTile, HiPerTile, HiPerTile});
    Tensor<double> out_hold({N, K, HTile, WTile, HoPerTile, WoPerTile});
Chao Liu's avatar
Chao Liu committed
194

Chao Liu's avatar
Chao Liu committed
195
196
    auto f_in_hold = [&](auto n, auto c, auto htile, auto wtile) {
        for(int j = 0; j < HiPerTile; ++j)
Chao Liu's avatar
Chao Liu committed
197
        {
Chao Liu's avatar
Chao Liu committed
198
199
            int hi = HoPerTile * htile + j - h_pad_low;
            for(int i = 0; i < WiPerTile; ++i)
Chao Liu's avatar
Chao Liu committed
200
            {
Chao Liu's avatar
Chao Liu committed
201
                int wi = WoPerTile * wtile + i - w_pad_low;
202
203
204
205

                if(hi >= 0 && hi < in_nchw.mDesc.GetLengths()[2] && wi >= 0 &&
                   wi < in_nchw.mDesc.GetLengths()[3])
                {
Chao Liu's avatar
Chao Liu committed
206
                    in_hold(n, c, htile, wtile, j, i) = in_nchw(n, c, hi, wi);
207
208
209
                }
                else
                {
210
                    in_hold(n, c, htile, wtile, j, i) = TIn(0);
211
                }
Chao Liu's avatar
Chao Liu committed
212
213
214
215
            }
        }
    };

Chao Liu's avatar
Chao Liu committed
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
    auto f_in_transform = [&](auto n, auto c, auto htile, auto wtile) {
        in_transform(n, c, htile, wtile, 0, 0) =
            in_hold(n, c, htile, wtile, 0, 0) - in_hold(n, c, htile, wtile, 0, 2) -
            in_hold(n, c, htile, wtile, 2, 0) + in_hold(n, c, htile, wtile, 2, 2);
        in_transform(n, c, htile, wtile, 0, 1) =
            in_hold(n, c, htile, wtile, 0, 1) + in_hold(n, c, htile, wtile, 0, 2) -
            in_hold(n, c, htile, wtile, 2, 1) - in_hold(n, c, htile, wtile, 2, 2);
        in_transform(n, c, htile, wtile, 0, 2) =
            -in_hold(n, c, htile, wtile, 0, 1) + in_hold(n, c, htile, wtile, 0, 2) +
            in_hold(n, c, htile, wtile, 2, 1) - in_hold(n, c, htile, wtile, 2, 2);
        in_transform(n, c, htile, wtile, 0, 3) =
            in_hold(n, c, htile, wtile, 0, 1) - in_hold(n, c, htile, wtile, 0, 3) -
            in_hold(n, c, htile, wtile, 2, 1) + in_hold(n, c, htile, wtile, 2, 3);

        in_transform(n, c, htile, wtile, 1, 0) =
            in_hold(n, c, htile, wtile, 1, 0) - in_hold(n, c, htile, wtile, 1, 2) +
            in_hold(n, c, htile, wtile, 2, 0) - in_hold(n, c, htile, wtile, 2, 2);
        in_transform(n, c, htile, wtile, 1, 1) =
            in_hold(n, c, htile, wtile, 1, 1) + in_hold(n, c, htile, wtile, 1, 2) +
            in_hold(n, c, htile, wtile, 2, 1) + in_hold(n, c, htile, wtile, 2, 2);
        in_transform(n, c, htile, wtile, 1, 2) =
            -in_hold(n, c, htile, wtile, 1, 1) + in_hold(n, c, htile, wtile, 1, 2) -
            in_hold(n, c, htile, wtile, 2, 1) + in_hold(n, c, htile, wtile, 2, 2);
        in_transform(n, c, htile, wtile, 1, 3) =
            in_hold(n, c, htile, wtile, 1, 1) - in_hold(n, c, htile, wtile, 1, 3) +
            in_hold(n, c, htile, wtile, 2, 1) - in_hold(n, c, htile, wtile, 2, 3);

        in_transform(n, c, htile, wtile, 2, 0) =
            -in_hold(n, c, htile, wtile, 1, 0) + in_hold(n, c, htile, wtile, 1, 2) +
            in_hold(n, c, htile, wtile, 2, 0) - in_hold(n, c, htile, wtile, 2, 2);
        in_transform(n, c, htile, wtile, 2, 1) =
            -in_hold(n, c, htile, wtile, 1, 1) - in_hold(n, c, htile, wtile, 1, 2) +
            in_hold(n, c, htile, wtile, 2, 1) + in_hold(n, c, htile, wtile, 2, 2);
        in_transform(n, c, htile, wtile, 2, 2) =
            in_hold(n, c, htile, wtile, 1, 1) - in_hold(n, c, htile, wtile, 1, 2) -
            in_hold(n, c, htile, wtile, 2, 1) + in_hold(n, c, htile, wtile, 2, 2);
        in_transform(n, c, htile, wtile, 2, 3) =
            -in_hold(n, c, htile, wtile, 1, 1) + in_hold(n, c, htile, wtile, 1, 3) +
            in_hold(n, c, htile, wtile, 2, 1) - in_hold(n, c, htile, wtile, 2, 3);

        in_transform(n, c, htile, wtile, 3, 0) =
            in_hold(n, c, htile, wtile, 1, 0) - in_hold(n, c, htile, wtile, 1, 2) -
            in_hold(n, c, htile, wtile, 3, 0) + in_hold(n, c, htile, wtile, 3, 2);
        in_transform(n, c, htile, wtile, 3, 1) =
            in_hold(n, c, htile, wtile, 1, 1) + in_hold(n, c, htile, wtile, 1, 2) -
            in_hold(n, c, htile, wtile, 3, 1) - in_hold(n, c, htile, wtile, 3, 2);
        in_transform(n, c, htile, wtile, 3, 2) =
            -in_hold(n, c, htile, wtile, 1, 1) + in_hold(n, c, htile, wtile, 1, 2) +
            in_hold(n, c, htile, wtile, 3, 1) - in_hold(n, c, htile, wtile, 3, 2);
        in_transform(n, c, htile, wtile, 3, 3) =
            in_hold(n, c, htile, wtile, 1, 1) - in_hold(n, c, htile, wtile, 1, 3) -
            in_hold(n, c, htile, wtile, 3, 1) + in_hold(n, c, htile, wtile, 3, 3);
Chao Liu's avatar
Chao Liu committed
268
269
270
    };

    auto f_wei_transform = [&](auto k, auto c) {
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
        wei_transform(k, c, 0, 0) = double(wei_kcyx(k, c, 0, 0));
        wei_transform(k, c, 0, 1) = 0.5 * double(wei_kcyx(k, c, 0, 0)) +
                                    0.5 * double(wei_kcyx(k, c, 0, 1)) +
                                    0.5 * double(wei_kcyx(k, c, 0, 2));
        wei_transform(k, c, 0, 2) = 0.5 * double(wei_kcyx(k, c, 0, 0)) -
                                    0.5 * double(wei_kcyx(k, c, 0, 1)) +
                                    0.5 * double(wei_kcyx(k, c, 0, 2));
        wei_transform(k, c, 0, 3) = double(wei_kcyx(k, c, 0, 2));

        wei_transform(k, c, 1, 0) = 0.5 * double(wei_kcyx(k, c, 0, 0)) +
                                    0.5 * double(wei_kcyx(k, c, 1, 0)) +
                                    0.5 * double(wei_kcyx(k, c, 2, 0));
        wei_transform(k, c, 1, 1) =
            0.25 * double(wei_kcyx(k, c, 0, 0)) + 0.25 * double(wei_kcyx(k, c, 0, 1)) +
            0.25 * double(wei_kcyx(k, c, 0, 2)) + 0.25 * double(wei_kcyx(k, c, 1, 0)) +
            0.25 * double(wei_kcyx(k, c, 1, 1)) + 0.25 * double(wei_kcyx(k, c, 1, 2)) +
            0.25 * double(wei_kcyx(k, c, 2, 0)) + 0.25 * double(wei_kcyx(k, c, 2, 1)) +
            0.25 * double(wei_kcyx(k, c, 2, 2));
        wei_transform(k, c, 1, 2) =
            0.25 * double(wei_kcyx(k, c, 0, 0)) - 0.25 * double(wei_kcyx(k, c, 0, 1)) +
            0.25 * double(wei_kcyx(k, c, 0, 2)) + 0.25 * double(wei_kcyx(k, c, 1, 0)) -
            0.25 * double(wei_kcyx(k, c, 1, 1)) + 0.25 * double(wei_kcyx(k, c, 1, 2)) +
            0.25 * double(wei_kcyx(k, c, 2, 0)) - 0.25 * double(wei_kcyx(k, c, 2, 1)) +
            0.25 * double(wei_kcyx(k, c, 2, 2));
        wei_transform(k, c, 1, 3) = 0.5 * double(wei_kcyx(k, c, 0, 2)) +
                                    0.5 * double(wei_kcyx(k, c, 1, 2)) +
                                    0.5 * double(wei_kcyx(k, c, 2, 2));

        wei_transform(k, c, 2, 0) = 0.5 * double(wei_kcyx(k, c, 0, 0)) -
                                    0.5 * double(wei_kcyx(k, c, 1, 0)) +
                                    0.5 * double(wei_kcyx(k, c, 2, 0));
        wei_transform(k, c, 2, 1) =
            0.25 * double(wei_kcyx(k, c, 0, 0)) + 0.25 * double(wei_kcyx(k, c, 0, 1)) +
            0.25 * double(wei_kcyx(k, c, 0, 2)) - 0.25 * double(wei_kcyx(k, c, 1, 0)) -
            0.25 * double(wei_kcyx(k, c, 1, 1)) - 0.25 * double(wei_kcyx(k, c, 1, 2)) +
            0.25 * double(wei_kcyx(k, c, 2, 0)) + 0.25 * double(wei_kcyx(k, c, 2, 1)) +
            0.25 * double(wei_kcyx(k, c, 2, 2));
        wei_transform(k, c, 2, 2) =
            0.25 * double(wei_kcyx(k, c, 0, 0)) - 0.25 * double(wei_kcyx(k, c, 0, 1)) +
            0.25 * double(wei_kcyx(k, c, 0, 2)) - 0.25 * double(wei_kcyx(k, c, 1, 0)) +
            0.25 * double(wei_kcyx(k, c, 1, 1)) - 0.25 * double(wei_kcyx(k, c, 1, 2)) +
            0.25 * double(wei_kcyx(k, c, 2, 0)) - 0.25 * double(wei_kcyx(k, c, 2, 1)) +
            0.25 * double(wei_kcyx(k, c, 2, 2));
        wei_transform(k, c, 2, 3) = 0.5 * double(wei_kcyx(k, c, 0, 2)) -
                                    0.5 * double(wei_kcyx(k, c, 1, 2)) +
                                    0.5 * double(wei_kcyx(k, c, 2, 2));

        wei_transform(k, c, 3, 0) = double(wei_kcyx(k, c, 2, 0));
        wei_transform(k, c, 3, 1) = 0.5 * double(wei_kcyx(k, c, 2, 0)) +
                                    0.5 * double(wei_kcyx(k, c, 2, 1)) +
                                    0.5 * double(wei_kcyx(k, c, 2, 2));
        wei_transform(k, c, 3, 2) = 0.5 * double(wei_kcyx(k, c, 2, 0)) -
                                    0.5 * double(wei_kcyx(k, c, 2, 1)) +
                                    0.5 * double(wei_kcyx(k, c, 2, 2));
        wei_transform(k, c, 3, 3) = double(wei_kcyx(k, c, 2, 2));
Chao Liu's avatar
Chao Liu committed
326
327
    };

Chao Liu's avatar
Chao Liu committed
328
329
    auto f_out_transform = [&](auto n, auto k, auto htile, auto wtile) {
        for(int j = 0; j < HiPerTile; ++j)
Chao Liu's avatar
Chao Liu committed
330
        {
Chao Liu's avatar
Chao Liu committed
331
            for(int i = 0; i < WiPerTile; ++i)
Chao Liu's avatar
Chao Liu committed
332
333
334
335
            {
                double v = 0;
                for(int c = 0; c < C; ++c)
                {
Chao Liu's avatar
Chao Liu committed
336
                    v += in_transform(n, c, htile, wtile, j, i) * wei_transform(k, c, j, i);
Chao Liu's avatar
Chao Liu committed
337
338
                }

Chao Liu's avatar
Chao Liu committed
339
                out_transform(n, k, htile, wtile, j, i) = v;
Chao Liu's avatar
Chao Liu committed
340
341
342
343
            }
        }
    };

Chao Liu's avatar
Chao Liu committed
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
    auto f_out_hold = [&](auto n, auto k, auto htile, auto wtile) {
        out_hold(n, k, htile, wtile, 0, 0) =
            out_transform(n, k, htile, wtile, 0, 0) + out_transform(n, k, htile, wtile, 0, 1) +
            out_transform(n, k, htile, wtile, 0, 2) + out_transform(n, k, htile, wtile, 1, 0) +
            out_transform(n, k, htile, wtile, 1, 1) + out_transform(n, k, htile, wtile, 1, 2) +
            out_transform(n, k, htile, wtile, 2, 0) + out_transform(n, k, htile, wtile, 2, 1) +
            out_transform(n, k, htile, wtile, 2, 2);
        out_hold(n, k, htile, wtile, 0, 1) =
            out_transform(n, k, htile, wtile, 0, 1) - out_transform(n, k, htile, wtile, 0, 2) -
            out_transform(n, k, htile, wtile, 0, 3) + out_transform(n, k, htile, wtile, 1, 1) -
            out_transform(n, k, htile, wtile, 1, 2) - out_transform(n, k, htile, wtile, 1, 3) +
            out_transform(n, k, htile, wtile, 2, 1) - out_transform(n, k, htile, wtile, 2, 2) -
            out_transform(n, k, htile, wtile, 2, 3);
        out_hold(n, k, htile, wtile, 1, 0) =
            out_transform(n, k, htile, wtile, 1, 0) + out_transform(n, k, htile, wtile, 1, 1) +
            out_transform(n, k, htile, wtile, 1, 2) - out_transform(n, k, htile, wtile, 2, 0) -
            out_transform(n, k, htile, wtile, 2, 1) - out_transform(n, k, htile, wtile, 2, 2) -
            out_transform(n, k, htile, wtile, 3, 0) - out_transform(n, k, htile, wtile, 3, 1) -
            out_transform(n, k, htile, wtile, 3, 2);
        out_hold(n, k, htile, wtile, 1, 1) =
            out_transform(n, k, htile, wtile, 1, 1) - out_transform(n, k, htile, wtile, 1, 2) -
            out_transform(n, k, htile, wtile, 1, 3) - out_transform(n, k, htile, wtile, 2, 1) +
            out_transform(n, k, htile, wtile, 2, 2) + out_transform(n, k, htile, wtile, 2, 3) -
            out_transform(n, k, htile, wtile, 3, 1) + out_transform(n, k, htile, wtile, 3, 2) +
            out_transform(n, k, htile, wtile, 3, 3);
Chao Liu's avatar
Chao Liu committed
369
370
    };

Chao Liu's avatar
Chao Liu committed
371
372
    auto f_out = [&](auto n, auto k, auto htile, auto wtile) {
        for(int j = 0; j < HoPerTile; ++j)
Chao Liu's avatar
Chao Liu committed
373
        {
Chao Liu's avatar
Chao Liu committed
374
375
            std::size_t ho = HoPerTile * htile + j;
            for(int i = 0; i < WoPerTile; ++i)
Chao Liu's avatar
Chao Liu committed
376
            {
377
                std::size_t wo         = WoPerTile * wtile + i;
378
                out_nkhw(n, k, ho, wo) = out_hold(n, k, htile, wtile, j, i);
Chao Liu's avatar
Chao Liu committed
379
380
381
382
383
384
            }
        }
    };

    std::size_t num_thread = std::thread::hardware_concurrency();

Chao Liu's avatar
Chao Liu committed
385
386
    make_ParallelTensorFunctor(f_in_hold, N, C, HTile, WTile)(num_thread);
    make_ParallelTensorFunctor(f_in_transform, N, C, HTile, WTile)(num_thread);
Chao Liu's avatar
Chao Liu committed
387
    make_ParallelTensorFunctor(f_wei_transform, K, C)(num_thread);
Chao Liu's avatar
Chao Liu committed
388
389
390
    make_ParallelTensorFunctor(f_out_transform, N, K, HTile, WTile)(num_thread);
    make_ParallelTensorFunctor(f_out_hold, N, K, HTile, WTile)(num_thread);
    make_ParallelTensorFunctor(f_out, N, K, HTile, WTile)(num_thread);
Chao Liu's avatar
Chao Liu committed
391
392
393
394
395
}

template <class T>
void check_error(const Tensor<T>& ref, const Tensor<T>& result)
{
396
397
    // printf("\n");

Chao Liu's avatar
Chao Liu committed
398
    float error     = 0;
Chao Liu's avatar
Chao Liu committed
399
    float max_diff  = -1;
Chao Liu's avatar
Chao Liu committed
400
401
402
    float ref_value = 0, result_value = 0;
    for(int i = 0; i < ref.mData.size(); ++i)
    {
403
404
        error += std::abs(double(ref.mData[i]) - double(result.mData[i]));
        float diff = std::abs(double(ref.mData[i]) - double(result.mData[i]));
Chao Liu's avatar
Chao Liu committed
405
406
407
408
409
410
        if(max_diff < diff)
        {
            max_diff     = diff;
            ref_value    = ref.mData[i];
            result_value = result.mData[i];
        }
411
412

        // printf("{%f, %f}", double(ref.mData[i]), double(result.mData[i]));
Chao Liu's avatar
Chao Liu committed
413
    }
414
    // printf("\n");
Chao Liu's avatar
Chao Liu committed
415
416
417
418
419

    std::cout << "error: " << error << std::endl;
    std::cout << "max_diff: " << max_diff << ", " << ref_value << ", " << result_value << std::endl;
}

Chao Liu's avatar
Chao Liu committed
420
int main(int argc, char* argv[])
Chao Liu's avatar
Chao Liu committed
421
{
Chao Liu's avatar
tuning  
Chao Liu committed
422
#if 0
423
    // 3x3, 34x34
Chao Liu's avatar
Chao Liu committed
424
425
426
427
    constexpr index_t N  = 64;
    constexpr index_t C  = 256;
    constexpr index_t HI = 34;
    constexpr index_t WI = 34;
428
    constexpr index_t K  = 128;
Chao Liu's avatar
Chao Liu committed
429
430
431
432
433
    constexpr index_t Y  = 3;
    constexpr index_t X  = 3;

    constexpr index_t HPad = 0;
    constexpr index_t WPad = 0;
Chao Liu's avatar
Chao Liu committed
434
#elif 0
435
    // 3x3, 56x56
436
437
    constexpr index_t N = 64;
    constexpr index_t C = 64;
Chao Liu's avatar
Chao Liu committed
438
439
    constexpr index_t HI = 56;
    constexpr index_t WI = 56;
440
441
442
    constexpr index_t K = 128;
    constexpr index_t Y = 3;
    constexpr index_t X = 3;
Chao Liu's avatar
Chao Liu committed
443
444
445

    constexpr index_t HPad = 0;
    constexpr index_t WPad = 0;
Chao Liu's avatar
Chao Liu committed
446
#elif 0
447
    // 3x3, 58x58
Chao Liu's avatar
Chao Liu committed
448
449
450
451
452
453
454
    constexpr index_t N  = 64;
    constexpr index_t C  = 64;
    constexpr index_t HI = 58;
    constexpr index_t WI = 58;
    constexpr index_t K  = 64;
    constexpr index_t Y  = 3;
    constexpr index_t X  = 3;
455
#elif 0
456
    // 3x3, 58x58
Chao Liu's avatar
Chao Liu committed
457
458
459
460
461
462
463
    constexpr index_t N  = 16;
    constexpr index_t C  = 128;
    constexpr index_t HI = 58;
    constexpr index_t WI = 58;
    constexpr index_t K  = 256;
    constexpr index_t Y  = 3;
    constexpr index_t X  = 3;
464
465
#elif 0
    // 3x3 filter, 58x58 image, 0x0 padding
Chao Liu's avatar
Chao Liu committed
466
467
468
469
470
471
472
473
474
475
    constexpr index_t N  = 16;
    constexpr index_t C  = 128;
    constexpr index_t HI = 58;
    constexpr index_t WI = 58;
    constexpr index_t K  = 256;
    constexpr index_t Y  = 3;
    constexpr index_t X  = 3;

    constexpr index_t HPad = 0;
    constexpr index_t WPad = 0;
Chao Liu's avatar
Chao Liu committed
476
#elif 0
477
    // 3x3 filter, 56x56 image, 1x1 padding
Chao Liu's avatar
Chao Liu committed
478
479
480
481
482
483
484
485
486
487
    constexpr index_t N  = 16;
    constexpr index_t C  = 128;
    constexpr index_t HI = 56;
    constexpr index_t WI = 56;
    constexpr index_t K  = 256;
    constexpr index_t Y  = 3;
    constexpr index_t X  = 3;

    constexpr index_t HPad = 1;
    constexpr index_t WPad = 1;
488
489
#elif 0
    // 3x3 filter, 28x28 image, 1x1 padding
Chao Liu's avatar
Chao Liu committed
490
491
492
493
494
495
496
497
498
499
    constexpr index_t N  = 16;
    constexpr index_t C  = 256;
    constexpr index_t HI = 28;
    constexpr index_t WI = 28;
    constexpr index_t K  = 512;
    constexpr index_t Y  = 3;
    constexpr index_t X  = 3;

    constexpr index_t HPad = 1;
    constexpr index_t WPad = 1;
Chao Liu's avatar
tuning  
Chao Liu committed
500
#elif 1
Chao Liu's avatar
Chao Liu committed
501
502
503
504
505
    // 3x3 filter, 28x28 image
    constexpr index_t N  = 128;
    constexpr index_t C  = 256;
    constexpr index_t HI = 28;
    constexpr index_t WI = 28;
506
    constexpr index_t K  = 128;
Chao Liu's avatar
Chao Liu committed
507
508
509
510
511
    constexpr index_t Y  = 3;
    constexpr index_t X  = 3;

    constexpr index_t HPad = 0;
    constexpr index_t WPad = 0;
Chao Liu's avatar
Chao Liu committed
512
#elif 0
Chao Liu's avatar
Chao Liu committed
513
    // 1x1 filter, 28x28 image
Chao Liu's avatar
Chao Liu committed
514
515
516
517
518
519
520
521
522
523
    constexpr index_t N  = 16;
    constexpr index_t C  = 256;
    constexpr index_t HI = 28;
    constexpr index_t WI = 28;
    constexpr index_t K  = 512;
    constexpr index_t Y  = 1;
    constexpr index_t X  = 1;

    constexpr index_t HPad = 0;
    constexpr index_t WPad = 0;
524
525
#elif 0
    // 3x3 filter, 20x84 image, 1x1 padding
Chao Liu's avatar
Chao Liu committed
526
527
528
529
530
531
532
533
534
535
    constexpr index_t N  = 16;
    constexpr index_t C  = 256;
    constexpr index_t HI = 20;
    constexpr index_t WI = 84;
    constexpr index_t K  = 256;
    constexpr index_t Y  = 3;
    constexpr index_t X  = 3;

    constexpr index_t HPad = 1;
    constexpr index_t WPad = 1;
Chao Liu's avatar
Chao Liu committed
536
537
#elif 0
    // 3x3 filter, 112x112 image, 1x1 padding
Chao Liu's avatar
Chao Liu committed
538
539
540
541
542
543
544
545
546
547
    constexpr index_t N  = 16;
    constexpr index_t C  = 64;
    constexpr index_t HI = 112;
    constexpr index_t WI = 112;
    constexpr index_t K  = 128;
    constexpr index_t Y  = 3;
    constexpr index_t X  = 3;

    constexpr index_t HPad = 1;
    constexpr index_t WPad = 1;
Chao Liu's avatar
Chao Liu committed
548
549
#elif 0
    // 5x5 filter, 20x86 image, 1x1 padding
Chao Liu's avatar
Chao Liu committed
550
551
552
553
554
555
556
557
558
559
    constexpr index_t N  = 16;
    constexpr index_t C  = 256;
    constexpr index_t HI = 20;
    constexpr index_t WI = 86;
    constexpr index_t K  = 512;
    constexpr index_t Y  = 5;
    constexpr index_t X  = 5;

    constexpr index_t HPad = 1;
    constexpr index_t WPad = 1;
Chao Liu's avatar
Chao Liu committed
560
561
#elif 0
    // 5x5 filter, 28x28 image, 2x2 padding
Chao Liu's avatar
Chao Liu committed
562
563
564
565
566
567
568
569
570
571
    constexpr index_t N  = 16;
    constexpr index_t C  = 192;
    constexpr index_t HI = 28;
    constexpr index_t WI = 28;
    constexpr index_t K  = 32;
    constexpr index_t Y  = 5;
    constexpr index_t X  = 5;

    constexpr index_t HPad = 2;
    constexpr index_t WPad = 2;
Chao Liu's avatar
Chao Liu committed
572
#elif 0
573
    // 1x1 filter, 32x32 image
Chao Liu's avatar
Chao Liu committed
574
575
576
577
578
579
580
581
582
583
    constexpr index_t N  = 64;
    constexpr index_t C  = 256;
    constexpr index_t HI = 32;
    constexpr index_t WI = 32;
    constexpr index_t K  = 512;
    constexpr index_t Y  = 1;
    constexpr index_t X  = 1;

    constexpr index_t HPad = 0;
    constexpr index_t WPad = 0;
584
#elif 1
Chao Liu's avatar
Chao Liu committed
585
586
587
588
589
590
591
592
593
594
595
    // 1x1 filter, 14x14 image, C = 2048
    constexpr index_t N  = 128;
    constexpr index_t C  = 2048;
    constexpr index_t HI = 14;
    constexpr index_t WI = 14;
    constexpr index_t K  = 512;
    constexpr index_t Y  = 1;
    constexpr index_t X  = 1;

    constexpr index_t HPad = 0;
    constexpr index_t WPad = 0;
596
#elif 1
Chao Liu's avatar
Chao Liu committed
597
    // 1x1 filter, 14x14 image, C = 512
Chao Liu's avatar
Chao Liu committed
598
599
600
601
602
603
604
605
606
607
    constexpr index_t N  = 128;
    constexpr index_t C  = 512;
    constexpr index_t HI = 14;
    constexpr index_t WI = 14;
    constexpr index_t K  = 512;
    constexpr index_t Y  = 1;
    constexpr index_t X  = 1;

    constexpr index_t HPad = 0;
    constexpr index_t WPad = 0;
Chao Liu's avatar
Chao Liu committed
608
#endif
Chao Liu's avatar
Chao Liu committed
609

610
611
612
    auto lower_pads = Sequence<HPad, WPad>{};
    auto upper_pads = Sequence<HPad, WPad>{};

Chao Liu's avatar
Chao Liu committed
613
    auto in_nchw_desc  = make_ConstantTensorDescriptor(Sequence<N, C, HI, WI>{});
Chao Liu's avatar
Chao Liu committed
614
    auto wei_kcyx_desc = make_ConstantTensorDescriptor(Sequence<K, C, Y, X>{});
615
    auto out_nkhw_desc = get_convolution_with_padding_output_default_4d_tensor_descriptor(
Chao Liu's avatar
Chao Liu committed
616
        in_nchw_desc, wei_kcyx_desc, lower_pads, upper_pads);
Chao Liu's avatar
Chao Liu committed
617

Chao Liu's avatar
Chao Liu committed
618
    ostream_ConstantTensorDescriptor(in_nchw_desc, std::cout << "in_nchw_desc: ");
Chao Liu's avatar
Chao Liu committed
619
    ostream_ConstantTensorDescriptor(wei_kcyx_desc, std::cout << "wei_kcyx_desc: ");
Chao Liu's avatar
Chao Liu committed
620
    ostream_ConstantTensorDescriptor(out_nkhw_desc, std::cout << "out_nkhw_desc: ");
Chao Liu's avatar
Chao Liu committed
621

Chao Liu's avatar
Chao Liu committed
622
623
    using in_data_t  = float;
    using out_data_t = float;
624
625
626
627
    Tensor<in_data_t> in_nchw(make_TensorDescriptor(in_nchw_desc));
    Tensor<in_data_t> wei_kcyx(make_TensorDescriptor(wei_kcyx_desc));
    Tensor<out_data_t> out_nkhw_host(make_TensorDescriptor(out_nkhw_desc));
    Tensor<out_data_t> out_nkhw_device(make_TensorDescriptor(out_nkhw_desc));
Chao Liu's avatar
Chao Liu committed
628

Chao Liu's avatar
Chao Liu committed
629
    std::size_t num_thread = std::thread::hardware_concurrency();
Chao Liu's avatar
Chao Liu committed
630

Chao Liu's avatar
Chao Liu committed
631
632
633
634
635
636
637
    if(argc != 3)
    {
        printf("arg1: do_verification, arg2: nrepeat\n");
        exit(1);
    }

    bool do_verification = atoi(argv[1]);
Chao Liu's avatar
Chao Liu committed
638
    index_t nrepeat      = atoi(argv[2]);
639
640
641

    if(do_verification)
    {
Chao Liu's avatar
Chao Liu committed
642
#if 0
643
        in_nchw.GenerateTensorValue(GeneratorTensor_1{}, num_thread);
Chao Liu's avatar
Chao Liu committed
644
        wei_kcyx.GenerateTensorValue(GeneratorTensor_1{}, num_thread);
Chao Liu's avatar
Chao Liu committed
645
#elif 1
646
        in_nchw.GenerateTensorValue(GeneratorTensor_2{-5, 5}, num_thread);
Chao Liu's avatar
Chao Liu committed
647
        wei_kcyx.GenerateTensorValue(GeneratorTensor_2{-5, 5}, num_thread);
648
#elif 1
649
650
651
652
653
654
        in_nchw.GenerateTensorValue(GeneratorTensor_2{1, 5}, num_thread);

        auto gen_wei = [](auto... is) {
            return GeneratorTensor_2{1, 5}(is...) * GeneratorTensor_Checkboard{}(is...);
        };
        wei_kcyx.GenerateTensorValue(gen_wei, num_thread);
Chao Liu's avatar
Chao Liu committed
655
#endif
656
    }
Chao Liu's avatar
Chao Liu committed
657

Chao Liu's avatar
Chao Liu committed
658
#if 1
Chao Liu's avatar
Chao Liu committed
659
#if 0
Chao Liu's avatar
Chao Liu committed
660
    device_direct_convolution_1
661
#elif 0
Chao Liu's avatar
Chao Liu committed
662
663
    device_direct_convolution_2_nchw_kcyx_nkhw
#elif 0
Chao Liu's avatar
Chao Liu committed
664
    device_direct_convolution_2_vectorized_nchw_kcyx_nkhw
665
#elif 1
Chao Liu's avatar
Chao Liu committed
666
    device_convolution_implicit_gemm_v1_chwn_cyxk_khwn
667
668
#elif 0
    device_convolution_implicit_gemm_v1_nchw_cyxk_khwn
669
#elif 0
Chao Liu's avatar
Chao Liu committed
670
    device_convolution_implicit_gemm_v2_chwn_cyxk_khwn
671
#endif
Chao Liu's avatar
Chao Liu committed
672
    (in_nchw_desc, in_nchw, wei_kcyx_desc, wei_kcyx, out_nkhw_desc, out_nkhw_device, nrepeat);
673

Chao Liu's avatar
Chao Liu committed
674
#elif 1
Chao Liu's avatar
Chao Liu committed
675
    device_implicit_gemm_convolution_1_chwn_cyxk_khwn_padded(in_nchw_desc,
Chao Liu's avatar
Chao Liu committed
676
                                                             in_nchw,
Chao Liu's avatar
Chao Liu committed
677
678
                                                             wei_kcyx_desc,
                                                             wei_kcyx,
Chao Liu's avatar
Chao Liu committed
679
680
681
682
683
                                                             out_nkhw_desc,
                                                             out_nkhw_device,
                                                             lower_pads,
                                                             upper_pads,
                                                             nrepeat);
684
#endif
Chao Liu's avatar
Chao Liu committed
685

686
    if(do_verification)
687
    {
Chao Liu's avatar
Chao Liu committed
688
        if(Y == 3 && X == 3)
689
        {
Chao Liu's avatar
Chao Liu committed
690
            host_winograd_3x3_convolution(in_nchw, wei_kcyx, out_nkhw_host, lower_pads, upper_pads);
691
692
693
        }
        else
        {
Chao Liu's avatar
Chao Liu committed
694
            host_direct_convolution(in_nchw, wei_kcyx, out_nkhw_host, lower_pads, upper_pads);
695
696
        }
        check_error(out_nkhw_host, out_nkhw_device);
Chao Liu's avatar
Chao Liu committed
697

Chao Liu's avatar
Chao Liu committed
698
#if 0
699
        LogRange(std::cout << "in_nchw : ", in_nchw.mData, ",") << std::endl;
Chao Liu's avatar
Chao Liu committed
700
        LogRange(std::cout << "wei_kcyx: ", wei_kcyx.mData, ",") << std::endl;
701
702
        LogRange(std::cout << "out_nkhw_host  : ", out_nkhw_host.mData, ",") << std::endl;
        LogRange(std::cout << "out_nkhw_device: ", out_nkhw_device.mData, ",") << std::endl;
Chao Liu's avatar
Chao Liu committed
703
#endif
704
    }
705
}