ck.hpp 7.23 KB
Newer Older
Chao Liu's avatar
Chao Liu committed
1
2
3
// SPDX-License-Identifier: MIT
// Copyright (c) 2018-2022, Advanced Micro Devices, Inc. All rights reserved.

Chao Liu's avatar
Chao Liu committed
4
#pragma once
5

JD's avatar
JD committed
6
#ifndef CK_DONT_USE_HIP_RUNTIME_HEADERS
7
8
#include "hip/hip_runtime.h"
#include "hip/hip_fp16.h"
Chao Liu's avatar
Chao Liu committed
9
#endif
10

Chao Liu's avatar
Chao Liu committed
11
12
#define CK_TIME_KERNEL 1

13
14
15
// constant address space for kernel parameter
// https://llvm.org/docs/AMDGPUUsage.html#address-spaces
#define CK_CONSTANT_ADDRESS_SPACE __attribute__((address_space(4)))
Chao Liu's avatar
Chao Liu committed
16
17

// launch bounds
zjing14's avatar
zjing14 committed
18
#define CK_USE_LAUNCH_BOUNDS 1
Chao Liu's avatar
Chao Liu committed
19
20

#ifdef CK_USE_LAUNCH_BOUNDS
21
// for most kernels
Chao Liu's avatar
Chao Liu committed
22
#define CK_MAX_THREAD_PER_BLOCK 256
Chao Liu's avatar
Chao Liu committed
23
#define CK_MIN_BLOCK_PER_CU 2
24
25
26
27

// for wavelet GEMM kernel
#define CK_WAVELET_MAX_THREAD_PER_BLOCK 512
#define CK_WAVELET_MIN_BLOCK_PER_CU 2
Chao Liu's avatar
Chao Liu committed
28
29
#endif

Chao Liu's avatar
Chao Liu committed
30
// buffer resource
31
32
33
#ifndef __HIP_DEVICE_COMPILE__ // for host code
#define CK_BUFFER_RESOURCE_3RD_DWORD -1
#elif defined(__gfx803__) || defined(__gfx900__) || defined(__gfx906__) || defined(__gfx908__) || \
34
    defined(__gfx90a__) || defined(__gfx940__) // for GPU code
Chao Liu's avatar
Chao Liu committed
35
#define CK_BUFFER_RESOURCE_3RD_DWORD 0x00020000
36
#elif defined(__gfx1030__) // for GPU code
Chao Liu's avatar
Chao Liu committed
37
#define CK_BUFFER_RESOURCE_3RD_DWORD 0x31014000
38
#elif defined(__gfx1100__) || defined(__gfx1101__) || defined(__gfx1102__) // for GPU code
aska-0096's avatar
aska-0096 committed
39
#define CK_BUFFER_RESOURCE_3RD_DWORD 0x31004000
Chao Liu's avatar
Chao Liu committed
40
41
#endif

42
// FMA instruction
43
44
#ifndef __HIP_DEVICE_COMPILE__                   // for host code, define nothing
#elif defined(__gfx803__) || defined(__gfx900__) // for GPU code
45
#define CK_USE_AMD_V_MAC_F32
46
47
#elif defined(__gfx906__) || defined(__gfx908__) || defined(__gfx90a__) || defined(__gfx1030__) || \
    defined(__gfx940__) // for GPU code
48
49
50
51
52
#define CK_USE_AMD_V_FMAC_F32
#define CK_USE_AMD_V_DOT2_F32_F16
#define CK_USE_AMD_V_DOT4_I32_I8
#endif

53
54
55
// MFMA instruction
#ifndef __HIP_DEVICE_COMPILE__ // for host code
#define CK_USE_AMD_MFMA
56
#elif defined(__gfx908__) || defined(__gfx90a__) || defined(__gfx940__) // for GPU code
57
#define CK_USE_AMD_MFMA
58
59
#endif

60
#if(defined(__gfx90a__) || defined(__gfx940__))
61
#define CK_USE_AMD_MFMA_BF16_1K_OP
Chao Liu's avatar
Chao Liu committed
62
63
#endif

64
65
66
67
#if defined(__gfx940__)
#define CK_USE_AMD_MFMA_GFX940
#endif

68
69
70
// WMMA instruction
#ifndef __HIP_DEVICE_COMPILE__ // for host code
#define CK_USE_AMD_WMMA
71
#elif defined(__gfx1100__) || defined(__gfx1101__) || defined(__gfx1102__) // for GPU code
72
73
74
#define CK_USE_AMD_WMMA
#endif

75
// TODO: enable buffer load when found correct 3rd dword
76
// buffer load
77
#define CK_USE_AMD_BUFFER_LOAD 1
78

79
// buffer store
80
#define CK_USE_AMD_BUFFER_STORE 1
Jianfeng Yan's avatar
Jianfeng Yan committed
81

82
83
// buffer atomic add: integer
#define CK_USE_AMD_BUFFER_ATOMIC_ADD_INTEGER 1
84

85
86
87
// buffer atomic add: floating point
#ifndef __HIP_DEVICE_COMPILE__ // for host code
#define CK_USE_AMD_BUFFER_ATOMIC_ADD_FLOAT 1
88
#elif defined(__gfx908__) || defined(__gfx90a__) || defined(__gfx940__) // for GPU code
89
90
91
#define CK_USE_AMD_BUFFER_ATOMIC_ADD_FLOAT 1
#else // for GPU code
#define CK_USE_AMD_BUFFER_ATOMIC_ADD_FLOAT 0
92
93
#endif

94
#if(defined(__gfx90a__) || defined(__gfx940__)) // for GPU code
rocking5566's avatar
rocking5566 committed
95
96
97
98
99
#define CK_USE_AMD_BUFFER_ATOMIC_MAX_FLOAT64 1
#else
#define CK_USE_AMD_BUFFER_ATOMIC_MAX_FLOAT64 0
#endif

100
101
102
103
104
105
// inline asm
#define CK_USE_AMD_INLINE_ASM 1

// inner product (DLOP)
#define CK_USE_AMD_INNER_PRODUCT_INLINE_ASM 1

Chao Liu's avatar
Chao Liu committed
106
// block synchronization only s_wait lgkmcnt(0), not vmcnt(0)
107
#define CK_EXPERIMENTAL_BLOCK_SYNC_LDS_WITHOUT_SYNC_VMEM 1
Chao Liu's avatar
Chao Liu committed
108

109
110
// experimental feature: multi index implemented as array
#define CK_EXPERIMENTAL_USE_DYNAMICALLY_INDEXED_MULTI_INDEX 0
Chao Liu's avatar
Chao Liu committed
111

112
113
// experimental feature: static tensor descriptor
#define CK_EXPERIMENTAL_STATIC_TENSOR_DESCRIPTOR 0
Chao Liu's avatar
Chao Liu committed
114

rocking5566's avatar
rocking5566 committed
115
// experimental feature: buffer load/store/atomic-add/ OOB trick
116
// This (ifndef) is a hack to use customized behavior for buffer load rather than using default
Po Yen Chen's avatar
Po Yen Chen committed
117
118
// setting. Don't use this hack unless absolutely necessary!
// FIXME: make the behavior of buffer load a configurable (template) parameter for each usage
119
#ifndef CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK
120
#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 0
121
#endif
122
#define CK_EXPERIMENTAL_USE_BUFFER_STORE_OOB_CHECK_OFFSET_TRICK 1
zjing14's avatar
zjing14 committed
123
#define CK_EXPERIMENTAL_USE_BUFFER_ATOMIC_ADD_OOB_CHECK_OFFSET_TRICK 1
rocking5566's avatar
rocking5566 committed
124
#define CK_EXPERIMENTAL_USE_BUFFER_ATOMIC_MAX_OOB_CHECK_OFFSET_TRICK 1
125

126
// experimental feature: in-regsiter sub-dword transpose
127
#define CK_EXPERIMENTAL_USE_IN_REGISTER_SUB_DWORD_TRANSPOSE 1
Chao Liu's avatar
Chao Liu committed
128

129
// experimental feature: merge transformation use magic number division
130
#define CK_EXPERIMENTAL_MERGE_USE_MAGIC_DIVISION 1
131

132
133
// experimental feature: use __builtin_memcpy instead of pointer cast to access a vector from
// pointer of scalar
134
135
#define CK_EXPERIMENTAL_USE_MEMCPY_FOR_VECTOR_ACCESS 0

136
// experimental feature: use __builtin_memcpy instead of union to do bit_cast
137
#define CK_EXPERIMENTAL_USE_MEMCPY_FOR_BIT_CAST 1
138

139
// experimental feature: optimize for inter-wave scheduling policy
140
#define CK_EXPERIMENTAL_INTER_WAVE_SCHEDULING 1
141
#define CK_EXPERIMENTAL_INTER_WAVE_SCHEDULING_MAC_CLUSTERS 1
142
143
144
145
146
147
// this will let make_default_loop_scheduler() return interwave scheduling flag by default
#define CK_EXPERIMENTAL_DEFAULT_TO_INTER_WAVE_SCHEDULING 0
// experimental feature: add instances using interwave scheduling
#define CK_EXPERIMENTAL_INTER_WAVE_INSTANCES 1
// experimental feature: add instances using pipeline v2
#define CK_EXPERIMENTAL_PIPELINE_V2_INSTANCES 1
148

Chao Liu's avatar
Chao Liu committed
149
150
151
152
// hack: have underlying assumption that need to be satsified, otherwise it's a bug
// hack for forcing register to keep idx_diff_low_const in SGPR. idx_diff_low_const must be
// thread-invariant, otherwise it's a bug
// TODO: separate index calculation into "compile-time", "global", "block", "wave", "thread"
Chao Liu's avatar
rename  
Chao Liu committed
153
#define CK_HACK_MERGE_CALCULATE_IDX_DIFF_LOW_CONST_USE_AMD_GCN_READ_FIRST_LANE 0
Chao Liu's avatar
Chao Liu committed
154

155
// workaround: compiler crash when compiling recursive lambda
Chao Liu's avatar
Chao Liu committed
156
#define CK_WORKAROUND_SWDEV_275126 1
157

158
// workaround: compiler crash when using buffer load/store for i8
159
160
#define CK_WORKAROUND_SWDEV_XXXXXX_INT8_BUFFER_LOAD_STORE_ISSUE 1

161
// workaround: compiler gnerating inefficient ds_write instructions
162
#define CK_WORKAROUND_SWDEV_XXXXXX_INT8_DS_WRITE_ISSUE 1
Chao Liu's avatar
Chao Liu committed
163

164
// workaround: verifaction failure, due to compiler regression, for conv bwd-data fp16 using some
165
// tuning parameter
166
#define CK_WORKAROUND_SWDEV_325164 0
167

168
169
170
// workaround: compiler not emiting reciprocal instruction frm __frcp_rn()
#define CK_WORKAROUND_SWDEV_383542 1

ltqin's avatar
ltqin committed
171
172
// workaround: compiler issue on gfx908
#define CK_WORKAROUND_SWDEV_388832 1
173
174
175
// flag to enable (1) or disable (0) the debugging output in some kernels
#define DEBUG_LOG 0

Haocong WANG's avatar
Haocong WANG committed
176
177
178
// denorm test fix, required to work around dissue
#ifndef CK_WORKAROUND_DENORM_FIX
#define CK_WORKAROUND_DENORM_FIX 0
179
180
181
182
#elif
// enable only on MI200
#define CK_WORKAROUND_DENORM_FIX = CK_WORKAROUND_DENORM_FIX && defined(__gfx90a__)
#endif // CK_WORKAROUND_DENORM_FIX
Haocong WANG's avatar
Haocong WANG committed
183

184
185
namespace ck {

186
enum struct InMemoryDataOperationEnum
Chao Liu's avatar
Chao Liu committed
187
{
Chao Liu's avatar
Chao Liu committed
188
    Set,
189
    AtomicAdd,
rocking5566's avatar
rocking5566 committed
190
    AtomicMax,
191
192
193
    Add
};

Chao Liu's avatar
Chao Liu committed
194
// FIXME: use regular Sequence and remove this
rocking5566's avatar
rocking5566 committed
195
196
197
198
199
200
201
202
203
204
205
206
207
template <InMemoryDataOperationEnum... Is>
struct InMemoryDataOperationEnumSequence
{
    static constexpr int mSize = sizeof...(Is);

    __host__ __device__ static constexpr InMemoryDataOperationEnum At(int I)
    {
        // the last dummy element is to prevent compiler complain about empty array, when mSize = 0
        const InMemoryDataOperationEnum mData[mSize + 1] = {Is..., InMemoryDataOperationEnum::Set};
        return mData[I];
    }
};

Chao Liu's avatar
Chao Liu committed
208
// index type
Jianfeng Yan's avatar
Jianfeng Yan committed
209
210
using index_t      = int32_t;
using long_index_t = int64_t;
211
212

} // namespace ck