ops.h 13.4 KB
Newer Older
1
2
#pragma once

3
#include <optional>
4
#include <torch/library.h>
5

6
7
#include "core/scalar_type.hpp"

8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
#include <vector>

torch::Tensor weak_ref_tensor(torch::Tensor& tensor) {
  // Ensure tensor is on CUDA
  if (!tensor.is_cuda()) {
    throw std::runtime_error("Tensor must be on CUDA device");
  }

  // Get the raw data pointer
  void* data_ptr = tensor.data_ptr();

  // Get tensor sizes and strides
  std::vector<int64_t> sizes = tensor.sizes().vec();
  std::vector<int64_t> strides = tensor.strides().vec();

  // Get tensor options (dtype, device)
  auto options = tensor.options();

  // Create a new tensor from the raw data pointer
  auto new_tensor = torch::from_blob(data_ptr, sizes, strides, options);

  return new_tensor;
}

32
33
void paged_attention_v1(
    torch::Tensor& out, torch::Tensor& query, torch::Tensor& key_cache,
34
35
    torch::Tensor& value_cache, int64_t num_kv_heads, double scale,
    torch::Tensor& block_tables, torch::Tensor& seq_lens, int64_t block_size,
36
    int64_t max_seq_len, const std::optional<torch::Tensor>& alibi_slopes,
37
38
39
    const std::string& kv_cache_dtype, torch::Tensor& k_scale,
    torch::Tensor& v_scale, const int64_t tp_rank,
    const int64_t blocksparse_local_blocks,
40
41
    const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
    const int64_t blocksparse_head_sliding_step);
42

43
44
45
void paged_attention_v2(
    torch::Tensor& out, torch::Tensor& exp_sums, torch::Tensor& max_logits,
    torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
46
47
    torch::Tensor& value_cache, int64_t num_kv_heads, double scale,
    torch::Tensor& block_tables, torch::Tensor& seq_lens, int64_t block_size,
48
    int64_t max_seq_len, const std::optional<torch::Tensor>& alibi_slopes,
49
50
51
    const std::string& kv_cache_dtype, torch::Tensor& k_scale,
    torch::Tensor& v_scale, const int64_t tp_rank,
    const int64_t blocksparse_local_blocks,
52
53
    const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
    const int64_t blocksparse_head_sliding_step);
54
55

void rms_norm(torch::Tensor& out, torch::Tensor& input, torch::Tensor& weight,
56
              double epsilon);
57
58

void fused_add_rms_norm(torch::Tensor& input, torch::Tensor& residual,
59
                        torch::Tensor& weight, double epsilon);
60

61
62
63
// void rms_norm_static_fp8_quant(torch::Tensor& out, torch::Tensor& input,
//                                torch::Tensor& weight, torch::Tensor& scale,
//                                double epsilon);
64

65
66
67
68
69
// void fused_add_rms_norm_static_fp8_quant(torch::Tensor& out,
//                                          torch::Tensor& input,
//                                          torch::Tensor& residual,
//                                          torch::Tensor& weight,
//                                          torch::Tensor& scale, double epsilon);
70

71
72
73
74
75
76
77
// void rms_norm_dynamic_per_token_quant(torch::Tensor& out,
//                                       torch::Tensor const& input,
//                                       torch::Tensor const& weight,
//                                       torch::Tensor& scales,
//                                       double const epsilon,
//                                       std::optional<torch::Tensor> scale_ub,
//                                       std::optional<torch::Tensor> residual);
78

79
void rotary_embedding(torch::Tensor& positions, torch::Tensor& query,
80
                      torch::Tensor& key, int64_t head_size,
81
82
83
                      torch::Tensor& cos_sin_cache, bool is_neox);

void batched_rotary_embedding(torch::Tensor& positions, torch::Tensor& query,
84
                              torch::Tensor& key, int64_t head_size,
85
                              torch::Tensor& cos_sin_cache, bool is_neox,
86
                              int64_t rot_dim,
87
88
89
90
                              torch::Tensor& cos_sin_cache_offsets);

void silu_and_mul(torch::Tensor& out, torch::Tensor& input);

91
92
void mul_and_silu(torch::Tensor& out, torch::Tensor& input);

93
94
95
96
void gelu_and_mul(torch::Tensor& out, torch::Tensor& input);

void gelu_tanh_and_mul(torch::Tensor& out, torch::Tensor& input);

97
98
99
void fatrelu_and_mul(torch::Tensor& out, torch::Tensor& input,
                     double threshold);

100
101
102
void gelu_new(torch::Tensor& out, torch::Tensor& input);

void gelu_fast(torch::Tensor& out, torch::Tensor& input);
103

104
105
void gelu_quick(torch::Tensor& out, torch::Tensor& input);

106
107
void trans_w16_gemm(torch::Tensor dst, torch::Tensor src, int64_t row, int64_t col);

108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
void advance_step_flashattn(int64_t num_seqs, int64_t num_queries,
                            int64_t block_size, torch::Tensor& input_tokens,
                            torch::Tensor& sampled_token_ids,
                            torch::Tensor& input_positions,
                            torch::Tensor& seq_lens,
                            torch::Tensor& slot_mapping,
                            torch::Tensor& block_tables);

void advance_step_flashinfer(
    int64_t num_seqs, int64_t num_queries, int64_t block_size,
    torch::Tensor& input_tokens, torch::Tensor& sampled_token_ids,
    torch::Tensor& input_positions, torch::Tensor& seq_lens,
    torch::Tensor& slot_mapping, torch::Tensor& block_tables,
    torch::Tensor& paged_kv_indices, torch::Tensor& paged_kv_indptr,
    torch::Tensor& paged_kv_last_page_len, torch::Tensor& block_table_bounds);
123

124
125
torch::Tensor get_cuda_view_from_cpu_tensor(torch::Tensor& cpu_tensor);

126
#ifndef USE_ROCM
127
128
129
torch::Tensor aqlm_gemm(const torch::Tensor& input, const torch::Tensor& codes,
                        const torch::Tensor& codebooks,
                        const torch::Tensor& scales,
130
                        const std::vector<int64_t>& codebook_partition_sizes,
131
132
                        const std::optional<torch::Tensor>& bias);

133
134
135
torch::Tensor aqlm_dequant(
    const torch::Tensor& codes, const torch::Tensor& codebooks,
    const std::vector<int64_t>& codebook_partition_sizes);
136
137
138

torch::Tensor awq_gemm(torch::Tensor _in_feats, torch::Tensor _kernel,
                       torch::Tensor _scaling_factors, torch::Tensor _zeros,
139
                       int64_t split_k_iters);
140
141
142

torch::Tensor awq_dequantize(torch::Tensor _kernel,
                             torch::Tensor _scaling_factors,
143
144
                             torch::Tensor _zeros, int64_t split_k_iters,
                             int64_t thx, int64_t thy);
145

146
torch::Tensor permute_cols(torch::Tensor const& A, torch::Tensor const& perm);
147
#endif
148

149
torch::Tensor ggml_dequantize(torch::Tensor W, int64_t type, int64_t m,
150
151
                              int64_t n,
                              std::optional<at::ScalarType> const& dtype);
152

153
154
torch::Tensor ggml_mul_mat_vec_a8(torch::Tensor W, torch::Tensor X,
                                  int64_t type, int64_t row);
155

156
torch::Tensor ggml_mul_mat_a8(torch::Tensor W, torch::Tensor X, int64_t type,
157
158
                              int64_t row);

159
160
161
162
163
164
165
166
torch::Tensor ggml_moe_a8(torch::Tensor X, torch::Tensor W,
                          torch::Tensor sorted_token_ids,
                          torch::Tensor expert_ids,
                          torch::Tensor num_tokens_post_padded, int64_t type,
                          int64_t row, int64_t top_k, int64_t tokens);

int64_t ggml_moe_get_block_size(int64_t type);

167
#ifndef USE_ROCM
168
169
170
171

bool cutlass_scaled_mm_supports_fp4(int64_t cuda_device_capability);
bool cutlass_scaled_mm_supports_fp8(int64_t cuda_device_capability);
bool cutlass_scaled_mm_supports_block_fp8(int64_t cuda_device_capability);
172
bool cutlass_group_gemm_supported(int64_t cuda_device_capability);
173

174
175
176
177
178
void cutlass_scaled_fp4_mm(torch::Tensor& D, torch::Tensor const& A,
                           torch::Tensor const& B, torch::Tensor const& A_sf,
                           torch::Tensor const& B_sf,
                           torch::Tensor const& alpha);

179
180
void cutlass_scaled_mm(torch::Tensor& out, torch::Tensor const& a,
                       torch::Tensor const& b, torch::Tensor const& a_scales,
181
                       torch::Tensor const& b_scales,
182
                       std::optional<torch::Tensor> const& bias);
183

184
185
186
187
188
189
190
191
192
193
194
195
196
void cutlass_moe_mm(
    torch::Tensor& out_tensors, torch::Tensor const& a_tensors,
    torch::Tensor const& b_tensors, torch::Tensor const& a_scales,
    torch::Tensor const& b_scales, torch::Tensor const& expert_offsets,
    torch::Tensor const& problem_sizes, torch::Tensor const& a_strides,
    torch::Tensor const& b_strides, torch::Tensor const& c_strides);

void get_cutlass_moe_mm_data(
    const torch::Tensor& topk_ids, torch::Tensor& expert_offsets,
    torch::Tensor& problem_sizes1, torch::Tensor& problem_sizes2,
    torch::Tensor& input_permutation, torch::Tensor& output_permutation,
    const int64_t num_experts, const int64_t n, const int64_t k);

197
198
199
200
201
void cutlass_scaled_mm_azp(torch::Tensor& out, torch::Tensor const& a,
                           torch::Tensor const& b,
                           torch::Tensor const& a_scales,
                           torch::Tensor const& b_scales,
                           torch::Tensor const& azp_adj,
202
203
                           std::optional<torch::Tensor> const& azp,
                           std::optional<torch::Tensor> const& bias);
204

205
206
bool cutlass_sparse_scaled_mm_supported(int64_t cuda_device_capability);

207
208
209
210
void cutlass_scaled_sparse_mm(torch::Tensor& out, torch::Tensor const& a,
                              torch::Tensor const& b, torch::Tensor const& e,
                              torch::Tensor const& a_scales,
                              torch::Tensor const& b_scales,
211
                              std::optional<torch::Tensor> const& bias);
212

213
std::vector<torch::Tensor> cutlass_sparse_compress(torch::Tensor const& a);
214
215
216
217

void scaled_fp4_quant(torch::Tensor& output, torch::Tensor const& input,
                      torch::Tensor& output_scale,
                      torch::Tensor const& input_scale);
218
#endif
219

220
void static_scaled_int8_quant(torch::Tensor& out, torch::Tensor const& input,
221
                              torch::Tensor const& scale,
222
                              std::optional<torch::Tensor> const& azp);
223

224
void dynamic_scaled_int8_quant(torch::Tensor& out, torch::Tensor const& input,
225
                               torch::Tensor& scales,
226
                               std::optional<torch::Tensor> const& azp);
227

228
229
230
torch::Tensor gptq_gemm(torch::Tensor a, torch::Tensor b_q_weight,
                        torch::Tensor b_gptq_qzeros,
                        torch::Tensor b_gptq_scales, torch::Tensor b_g_idx,
231
                        bool use_exllama, int64_t bit);
232

233
void gptq_shuffle(torch::Tensor q_weight, torch::Tensor q_perm, int64_t bit);
234

235
236
// void static_scaled_fp8_quant(torch::Tensor& out, torch::Tensor const& input,
//                              torch::Tensor const& scale);
237

238
239
// void dynamic_scaled_fp8_quant(torch::Tensor& out, torch::Tensor const& input,
//                               torch::Tensor& scale);
240

241
242
243
// void dynamic_per_token_scaled_fp8_quant(
//     torch::Tensor& out, torch::Tensor const& input, torch::Tensor& scale,
//     std::optional<torch::Tensor> const& scale_ub);
244

245
246
247
void selective_scan_fwd(const torch::Tensor& u, const torch::Tensor& delta,
                        const torch::Tensor& A, const torch::Tensor& B,
                        const torch::Tensor& C,
248
249
250
                        const std::optional<torch::Tensor>& D_,
                        const std::optional<torch::Tensor>& z_,
                        const std::optional<torch::Tensor>& delta_bias_,
251
                        bool delta_softplus,
252
253
254
                        const std::optional<torch::Tensor>& query_start_loc,
                        const std::optional<torch::Tensor>& cache_indices,
                        const std::optional<torch::Tensor>& has_initial_state,
255
256
257
258
                        const torch::Tensor& ssm_states, int64_t pad_slot_id);

void causal_conv1d_update(const at::Tensor& x, const at::Tensor& conv_state,
                          const at::Tensor& weight,
259
                          const std::optional<at::Tensor>& bias_,
260
                          bool silu_activation,
261
262
                          const std::optional<at::Tensor>& cache_seqlens_,
                          const std::optional<at::Tensor>& conv_state_indices_,
263
264
265
                          int64_t pad_slot_id);

void causal_conv1d_fwd(const at::Tensor& x, const at::Tensor& weight,
266
267
268
269
270
                       const std::optional<at::Tensor>& bias_,
                       const std::optional<at::Tensor>& conv_states,
                       const std::optional<at::Tensor>& query_start_loc,
                       const std::optional<at::Tensor>& cache_indices,
                       const std::optional<at::Tensor>& has_initial_state,
271
                       bool silu_activation, int64_t pad_slot_id);
272

273
using fptr_t = int64_t;
274
fptr_t init_custom_ar(const std::vector<int64_t>& fake_ipc_ptrs,
275
276
                      torch::Tensor& rank_data, int64_t rank,
                      bool fully_connected);
277
278
void all_reduce(fptr_t _fa, torch::Tensor& inp, torch::Tensor& out,
                fptr_t reg_buffer, int64_t reg_buffer_sz_bytes);
279
void dispose(fptr_t _fa);
280
int64_t meta_size();
281
282
283
284
285
void register_buffer(fptr_t _fa, const std::vector<int64_t>& fake_ipc_ptrs);
std::tuple<std::vector<int64_t>, std::vector<int64_t>>
get_graph_buffer_ipc_meta(fptr_t _fa);
void register_graph_buffers(fptr_t _fa,
                            const std::vector<std::vector<int64_t>>& handles,
286
                            const std::vector<std::vector<int64_t>>& offsets);
287
288
289
290
std::tuple<int64_t, torch::Tensor> allocate_shared_buffer_and_handle(
    int64_t size);
int64_t open_mem_handle(torch::Tensor& mem_handle);
void free_shared_buffer(int64_t buffer);