CUDNN Frontend API 8.3.0

cudnn_frontend_Tensor.h
Go to the documentation of this file.
1 /*
2 * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23 #pragma once
24
25 #include <algorithm>
26 #include <array>
27 #include <functional>
28 #include <memory>
29 #include <sstream>
30 #include <utility>
31
32 #include "cudnn_frontend_utils.h"
33
34 namespace cudnn_frontend {
35
52 class Tensor_v8 : public BackendDescriptor {
53 public:
54 friend class TensorBuilder_v8;
55 std::string
56 describe() const override {
57 std::stringstream ss;
58 char sep = ' ';
59 ss << "CUDNN_BACKEND_TENSOR_DESCRIPTOR :"
60 << " Datatype: " << to_string(data_type) << " Id: " << std::to_string(id) << " Alignment: " << alignment
61 << " nDims " << nDims << " VectorCount: " << vectorCount << " vectorDimension " << vectorDimension;
62 ss << " Dim [";
63 for (auto i = 0; i < nDims; i++) {
64 ss << sep << btensor_dimA[i];
65 sep = ',';
66 }
67 ss << " ] Str [";
68 for (auto i = 0; i < nDims; i++) {
69 ss << sep << btensor_strA[i];
70 sep = ',';
71 }
72 ss << "]";
73 ss << " isVirtual: " << std::to_string(isVirtual) << " isByValue: " << std::to_string(isByValue);
74 #if (CUDNN_VERSION >= 8300)
75 ss << " reorder_type: " << reorder_type;
76 #endif
77 return ss.str();
78 }
79
80 int64_t
81 getPackedElementCount() const {
82 int64_t count = vectorCount;
83 for (auto i = 0; i < nDims; i++) {
84 count = count * btensor_dimA[i];
85 }
86 return count;
87 };
88
89 int64_t
90 getDimensionCount() const {
91 return nDims;
92 }
93
94 int64_t const *
95 getDimArray() const {
96 return btensor_dimA;
97 }
98
99 int64_t const *
100 getStrideArray() const {
101 return btensor_strA;
102 }
103
104 int64_t
105 getDataType() const {
106 return static_cast<int64_t>(data_type);
107 }
108
109 Tensor_v8(Tensor_v8 &&from) = default;
110 Tensor_v8 &
111 operator=(Tensor_v8 &&) = default;
112
113 ~Tensor_v8() = default;
114
115 private:
116 Tensor_v8() = default;
117 Tensor_v8(Tensor_v8 const &) = delete;
118 Tensor_v8 &
119 operator=(Tensor_v8 const &) = delete;
120
121 cudnnDataType_t data_type = CUDNN_DATA_FLOAT;
122 int64_t btensor_dimA[CUDNN_DIM_MAX + 1] = {-1};
123 int64_t btensor_strA[CUDNN_DIM_MAX + 1] = {-1};
124 int64_t id = -1;
125 int64_t alignment = -1;
126 int64_t nDims = -1;
128 int64_t vectorDimension = -1;
129 int64_t vectorCount = 1;
130 bool isVirtual = false;
131 bool isByValue = false;
132 #if (CUDNN_VERSION >= 8300)
133 cudnnBackendTensorReordering_t reorder_type = CUDNN_TENSOR_REORDERING_NONE;
134 #endif
135 };
136
140 class TensorBuilder_v8 {
141 public:
146 auto
148 setDataType(cudnnDataType_t data_type_) -> TensorBuilder_v8 & {
149 m_tensor.data_type = data_type_;
150 return *this;
151 }
153 auto
154 setDim(int64_t ndim, int64_t const *dim) -> TensorBuilder_v8 & {
155 std::copy((dim), dim + ndim, m_tensor.btensor_dimA);
156 m_tensor.nDims = ndim;
157 return *this;
158 }
160 auto
161 setStrides(int64_t ndim, int64_t const *strides) -> TensorBuilder_v8 & {
162 std::copy(strides, strides + ndim, m_tensor.btensor_strA);
163 return *this;
164 }
166 auto
167 setId(int64_t id_) -> TensorBuilder_v8 & {
168 m_tensor.id = id_;
169 return *this;
170 }
172 auto
173 setAlignment(int64_t alignment_) -> TensorBuilder_v8 & {
174 m_tensor.alignment = alignment_;
175 return *this;
176 }
178 auto
179 setVirtual(bool virtual_ = true) -> TensorBuilder_v8 & {
180 m_tensor.isVirtual = virtual_;
181 return *this;
182 }
183 auto
184 setByValue(bool isByValue_ = true) -> TensorBuilder_v8 & {
185 m_tensor.isByValue = isByValue_;
186 return *this;
187 }
188 auto
189 setVectorCountAndDimension(int64_t vectorCount_, int64_t vectorDimension_) -> TensorBuilder_v8 & {
190 m_tensor.vectorCount = vectorCount_;
191 m_tensor.vectorDimension = vectorDimension_;
192 return *this;
193 }
194
195 #if (CUDNN_VERSION >= 8300)
196 auto
197 setReorderType(cudnnBackendTensorReordering_t type_) -> TensorBuilder_v8 & {
198 m_tensor.reorder_type = type_;
199 return *this;
200 }
201 #endif
202
204 Tensor_v8 &&
207 build() {
208 // Sanity check if non-default fields have been set correctly.
209 if (m_tensor.alignment <= 0) {
210 set_error_and_throw_exception(
211 &m_tensor,
212 CUDNN_STATUS_BAD_PARAM,
213 "CUDNN_BACKEND_TENSOR_DESCRIPTOR: Check and Set the CUDNN_ATTR_TENSOR_BYTE_ALIGNMENT field");
214 return std::move(m_tensor);
215 }
216 if (m_tensor.id <= 0) {
217 set_error_and_throw_exception(
218 &m_tensor,
219 CUDNN_STATUS_BAD_PARAM,
220 "CUDNN_BACKEND_TENSOR_DESCRIPTOR: Check and Set the CUDNN_ATTR_TENSOR_UNIQUE_ID as a valid value");
221 return std::move(m_tensor);
222 }
223 if (m_tensor.btensor_strA[0] <= 0) {
224 set_error_and_throw_exception(
225 &m_tensor,
226 CUDNN_STATUS_BAD_PARAM,
227 "CUDNN_BACKEND_TENSOR_DESCRIPTOR: Check and Set the CUDNN_ATTR_TENSOR_STRIDES Correctly");
228 return std::move(m_tensor);
229 }
230 if (m_tensor.btensor_dimA[0] <= 0) {
231 set_error_and_throw_exception(
232 &m_tensor,
233 CUDNN_STATUS_BAD_PARAM,
234 "CUDNN_BACKEND_TENSOR_DESCRIPTOR: Check and Set the CUDNN_ATTR_TENSOR_DIMENSIONS Correctly");
235 return std::move(m_tensor);
236 }
237 if (m_tensor.pointer != nullptr) {
238 set_error_and_throw_exception(&m_tensor,
239 CUDNN_STATUS_BAD_PARAM,
240 "CUDNN_BACKEND_TENSOR_DESCRIPTOR: Bad tensor created. The tensor already "
241 "seems to be pointing to something");
242 return std::move(m_tensor);
243 }
244
245 // Create a descriptor. Memory allocation happens here.
246 auto status = m_tensor.initialize_managed_backend_pointer(CUDNN_BACKEND_TENSOR_DESCRIPTOR);
247 if (status != CUDNN_STATUS_SUCCESS) {
248 set_error_and_throw_exception(
249 &m_tensor, status, "CUDNN_BACKEND_TENSOR_DESCRIPTOR: cudnnCreate Descriptor Failed");
250 return std::move(m_tensor);
251 }
252
253 // Once Created lets set the descriptor parameters.
254 status = cudnnBackendSetAttribute(m_tensor.pointer->get_backend_descriptor(),
255 CUDNN_ATTR_TENSOR_DATA_TYPE,
256 CUDNN_TYPE_DATA_TYPE,
257 1,
258 &m_tensor.data_type);
259 if (status != CUDNN_STATUS_SUCCESS) {
260 set_error_and_throw_exception(
261 &m_tensor, status, "CUDNN_BACKEND_TENSOR_DESCRIPTOR: SetAttribute CUDNN_ATTR_TENSOR_DATA_TYPE Failed");
262 return std::move(m_tensor);
263 }
264 status = cudnnBackendSetAttribute(m_tensor.pointer->get_backend_descriptor(),
265 CUDNN_ATTR_TENSOR_DIMENSIONS,
266 CUDNN_TYPE_INT64,
267 m_tensor.nDims,
268 m_tensor.btensor_dimA);
269 if (status != CUDNN_STATUS_SUCCESS) {
270 set_error_and_throw_exception(
271 &m_tensor, status, "CUDNN_BACKEND_TENSOR_DESCRIPTOR: SetAttribute CUDNN_ATTR_TENSOR_DIMENSIONS Failed");
272 return std::move(m_tensor);
273 }
274 status = cudnnBackendSetAttribute(m_tensor.pointer->get_backend_descriptor(),
275 CUDNN_ATTR_TENSOR_STRIDES,
276 CUDNN_TYPE_INT64,
277 m_tensor.nDims,
278 m_tensor.btensor_strA);
279 if (status != CUDNN_STATUS_SUCCESS) {
280 set_error_and_throw_exception(
281 &m_tensor, status, "CUDNN_BACKEND_TENSOR_DESCRIPTOR: SetAttribute CUDNN_ATTR_TENSOR_STRIDES Failed");
282 return std::move(m_tensor);
283 }
284 status = cudnnBackendSetAttribute(
285 m_tensor.pointer->get_backend_descriptor(), CUDNN_ATTR_TENSOR_UNIQUE_ID, CUDNN_TYPE_INT64, 1, &m_tensor.id);
286 if (status != CUDNN_STATUS_SUCCESS) {
287 set_error_and_throw_exception(
288 &m_tensor, status, "CUDNN_BACKEND_TENSOR_DESCRIPTOR: SetAttribute CUDNN_ATTR_TENSOR_UNIQUE_ID Failed");
289 return std::move(m_tensor);
290 }
291 cudnnBackendSetAttribute(m_tensor.pointer->get_backend_descriptor(),
292 CUDNN_ATTR_TENSOR_BYTE_ALIGNMENT,
293 CUDNN_TYPE_INT64,
294 1,
295 &m_tensor.alignment);
296 if (status != CUDNN_STATUS_SUCCESS) {
297 set_error_and_throw_exception(
298 &m_tensor,
299 status,
300 "CUDNN_BACKEND_TENSOR_DESCRIPTOR: SetAttribute CUDNN_ATTR_TENSOR_BYTE_ALIGNMENT Failed");
301 return std::move(m_tensor);
302 }
303 if (m_tensor.isVirtual) {
304 cudnnBackendSetAttribute(m_tensor.pointer->get_backend_descriptor(),
305 CUDNN_ATTR_TENSOR_IS_VIRTUAL,
306 CUDNN_TYPE_BOOLEAN,
307 1,
308 &m_tensor.isVirtual);
309 if (status != CUDNN_STATUS_SUCCESS) {
310 set_error_and_throw_exception(
311 &m_tensor,
312 status,
313 "CUDNN_BACKEND_TENSOR_DESCRIPTOR: SetAttribute CUDNN_ATTR_TENSOR_BYTE_ALIGNMENT Failed");
314 return std::move(m_tensor);
315 }
316 }
317 if (m_tensor.isByValue) {
318 cudnnBackendSetAttribute(m_tensor.pointer->get_backend_descriptor(),
319 CUDNN_ATTR_TENSOR_IS_BY_VALUE,
320 CUDNN_TYPE_BOOLEAN,
321 1,
322 &m_tensor.isByValue);
323 if (status != CUDNN_STATUS_SUCCESS) {
324 set_error_and_throw_exception(
325 &m_tensor,
326 status,
327 "CUDNN_BACKEND_TENSOR_DESCRIPTOR: SetAttribute CUDNN_ATTR_TENSOR_IS_BY_VALUE Failed");
328 return std::move(m_tensor);
329 }
330 }
331
332 if (m_tensor.vectorCount > 1) {
333 cudnnBackendSetAttribute(m_tensor.pointer->get_backend_descriptor(),
334 CUDNN_ATTR_TENSOR_VECTOR_COUNT,
335 CUDNN_TYPE_INT64,
336 1,
337 &m_tensor.vectorCount);
338 if (status != CUDNN_STATUS_SUCCESS) {
339 set_error_and_throw_exception(
340 &m_tensor,
341 status,
342 "CUDNN_BACKEND_TENSOR_DESCRIPTOR: SetAttribute CUDNN_ATTR_TENSOR_VECTOR_COUNT Failed");
343 return std::move(m_tensor);
344 }
345 }
346 if (m_tensor.vectorDimension >= 0) {
347 cudnnBackendSetAttribute(m_tensor.pointer->get_backend_descriptor(),
348 CUDNN_ATTR_TENSOR_VECTORIZED_DIMENSION,
349 CUDNN_TYPE_INT64,
350 1,
351 &m_tensor.vectorDimension);
352 if (status != CUDNN_STATUS_SUCCESS) {
353 set_error_and_throw_exception(
354 &m_tensor,
355 status,
356 "CUDNN_BACKEND_TENSOR_DESCRIPTOR: SetAttribute CUDNN_ATTR_TENSOR_VECTORIZED_DIMENSION Failed");
357 return std::move(m_tensor);
358 }
359 }
360
361 // Set the reorder_type
362 #if (CUDNN_VERSION >= 8300)
363 if (m_tensor.reorder_type != CUDNN_TENSOR_REORDERING_NONE) {
364 cudnnBackendSetAttribute(m_tensor.pointer->get_backend_descriptor(),
365 CUDNN_ATTR_TENSOR_REORDERING_MODE,
366 CUDNN_TYPE_TENSOR_REORDERING_MODE,
367 1,
368 &m_tensor.reorder_type);
369 if (status != CUDNN_STATUS_SUCCESS) {
370 set_error_and_throw_exception(
371 &m_tensor,
372 status,
373 "CUDNN_BACKEND_TENSOR_DESCRIPTOR: SetAttribute CUDNN_ATTR_TENSOR_REORDERING_MODE Failed");
374 return std::move(m_tensor);
375 }
376 }
377 #endif
378 // Finalizing the descriptor
379 status = cudnnBackendFinalize(m_tensor.pointer->get_backend_descriptor());
380 if (status != CUDNN_STATUS_SUCCESS) {
381 set_error_and_throw_exception(&m_tensor, status, "CUDNN_BACKEND_TENSOR_DESCRIPTOR cudnnFinalize failed");
382 return std::move(m_tensor);
383 }
384 getLogger() << "[cudnn_frontend] " << m_tensor << std::endl;
385 return std::move(m_tensor);
386 }
387
388 explicit TensorBuilder_v8() = default;
389 ~TensorBuilder_v8() = default;
390 TensorBuilder_v8(TensorBuilder_v8 &&) = delete;
391 TensorBuilder_v8(TensorBuilder_v8 const &) = delete;
392 TensorBuilder_v8 &
393 operator=(TensorBuilder_v8 const &) = delete;
394
395 private:
396 Tensor_v8 m_tensor;
397 };
398
399 }
cudnn_frontend::Tensor_v8::Tensor_v8
Tensor_v8()=default
cudnn_frontend::Tensor_v8::isByValue
bool isByValue
Whether it is an intermediate tensor of an op graph.
Definition: cudnn_frontend_Tensor.h:131
cudnn_frontend::TensorBuilder_v8::setVirtual
auto setVirtual(bool virtual_=true) -> TensorBuilder_v8 &
Set Alignment of the tensor.
Definition: cudnn_frontend_Tensor.h:179
cudnn_frontend::TensorBuilder_v8::build
Tensor_v8 && build()
Definition: cudnn_frontend_Tensor.h:207
cudnn_frontend::Tensor_v8::getDataType
int64_t getDataType() const
Definition: cudnn_frontend_Tensor.h:105
cudnn_frontend::getLogger
ConditionalStreamer & getLogger()
Definition: cudnn_frontend_Logging.h:72
cudnn_frontend::set_error_and_throw_exception
static void set_error_and_throw_exception(BackendDescriptor const *desc, cudnnStatus_t status, const char *message)
Definition: cudnn_frontend_utils.h:138
nlohmann::to_string
NLOHMANN_BASIC_JSON_TPL_DECLARATION std::string to_string(const NLOHMANN_BASIC_JSON_TPL &j)
user-defined to_string function for JSON values
Definition: json.hpp:25855
cudnn_frontend::Tensor_v8::btensor_strA
int64_t btensor_strA[CUDNN_DIM_MAX+1]
n, g, c, d, h, w
Definition: cudnn_frontend_Tensor.h:123
cudnn_frontend::Tensor_v8::operator=
Tensor_v8 & operator=(Tensor_v8 &&)=default
cudnn_frontend::TensorBuilder_v8::setByValue
auto setByValue(bool isByValue_=true) -> TensorBuilder_v8 &
Definition: cudnn_frontend_Tensor.h:184
cudnn_frontend::Tensor_v8::getDimensionCount
int64_t getDimensionCount() const
Definition: cudnn_frontend_Tensor.h:90
cudnn_frontend::BackendDescriptor
Definition: cudnn_backend_base.h:99
cudnn_frontend::Tensor_v8::alignment
int64_t alignment
Unique id of the tensor.
Definition: cudnn_frontend_Tensor.h:125
cudnn_frontend::Tensor_v8::TensorBuilder_v8
friend class TensorBuilder_v8
Definition: cudnn_frontend_Tensor.h:54
cudnn_frontend::Tensor_v8::isVirtual
bool isVirtual
What is the vectorization count (4 or 32)
Definition: cudnn_frontend_Tensor.h:130
cudnn_frontend::TensorBuilder_v8::setAlignment
auto setAlignment(int64_t alignment_) -> TensorBuilder_v8 &
Set Alignment of the tensor.
Definition: cudnn_frontend_Tensor.h:173
cudnn_frontend::Tensor_v8::getDimArray
int64_t const * getDimArray() const
Definition: cudnn_frontend_Tensor.h:95
cudnn_frontend::Tensor_v8::vectorCount
int64_t vectorCount
Which dimension of the tensor is vectorized (Generally the c dim)
Definition: cudnn_frontend_Tensor.h:129
cudnn_frontend::Tensor_v8::getStrideArray
int64_t const * getStrideArray() const
Definition: cudnn_frontend_Tensor.h:100
cudnn_frontend_utils.h
cudnn_frontend::Tensor_v8::nDims
int64_t nDims
Certain engine config expect minimum alignment of 16B.
Definition: cudnn_frontend_Tensor.h:127
cudnn_frontend::Tensor_v8::btensor_dimA
int64_t btensor_dimA[CUDNN_DIM_MAX+1]
Datatype of the elements.
Definition: cudnn_frontend_Tensor.h:122
cudnn_frontend::Tensor_v8::describe
std::string describe() const override
Return a string describing the backend Descriptor.
Definition: cudnn_frontend_Tensor.h:56
cudnn_frontend::TensorBuilder_v8::setVectorCountAndDimension
auto setVectorCountAndDimension(int64_t vectorCount_, int64_t vectorDimension_) -> TensorBuilder_v8 &
Definition: cudnn_frontend_Tensor.h:189
cudnn_frontend::Tensor_v8::~Tensor_v8
~Tensor_v8()=default
cudnn_frontend
Definition: cudnn_backend_base.h:29
cudnn_frontend::Tensor_v8
Definition: cudnn_frontend_Tensor.h:52
cudnn_frontend::TensorBuilder_v8::setStrides
auto setStrides(int64_t ndim, int64_t const *strides) -> TensorBuilder_v8 &
Set Strides of the tensor.
Definition: cudnn_frontend_Tensor.h:161
cudnn_frontend::to_string
static std::string to_string(cudnnDataType_t type)
Definition: cudnn_frontend_utils.h:72
cudnn_frontend::Tensor_v8::vectorDimension
int64_t vectorDimension
Number of Dimensions of the tensor.
Definition: cudnn_frontend_Tensor.h:128
cudnn_frontend::Tensor_v8::getPackedElementCount
int64_t getPackedElementCount() const
Definition: cudnn_frontend_Tensor.h:81
cudnn_frontend::TensorBuilder_v8::setId
auto setId(int64_t id_) -> TensorBuilder_v8 &
Set Unique Id of the tensor.
Definition: cudnn_frontend_Tensor.h:167
cudnn_frontend::TensorBuilder_v8::setDim
auto setDim(int64_t ndim, int64_t const *dim) -> TensorBuilder_v8 &
Set Dimensions of the tensor.
Definition: cudnn_frontend_Tensor.h:154
cudnn_frontend::Tensor_v8::data_type
cudnnDataType_t data_type
Definition: cudnn_frontend_Tensor.h:121
cudnn_frontend::TensorBuilder_v8
Definition: cudnn_frontend_Tensor.h:140
cudnn_frontend::TensorBuilder_v8::setDataType
auto setDataType(cudnnDataType_t data_type_) -> TensorBuilder_v8 &
Set Datatype for the Tensor_v8.
Definition: cudnn_frontend_Tensor.h:148
cudnn_frontend::BackendDescriptor::status
cudnnStatus_t status
Shared pointer of the OpaqueBackendPointer.
Definition: cudnn_backend_base.h:160
cudnn_frontend::TensorBuilder_v8::m_tensor
Tensor_v8 m_tensor
Definition: cudnn_frontend_Tensor.h:396