CUDNN Frontend API 8.3.0

cudnn_frontend_Engine.h
Go to the documentation of this file.
1 /*
2 * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23 #pragma once
24
25 #include <algorithm>
26 #include <array>
27 #include <functional>
28 #include <memory>
29 #include <sstream>
30 #include <utility>
31 #include <vector>
32
33 #include <cudnn.h>
34 #include <cudnn_backend.h>
35
36 #include "cudnn_frontend_OperationGraph.h"
37 #include "cudnn_frontend_utils.h"
38
39 namespace cudnn_frontend {
40
52 class Engine_v8 : public BackendDescriptor {
53 private:
54 Engine_v8() = default;
55 Engine_v8(Engine_v8 const &) = delete;
56 Engine_v8 &
57 operator=(Engine_v8 const &) = delete;
58
62 class Knob {
63 public:
64 Knob(cudnnBackendKnobType_t type_, int64_t max, int64_t min, int64_t stride_)
65 : knobType(type_), maxValue(max), minValue(min), stride(stride_) {}
66
67 std::string
68 describe() const {
69 std::stringstream ss;
70 ss << "Knob:" << knobType;
71 ss << " Min: " << minValue;
72 ss << " Max: " << maxValue;
73 ss << " Stride: " << stride;
74 return ss.str();
75 }
76
77 void
78 setChoice(uint64_t val_) {
79 choice = val_;
80 }
81
82 int64_t
83 getChoice() const {
84 return choice;
85 }
86
87 cudnnBackendKnobType_t
88 getKnobType() const {
89 return knobType;
90 }
91
92 int64_t
93 getMinValue() const {
94 return minValue;
95 }
96
97 int64_t
98 getMaxValue() const {
99 return minValue;
100 }
101
102 int64_t
103 getStride() const {
104 return stride;
105 }
106
107 private:
108 cudnnBackendKnobType_t knobType = CUDNN_KNOB_TYPE_COUNTS;
109 int64_t maxValue = 0, minValue = 0, stride = 0;
110 int64_t choice = -1;
111 };
112
113 ManagedOpaqueDescriptor opGraph = nullptr;
114 int64_t idx = -1;
115 int64_t numKnobs = 0;
116 std::array<ManagedOpaqueDescriptor, CUDNN_KNOB_TYPE_COUNTS> bKnobs = {};
117 std::vector<Knob> knobs;
118 std::string opGraphTag;
119
121 void
122 buildKnobs() {
123 cudnnStatus_t status;
124 for (auto i = 0; i < numKnobs; i++) {
125 auto bKnob = bKnobs[i]->get_backend_descriptor();
126 cudnnBackendKnobType_t type;
127 int64_t maxValue, minValue, stride, elemCount;
128 status =
129 cudnnBackendGetAttribute(bKnob, CUDNN_ATTR_KNOB_INFO_TYPE, CUDNN_TYPE_KNOB_TYPE, 1, &elemCount, &type);
130 if (status != CUDNN_STATUS_SUCCESS) {
131 set_error_and_throw_exception(this,
132 status,
133 "CUDNN_BACKEND_ENGINE_DESCRIPTOR: CUDNN_BACKEND_KNOB_INFO_DESCRIPTOR "
134 "GetAttribute CUDNN_ATTR_KNOB_INFO_TYPE failed");
135 }
136 status = cudnnBackendGetAttribute(
137 bKnob, CUDNN_ATTR_KNOB_INFO_MAXIMUM_VALUE, CUDNN_TYPE_INT64, 1, &elemCount, &maxValue);
138 if (status != CUDNN_STATUS_SUCCESS) {
139 set_error_and_throw_exception(this,
140 status,
141 "CUDNN_BACKEND_ENGINE_DESCRIPTOR: CUDNN_BACKEND_KNOB_INFO_DESCRIPTOR "
142 "GetAttribute CUDNN_ATTR_KNOB_INFO_MAXIMUM_VALUE Failed");
143 }
144 status = cudnnBackendGetAttribute(
145 bKnob, CUDNN_ATTR_KNOB_INFO_MINIMUM_VALUE, CUDNN_TYPE_INT64, 1, &elemCount, &minValue);
146 if (status != CUDNN_STATUS_SUCCESS) {
147 set_error_and_throw_exception(this,
148 status,
149 "CUDNN_BACKEND_ENGINE_DESCRIPTOR: CUDNN_BACKEND_KNOB_INFO_DESCRIPTOR "
150 "GetAttribute CUDNN_ATTR_KNOB_INFO_MINIMUM_VALUE Failed");
151 }
152 status =
153 cudnnBackendGetAttribute(bKnob, CUDNN_ATTR_KNOB_INFO_STRIDE, CUDNN_TYPE_INT64, 1, &elemCount, &stride);
154 if (status != CUDNN_STATUS_SUCCESS) {
155 set_error_and_throw_exception(this,
156 status,
157 "CUDNN_BACKEND_ENGINE_DESCRIPTOR: CUDNN_BACKEND_KNOB_INFO_DESCRIPTOR "
158 "GetAttribute CUDNN_ATTR_KNOB_INFO_STRIDE Failed");
159 }
160 knobs.emplace_back(Knob(type, maxValue, minValue, stride));
161 }
162 }
163
164 public:
165 friend class EngineBuilder_v8;
166 std::string
167 describe() const override {
168 std::stringstream ss;
169 ss << "CUDNN_BACKEND_ENGINE_DESCRIPTOR :";
170 ss << " ID: " << idx;
171 ss << " Has " << numKnobs << " knobs";
172 return ss.str();
173 }
174 Engine_v8(Engine_v8 &&from)
175 : BackendDescriptor(from.get_desc(), from.get_status(), from.get_error()),
176 opGraph(from.opGraph),
177 idx(from.idx),
178 opGraphTag(from.opGraphTag) {
179 cudnnStatus_t status;
180 for (uint64_t i = 0; i < bKnobs.size(); i++) {
181 bKnobs[i] = make_shared_backend_pointer(CUDNN_BACKEND_KNOB_INFO_DESCRIPTOR);
182 if (bKnobs[i]->is_good() == false) {
183 status = bKnobs[i]->get_status();
184 set_error_and_throw_exception(
185 this,
186 status,
187 "CUDNN_BACKEND_ENGINE_DESCRIPTOR: CUDNN_BACKEND_KNOB_INFO_DESCRIPTOR cudnnCreate Failed");
188 }
189 }
190
191 std::array<cudnnBackendDescriptor_t, CUDNN_KNOB_TYPE_COUNTS> bKnobs_ =
192 {};
193 for (std::uint32_t i = 0; i < bKnobs.size(); i++) {
194 bKnobs_[i] = bKnobs[i]->get_backend_descriptor();
195 }
196 status = cudnnBackendGetAttribute(pointer->get_backend_descriptor(),
197 CUDNN_ATTR_ENGINE_KNOB_INFO,
198 CUDNN_TYPE_BACKEND_DESCRIPTOR,
199 CUDNN_KNOB_TYPE_COUNTS,
200 &numKnobs,
201 bKnobs_.data());
202 if (status != CUDNN_STATUS_SUCCESS) {
203 set_error_and_throw_exception(
204 this, status, "CUDNN_BACKEND_ENGINE_DESCRIPTOR: GetAttribute CUDNN_ATTR_ENGINE_KNOB_INFO Query Failed");
205 }
206 buildKnobs();
207 }
208 Engine_v8 &
209 operator=(Engine_v8 &&) = default;
210 ~Engine_v8() = default;
211
212 std::string const &
213 getTag() const {
214 return opGraphTag;
215 }
216
218 std::vector<Knob> &
219 getSupportedKnobs() {
220 return knobs;
221 }
222
224 std::vector<Knob> const &
225 getFinalizedKnobs() const {
226 return knobs;
227 }
228
229 bool knobs_set() const {
230 bool is_knob_set = false;
231 for (auto i = 0; i < numKnobs; i++) {
232 if(knobs[i].getChoice() != -1) {
233 is_knob_set = true;
234 break;
235 }
236 }
237 return is_knob_set;
238 }
239 };
240
244 class EngineBuilder_v8 {
245 public:
250 auto
252 setOperationGraph(OperationGraph_v8 const &opGraph_) -> EngineBuilder_v8 & {
253 m_engine.opGraph = opGraph_.get_desc();
254 m_engine.opGraphTag = opGraph_.getTag();
255 return *this;
256 }
257
259 auto
260 setOperationGraph(ManagedOpaqueDescriptor desc_) -> EngineBuilder_v8 & {
261 m_engine.opGraph = desc_;
262 return *this;
263 }
265 auto
266 setGlobalEngineIdx(int64_t idx_) -> EngineBuilder_v8 & {
267 m_engine.idx = idx_;
268 return *this;
269 }
272 Engine_v8 &&
275 build() {
276 if (m_engine.idx < 0) {
277 set_error_and_throw_exception(
278 &m_engine,
279 CUDNN_STATUS_BAD_PARAM,
280 "CUDNN_BACKEND_ENGINE_DESCRIPTOR: Check and Set the CUDNN_ATTR_ENGINE_GLOBAL_INDEX to valid value");
281 return std::move(m_engine);
282 }
283 if (m_engine.opGraph == nullptr) {
284 set_error_and_throw_exception(
285 &m_engine,
286 CUDNN_STATUS_BAD_PARAM,
287 "CUDNN_BACKEND_ENGINE_DESCRIPTOR: Check and Set CUDNN_ATTR_ENGINE_OPERATION_GRAPH to valid value");
288 return std::move(m_engine);
289 }
290
291 // Create a descriptor. Memory allocation happens here.
292 auto status = m_engine.initialize_managed_backend_pointer(CUDNN_BACKEND_ENGINE_DESCRIPTOR);
293 if (status != CUDNN_STATUS_SUCCESS) {
294 set_error_and_throw_exception(
295 &m_engine, status, "CUDNN_BACKEND_ENGINE_DESCRIPTOR: cudnnCreate Descriptor Failed");
296 return std::move(m_engine);
297 }
298
299 status = cudnnBackendSetAttribute(m_engine.pointer->get_backend_descriptor(),
300 CUDNN_ATTR_ENGINE_OPERATION_GRAPH,
301 CUDNN_TYPE_BACKEND_DESCRIPTOR,
302 1,
303 &(m_engine.opGraph->get_backend_descriptor()));
304 if (status != CUDNN_STATUS_SUCCESS) {
305 set_error_and_throw_exception(
306 &m_engine,
307 status,
308 "CUDNN_BACKEND_ENGINE_DESCRIPTOR: SetAttribute CUDNN_ATTR_ENGINE_OPERATION_GRAPH Failed");
309 return std::move(m_engine);
310 }
311
312 status = cudnnBackendSetAttribute(m_engine.pointer->get_backend_descriptor(),
313 CUDNN_ATTR_ENGINE_GLOBAL_INDEX,
314 CUDNN_TYPE_INT64,
315 1,
316 &m_engine.idx);
317 if (status != CUDNN_STATUS_SUCCESS) {
318 set_error_and_throw_exception(
319 &m_engine,
320 status,
321 "CUDNN_BACKEND_ENGINE_DESCRIPTOR: SetAttribute CUDNN_ATTR_ENGINE_GLOBAL_INDEX Failed");
322 return std::move(m_engine);
323 }
324
325 // Finalizing the descriptor
326 status = cudnnBackendFinalize(m_engine.pointer->get_backend_descriptor());
327 if (status != CUDNN_STATUS_SUCCESS) {
328 set_error_and_throw_exception(&m_engine, status, "CUDNN_BACKEND_ENGINE_DESCRIPTOR: cudnnFinalize Failed");
329 return std::move(m_engine);
330 }
331
332 getLogger() << "[cudnn_frontend] " << m_engine << std::endl;
333 return std::move(m_engine);
334 }
335
336 explicit EngineBuilder_v8() = default;
337 ~EngineBuilder_v8() = default;
338 EngineBuilder_v8(EngineBuilder_v8 &&) = delete;
339 EngineBuilder_v8(EngineBuilder_v8 const &) = delete;
340 EngineBuilder_v8 &
341 operator=(EngineBuilder_v8 const &) = delete;
342
343 private:
344 Engine_v8 m_engine;
345 };
346 }
cudnn_frontend::Engine_v8::Knob::choice
int64_t choice
Choice set by the user.
Definition: cudnn_frontend_Engine.h:110
cudnn_frontend::Engine_v8::opGraphTag
std::string opGraphTag
Definition: cudnn_frontend_Engine.h:118
cudnn_frontend::Engine_v8::operator=
Engine_v8 & operator=(Engine_v8 const &)=delete
cudnn_frontend::EngineBuilder_v8::setOperationGraph
auto setOperationGraph(ManagedOpaqueDescriptor desc_) -> EngineBuilder_v8 &
Set operationGraph for the engine.
Definition: cudnn_frontend_Engine.h:260
cudnn_frontend::Engine_v8::Knob::maxValue
int64_t maxValue
Definition: cudnn_frontend_Engine.h:109
cudnn_frontend::getLogger
ConditionalStreamer & getLogger()
Definition: cudnn_frontend_Logging.h:72
cudnn_frontend::Engine_v8::Knob::stride
int64_t stride
min, max and stride of the knob value
Definition: cudnn_frontend_Engine.h:109
cudnn_frontend::set_error_and_throw_exception
static void set_error_and_throw_exception(BackendDescriptor const *desc, cudnnStatus_t status, const char *message)
Definition: cudnn_frontend_utils.h:138
cudnn_frontend::Engine_v8::Knob::getKnobType
cudnnBackendKnobType_t getKnobType() const
Definition: cudnn_frontend_Engine.h:88
cudnn_frontend::EngineBuilder_v8::setGlobalEngineIdx
auto setGlobalEngineIdx(int64_t idx_) -> EngineBuilder_v8 &
Set engine index for the engine.
Definition: cudnn_frontend_Engine.h:266
cudnn_frontend::Engine_v8::EngineBuilder_v8
friend class EngineBuilder_v8
Definition: cudnn_frontend_Engine.h:165
cudnn_frontend::Engine_v8::Knob::getStride
int64_t getStride() const
Definition: cudnn_frontend_Engine.h:103
cudnn_frontend::Engine_v8::getTag
std::string const & getTag() const
Definition: cudnn_frontend_Engine.h:213
cudnn_frontend::BackendDescriptor
Definition: cudnn_backend_base.h:99
cudnn_frontend::Engine_v8::idx
int64_t idx
Global Index of the engine for the given operationGraph.
Definition: cudnn_frontend_Engine.h:114
cudnn_frontend_OperationGraph.h
cudnn_frontend::make_shared_backend_pointer
static ManagedOpaqueDescriptor make_shared_backend_pointer(cudnnBackendDescriptorType_t type)
Definition: cudnn_backend_base.h:88
cudnn_frontend::EngineBuilder_v8::build
Engine_v8 && build()
Definition: cudnn_frontend_Engine.h:275
cudnn_frontend::Engine_v8::Engine_v8
Engine_v8()=default
cudnn_frontend::Engine_v8::Knob::getMinValue
int64_t getMinValue() const
Definition: cudnn_frontend_Engine.h:93
cudnn_frontend::Engine_v8::Knob::describe
std::string describe() const
Definition: cudnn_frontend_Engine.h:68
cudnn_frontend::Engine_v8::buildKnobs
void buildKnobs()
Called from the constructor builds the internal knobs vector.
Definition: cudnn_frontend_Engine.h:122
cudnn_frontend::Engine_v8::opGraph
ManagedOpaqueDescriptor opGraph
Definition: cudnn_frontend_Engine.h:113
cudnn_frontend::BackendDescriptor::get_desc
ManagedOpaqueDescriptor get_desc() const
Returns a copy of underlying managed descriptor.
Definition: cudnn_backend_base.h:138
cudnn_frontend::Engine_v8::Knob::knobType
cudnnBackendKnobType_t knobType
Definition: cudnn_frontend_Engine.h:108
cudnn_frontend_utils.h
cudnn_frontend::Engine_v8::bKnobs
std::array< ManagedOpaqueDescriptor, CUDNN_KNOB_TYPE_COUNTS > bKnobs
Opaque pointer to the backend knobs.
Definition: cudnn_frontend_Engine.h:116
cudnn_frontend::EngineBuilder_v8::setOperationGraph
auto setOperationGraph(OperationGraph_v8 const &opGraph_) -> EngineBuilder_v8 &
Set operationGraph for the engine.
Definition: cudnn_frontend_Engine.h:252
cudnn_frontend::Engine_v8::Knob::getMaxValue
int64_t getMaxValue() const
Definition: cudnn_frontend_Engine.h:98
cudnn_frontend::Engine_v8::Knob::minValue
int64_t minValue
Definition: cudnn_frontend_Engine.h:109
cudnn_frontend::Engine_v8::Knob::getChoice
int64_t getChoice() const
Definition: cudnn_frontend_Engine.h:83
cudnn_frontend::BackendDescriptor::get_status
cudnnStatus_t get_status() const
Current status of the descriptor.
Definition: cudnn_backend_base.h:114
cudnn_frontend::Engine_v8::getFinalizedKnobs
std::vector< Knob > const & getFinalizedKnobs() const
Returns a final vector of knobs. Used in EngineConfigBuilder.
Definition: cudnn_frontend_Engine.h:225
cudnn_frontend::Engine_v8::getSupportedKnobs
std::vector< Knob > & getSupportedKnobs()
Returns a vector of knobs to the user for modification.
Definition: cudnn_frontend_Engine.h:219
cudnn_frontend::Engine_v8
Definition: cudnn_frontend_Engine.h:52
cudnn_frontend::ManagedOpaqueDescriptor
std::shared_ptr< OpaqueBackendPointer > ManagedOpaqueDescriptor
Definition: cudnn_backend_base.h:84
cudnn_frontend::Engine_v8::~Engine_v8
~Engine_v8()=default
cudnn_frontend
Definition: cudnn_backend_base.h:29
cudnn_frontend::Engine_v8::describe
std::string describe() const override
Return a string describing the backend Descriptor.
Definition: cudnn_frontend_Engine.h:167
cudnn_frontend::BackendDescriptor::get_error
const char * get_error() const
Diagonistic error message if any.
Definition: cudnn_backend_base.h:132
cudnn_frontend::Engine_v8::Engine_v8
Engine_v8(Engine_v8 &&from)
Definition: cudnn_frontend_Engine.h:174
type
cudnn_frontend::Engine_v8::Knob::setChoice
void setChoice(uint64_t val_)
Definition: cudnn_frontend_Engine.h:78
cudnn_frontend::EngineBuilder_v8::m_engine
Engine_v8 m_engine
Definition: cudnn_frontend_Engine.h:344
cudnn_frontend::Engine_v8::Knob
Definition: cudnn_frontend_Engine.h:62
cudnn_frontend::Engine_v8::knobs
std::vector< Knob > knobs
Definition: cudnn_frontend_Engine.h:117
cudnn_frontend::EngineBuilder_v8
Definition: cudnn_frontend_Engine.h:244
cudnn_frontend::Engine_v8::Knob::Knob
Knob(cudnnBackendKnobType_t type_, int64_t max, int64_t min, int64_t stride_)
Definition: cudnn_frontend_Engine.h:64
cudnn_frontend::OperationGraph_v8
Definition: cudnn_frontend_OperationGraph.h:52
cudnn_frontend::Engine_v8::knobs_set
bool knobs_set() const
Definition: cudnn_frontend_Engine.h:229
cudnn_frontend::BackendDescriptor::status
cudnnStatus_t status
Shared pointer of the OpaqueBackendPointer.
Definition: cudnn_backend_base.h:160
cudnn_frontend::BackendDescriptor::pointer
ManagedOpaqueDescriptor pointer
Definition: cudnn_backend_base.h:158
cudnn_frontend::Engine_v8::numKnobs
int64_t numKnobs
Count of the backend knobs in the engine.
Definition: cudnn_frontend_Engine.h:115