CUDNN Frontend API 8.3.0

cudnn_frontend_ExecutionPlan.h
Go to the documentation of this file.
1 /*
2 * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23 #pragma once
24
25 #include <algorithm>
26 #include <array>
27 #include <functional>
28 #include <memory>
29 #include <sstream>
30 #include <utility>
31
32 #include <cudnn.h>
33 #include <cudnn_backend.h>
34
35 #include "cudnn_frontend_Engine.h"
36 #include "cudnn_frontend_utils.h"
37
38 namespace cudnn_frontend {
51 class ExecutionPlan_v8 : public BackendDescriptor {
52 public:
53 friend class ExecutionPlanBuilder_v8;
54
55 ExecutionPlan_v8(ExecutionPlan_v8 &&from) = default;
56 ExecutionPlan_v8 &
57 operator=(ExecutionPlan_v8 &&) = default;
58
59 ~ExecutionPlan_v8() = default;
64 auto
66 getWorkspaceSize(void) const -> int64_t {
67 return workSpaceSize;
68 }
69
70 std::string
71 describe() const override {
72 std::stringstream ss;
73 ss << "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR : ";
74 ss << getTag() << ", ";
75 ss << "numeric_notes:" << " ";
76 for (auto note : numeric_notes)
77 ss << note << " ";
78 #if (CUDNN_VERSION >= 8200)
79 ss << "behavior_notes:" << " ";
80 for (auto note : behavior_notes)
81 ss << note << " ";
82 #endif
83 ss << "workSpaceSize: " << workSpaceSize;
84 return ss.str();
85 }
86
87 std::string const &
88 getTag() const {
89 return planTag;
90 }
91
92 void
93 setExecutionTime(float time_) {
94 execution_time_ms = time_;
95 }
96
97 float
98 getExecutionTime() const {
99 return execution_time_ms;
100 }
101
102 std::array<cudnnBackendNumericalNote_t,CUDNN_NUMERICAL_NOTE_TYPE_COUNT> const &
103 getNumericNotes() const {
104 return numeric_notes;
105 }
106
107 #if (CUDNN_VERSION >= 8200)
108 std::array<cudnnBackendBehaviorNote_t, CUDNN_BEHAVIOR_NOTE_TYPE_COUNT> const &
109 getBehaviorNotes() const {
110 return behavior_notes;
111 }
112 #endif
113
114 ExecutionPlan_v8(ExecutionPlan_v8 const &) = default;
115 ExecutionPlan_v8 &
116 operator=(ExecutionPlan_v8 const &) = default;
117 private:
118 void
119 fetchNotes(ManagedOpaqueDescriptor &extractedEngine) {
120 auto status = CUDNN_STATUS_SUCCESS;
121 int64_t elem_count = 0;
122 cudnnBackendDescriptor_t extractedEngine_ = extractedEngine->get_backend_descriptor();
123 status = cudnnBackendGetAttribute(extractedEngine_,
124 CUDNN_ATTR_ENGINE_NUMERICAL_NOTE,
125 CUDNN_TYPE_NUMERICAL_NOTE,
126 CUDNN_NUMERICAL_NOTE_TYPE_COUNT,
127 &elem_count,
128 numeric_notes.data());
129 if (status != CUDNN_STATUS_SUCCESS) {
130 set_error_and_throw_exception(this,
131 status,
132 "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: GetAttribute "
133 "CUDNN_ATTR_ENGINE_NUMERICAL_NOTE Failed");
134 }
135 #if (CUDNN_VERSION >= 8200)
136 status = cudnnBackendGetAttribute(extractedEngine_,
137 CUDNN_ATTR_ENGINE_BEHAVIOR_NOTE,
138 CUDNN_TYPE_BEHAVIOR_NOTE,
139 CUDNN_BEHAVIOR_NOTE_TYPE_COUNT,
140 &elem_count,
141 behavior_notes.data());
142 if (status != CUDNN_STATUS_SUCCESS) {
143 set_error_and_throw_exception(this,
144 status,
145 "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: GetAttribute "
146 "CUDNN_ATTR_ENGINE_BEHAVIOR_NOTE Failed");
147 }
148 #endif
149 }
150
151 void
152 buildTag(ManagedOpaqueDescriptor &extractedEngine) {
153 // Compute a unique tag for execution plan:
154 auto status = CUDNN_STATUS_SUCCESS;
155 std::stringstream tag{""};
156 int64_t elemCount = 0, engineId = 0, numKnobs = 0;
157
158 std::array<ManagedOpaqueDescriptor, CUDNN_KNOB_TYPE_COUNTS> extractedKnobs{{nullptr}};
159 for (auto &knob : extractedKnobs) {
160 knob = make_shared_backend_pointer(CUDNN_BACKEND_KNOB_CHOICE_DESCRIPTOR);
161 status = knob->get_status();
162 if (status != CUDNN_STATUS_SUCCESS) {
163 set_error_and_throw_exception(
164 this, status, "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: cudnnCreate Failed when compute tag");
165 }
166 }
167
168 cudnnBackendDescriptor_t extractedEngine_ = extractedEngine->get_backend_descriptor();
169 std::array<cudnnBackendDescriptor_t, CUDNN_KNOB_TYPE_COUNTS> extractedKnobs_{{nullptr}};
170 for (std::uint32_t i = 0; i < extractedKnobs.size(); i++) {
171 extractedKnobs_[i] = extractedKnobs[i]->get_backend_descriptor();
172 }
173
174 status = cudnnBackendGetAttribute(
175 extractedEngine_, CUDNN_ATTR_ENGINE_GLOBAL_INDEX, CUDNN_TYPE_INT64, 1, &elemCount, &engineId);
176 if (status != CUDNN_STATUS_SUCCESS) {
177 set_error_and_throw_exception(this,
178 status,
179 "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: GetAttribute "
180 "CUDNN_ATTR_ENGINE_GLOBAL_INDEX Failed");
181 }
182 tag << "eng" << engineId;
183
184 status = cudnnBackendGetAttribute(engine_config->get_backend_descriptor(),
185 CUDNN_ATTR_ENGINECFG_KNOB_CHOICES,
186 CUDNN_TYPE_BACKEND_DESCRIPTOR,
187 CUDNN_KNOB_TYPE_COUNTS,
188 &numKnobs,
189 &(extractedKnobs_[0]));
190 if (status != CUDNN_STATUS_SUCCESS) {
191 set_error_and_throw_exception(this,
192 status,
193 "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: GetAttribute "
194 "CUDNN_ATTR_ENGINECFG_KNOB_CHOICES Failed");
195 }
196 if (numKnobs > CUDNN_KNOB_TYPE_COUNTS) {
197 set_error_and_throw_exception(this,
198 status,
199 "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: GetAttribute "
200 "numKnobs exceed the CUDNN_KNOB_TYPE_COUNTS");
201 }
202 for (int64_t idx = 0; idx < numKnobs; ++idx) {
203 const cudnnBackendDescriptor_t &knob = extractedKnobs_[idx];
204 cudnnBackendKnobType_t type = CUDNN_KNOB_TYPE_COUNTS;
205 int64_t choice = -2;
206 status = cudnnBackendGetAttribute(
207 knob, CUDNN_ATTR_KNOB_CHOICE_KNOB_TYPE, CUDNN_TYPE_KNOB_TYPE, 1, nullptr, &type);
208 if (status != CUDNN_STATUS_SUCCESS) {
209 set_error_and_throw_exception(this,
210 status,
211 "computeTag CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: GetAttribute "
212 "CUDNN_ATTR_KNOB_CHOICE_KNOB_TYPE Failed");
213 }
214 status = cudnnBackendGetAttribute(
215 knob, CUDNN_ATTR_KNOB_CHOICE_KNOB_VALUE, CUDNN_TYPE_INT64, 1, nullptr, &choice);
216 if (status != CUDNN_STATUS_SUCCESS) {
217 set_error_and_throw_exception(this,
218 status,
219 "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: GetAttribute "
220 "CUDNN_ATTR_KNOB_CHOICE_KNOB_VALUE Failed");
221 }
222 tag << "_k" << type << "=" << choice;
223 }
224 planTag += tag.str();
225
226 }
227
228 void
229 computeWorkSpaceSize() {
230 auto status = cudnnBackendGetAttribute(pointer->get_backend_descriptor(),
231 CUDNN_ATTR_EXECUTION_PLAN_WORKSPACE_SIZE,
232 CUDNN_TYPE_INT64,
233 1,
234 NULL,
235 &workSpaceSize);
236 if (status != CUDNN_STATUS_SUCCESS) {
237 set_error_and_throw_exception(this,
238 status,
239 "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: GetAttribute "
240 "CUDNN_ATTR_EXECUTION_PLAN_WORKSPACE_SIZE Failed");
241 }
242 if (workSpaceSize < 0) {
243 set_error_and_throw_exception(
244 this, status, "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: GetAttribute Workspace Size Invalid");
245 }
246 }
247
248 ExecutionPlan_v8() = default;
249 ManagedOpaqueDescriptor engine_config = nullptr;
250 cudnnHandle_t handle = nullptr;
251 std::string planTag;
252
253 std::int64_t workSpaceSize = 0;
254 std::array<cudnnBackendNumericalNote_t,CUDNN_NUMERICAL_NOTE_TYPE_COUNT> numeric_notes;
255 #if (CUDNN_VERSION >= 8200)
256 std::array<cudnnBackendBehaviorNote_t, CUDNN_BEHAVIOR_NOTE_TYPE_COUNT> behavior_notes;
257 #endif
258
259 float execution_time_ms = 0.0f;
260
261 };
262
266 class ExecutionPlanBuilder_v8 {
267 public:
272 auto
274 setHandle(cudnnHandle_t handle_) -> ExecutionPlanBuilder_v8 & {
275 m_execution_plan.handle = handle_;
276 return *this;
277 }
279 auto
280 setEngineConfig(EngineConfig_v8 const &engine_config_) -> ExecutionPlanBuilder_v8 & {
281 m_execution_plan.engine_config = engine_config_.get_desc();
282 m_execution_plan.planTag = engine_config_.getTag();
283 return *this;
284 }
285
287 auto
288 setEngineConfig(ManagedOpaqueDescriptor &desc, std::string const &opGraphTag_ = "") -> ExecutionPlanBuilder_v8 & {
289 m_execution_plan.engine_config = desc;
290 m_execution_plan.planTag = opGraphTag_;
291 return *this;
292 }
295 ExecutionPlan_v8 &&
298 build() {
299 if (m_execution_plan.handle == nullptr) {
300 set_error_and_throw_exception(
301 &m_execution_plan,
302 CUDNN_STATUS_BAD_PARAM,
303 "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: Check and Set the CUDNN_ATTR_EXECUTION_PLAN_HANDLE");
304 return std::move(m_execution_plan);
305 };
306 if (m_execution_plan.engine_config == nullptr) {
307 set_error_and_throw_exception(
308 &m_execution_plan,
309 CUDNN_STATUS_BAD_PARAM,
310 "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: Check and Set the CUDNN_ATTR_EXECUTION_PLAN_ENGINE_CONFIG");
311 return std::move(m_execution_plan);
312 };
313
314 // Create a descriptor. Memory allocation happens here.
315 auto status = m_execution_plan.initialize_managed_backend_pointer(CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR);
316 if (status != CUDNN_STATUS_SUCCESS) {
317 set_error_and_throw_exception(
318 &m_execution_plan, status, "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: cudnnCreate Failed");
319 return std::move(m_execution_plan);
320 }
321
322 status = cudnnBackendSetAttribute(m_execution_plan.pointer->get_backend_descriptor(),
323 CUDNN_ATTR_EXECUTION_PLAN_ENGINE_CONFIG,
324 CUDNN_TYPE_BACKEND_DESCRIPTOR,
325 1,
326 &(m_execution_plan.engine_config->get_backend_descriptor()));
327 if (status != CUDNN_STATUS_SUCCESS) {
328 set_error_and_throw_exception(
329 &m_execution_plan,
330 status,
331 "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: SetAttribute CUDNN_ATTR_EXECUTION_PLAN_ENGINE_CONFIG Failed");
332 return std::move(m_execution_plan);
333 }
334 status = cudnnBackendSetAttribute(m_execution_plan.pointer->get_backend_descriptor(),
335 CUDNN_ATTR_EXECUTION_PLAN_HANDLE,
336 CUDNN_TYPE_HANDLE,
337 1,
338 &m_execution_plan.handle);
339 if (status != CUDNN_STATUS_SUCCESS) {
340 set_error_and_throw_exception(
341 &m_execution_plan,
342 status,
343 "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: SetAttribute CUDNN_ATTR_EXECUTION_PLAN_HANDLE Failed");
344 return std::move(m_execution_plan);
345 }
346 // Finalizing the descriptor
347 status = cudnnBackendFinalize(m_execution_plan.pointer->get_backend_descriptor());
348 if (status != CUDNN_STATUS_SUCCESS) {
349 set_error_and_throw_exception(
350 &m_execution_plan, status, "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: cudnnFinalize Descriptor Failed");
351 return std::move(m_execution_plan);
352 }
353
354 ManagedOpaqueDescriptor extractedEngine = make_shared_backend_pointer(CUDNN_BACKEND_ENGINE_DESCRIPTOR);
355 status = extractedEngine->get_status();
356 if (status != CUDNN_STATUS_SUCCESS) {
357 set_error_and_throw_exception(
358 &m_execution_plan, status, "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: cudnnCreate of CUDNN_BACKEND_ENGINE_DESCRIPTOR failed when compute tag");
359 return std::move(m_execution_plan);
360 }
361 cudnnBackendDescriptor_t extractedEngine_ = extractedEngine->get_backend_descriptor();
362 int64_t elemCount = 0;
363 status = cudnnBackendGetAttribute(m_execution_plan.engine_config->get_backend_descriptor(),
364 CUDNN_ATTR_ENGINECFG_ENGINE,
365 CUDNN_TYPE_BACKEND_DESCRIPTOR,
366 1,
367 &elemCount,
368 &extractedEngine_);
369 if (status != CUDNN_STATUS_SUCCESS) {
370 set_error_and_throw_exception(&m_execution_plan,
371 status,
372 "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: GetAttribute "
373 "CUDNN_ATTR_ENGINECFG_ENGINE Failed");
374 return std::move(m_execution_plan);
375 }
376
377 m_execution_plan.buildTag(extractedEngine);
378 m_execution_plan.fetchNotes(extractedEngine);
379 m_execution_plan.computeWorkSpaceSize();
380
381 getLogger() << "[cudnn_frontend] " << m_execution_plan << std::endl;
382 return std::move(m_execution_plan);
383 }
384
385 explicit ExecutionPlanBuilder_v8() = default;
386 ~ExecutionPlanBuilder_v8() = default;
387 ExecutionPlanBuilder_v8(ExecutionPlanBuilder_v8 &&) = delete;
388 ExecutionPlanBuilder_v8(ExecutionPlanBuilder_v8 const &) = delete;
389 ExecutionPlanBuilder_v8 &
390 operator=(ExecutionPlanBuilder_v8 const &) = delete;
391
392 private:
393 ExecutionPlan_v8 m_execution_plan;
394 };
395
396 using ExecutionPlan = ExecutionPlan_v8;
397 using ExecutionPlanBuilder = ExecutionPlanBuilder_v8;
398 }
cudnn_frontend::ExecutionPlanBuilder_v8::setEngineConfig
auto setEngineConfig(EngineConfig_v8 const &engine_config_) -> ExecutionPlanBuilder_v8 &
Set engine Config for the Plan.
Definition: cudnn_frontend_ExecutionPlan.h:280
cudnn_frontend::getLogger
ConditionalStreamer & getLogger()
Definition: cudnn_frontend_Logging.h:72
cudnn_frontend::set_error_and_throw_exception
static void set_error_and_throw_exception(BackendDescriptor const *desc, cudnnStatus_t status, const char *message)
Definition: cudnn_frontend_utils.h:138
cudnn_frontend::ExecutionPlan_v8
Definition: cudnn_frontend_ExecutionPlan.h:51
cudnn_frontend::BackendDescriptor
Definition: cudnn_backend_base.h:99
cudnn_frontend::make_shared_backend_pointer
static ManagedOpaqueDescriptor make_shared_backend_pointer(cudnnBackendDescriptorType_t type)
Definition: cudnn_backend_base.h:88
cudnn_frontend::EngineConfig_v8
Definition: cudnn_frontend_EngineConfig.h:51
cudnn_frontend::ExecutionPlan_v8::buildTag
void buildTag(ManagedOpaqueDescriptor &extractedEngine)
Definition: cudnn_frontend_ExecutionPlan.h:152
cudnn_frontend::ExecutionPlan_v8::getTag
std::string const & getTag() const
Definition: cudnn_frontend_ExecutionPlan.h:88
cudnn_frontend::ExecutionPlan_v8::workSpaceSize
std::int64_t workSpaceSize
Definition: cudnn_frontend_ExecutionPlan.h:253
cudnn_frontend::ExecutionPlanBuilder_v8
Definition: cudnn_frontend_ExecutionPlan.h:266
cudnn_frontend_utils.h
cudnn_frontend::ExecutionPlanBuilder_v8::setEngineConfig
auto setEngineConfig(ManagedOpaqueDescriptor &desc, std::string const &opGraphTag_="") -> ExecutionPlanBuilder_v8 &
Set engine Config for the Plan.
Definition: cudnn_frontend_ExecutionPlan.h:288
cudnn_frontend::ExecutionPlan_v8::fetchNotes
void fetchNotes(ManagedOpaqueDescriptor &extractedEngine)
Definition: cudnn_frontend_ExecutionPlan.h:119
cudnn_frontend::ExecutionPlan_v8::computeWorkSpaceSize
void computeWorkSpaceSize()
Definition: cudnn_frontend_ExecutionPlan.h:229
cudnn_frontend::ExecutionPlan_v8::execution_time_ms
float execution_time_ms
Definition: cudnn_frontend_ExecutionPlan.h:259
cudnn_frontend::ExecutionPlanBuilder_v8::build
ExecutionPlan_v8 && build()
Definition: cudnn_frontend_ExecutionPlan.h:298
cudnn_frontend::ExecutionPlan_v8::planTag
std::string planTag
Definition: cudnn_frontend_ExecutionPlan.h:251
cudnn_frontend::ExecutionPlanBuilder_v8::setHandle
auto setHandle(cudnnHandle_t handle_) -> ExecutionPlanBuilder_v8 &
Set engine for the ExecutionPlan_v8.
Definition: cudnn_frontend_ExecutionPlan.h:274
cudnn_frontend::ExecutionPlanBuilder_v8::m_execution_plan
ExecutionPlan_v8 m_execution_plan
Definition: cudnn_frontend_ExecutionPlan.h:393
cudnn_frontend::ExecutionPlan_v8::handle
cudnnHandle_t handle
Definition: cudnn_frontend_ExecutionPlan.h:250
cudnn_frontend::ManagedOpaqueDescriptor
std::shared_ptr< OpaqueBackendPointer > ManagedOpaqueDescriptor
Definition: cudnn_backend_base.h:84
cudnn_frontend::ExecutionPlan_v8::describe
std::string describe() const override
Return a string describing the backend Descriptor.
Definition: cudnn_frontend_ExecutionPlan.h:71
cudnn_frontend::ExecutionPlan_v8::engine_config
ManagedOpaqueDescriptor engine_config
Definition: cudnn_frontend_ExecutionPlan.h:249
cudnn_frontend
Definition: cudnn_backend_base.h:29
cudnn_frontend::ExecutionPlan_v8::operator=
ExecutionPlan_v8 & operator=(ExecutionPlan_v8 &&)=default
type
cudnn_frontend::ExecutionPlan_v8::getNumericNotes
std::array< cudnnBackendNumericalNote_t, CUDNN_NUMERICAL_NOTE_TYPE_COUNT > const & getNumericNotes() const
Definition: cudnn_frontend_ExecutionPlan.h:103
cudnn_frontend::ExecutionPlan_v8::numeric_notes
std::array< cudnnBackendNumericalNote_t, CUDNN_NUMERICAL_NOTE_TYPE_COUNT > numeric_notes
Definition: cudnn_frontend_ExecutionPlan.h:254
cudnn_frontend::ExecutionPlan_v8::getExecutionTime
float getExecutionTime() const
Definition: cudnn_frontend_ExecutionPlan.h:98
cudnn_frontend_Engine.h
cudnn_frontend::ExecutionPlan_v8::ExecutionPlanBuilder_v8
friend class ExecutionPlanBuilder_v8
Definition: cudnn_frontend_ExecutionPlan.h:53
cudnn_frontend::ExecutionPlan_v8::ExecutionPlan_v8
ExecutionPlan_v8()=default
cudnn_frontend::ExecutionPlan_v8::~ExecutionPlan_v8
~ExecutionPlan_v8()=default
cudnn_frontend::ExecutionPlan_v8::setExecutionTime
void setExecutionTime(float time_)
Definition: cudnn_frontend_ExecutionPlan.h:93
cudnn_frontend::BackendDescriptor::status
cudnnStatus_t status
Shared pointer of the OpaqueBackendPointer.
Definition: cudnn_backend_base.h:160
cudnn_frontend::BackendDescriptor::pointer
ManagedOpaqueDescriptor pointer
Definition: cudnn_backend_base.h:158
cudnn_frontend::ExecutionPlan_v8::getWorkspaceSize
auto getWorkspaceSize(void) const -> int64_t
Query the workspace requirement for the given plan.
Definition: cudnn_frontend_ExecutionPlan.h:66