CUDNN Frontend API 8.3.0

cudnn_frontend_Heuristics.h
Go to the documentation of this file.
1 /*
2 * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23 #pragma once
24
25 #include <vector>
26
27 #include <cudnn.h>
28 #include <cudnn_backend.h>
29
30 #include "cudnn_frontend_OperationGraph.h"
31 #include "cudnn_frontend_EngineConfig.h"
32 #include "cudnn_frontend_utils.h"
33 #include "cudnn_frontend_Filters.h"
34
35 namespace cudnn_frontend {
47 class EngineHeuristics_v8 : public BackendDescriptor {
48 public:
49 friend class EngineHeuristicsBuilder_v8;
50 std::string
51 describe() const override {
52 std::stringstream ss;
53 ss << "CUDNN_BACKEND_ENGINEHEUR_DESCRIPTOR :";
54 return ss.str();
55 }
56
57 EngineHeuristics_v8(EngineHeuristics_v8 &&from) = default;
58 EngineHeuristics_v8 &
59 operator= (EngineHeuristics_v8 &&from) = default;
60
61 ~EngineHeuristics_v8() = default;
62
67 auto
69 getEngineConfig(int64_t count = 1) -> std::vector<ManagedOpaqueDescriptor> & {
70 cudnnStatus_t status;
71 for (auto i = 0u; i < count; ++i) {
72 ManagedOpaqueDescriptor engConfig = nullptr;
73 engConfig = make_shared_backend_pointer(CUDNN_BACKEND_ENGINECFG_DESCRIPTOR);
74 if (engConfig->is_good() == false) {
75 set_error_and_throw_exception(
76 this,
77 engConfig->get_status(),
78 "CUDNN_BACKEND_ENGINEHEUR_DESCRIPTOR: CUDNN_BACKEND_ENGINECFG_DESCRIPTOR cudnnCreate Failed");
79 return m_heuristic_results;
80 };
81 m_heuristic_results.emplace_back(engConfig);
82 }
83 std::vector<cudnnBackendDescriptor_t> heuristic_results_;
84 for (std::uint32_t i = 0; i < m_heuristic_results.size(); i++) {
85 heuristic_results_.emplace_back(m_heuristic_results[i]->get_backend_descriptor());
86 }
87 int64_t result = -1;
88 status = cudnnBackendGetAttribute(pointer->get_backend_descriptor(),
89 CUDNN_ATTR_ENGINEHEUR_RESULTS,
90 CUDNN_TYPE_BACKEND_DESCRIPTOR,
91 count,
92 &result,
93 heuristic_results_.data());
94 if (status != CUDNN_STATUS_SUCCESS) {
95 set_error_and_throw_exception(
96 this, status, "CUDNN_BACKEND_ENGINEHEUR_DESCRIPTOR: GetAttribute CUDNN_ATTR_ENGINEHEUR_RESULTS Failed");
97 };
98 return m_heuristic_results;
99 }
100
102 auto
103 getEngineConfigCount(void) const -> int64_t {
104 cudnnStatus_t status;
105 int64_t count = -1;
106 status = cudnnBackendGetAttribute(pointer->get_backend_descriptor(),
107 CUDNN_ATTR_ENGINEHEUR_RESULTS,
108 CUDNN_TYPE_BACKEND_DESCRIPTOR,
109 0,
110 &count,
111 nullptr);
112 if (status != CUDNN_STATUS_SUCCESS) {
113 set_error_and_throw_exception(
114 this,
115 status,
116 "CUDNN_BACKEND_ENGINEHEUR_DESCRIPTOR: GetAttribute CUDNN_ATTR_ENGINEHEUR_RESULTS Count Failed");
117 };
118 return count;
119 }
122 private:
123 EngineHeuristics_v8() = default;
124 EngineHeuristics_v8(EngineHeuristics_v8 const &) = delete;
125 EngineHeuristics_v8 &
126 operator=(EngineHeuristics_v8 const &) = delete;
127
128 cudnnBackendHeurMode_t mode = CUDNN_HEUR_MODE_INSTANT;
129 ManagedOpaqueDescriptor opGraph = nullptr;
130 std::vector<ManagedOpaqueDescriptor> m_heuristic_results;
131 std::string opGraphTag;
132 };
133
137 class EngineHeuristicsBuilder_v8 {
138 public:
143 auto
145 setOperationGraph(OperationGraph_v8 &opGraph_) -> EngineHeuristicsBuilder_v8 & {
146 m_heuristics.opGraph = opGraph_.get_desc();
147 m_heuristics.opGraphTag = opGraph_.getTag();
148 return *this;
149 }
151 auto
152 setHeurMode(cudnnBackendHeurMode_t mode_) -> EngineHeuristicsBuilder_v8 & {
153 m_heuristics.mode = mode_;
154 return *this;
155 }
158 EngineHeuristics_v8 &&
161 build() {
162 if (m_heuristics.opGraph == nullptr) {
163 set_error_and_throw_exception(&m_heuristics,
164 CUDNN_STATUS_BAD_PARAM,
165 "CUDNN_BACKEND_ENGINEHEUR_DESCRIPTOR: Check and Set the "
166 "CUDNN_ATTR_ENGINEHEUR_OPERATION_GRAPH field for heuristic");
167 return std::move(m_heuristics);
168 };
169
170 // Create a descriptor. Memory allocation happens here.
171 auto status = m_heuristics.initialize_managed_backend_pointer(CUDNN_BACKEND_ENGINEHEUR_DESCRIPTOR);
172 if (status != CUDNN_STATUS_SUCCESS) {
173 set_error_and_throw_exception(
174 &m_heuristics, status, "CUDNN_BACKEND_ENGINEHEUR_DESCRIPTOR: cudnnCreate Failed");
175 return std::move(m_heuristics);
176 };
177
178 status = cudnnBackendSetAttribute(m_heuristics.pointer->get_backend_descriptor(),
179 CUDNN_ATTR_ENGINEHEUR_OPERATION_GRAPH,
180 CUDNN_TYPE_BACKEND_DESCRIPTOR,
181 1,
182 &(m_heuristics.opGraph->get_backend_descriptor()));
183 if (status != CUDNN_STATUS_SUCCESS) {
184 set_error_and_throw_exception(
185 &m_heuristics,
186 status,
187 "CUDNN_BACKEND_ENGINEHEUR_DESCRIPTOR: SetAttribute CUDNN_ATTR_ENGINEHEUR_OPERATION_GRAPH Failed");
188 return std::move(m_heuristics);
189 };
190 status = cudnnBackendSetAttribute(m_heuristics.pointer->get_backend_descriptor(),
191 CUDNN_ATTR_ENGINEHEUR_MODE,
192 CUDNN_TYPE_HEUR_MODE,
193 1,
194 &m_heuristics.mode);
195 if (status != CUDNN_STATUS_SUCCESS) {
196 set_error_and_throw_exception(
197 &m_heuristics,
198 status,
199 "CUDNN_BACKEND_ENGINEHEUR_DESCRIPTOR: SetAttribute CUDNN_ATTR_ENGINEHEUR_MODE Failed");
200 return std::move(m_heuristics);
201 };
202
203 // Finalizing the descriptor
204 status = cudnnBackendFinalize(m_heuristics.pointer->get_backend_descriptor());
205 if (status != CUDNN_STATUS_SUCCESS) {
206 set_error_and_throw_exception(
207 &m_heuristics, status, "CUDNN_BACKEND_ENGINEHEUR_DESCRIPTOR: cudnn Finalize failed");
208 return std::move(m_heuristics);
209 };
210
211 getLogger() << "[cudnn_frontend] " << m_heuristics << std::endl;
212 return std::move(m_heuristics);
213 }
214
215 explicit EngineHeuristicsBuilder_v8() = default;
216 ~EngineHeuristicsBuilder_v8() = default;
217 EngineHeuristicsBuilder_v8(EngineHeuristicsBuilder_v8 &&) = delete;
218 EngineHeuristicsBuilder_v8(EngineHeuristicsBuilder_v8 const &) = delete;
219 EngineHeuristicsBuilder_v8 &
220 operator=(EngineHeuristicsBuilder_v8 const &) = delete;
221
222 private:
223 EngineHeuristics_v8 m_heuristics;
224 };
225
226 template<std::size_t SIZE>
227 EngineConfigList
228 get_heuristics_list(std::array<cudnnBackendHeurMode_t, SIZE> modes,
229 OperationGraph_v8 &opGraph,
230 std::function<bool(cudnnBackendDescriptor_t)> filter_fn) {
231 (void) modes;
232 EngineConfigList filtered_configs;
233
234
235 for (auto mode : modes) {
236 auto heuristics = EngineHeuristicsBuilder_v8()
237 .setOperationGraph(opGraph)
238 .setHeurMode(mode)
239 .build();
240
241 auto& engine_config = heuristics.getEngineConfig(heuristics.getEngineConfigCount());
242 cudnn_frontend::filter(engine_config, filtered_configs, filter_fn);
243 }
244
245 return filtered_configs;
246 }
247 }
cudnn_frontend::EngineHeuristicsBuilder_v8::setHeurMode
auto setHeurMode(cudnnBackendHeurMode_t mode_) -> EngineHeuristicsBuilder_v8 &
Set cudnnHandle for the operations.
Definition: cudnn_frontend_Heuristics.h:152
cudnn_frontend_Filters.h
cudnn_frontend::getLogger
ConditionalStreamer & getLogger()
Definition: cudnn_frontend_Logging.h:72
cudnn_frontend::set_error_and_throw_exception
static void set_error_and_throw_exception(BackendDescriptor const *desc, cudnnStatus_t status, const char *message)
Definition: cudnn_frontend_utils.h:138
cudnn_frontend::EngineHeuristics_v8::EngineHeuristicsBuilder_v8
friend class EngineHeuristicsBuilder_v8
Definition: cudnn_frontend_Heuristics.h:49
cudnn_frontend::EngineHeuristicsBuilder_v8::build
EngineHeuristics_v8 && build()
Definition: cudnn_frontend_Heuristics.h:161
cudnn_frontend::EngineHeuristics_v8::getEngineConfig
auto getEngineConfig(int64_t count=1) -> std::vector< ManagedOpaqueDescriptor > &
Query the total count of the engines for the Operation Set.
Definition: cudnn_frontend_Heuristics.h:69
cudnn_frontend::BackendDescriptor
Definition: cudnn_backend_base.h:99
cudnn_frontend::filter
static auto filter(Predicate pred, executionPlans_t &plans) -> executionPlans_t
Definition: cudnn_frontend_EngineConfigGenerator.h:106
cudnn_frontend_OperationGraph.h
cudnn_frontend::make_shared_backend_pointer
static ManagedOpaqueDescriptor make_shared_backend_pointer(cudnnBackendDescriptorType_t type)
Definition: cudnn_backend_base.h:88
cudnn_frontend::EngineHeuristicsBuilder_v8::setOperationGraph
auto setOperationGraph(OperationGraph_v8 &opGraph_) -> EngineHeuristicsBuilder_v8 &
Set operationGraph for the engine (opGraph is not destroyed)
Definition: cudnn_frontend_Heuristics.h:145
cudnn_frontend::EngineHeuristics_v8::mode
cudnnBackendHeurMode_t mode
Definition: cudnn_frontend_Heuristics.h:128
cudnn_frontend::EngineHeuristics_v8
Definition: cudnn_frontend_Heuristics.h:47
cudnn_frontend::EngineHeuristics_v8::opGraph
ManagedOpaqueDescriptor opGraph
Definition: cudnn_frontend_Heuristics.h:129
cudnn_frontend_utils.h
cudnn_frontend::EngineHeuristics_v8::opGraphTag
std::string opGraphTag
storage of heuristic results
Definition: cudnn_frontend_Heuristics.h:131
cudnn_frontend_EngineConfig.h
cudnn_frontend::EngineHeuristics_v8::getEngineConfigCount
auto getEngineConfigCount(void) const -> int64_t
Query the total count of the engine config for the Operation Set.
Definition: cudnn_frontend_Heuristics.h:103
cudnn_frontend::EngineHeuristics_v8::~EngineHeuristics_v8
~EngineHeuristics_v8()=default
cudnn_frontend::get_heuristics_list
EngineConfigList get_heuristics_list(std::array< cudnnBackendHeurMode_t, SIZE > modes, OperationGraph_v8 &opGraph, std::function< bool(cudnnBackendDescriptor_t)> filter_fn)
Definition: cudnn_frontend_Heuristics.h:228
nlohmann::detail::void
j template void())
Definition: json.hpp:4061
cudnn_frontend::ManagedOpaqueDescriptor
std::shared_ptr< OpaqueBackendPointer > ManagedOpaqueDescriptor
Definition: cudnn_backend_base.h:84
cudnn_frontend
Definition: cudnn_backend_base.h:29
cudnn_frontend::EngineHeuristics_v8::EngineHeuristics_v8
EngineHeuristics_v8()=default
cudnn_frontend::EngineHeuristics_v8::operator=
EngineHeuristics_v8 & operator=(EngineHeuristics_v8 &&from)=default
cudnn_frontend::EngineHeuristics_v8::m_heuristic_results
std::vector< ManagedOpaqueDescriptor > m_heuristic_results
Definition: cudnn_frontend_Heuristics.h:130
cudnn_frontend::EngineHeuristicsBuilder_v8
Definition: cudnn_frontend_Heuristics.h:137
cudnn_frontend::EngineHeuristicsBuilder_v8::m_heuristics
EngineHeuristics_v8 m_heuristics
Definition: cudnn_frontend_Heuristics.h:223
cudnn_frontend::EngineConfigList
std::vector< ManagedOpaqueDescriptor > EngineConfigList
Definition: cudnn_frontend_EngineConfig.h:248
cudnn_frontend::EngineHeuristics_v8::describe
std::string describe() const override
Return a string describing the backend Descriptor.
Definition: cudnn_frontend_Heuristics.h:51
cudnn_frontend::OperationGraph_v8
Definition: cudnn_frontend_OperationGraph.h:52
cudnn_frontend::BackendDescriptor::status
cudnnStatus_t status
Shared pointer of the OpaqueBackendPointer.
Definition: cudnn_backend_base.h:160
cudnn_frontend::BackendDescriptor::pointer
ManagedOpaqueDescriptor pointer
Definition: cudnn_backend_base.h:158