CUDNN Frontend API 8.3.0

cudnn_frontend_ExecutionPlanCache.h
Go to the documentation of this file.
1 /*
2 * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23 #include <tuple>
24 #include <unordered_map>
25 #include <map>
26 #include <memory>
27 #include <mutex>
28
29 #include <cudnn_frontend_OperationGraph.h>
30 #include <cudnn_frontend_ExecutionPlan.h>
31
37
40 /***
41 * device_id_0 Operation_Graph0 (conv_fprop)
42 * -------------------------------------------------------------------------------
43 * | Feature_vector_type0_val0 | Plan A0 |
44 * | Feature_vector_type0_val1 | Plan B0 |
45 * ===============================================================================
46 *
47 * device_id_0 Operation_Graph1 (dgrad)
48 * -------------------------------------------------------------------------------
49 * | Feature_vector_type1_val0 | Plan A1 |
50 * | Feature_vector_type1_val1 | Plan B1 |
51 * ===============================================================================
52 *
53 * device_id_0 Operation_Graph2 (wgrad)
54 * -------------------------------------------------------------------------------
55 * | Feature_vector_type2_val0 | Plan B2 |
56 * ===============================================================================
57 *
58 * device_id_1 Operation_Graph0 (conv_fprop)
59 * -------------------------------------------------------------------------------
60 * | Feature_vector_type0_val0 | Plan A0 |
61 * | Feature_vector_type0_val1 | Plan B0 |
62 * ===============================================================================
63 *
64 * device_id_1 Operation_Graph1 (dgrad)
65 * -------------------------------------------------------------------------------
66 * | Feature_vector_type1_val0 | Plan A1 |
67 * | Feature_vector_type1_val1 | Plan B1 |
68 * ===============================================================================
69 *
70 * device_id_1 Operation_Graph2 (wgrad)
71 * -------------------------------------------------------------------------------
72 * | Feature_vector_type2_val0 | Plan B2 |
73 * ===============================================================================
74 */
75
76 namespace cudnn_frontend {
77
79 class ExecutionPlanCache_v1 {
80
81 protected:
82 struct compare {
83 bool operator ()(const feature_vector_t & fv1, const feature_vector_t &fv2) const {
84 for (uint32_t i = 0u; i < fv1.size(); i++) {
85 if (fv1[i] < fv2[i]) {
86 return true;
87 }
88 }
89 }
90 };
91
92 std::string name = "plan_cache_[unnamed]";
93
96 using FeatureVectorToPlanMap = std::map<cudnn_frontend::feature_vector_t, cudnn_frontend::ExecutionPlan, cudnn_frontend::ExecutionPlanCache_v1::compare>;
97 FeatureVectorToPlanMap cache;
98
99 mutable std::mutex cache_mutex;
100
101 public:
102 virtual bool
103 is_fastest_plan_stable(const cudnn_frontend::OperationGraph &op_graph,
104 const std::string & tag) {
105 return true;
106 }
107
108 void add_plan_to_cache(const cudnn_frontend::OperationGraph &op_graph,
109 const cudnn_frontend::ExecutionPlan &plan) {
110 std::lock_guard<std::mutex> guard(cache_mutex);
111 cache.insert(std::make_pair(op_graph.getFeatureVector(),plan));
112 getLogger() << "[cudnn_frontend] Added to " << name << " " << op_graph.getTag() << std::endl;
113 }
114
115 ExecutionPlanCache_v1(const char * name_) {
116 name = name_;
117 }
118
119 const std::string & get_name() const {
120 return name;
121 }
122
123 // Plan is the output here.
124 bool get_plan_from_cache(const cudnn_frontend::OperationGraph &op_graph,
125 const cudnn_frontend::ExecutionPlan *&plan) const {
126 {
127 std::lock_guard<std::mutex> guard(cache_mutex);
128 auto it = cache.find(op_graph.getFeatureVector());
129
130 if (it == cache.end()) {
131 getLogger() << "[cudnn_frontend] Cached Plan Not Found in " << name << std::endl;
132 return false;
133 }
134 plan = &(it->second);
135 }
136 getLogger() << "[cudnn_frontend] Cached Plan Found in " << name << std::endl;
137 return true;
138 }
139 };
140
141 class ExecutionPlanCache_v2 : public ExecutionPlanCache_v1 {
142
143 using SaturationTracker = std::map<std::pair<cudnn_frontend::feature_vector_t, std::string>, int32_t>;
144 SaturationTracker tracker;
145
146 int32_t saturationCount = 1;
147
148 public:
149 virtual bool
150 is_fastest_plan_stable(const cudnn_frontend::OperationGraph &op_graph,
151 const std::string & tag) {
152 if (saturationCount == 1) {return true;} // Special case. Always add to the cache.
153
154 // If plan cache is already created for the op_graph no need to update.
155 // Ideally, one will auto-tune only if the plan cache has no plan for the op_graph.
156 cudnn_frontend::ExecutionPlan const *plan = nullptr;
157 if (get_plan_from_cache(op_graph, plan)) {
158 getLogger() << "[cudnn_frontend] SaturationTracker " << name << " " << op_graph.getTag() << " " << tag << " plan already in cache." << std::endl;
159 return false;
160 }
161
162 // Lock the cache and increase the count till we saturate
163 std::lock_guard<std::mutex> guard(cache_mutex);
164 auto cnt = tracker[std::make_pair(op_graph.getFeatureVector(),tag)] += 1;
165 getLogger() << "[cudnn_frontend] SaturationTracker " << name << " " << op_graph.getTag() << " " << tag << " " << cnt << std::endl;
166 return cnt >= saturationCount;
167 }
168
169 void set_saturation_count(int32_t count) {
170 saturationCount = count;
171 }
172
173 ExecutionPlanCache_v2(const char * name_) : ExecutionPlanCache_v1(name_) {
174 }
175
176 };
177
178 using ExecutionPlanCache = ExecutionPlanCache_v2;
179
180 }
cudnn_frontend::feature_vector_t
std::vector< int64_t > feature_vector_t
Detailed feature_vector. Generally the Tensor and Operation properties.
Definition: cudnn_frontend_utils.h:34
cudnn_frontend::getLogger
ConditionalStreamer & getLogger()
Definition: cudnn_frontend_Logging.h:72
cudnn_frontend::ExecutionPlanCache_v1::ExecutionPlanCache_v1
ExecutionPlanCache_v1(const char *name_)
Definition: cudnn_frontend_ExecutionPlanCache.h:115
cudnn_frontend::ExecutionPlan_v8
Definition: cudnn_frontend_ExecutionPlan.h:51
cudnn_frontend::ExecutionPlanCache_v1
Plan Cache structure for the above table.
Definition: cudnn_frontend_ExecutionPlanCache.h:79
cudnn_frontend_OperationGraph.h
cudnn_frontend::ExecutionPlanCache_v1::is_fastest_plan_stable
virtual bool is_fastest_plan_stable(const cudnn_frontend::OperationGraph &op_graph, const std::string &tag)
Definition: cudnn_frontend_ExecutionPlanCache.h:103
cudnn_frontend::ExecutionPlanCache_v1::compare::operator()
bool operator()(const feature_vector_t &fv1, const feature_vector_t &fv2) const
Definition: cudnn_frontend_ExecutionPlanCache.h:83
cudnn_frontend::ExecutionPlanCache_v1::cache
FeatureVectorToPlanMap cache
Definition: cudnn_frontend_ExecutionPlanCache.h:97
cudnn_frontend::ExecutionPlanCache_v2::SaturationTracker
std::map< std::pair< cudnn_frontend::feature_vector_t, std::string >, int32_t > SaturationTracker
Definition: cudnn_frontend_ExecutionPlanCache.h:143
cudnn_frontend::OperationGraph_v8::getTag
std::string const & getTag() const
Definition: cudnn_frontend_OperationGraph.h:98
cudnn_frontend::OperationGraph_v8::getFeatureVector
feature_vector_t getFeatureVector() const
Definition: cudnn_frontend_OperationGraph.h:103
cudnn_frontend::ExecutionPlanCache_v1::cache_mutex
std::mutex cache_mutex
Definition: cudnn_frontend_ExecutionPlanCache.h:99
cudnn_frontend::ExecutionPlanCache_v1::add_plan_to_cache
void add_plan_to_cache(const cudnn_frontend::OperationGraph &op_graph, const cudnn_frontend::ExecutionPlan &plan)
Definition: cudnn_frontend_ExecutionPlanCache.h:108
cudnn_frontend::ExecutionPlanCache_v2::is_fastest_plan_stable
virtual bool is_fastest_plan_stable(const cudnn_frontend::OperationGraph &op_graph, const std::string &tag)
Definition: cudnn_frontend_ExecutionPlanCache.h:150
cudnn_frontend::ExecutionPlanCache_v1::get_name
const std::string & get_name() const
Definition: cudnn_frontend_ExecutionPlanCache.h:119
cudnn_frontend
Definition: cudnn_backend_base.h:29
cudnn_frontend::ExecutionPlanCache_v2::tracker
SaturationTracker tracker
Definition: cudnn_frontend_ExecutionPlanCache.h:144
cudnn_frontend::ExecutionPlanCache_v1::get_plan_from_cache
bool get_plan_from_cache(const cudnn_frontend::OperationGraph &op_graph, const cudnn_frontend::ExecutionPlan *&plan) const
Definition: cudnn_frontend_ExecutionPlanCache.h:124
cudnn_frontend_ExecutionPlan.h
cudnn_frontend::ExecutionPlanCache_v1::name
std::string name
Definition: cudnn_frontend_ExecutionPlanCache.h:92
cudnn_frontend::ExecutionPlanCache_v2::set_saturation_count
void set_saturation_count(int32_t count)
Definition: cudnn_frontend_ExecutionPlanCache.h:169
cudnn_frontend::ExecutionPlanCache_v2::ExecutionPlanCache_v2
ExecutionPlanCache_v2(const char *name_)
Definition: cudnn_frontend_ExecutionPlanCache.h:173
cudnn_frontend::ExecutionPlanCache_v1::compare
Definition: cudnn_frontend_ExecutionPlanCache.h:82
cudnn_frontend::OperationGraph_v8
Definition: cudnn_frontend_OperationGraph.h:52
cudnn_frontend::ExecutionPlanCache_v2
Definition: cudnn_frontend_ExecutionPlanCache.h:141
cudnn_frontend::ExecutionPlanCache_v1::FeatureVectorToPlanMap
std::map< cudnn_frontend::feature_vector_t, cudnn_frontend::ExecutionPlan, cudnn_frontend::ExecutionPlanCache_v1::compare > FeatureVectorToPlanMap
Definition: cudnn_frontend_ExecutionPlanCache.h:96