cudnn_frontend_ExecutionPlan.h
Go to the documentation of this file.
/*
* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#pragma once
#include <algorithm>
#include <array>
#include <functional>
#include <memory>
#include <sstream>
#include <utility>
#include <cudnn.h>
#include <cudnn_backend.h>
#include "cudnn_frontend_Engine.h"
#include "cudnn_frontend_utils.h"
namespace cudnn_frontend {
class ExecutionPlan_v8 : public BackendDescriptor {
public:
friend class ExecutionPlanBuilder_v8;
ExecutionPlan_v8(ExecutionPlan_v8 &&from) = default;
ExecutionPlan_v8 &
operator=(ExecutionPlan_v8 &&) = default;
~ExecutionPlan_v8() = default;
auto
getWorkspaceSize(void) const -> int64_t {
return workSpaceSize;
}
std::string
describe() const override {
std::stringstream ss;
ss << "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR : ";
ss << getTag() << ", ";
ss << "numeric_notes:" << " ";
for (auto note : numeric_notes)
ss << note << " ";
#if (CUDNN_VERSION >= 8200)
ss << "behavior_notes:" << " ";
for (auto note : behavior_notes)
ss << note << " ";
#endif
ss << "workSpaceSize: " << workSpaceSize;
return ss.str();
}
std::string const &
getTag() const {
return planTag;
}
void
setExecutionTime(float time_) {
execution_time_ms = time_;
}
float
getExecutionTime() const {
return execution_time_ms;
}
std::array<cudnnBackendNumericalNote_t,CUDNN_NUMERICAL_NOTE_TYPE_COUNT> const &
getNumericNotes() const {
return numeric_notes;
}
#if (CUDNN_VERSION >= 8200)
std::array<cudnnBackendBehaviorNote_t, CUDNN_BEHAVIOR_NOTE_TYPE_COUNT> const &
getBehaviorNotes() const {
return behavior_notes;
}
#endif
ExecutionPlan_v8(ExecutionPlan_v8 const &) = default;
ExecutionPlan_v8 &
operator=(ExecutionPlan_v8 const &) = default;
private:
void
fetchNotes(ManagedOpaqueDescriptor &extractedEngine) {
auto status = CUDNN_STATUS_SUCCESS;
int64_t elem_count = 0;
cudnnBackendDescriptor_t extractedEngine_ = extractedEngine->get_backend_descriptor();
status = cudnnBackendGetAttribute(extractedEngine_,
CUDNN_ATTR_ENGINE_NUMERICAL_NOTE,
CUDNN_TYPE_NUMERICAL_NOTE,
CUDNN_NUMERICAL_NOTE_TYPE_COUNT,
&elem_count,
numeric_notes.data());
if (status != CUDNN_STATUS_SUCCESS) {
set_error_and_throw_exception(this,
status,
"CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: GetAttribute "
"CUDNN_ATTR_ENGINE_NUMERICAL_NOTE Failed");
}
#if (CUDNN_VERSION >= 8200)
status = cudnnBackendGetAttribute(extractedEngine_,
CUDNN_ATTR_ENGINE_BEHAVIOR_NOTE,
CUDNN_TYPE_BEHAVIOR_NOTE,
CUDNN_BEHAVIOR_NOTE_TYPE_COUNT,
&elem_count,
behavior_notes.data());
if (status != CUDNN_STATUS_SUCCESS) {
set_error_and_throw_exception(this,
status,
"CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: GetAttribute "
"CUDNN_ATTR_ENGINE_BEHAVIOR_NOTE Failed");
}
#endif
}
void
buildTag(ManagedOpaqueDescriptor &extractedEngine) {
// Compute a unique tag for execution plan:
auto status = CUDNN_STATUS_SUCCESS;
std::stringstream tag{""};
int64_t elemCount = 0, engineId = 0, numKnobs = 0;
std::array<ManagedOpaqueDescriptor, CUDNN_KNOB_TYPE_COUNTS> extractedKnobs{{nullptr}};
for (auto &knob : extractedKnobs) {
knob = make_shared_backend_pointer(CUDNN_BACKEND_KNOB_CHOICE_DESCRIPTOR);
status = knob->get_status();
if (status != CUDNN_STATUS_SUCCESS) {
set_error_and_throw_exception(
this, status, "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: cudnnCreate Failed when compute tag");
}
}
cudnnBackendDescriptor_t extractedEngine_ = extractedEngine->get_backend_descriptor();
std::array<cudnnBackendDescriptor_t, CUDNN_KNOB_TYPE_COUNTS> extractedKnobs_{{nullptr}};
for (std::uint32_t i = 0; i < extractedKnobs.size(); i++) {
extractedKnobs_[i] = extractedKnobs[i]->get_backend_descriptor();
}
status = cudnnBackendGetAttribute(
extractedEngine_, CUDNN_ATTR_ENGINE_GLOBAL_INDEX, CUDNN_TYPE_INT64, 1, &elemCount, &engineId);
if (status != CUDNN_STATUS_SUCCESS) {
set_error_and_throw_exception(this,
status,
"CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: GetAttribute "
"CUDNN_ATTR_ENGINE_GLOBAL_INDEX Failed");
}
tag << "eng" << engineId;
status = cudnnBackendGetAttribute(engine_config->get_backend_descriptor(),
CUDNN_ATTR_ENGINECFG_KNOB_CHOICES,
CUDNN_TYPE_BACKEND_DESCRIPTOR,
CUDNN_KNOB_TYPE_COUNTS,
&numKnobs,
&(extractedKnobs_[0]));
if (status != CUDNN_STATUS_SUCCESS) {
set_error_and_throw_exception(this,
status,
"CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: GetAttribute "
"CUDNN_ATTR_ENGINECFG_KNOB_CHOICES Failed");
}
if (numKnobs > CUDNN_KNOB_TYPE_COUNTS) {
set_error_and_throw_exception(this,
status,
"CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: GetAttribute "
"numKnobs exceed the CUDNN_KNOB_TYPE_COUNTS");
}
for (int64_t idx = 0; idx < numKnobs; ++idx) {
const cudnnBackendDescriptor_t &knob = extractedKnobs_[idx];
cudnnBackendKnobType_t type = CUDNN_KNOB_TYPE_COUNTS;
int64_t choice = -2;
status = cudnnBackendGetAttribute(
knob, CUDNN_ATTR_KNOB_CHOICE_KNOB_TYPE, CUDNN_TYPE_KNOB_TYPE, 1, nullptr, &type);
if (status != CUDNN_STATUS_SUCCESS) {
set_error_and_throw_exception(this,
status,
"computeTag CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: GetAttribute "
"CUDNN_ATTR_KNOB_CHOICE_KNOB_TYPE Failed");
}
status = cudnnBackendGetAttribute(
knob, CUDNN_ATTR_KNOB_CHOICE_KNOB_VALUE, CUDNN_TYPE_INT64, 1, nullptr, &choice);
if (status != CUDNN_STATUS_SUCCESS) {
set_error_and_throw_exception(this,
status,
"CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: GetAttribute "
"CUDNN_ATTR_KNOB_CHOICE_KNOB_VALUE Failed");
}
tag << "_k" << type << "=" << choice;
}
planTag += tag.str();
}
void
computeWorkSpaceSize() {
auto status = cudnnBackendGetAttribute(pointer->get_backend_descriptor(),
CUDNN_ATTR_EXECUTION_PLAN_WORKSPACE_SIZE,
CUDNN_TYPE_INT64,
1,
NULL,
&workSpaceSize);
if (status != CUDNN_STATUS_SUCCESS) {
set_error_and_throw_exception(this,
status,
"CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: GetAttribute "
"CUDNN_ATTR_EXECUTION_PLAN_WORKSPACE_SIZE Failed");
}
if (workSpaceSize < 0) {
set_error_and_throw_exception(
this, status, "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: GetAttribute Workspace Size Invalid");
}
}
ExecutionPlan_v8() = default;
ManagedOpaqueDescriptor engine_config = nullptr;
cudnnHandle_t handle = nullptr;
std::string planTag;
std::int64_t workSpaceSize = 0;
std::array<cudnnBackendNumericalNote_t,CUDNN_NUMERICAL_NOTE_TYPE_COUNT> numeric_notes;
#if (CUDNN_VERSION >= 8200)
std::array<cudnnBackendBehaviorNote_t, CUDNN_BEHAVIOR_NOTE_TYPE_COUNT> behavior_notes;
#endif
float execution_time_ms = 0.0f;
};
class ExecutionPlanBuilder_v8 {
public:
auto
setHandle(cudnnHandle_t handle_) -> ExecutionPlanBuilder_v8 & {
m_execution_plan.handle = handle_;
return *this;
}
auto
setEngineConfig(EngineConfig_v8 const &engine_config_) -> ExecutionPlanBuilder_v8 & {
m_execution_plan.engine_config = engine_config_.get_desc();
m_execution_plan.planTag = engine_config_.getTag();
return *this;
}
auto
setEngineConfig(ManagedOpaqueDescriptor &desc, std::string const &opGraphTag_ = "") -> ExecutionPlanBuilder_v8 & {
m_execution_plan.engine_config = desc;
m_execution_plan.planTag = opGraphTag_;
return *this;
}
ExecutionPlan_v8 &&
build() {
if (m_execution_plan.handle == nullptr) {
set_error_and_throw_exception(
&m_execution_plan,
CUDNN_STATUS_BAD_PARAM,
"CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: Check and Set the CUDNN_ATTR_EXECUTION_PLAN_HANDLE");
return std::move(m_execution_plan);
};
if (m_execution_plan.engine_config == nullptr) {
set_error_and_throw_exception(
&m_execution_plan,
CUDNN_STATUS_BAD_PARAM,
"CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: Check and Set the CUDNN_ATTR_EXECUTION_PLAN_ENGINE_CONFIG");
return std::move(m_execution_plan);
};
// Create a descriptor. Memory allocation happens here.
auto status = m_execution_plan.initialize_managed_backend_pointer(CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR);
if (status != CUDNN_STATUS_SUCCESS) {
set_error_and_throw_exception(
&m_execution_plan, status, "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: cudnnCreate Failed");
return std::move(m_execution_plan);
}
status = cudnnBackendSetAttribute(m_execution_plan.pointer->get_backend_descriptor(),
CUDNN_ATTR_EXECUTION_PLAN_ENGINE_CONFIG,
CUDNN_TYPE_BACKEND_DESCRIPTOR,
1,
&(m_execution_plan.engine_config->get_backend_descriptor()));
if (status != CUDNN_STATUS_SUCCESS) {
set_error_and_throw_exception(
&m_execution_plan,
status,
"CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: SetAttribute CUDNN_ATTR_EXECUTION_PLAN_ENGINE_CONFIG Failed");
return std::move(m_execution_plan);
}
status = cudnnBackendSetAttribute(m_execution_plan.pointer->get_backend_descriptor(),
CUDNN_ATTR_EXECUTION_PLAN_HANDLE,
CUDNN_TYPE_HANDLE,
1,
&m_execution_plan.handle);
if (status != CUDNN_STATUS_SUCCESS) {
set_error_and_throw_exception(
&m_execution_plan,
status,
"CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: SetAttribute CUDNN_ATTR_EXECUTION_PLAN_HANDLE Failed");
return std::move(m_execution_plan);
}
// Finalizing the descriptor
status = cudnnBackendFinalize(m_execution_plan.pointer->get_backend_descriptor());
if (status != CUDNN_STATUS_SUCCESS) {
set_error_and_throw_exception(
&m_execution_plan, status, "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: cudnnFinalize Descriptor Failed");
return std::move(m_execution_plan);
}
ManagedOpaqueDescriptor extractedEngine = make_shared_backend_pointer(CUDNN_BACKEND_ENGINE_DESCRIPTOR);
status = extractedEngine->get_status();
if (status != CUDNN_STATUS_SUCCESS) {
set_error_and_throw_exception(
&m_execution_plan, status, "CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: cudnnCreate of CUDNN_BACKEND_ENGINE_DESCRIPTOR failed when compute tag");
return std::move(m_execution_plan);
}
cudnnBackendDescriptor_t extractedEngine_ = extractedEngine->get_backend_descriptor();
int64_t elemCount = 0;
status = cudnnBackendGetAttribute(m_execution_plan.engine_config->get_backend_descriptor(),
CUDNN_ATTR_ENGINECFG_ENGINE,
CUDNN_TYPE_BACKEND_DESCRIPTOR,
1,
&elemCount,
&extractedEngine_);
if (status != CUDNN_STATUS_SUCCESS) {
set_error_and_throw_exception(&m_execution_plan,
status,
"CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: GetAttribute "
"CUDNN_ATTR_ENGINECFG_ENGINE Failed");
return std::move(m_execution_plan);
}
m_execution_plan.buildTag(extractedEngine);
m_execution_plan.fetchNotes(extractedEngine);
m_execution_plan.computeWorkSpaceSize();
getLogger() << "[cudnn_frontend] " << m_execution_plan << std::endl;
return std::move(m_execution_plan);
}
explicit ExecutionPlanBuilder_v8() = default;
~ExecutionPlanBuilder_v8() = default;
ExecutionPlanBuilder_v8(ExecutionPlanBuilder_v8 &&) = delete;
ExecutionPlanBuilder_v8(ExecutionPlanBuilder_v8 const &) = delete;
ExecutionPlanBuilder_v8 &
operator=(ExecutionPlanBuilder_v8 const &) = delete;
private:
ExecutionPlan_v8 m_execution_plan;
};
using ExecutionPlan = ExecutionPlan_v8;
using ExecutionPlanBuilder = ExecutionPlanBuilder_v8;
}
cudnn_frontend::ExecutionPlanBuilder_v8::setEngineConfig
auto setEngineConfig(EngineConfig_v8 const &engine_config_) -> ExecutionPlanBuilder_v8 &
Set engine Config for the Plan.
Definition: cudnn_frontend_ExecutionPlan.h:280
cudnn_frontend::getLogger
ConditionalStreamer & getLogger()
Definition: cudnn_frontend_Logging.h:72
cudnn_frontend::set_error_and_throw_exception
static void set_error_and_throw_exception(BackendDescriptor const *desc, cudnnStatus_t status, const char *message)
Definition: cudnn_frontend_utils.h:138
cudnn_frontend::ExecutionPlan_v8
Definition: cudnn_frontend_ExecutionPlan.h:51
cudnn_frontend::BackendDescriptor
Definition: cudnn_backend_base.h:99
cudnn_frontend::make_shared_backend_pointer
static ManagedOpaqueDescriptor make_shared_backend_pointer(cudnnBackendDescriptorType_t type)
Definition: cudnn_backend_base.h:88
cudnn_frontend::EngineConfig_v8
Definition: cudnn_frontend_EngineConfig.h:51
cudnn_frontend::ExecutionPlan_v8::buildTag
void buildTag(ManagedOpaqueDescriptor &extractedEngine)
Definition: cudnn_frontend_ExecutionPlan.h:152
cudnn_frontend::ExecutionPlan_v8::getTag
std::string const & getTag() const
Definition: cudnn_frontend_ExecutionPlan.h:88
cudnn_frontend::ExecutionPlan_v8::workSpaceSize
std::int64_t workSpaceSize
Definition: cudnn_frontend_ExecutionPlan.h:253
cudnn_frontend::ExecutionPlanBuilder_v8
Definition: cudnn_frontend_ExecutionPlan.h:266
cudnn_frontend_utils.h
cudnn_frontend::ExecutionPlanBuilder_v8::setEngineConfig
auto setEngineConfig(ManagedOpaqueDescriptor &desc, std::string const &opGraphTag_="") -> ExecutionPlanBuilder_v8 &
Set engine Config for the Plan.
Definition: cudnn_frontend_ExecutionPlan.h:288
cudnn_frontend::ExecutionPlan_v8::fetchNotes
void fetchNotes(ManagedOpaqueDescriptor &extractedEngine)
Definition: cudnn_frontend_ExecutionPlan.h:119
cudnn_frontend::ExecutionPlan_v8::computeWorkSpaceSize
void computeWorkSpaceSize()
Definition: cudnn_frontend_ExecutionPlan.h:229
cudnn_frontend::ExecutionPlan_v8::execution_time_ms
float execution_time_ms
Definition: cudnn_frontend_ExecutionPlan.h:259
cudnn_frontend::ExecutionPlanBuilder_v8::build
ExecutionPlan_v8 && build()
Definition: cudnn_frontend_ExecutionPlan.h:298
cudnn_frontend::ExecutionPlan_v8::planTag
std::string planTag
Definition: cudnn_frontend_ExecutionPlan.h:251
cudnn_frontend::ExecutionPlanBuilder_v8::setHandle
auto setHandle(cudnnHandle_t handle_) -> ExecutionPlanBuilder_v8 &
Set engine for the ExecutionPlan_v8.
Definition: cudnn_frontend_ExecutionPlan.h:274
cudnn_frontend::ExecutionPlanBuilder_v8::m_execution_plan
ExecutionPlan_v8 m_execution_plan
Definition: cudnn_frontend_ExecutionPlan.h:393
cudnn_frontend::ExecutionPlan_v8::handle
cudnnHandle_t handle
Definition: cudnn_frontend_ExecutionPlan.h:250
cudnn_frontend::ManagedOpaqueDescriptor
std::shared_ptr< OpaqueBackendPointer > ManagedOpaqueDescriptor
Definition: cudnn_backend_base.h:84
cudnn_frontend::ExecutionPlan_v8::describe
std::string describe() const override
Return a string describing the backend Descriptor.
Definition: cudnn_frontend_ExecutionPlan.h:71
cudnn_frontend::ExecutionPlan_v8::engine_config
ManagedOpaqueDescriptor engine_config
Definition: cudnn_frontend_ExecutionPlan.h:249
cudnn_frontend
Definition: cudnn_backend_base.h:29
cudnn_frontend::ExecutionPlan_v8::operator=
ExecutionPlan_v8 & operator=(ExecutionPlan_v8 &&)=default
type
cudnn_frontend::ExecutionPlan_v8::getNumericNotes
std::array< cudnnBackendNumericalNote_t, CUDNN_NUMERICAL_NOTE_TYPE_COUNT > const & getNumericNotes() const
Definition: cudnn_frontend_ExecutionPlan.h:103
cudnn_frontend::ExecutionPlan_v8::numeric_notes
std::array< cudnnBackendNumericalNote_t, CUDNN_NUMERICAL_NOTE_TYPE_COUNT > numeric_notes
Definition: cudnn_frontend_ExecutionPlan.h:254
cudnn_frontend::ExecutionPlan_v8::getExecutionTime
float getExecutionTime() const
Definition: cudnn_frontend_ExecutionPlan.h:98
cudnn_frontend_Engine.h
cudnn_frontend::ExecutionPlan_v8::ExecutionPlanBuilder_v8
friend class ExecutionPlanBuilder_v8
Definition: cudnn_frontend_ExecutionPlan.h:53
cudnn_frontend::ExecutionPlan_v8::ExecutionPlan_v8
ExecutionPlan_v8()=default
cudnn_frontend::ExecutionPlan_v8::~ExecutionPlan_v8
~ExecutionPlan_v8()=default
cudnn_frontend::ExecutionPlan_v8::setExecutionTime
void setExecutionTime(float time_)
Definition: cudnn_frontend_ExecutionPlan.h:93
cudnn_frontend::BackendDescriptor::status
cudnnStatus_t status
Shared pointer of the OpaqueBackendPointer.
Definition: cudnn_backend_base.h:160
cudnn_frontend::BackendDescriptor::pointer
ManagedOpaqueDescriptor pointer
Definition: cudnn_backend_base.h:158
cudnn_frontend::ExecutionPlan_v8::getWorkspaceSize
auto getWorkspaceSize(void) const -> int64_t
Query the workspace requirement for the given plan.
Definition: cudnn_frontend_ExecutionPlan.h:66