cudnn_frontend_Engine.h
Go to the documentation of this file.
/*
* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#pragma once
#include <algorithm>
#include <array>
#include <functional>
#include <memory>
#include <sstream>
#include <utility>
#include <vector>
#include <cudnn.h>
#include <cudnn_backend.h>
#include "cudnn_frontend_OperationGraph.h"
#include "cudnn_frontend_utils.h"
namespace cudnn_frontend {
class Engine_v8 : public BackendDescriptor {
private:
Engine_v8() = default;
Engine_v8(Engine_v8 const &) = delete;
Engine_v8 &
operator=(Engine_v8 const &) = delete;
class Knob {
public:
Knob(cudnnBackendKnobType_t type_, int64_t max, int64_t min, int64_t stride_)
: knobType(type_), maxValue(max), minValue(min), stride(stride_) {}
std::string
describe() const {
std::stringstream ss;
ss << "Knob:" << knobType;
ss << " Min: " << minValue;
ss << " Max: " << maxValue;
ss << " Stride: " << stride;
return ss.str();
}
void
setChoice(uint64_t val_) {
choice = val_;
}
int64_t
getChoice() const {
return choice;
}
cudnnBackendKnobType_t
getKnobType() const {
return knobType;
}
int64_t
getMinValue() const {
return minValue;
}
int64_t
getMaxValue() const {
return minValue;
}
int64_t
getStride() const {
return stride;
}
private:
cudnnBackendKnobType_t knobType = CUDNN_KNOB_TYPE_COUNTS;
int64_t maxValue = 0, minValue = 0, stride = 0;
int64_t choice = -1;
};
ManagedOpaqueDescriptor opGraph = nullptr;
int64_t idx = -1;
int64_t numKnobs = 0;
std::array<ManagedOpaqueDescriptor, CUDNN_KNOB_TYPE_COUNTS> bKnobs = {};
std::vector<Knob> knobs;
std::string opGraphTag;
void
buildKnobs() {
cudnnStatus_t status;
for (auto i = 0; i < numKnobs; i++) {
auto bKnob = bKnobs[i]->get_backend_descriptor();
cudnnBackendKnobType_t type;
int64_t maxValue, minValue, stride, elemCount;
status =
cudnnBackendGetAttribute(bKnob, CUDNN_ATTR_KNOB_INFO_TYPE, CUDNN_TYPE_KNOB_TYPE, 1, &elemCount, &type);
if (status != CUDNN_STATUS_SUCCESS) {
set_error_and_throw_exception(this,
status,
"CUDNN_BACKEND_ENGINE_DESCRIPTOR: CUDNN_BACKEND_KNOB_INFO_DESCRIPTOR "
"GetAttribute CUDNN_ATTR_KNOB_INFO_TYPE failed");
}
status = cudnnBackendGetAttribute(
bKnob, CUDNN_ATTR_KNOB_INFO_MAXIMUM_VALUE, CUDNN_TYPE_INT64, 1, &elemCount, &maxValue);
if (status != CUDNN_STATUS_SUCCESS) {
set_error_and_throw_exception(this,
status,
"CUDNN_BACKEND_ENGINE_DESCRIPTOR: CUDNN_BACKEND_KNOB_INFO_DESCRIPTOR "
"GetAttribute CUDNN_ATTR_KNOB_INFO_MAXIMUM_VALUE Failed");
}
status = cudnnBackendGetAttribute(
bKnob, CUDNN_ATTR_KNOB_INFO_MINIMUM_VALUE, CUDNN_TYPE_INT64, 1, &elemCount, &minValue);
if (status != CUDNN_STATUS_SUCCESS) {
set_error_and_throw_exception(this,
status,
"CUDNN_BACKEND_ENGINE_DESCRIPTOR: CUDNN_BACKEND_KNOB_INFO_DESCRIPTOR "
"GetAttribute CUDNN_ATTR_KNOB_INFO_MINIMUM_VALUE Failed");
}
status =
cudnnBackendGetAttribute(bKnob, CUDNN_ATTR_KNOB_INFO_STRIDE, CUDNN_TYPE_INT64, 1, &elemCount, &stride);
if (status != CUDNN_STATUS_SUCCESS) {
set_error_and_throw_exception(this,
status,
"CUDNN_BACKEND_ENGINE_DESCRIPTOR: CUDNN_BACKEND_KNOB_INFO_DESCRIPTOR "
"GetAttribute CUDNN_ATTR_KNOB_INFO_STRIDE Failed");
}
knobs.emplace_back(Knob(type, maxValue, minValue, stride));
}
}
public:
friend class EngineBuilder_v8;
std::string
describe() const override {
std::stringstream ss;
ss << "CUDNN_BACKEND_ENGINE_DESCRIPTOR :";
ss << " ID: " << idx;
ss << " Has " << numKnobs << " knobs";
return ss.str();
}
Engine_v8(Engine_v8 &&from)
: BackendDescriptor(from.get_desc(), from.get_status(), from.get_error()),
opGraph(from.opGraph),
idx(from.idx),
opGraphTag(from.opGraphTag) {
cudnnStatus_t status;
for (uint64_t i = 0; i < bKnobs.size(); i++) {
bKnobs[i] = make_shared_backend_pointer(CUDNN_BACKEND_KNOB_INFO_DESCRIPTOR);
if (bKnobs[i]->is_good() == false) {
status = bKnobs[i]->get_status();
set_error_and_throw_exception(
this,
status,
"CUDNN_BACKEND_ENGINE_DESCRIPTOR: CUDNN_BACKEND_KNOB_INFO_DESCRIPTOR cudnnCreate Failed");
}
}
std::array<cudnnBackendDescriptor_t, CUDNN_KNOB_TYPE_COUNTS> bKnobs_ =
{};
for (std::uint32_t i = 0; i < bKnobs.size(); i++) {
bKnobs_[i] = bKnobs[i]->get_backend_descriptor();
}
status = cudnnBackendGetAttribute(pointer->get_backend_descriptor(),
CUDNN_ATTR_ENGINE_KNOB_INFO,
CUDNN_TYPE_BACKEND_DESCRIPTOR,
CUDNN_KNOB_TYPE_COUNTS,
&numKnobs,
bKnobs_.data());
if (status != CUDNN_STATUS_SUCCESS) {
set_error_and_throw_exception(
this, status, "CUDNN_BACKEND_ENGINE_DESCRIPTOR: GetAttribute CUDNN_ATTR_ENGINE_KNOB_INFO Query Failed");
}
buildKnobs();
}
Engine_v8 &
operator=(Engine_v8 &&) = default;
~Engine_v8() = default;
std::string const &
getTag() const {
return opGraphTag;
}
std::vector<Knob> &
getSupportedKnobs() {
return knobs;
}
std::vector<Knob> const &
getFinalizedKnobs() const {
return knobs;
}
bool knobs_set() const {
bool is_knob_set = false;
for (auto i = 0; i < numKnobs; i++) {
if(knobs[i].getChoice() != -1) {
is_knob_set = true;
break;
}
}
return is_knob_set;
}
};
class EngineBuilder_v8 {
public:
auto
setOperationGraph(OperationGraph_v8 const &opGraph_) -> EngineBuilder_v8 & {
m_engine.opGraph = opGraph_.get_desc();
m_engine.opGraphTag = opGraph_.getTag();
return *this;
}
auto
setOperationGraph(ManagedOpaqueDescriptor desc_) -> EngineBuilder_v8 & {
m_engine.opGraph = desc_;
return *this;
}
auto
setGlobalEngineIdx(int64_t idx_) -> EngineBuilder_v8 & {
m_engine.idx = idx_;
return *this;
}
Engine_v8 &&
build() {
if (m_engine.idx < 0) {
set_error_and_throw_exception(
&m_engine,
CUDNN_STATUS_BAD_PARAM,
"CUDNN_BACKEND_ENGINE_DESCRIPTOR: Check and Set the CUDNN_ATTR_ENGINE_GLOBAL_INDEX to valid value");
return std::move(m_engine);
}
if (m_engine.opGraph == nullptr) {
set_error_and_throw_exception(
&m_engine,
CUDNN_STATUS_BAD_PARAM,
"CUDNN_BACKEND_ENGINE_DESCRIPTOR: Check and Set CUDNN_ATTR_ENGINE_OPERATION_GRAPH to valid value");
return std::move(m_engine);
}
// Create a descriptor. Memory allocation happens here.
auto status = m_engine.initialize_managed_backend_pointer(CUDNN_BACKEND_ENGINE_DESCRIPTOR);
if (status != CUDNN_STATUS_SUCCESS) {
set_error_and_throw_exception(
&m_engine, status, "CUDNN_BACKEND_ENGINE_DESCRIPTOR: cudnnCreate Descriptor Failed");
return std::move(m_engine);
}
status = cudnnBackendSetAttribute(m_engine.pointer->get_backend_descriptor(),
CUDNN_ATTR_ENGINE_OPERATION_GRAPH,
CUDNN_TYPE_BACKEND_DESCRIPTOR,
1,
&(m_engine.opGraph->get_backend_descriptor()));
if (status != CUDNN_STATUS_SUCCESS) {
set_error_and_throw_exception(
&m_engine,
status,
"CUDNN_BACKEND_ENGINE_DESCRIPTOR: SetAttribute CUDNN_ATTR_ENGINE_OPERATION_GRAPH Failed");
return std::move(m_engine);
}
status = cudnnBackendSetAttribute(m_engine.pointer->get_backend_descriptor(),
CUDNN_ATTR_ENGINE_GLOBAL_INDEX,
CUDNN_TYPE_INT64,
1,
&m_engine.idx);
if (status != CUDNN_STATUS_SUCCESS) {
set_error_and_throw_exception(
&m_engine,
status,
"CUDNN_BACKEND_ENGINE_DESCRIPTOR: SetAttribute CUDNN_ATTR_ENGINE_GLOBAL_INDEX Failed");
return std::move(m_engine);
}
// Finalizing the descriptor
status = cudnnBackendFinalize(m_engine.pointer->get_backend_descriptor());
if (status != CUDNN_STATUS_SUCCESS) {
set_error_and_throw_exception(&m_engine, status, "CUDNN_BACKEND_ENGINE_DESCRIPTOR: cudnnFinalize Failed");
return std::move(m_engine);
}
getLogger() << "[cudnn_frontend] " << m_engine << std::endl;
return std::move(m_engine);
}
explicit EngineBuilder_v8() = default;
~EngineBuilder_v8() = default;
EngineBuilder_v8(EngineBuilder_v8 &&) = delete;
EngineBuilder_v8(EngineBuilder_v8 const &) = delete;
EngineBuilder_v8 &
operator=(EngineBuilder_v8 const &) = delete;
private:
Engine_v8 m_engine;
};
}
cudnn_frontend::Engine_v8::Knob::choice
int64_t choice
Choice set by the user.
Definition: cudnn_frontend_Engine.h:110
cudnn_frontend::Engine_v8::opGraphTag
std::string opGraphTag
Definition: cudnn_frontend_Engine.h:118
cudnn_frontend::Engine_v8::operator=
Engine_v8 & operator=(Engine_v8 const &)=delete
cudnn_frontend::EngineBuilder_v8::setOperationGraph
auto setOperationGraph(ManagedOpaqueDescriptor desc_) -> EngineBuilder_v8 &
Set operationGraph for the engine.
Definition: cudnn_frontend_Engine.h:260
cudnn_frontend::Engine_v8::Knob::maxValue
int64_t maxValue
Definition: cudnn_frontend_Engine.h:109
cudnn_frontend::getLogger
ConditionalStreamer & getLogger()
Definition: cudnn_frontend_Logging.h:72
cudnn_frontend::Engine_v8::Knob::stride
int64_t stride
min, max and stride of the knob value
Definition: cudnn_frontend_Engine.h:109
cudnn_frontend::set_error_and_throw_exception
static void set_error_and_throw_exception(BackendDescriptor const *desc, cudnnStatus_t status, const char *message)
Definition: cudnn_frontend_utils.h:138
cudnn_frontend::Engine_v8::Knob::getKnobType
cudnnBackendKnobType_t getKnobType() const
Definition: cudnn_frontend_Engine.h:88
cudnn_frontend::EngineBuilder_v8::setGlobalEngineIdx
auto setGlobalEngineIdx(int64_t idx_) -> EngineBuilder_v8 &
Set engine index for the engine.
Definition: cudnn_frontend_Engine.h:266
cudnn_frontend::Engine_v8::EngineBuilder_v8
friend class EngineBuilder_v8
Definition: cudnn_frontend_Engine.h:165
cudnn_frontend::Engine_v8::Knob::getStride
int64_t getStride() const
Definition: cudnn_frontend_Engine.h:103
cudnn_frontend::Engine_v8::getTag
std::string const & getTag() const
Definition: cudnn_frontend_Engine.h:213
cudnn_frontend::BackendDescriptor
Definition: cudnn_backend_base.h:99
cudnn_frontend::Engine_v8::idx
int64_t idx
Global Index of the engine for the given operationGraph.
Definition: cudnn_frontend_Engine.h:114
cudnn_frontend_OperationGraph.h
cudnn_frontend::make_shared_backend_pointer
static ManagedOpaqueDescriptor make_shared_backend_pointer(cudnnBackendDescriptorType_t type)
Definition: cudnn_backend_base.h:88
cudnn_frontend::EngineBuilder_v8::build
Engine_v8 && build()
Definition: cudnn_frontend_Engine.h:275
cudnn_frontend::Engine_v8::Engine_v8
Engine_v8()=default
cudnn_frontend::Engine_v8::Knob::getMinValue
int64_t getMinValue() const
Definition: cudnn_frontend_Engine.h:93
cudnn_frontend::Engine_v8::Knob::describe
std::string describe() const
Definition: cudnn_frontend_Engine.h:68
cudnn_frontend::Engine_v8::buildKnobs
void buildKnobs()
Called from the constructor builds the internal knobs vector.
Definition: cudnn_frontend_Engine.h:122
cudnn_frontend::Engine_v8::opGraph
ManagedOpaqueDescriptor opGraph
Definition: cudnn_frontend_Engine.h:113
cudnn_frontend::BackendDescriptor::get_desc
ManagedOpaqueDescriptor get_desc() const
Returns a copy of underlying managed descriptor.
Definition: cudnn_backend_base.h:138
cudnn_frontend::Engine_v8::Knob::knobType
cudnnBackendKnobType_t knobType
Definition: cudnn_frontend_Engine.h:108
cudnn_frontend_utils.h
cudnn_frontend::Engine_v8::bKnobs
std::array< ManagedOpaqueDescriptor, CUDNN_KNOB_TYPE_COUNTS > bKnobs
Opaque pointer to the backend knobs.
Definition: cudnn_frontend_Engine.h:116
cudnn_frontend::EngineBuilder_v8::setOperationGraph
auto setOperationGraph(OperationGraph_v8 const &opGraph_) -> EngineBuilder_v8 &
Set operationGraph for the engine.
Definition: cudnn_frontend_Engine.h:252
cudnn_frontend::Engine_v8::Knob::getMaxValue
int64_t getMaxValue() const
Definition: cudnn_frontend_Engine.h:98
cudnn_frontend::Engine_v8::Knob::minValue
int64_t minValue
Definition: cudnn_frontend_Engine.h:109
cudnn_frontend::Engine_v8::Knob::getChoice
int64_t getChoice() const
Definition: cudnn_frontend_Engine.h:83
cudnn_frontend::BackendDescriptor::get_status
cudnnStatus_t get_status() const
Current status of the descriptor.
Definition: cudnn_backend_base.h:114
cudnn_frontend::Engine_v8::getFinalizedKnobs
std::vector< Knob > const & getFinalizedKnobs() const
Returns a final vector of knobs. Used in EngineConfigBuilder.
Definition: cudnn_frontend_Engine.h:225
cudnn_frontend::Engine_v8::getSupportedKnobs
std::vector< Knob > & getSupportedKnobs()
Returns a vector of knobs to the user for modification.
Definition: cudnn_frontend_Engine.h:219
cudnn_frontend::Engine_v8
Definition: cudnn_frontend_Engine.h:52
cudnn_frontend::ManagedOpaqueDescriptor
std::shared_ptr< OpaqueBackendPointer > ManagedOpaqueDescriptor
Definition: cudnn_backend_base.h:84
cudnn_frontend::Engine_v8::~Engine_v8
~Engine_v8()=default
cudnn_frontend
Definition: cudnn_backend_base.h:29
cudnn_frontend::Engine_v8::describe
std::string describe() const override
Return a string describing the backend Descriptor.
Definition: cudnn_frontend_Engine.h:167
cudnn_frontend::BackendDescriptor::get_error
const char * get_error() const
Diagonistic error message if any.
Definition: cudnn_backend_base.h:132
cudnn_frontend::Engine_v8::Engine_v8
Engine_v8(Engine_v8 &&from)
Definition: cudnn_frontend_Engine.h:174
type
cudnn_frontend::Engine_v8::Knob::setChoice
void setChoice(uint64_t val_)
Definition: cudnn_frontend_Engine.h:78
cudnn_frontend::EngineBuilder_v8::m_engine
Engine_v8 m_engine
Definition: cudnn_frontend_Engine.h:344
cudnn_frontend::Engine_v8::Knob
Definition: cudnn_frontend_Engine.h:62
cudnn_frontend::Engine_v8::knobs
std::vector< Knob > knobs
Definition: cudnn_frontend_Engine.h:117
cudnn_frontend::EngineBuilder_v8
Definition: cudnn_frontend_Engine.h:244
cudnn_frontend::Engine_v8::Knob::Knob
Knob(cudnnBackendKnobType_t type_, int64_t max, int64_t min, int64_t stride_)
Definition: cudnn_frontend_Engine.h:64
cudnn_frontend::OperationGraph_v8
Definition: cudnn_frontend_OperationGraph.h:52
cudnn_frontend::Engine_v8::knobs_set
bool knobs_set() const
Definition: cudnn_frontend_Engine.h:229
cudnn_frontend::BackendDescriptor::status
cudnnStatus_t status
Shared pointer of the OpaqueBackendPointer.
Definition: cudnn_backend_base.h:160
cudnn_frontend::BackendDescriptor::pointer
ManagedOpaqueDescriptor pointer
Definition: cudnn_backend_base.h:158
cudnn_frontend::Engine_v8::numKnobs
int64_t numKnobs
Count of the backend knobs in the engine.
Definition: cudnn_frontend_Engine.h:115