cudnn_frontend
cudnn_frontend::BackendDescriptor
cudnn_frontend::ConditionalStreamer
cudnn_frontend::ConvDesc_v8
cudnn_frontend::ConvDescBuilder_v8
cudnn_frontend::cudnnException
cudnn_frontend::Engine_v8
cudnn_frontend::EngineBuilder_v8
cudnn_frontend::EngineConfig_v8
cudnn_frontend::EngineConfigBuilder_v8
cudnn_frontend::EngineConfigGenerator
cudnn_frontend::EngineFallbackList_v8
cudnn_frontend::EngineFallbackListBuilder_v8
cudnn_frontend::EngineHeuristics_v8
cudnn_frontend::EngineHeuristicsBuilder_v8
cudnn_frontend::ExecutionPlan_v8
cudnn_frontend::ExecutionPlanBuilder_v8
cudnn_frontend::ExecutionPlanCache_v1
cudnn_frontend::ExecutionPlanCache_v2
cudnn_frontend::MatMulDesc_v8
cudnn_frontend::MatMulDescBuilder_v8
cudnn_frontend::OpaqueBackendPointer
cudnn_frontend::Operation_v8
cudnn_frontend::OperationBuilder_v8
cudnn_frontend::OperationGraph_v8
cudnn_frontend::OperationGraphBuilder_v8
cudnn_frontend::PointWiseDesc_v8
cudnn_frontend::PointWiseDescBuilder_v8
cudnn_frontend::ReductionDesc_v8
cudnn_frontend::ReductionDescBuilder_v8
cudnn_frontend::Tensor_v8
cudnn_frontend::TensorBuilder_v8
cudnn_frontend::VariantPack_v8
cudnn_frontend::VariantPackBuilder_v8
CudnnFindSamplingTechnique
CUDNN_FIND_SAMPLE_ONCE

Sample once quick but may have unstable values.

CUDNN_FIND_SAMPLE_MEDIAN_OF_THREE

Sample 3 times and take median.

CUDNN_FIND_SAMPLE_TILL_STABLE

Sample multiple times till stable.

std::shared_ptr< OpaqueBackendPointer > using cudnn_frontend::ManagedOpaqueDescriptor = typedef std::shared_ptr<OpaqueBackendPointer> ManagedOpaqueDescriptor
Tensor_v8 using cudnn_frontend::Tensor = typedef Tensor_v8 Tensor
TensorBuilder_v8 using cudnn_frontend::TensorBuilder = typedef TensorBuilder_v8 TensorBuilder
ConvDesc_v8 using cudnn_frontend::ConvDesc = typedef ConvDesc_v8 ConvDesc
ConvDescBuilder_v8 using cudnn_frontend::ConvDescBuilder = typedef ConvDescBuilder_v8 ConvDescBuilder
PointWiseDescBuilder_v8 using cudnn_frontend::PointWiseDescBuilder = typedef PointWiseDescBuilder_v8 PointWiseDescBuilder
PointWiseDesc_v8 using cudnn_frontend::PointWiseDesc = typedef PointWiseDesc_v8 PointWiseDesc
MatMulDesc_v8 using cudnn_frontend::MatMulDesc = typedef MatMulDesc_v8 MatMulDesc
MatMulDescBuilder_v8 using cudnn_frontend::MatMulDescBuilder = typedef MatMulDescBuilder_v8 MatMulDescBuilder
ReductionDesc_v8 using cudnn_frontend::ReductionDesc = typedef ReductionDesc_v8 ReductionDesc
ReductionDescBuilder_v8 using cudnn_frontend::ReductionDescBuilder = typedef ReductionDescBuilder_v8 ReductionDescBuilder
Operation_v8 using cudnn_frontend::Operation = typedef Operation_v8 Operation
OperationBuilder_v8 using cudnn_frontend::OperationBuilder = typedef OperationBuilder_v8 OperationBuilder
EngineHeuristicsBuilder_v8 using cudnn_frontend::EngineHeuristicsBuilder = typedef EngineHeuristicsBuilder_v8 EngineHeuristicsBuilder
EngineHeuristics_v8 using cudnn_frontend::EngineHeuristics = typedef EngineHeuristics_v8 EngineHeuristics
EngineBuilder_v8 using cudnn_frontend::EngineBuilder = typedef EngineBuilder_v8 EngineBuilder
Engine_v8 using cudnn_frontend::Engine = typedef Engine_v8 Engine
EngineConfig_v8 using cudnn_frontend::EngineConfig = typedef EngineConfig_v8 EngineConfig
EngineConfigBuilder_v8 using cudnn_frontend::EngineConfigBuilder = typedef EngineConfigBuilder_v8 EngineConfigBuilder
VariantPack_v8 using cudnn_frontend::VariantPack = typedef VariantPack_v8 VariantPack
VariantPackBuilder_v8 using cudnn_frontend::VariantPackBuilder = typedef VariantPackBuilder_v8 VariantPackBuilder
EngineFallbackList_v8 using cudnn_frontend::EngineFallbackList = typedef EngineFallbackList_v8 EngineFallbackList
EngineFallbackListBuilder_v8 using cudnn_frontend::EngineFallbackListBuilder = typedef EngineFallbackListBuilder_v8 EngineFallbackListBuilder
std::vector< ManagedOpaqueDescriptor > using cudnn_frontend::EngineConfigList = typedef std::vector<ManagedOpaqueDescriptor> EngineConfigList

EngineConfigList class This is a RAII type class that holds naked EngineConfig backendDescriptor. The purpose of this class is to provide an easy interface to store the EngineConfigs generated from various source and apply a filter.

std::vector< cudnn_frontend::ExecutionPlan > using cudnn_frontend::executionPlans_t = typedef std::vector<cudnn_frontend::ExecutionPlan> executionPlans_t

Variety of renames.

std::function< bool(cudnn_frontend::ExecutionPlan const &plan)> using cudnn_frontend::Predicate = typedef std::function<bool(cudnn_frontend::ExecutionPlan const &plan)> Predicate
std::function< cudnn_frontend::EngineConfigList(cudnn_frontend::OperationGraph &)> using cudnn_frontend::GeneratorSource = typedef std::function<cudnn_frontend::EngineConfigList(cudnn_frontend::OperationGraph &)> GeneratorSource
ExecutionPlan_v8 using cudnn_frontend::ExecutionPlan = typedef ExecutionPlan_v8 ExecutionPlan
ExecutionPlanBuilder_v8 using cudnn_frontend::ExecutionPlanBuilder = typedef ExecutionPlanBuilder_v8 ExecutionPlanBuilder
ExecutionPlanCache_v2 using cudnn_frontend::ExecutionPlanCache = typedef ExecutionPlanCache_v2 ExecutionPlanCache
OperationGraph_v8 using cudnn_frontend::OperationGraph = typedef OperationGraph_v8 OperationGraph
OperationGraphBuilder_v8 using cudnn_frontend::OperationGraphBuilder = typedef OperationGraphBuilder_v8 OperationGraphBuilder
std::vector< int64_t > using cudnn_frontend::feature_vector_t = typedef std::vector<int64_t> feature_vector_t

Detailed feature_vector. Generally the Tensor and Operation properties.

ManagedOpaqueDescriptor static ManagedOpaqueDescriptor cudnn_frontend::make_shared_backend_pointer (cudnnBackendDescriptorType_t type) make_shared_backend_pointer cudnnBackendDescriptorType_t type cudnn_frontend::ExecutionPlanBuilder_v8::build cudnn_frontend::ExecutionPlan_v8::buildTag cudnn_frontend::Engine_v8::Engine_v8 cudnn_frontend::EngineConfig_v8::EngineConfig_v8 cudnn_frontend::EngineHeuristics_v8::getEngineConfig hasNumericalNote cudnn_frontend::BackendDescriptor::initialize_managed_backend_pointer
auto static auto cudnn_frontend::filter (Predicate pred, executionPlans_t &plans) -> executionPlans_t filter Predicate pred executionPlans_t & plans

Filter out the execution plan based on the prerequisite conditions. Goes through vector of execution plans and if the predicate returns not to block (false), it is inserted into the filtered plans.

getLogger
cudnn_frontend::EngineConfigGenerator::cudnnGetPlan
get_heuristics_list
auto static auto cudnn_frontend::get_fallback_engine_list (cudnnBackendDescriptorType_t mode, const std::string &opGraphTag) -> std::vector< int > get_fallback_engine_list cudnnBackendDescriptorType_t mode const std::string & opGraphTag

Here we are using the term "bias" in the operationGraph as a proxy for the conv*bias* operation graph. We are not strictly checking the order of the operations in the graph. We propose this as a temporary workaround until the backend API supports querying the fallback list directly from cudnn

cudnn_frontend::EngineFallbackListBuilder_v8::build
bool static bool cudnn_frontend::load_from_config (json &json_handle, const std::string &errata_json) load_from_config json & json_handle const std::string & errata_json
typename T
bool static bool cudnn_frontend::check_rule (const json &json_handle, const std::string &executionPlanTag, cudnnHandle_t handle, T fn) check_rule const json & json_handle const std::string & executionPlanTag cudnnHandle_t handle T fn nlohmann::basic_json< ObjectType, ArrayType, StringType, BooleanType, NumberIntegerType, NumberUnsignedType, NumberFloatType, AllocatorType, JSONSerializer, BinaryType >::contains nlohmann::to_string nlohmann::detail::void
typename T
bool static bool cudnn_frontend::check_errata (const json &json_handle, const std::string &executionPlanTag, cudnnHandle_t handle, T fn) check_errata const json & json_handle const std::string & executionPlanTag cudnnHandle_t handle T fn getLogger
void static void cudnn_frontend::filter (EngineConfigList &from, EngineConfigList &to, std::function< bool(cudnnBackendDescriptor_t)> filter_fn) filter EngineConfigList & from EngineConfigList & to std::function< bool(cudnnBackendDescriptor_t)> filter_fn
cudnnBackendNumericalNote_t NUMERIC_NOTE NUMERIC_NOTE
bool bool cudnn_frontend::hasNumericalNote (cudnnBackendDescriptor_t engine_config) hasNumericalNote cudnnBackendDescriptor_t engine_config make_shared_backend_pointer
CudnnFindSamplingTechnique samplingTechnique samplingTechnique
auto auto cudnn_frontend::time_sorted_plan (cudnnHandle_t handle, executionPlans_t plans, VariantPack const &variantPack) -> executionPlans_t time_sorted_plan cudnnHandle_t handle executionPlans_t plans VariantPack const & variantPack

Sorts the execution plans by their run time. The run time of plan may not trivial and hence we run it multiple times till we get a stable value. We have an additional dry-run which helps stabilize the time further.

CUDNN_FIND_SAMPLE_MEDIAN_OF_THREE
CUDNN_FIND_SAMPLE_ONCE
CUDNN_FIND_SAMPLE_TILL_STABLE
cudnn_frontend::ExecutionPlan_v8::getExecutionTime
getLogger
to_string
std::size_t SIZE SIZE
EngineConfigList EngineConfigList cudnn_frontend::get_heuristics_list (std::array< cudnnBackendHeurMode_t, SIZE > modes, OperationGraph_v8 &opGraph, std::function< bool(cudnnBackendDescriptor_t)> filter_fn) get_heuristics_list std::array< cudnnBackendHeurMode_t, SIZE > modes OperationGraph_v8 & opGraph std::function< bool(cudnnBackendDescriptor_t)> filter_fn cudnn_frontend::EngineHeuristics_v8::EngineHeuristicsBuilder_v8 filter cudnn_frontend::EngineHeuristics_v8::mode nlohmann::detail::void
bool & bool& cudnn_frontend::isLoggingEnabled () isLoggingEnabled getStream cudnn_frontend::ConditionalStreamer::operator<< operator<<
std::ostream & std::ostream& cudnn_frontend::getStream () getStream isLoggingEnabled getLogger
ConditionalStreamer & ConditionalStreamer& cudnn_frontend::getLogger () getLogger getStream cudnn_frontend::ExecutionPlanCache_v1::add_plan_to_cache cudnn_frontend::MatMulDescBuilder_v8::build cudnn_frontend::ReductionDescBuilder_v8::build cudnn_frontend::VariantPackBuilder_v8::build cudnn_frontend::EngineFallbackListBuilder_v8::build cudnn_frontend::OperationGraphBuilder_v8::build cudnn_frontend::EngineHeuristicsBuilder_v8::build cudnn_frontend::EngineConfigBuilder_v8::build cudnn_frontend::ConvDescBuilder_v8::build cudnn_frontend::TensorBuilder_v8::build cudnn_frontend::PointWiseDescBuilder_v8::build cudnn_frontend::EngineBuilder_v8::build cudnn_frontend::ExecutionPlanBuilder_v8::build cudnn_frontend::OperationBuilder_v8::build cudnn_frontend::OperationBuilder_v8::build_conv_backward_data cudnn_frontend::OperationBuilder_v8::build_conv_backward_filter cudnn_frontend::OperationBuilder_v8::build_conv_forward check_errata cudnn_frontend::EngineConfigGenerator::cudnnGetPlan filter cudnn_frontend::EngineConfigGenerator::generate_engine_config cudnn_frontend::ExecutionPlanCache_v1::get_plan_from_cache cudnn_frontend::ExecutionPlanCache_v2::is_fastest_plan_stable time_sorted_plan
std::ostream & static std::ostream& cudnn_frontend::operator<< (std::ostream &os, const BackendDescriptor &desc) operator<< std::ostream & os const BackendDescriptor & desc cudnn_frontend::BackendDescriptor::describe isLoggingEnabled
cudnnStatus_t static cudnnStatus_t cudnn_frontend::cudnnReorderFilterAndBiasInt8x32 (cudnnHandle_t handle, const Tensor_v8 &tensor, const ConvDesc_v8 &conv_desc, void *dev_filter_ptr, void *reordered_filter_ptr, void *dev_bias_ptr, void *reordered_bias_ptr) cudnnReorderFilterAndBiasInt8x32 cudnnHandle_t handle const Tensor_v8 & tensor const ConvDesc_v8 & conv_desc void * dev_filter_ptr void * reordered_filter_ptr void * dev_bias_ptr void * reordered_bias_ptr cudnn_frontend::Tensor_v8::getDimArray cudnn_frontend::Tensor_v8::getDimensionCount cudnn_frontend::ConvDesc_v8::getDimensionCount
void static void cudnn_frontend::throw_if (std::function< bool()> expr, const char *message, cudnnStatus_t status) throw_if std::function< bool()> expr const char * message cudnnStatus_t status cudnn_frontend::cudnnException::cudnnException
void static void cudnn_frontend::throw_if (bool expr, const char *message, cudnnStatus_t status) throw_if bool expr const char * message cudnnStatus_t status cudnn_frontend::cudnnException::cudnnException
std::string static std::string cudnn_frontend::to_string (cudnnDataType_t type) to_string cudnnDataType_t type cudnn_frontend::EngineConfigGenerator::cudnnGetPlan cudnn_frontend::Tensor_v8::describe cudnn_frontend::ConvDesc_v8::describe set_error_and_throw_exception time_sorted_plan
std::string static std::string cudnn_frontend::to_string (cudnnStatus_t status) to_string cudnnStatus_t status
void static void cudnn_frontend::set_error_and_throw_exception (BackendDescriptor const *desc, cudnnStatus_t status, const char *message) set_error_and_throw_exception BackendDescriptor const * desc cudnnStatus_t status const char * message cudnn_frontend::cudnnException::cudnnException cudnn_frontend::BackendDescriptor::set_error cudnn_frontend::BackendDescriptor::set_status to_string cudnn_frontend::MatMulDescBuilder_v8::build cudnn_frontend::ReductionDescBuilder_v8::build cudnn_frontend::VariantPackBuilder_v8::build cudnn_frontend::EngineFallbackListBuilder_v8::build cudnn_frontend::OperationGraphBuilder_v8::build cudnn_frontend::EngineHeuristicsBuilder_v8::build cudnn_frontend::EngineConfigBuilder_v8::build cudnn_frontend::ConvDescBuilder_v8::build cudnn_frontend::TensorBuilder_v8::build cudnn_frontend::PointWiseDescBuilder_v8::build cudnn_frontend::EngineBuilder_v8::build cudnn_frontend::ExecutionPlanBuilder_v8::build cudnn_frontend::OperationBuilder_v8::build cudnn_frontend::OperationBuilder_v8::build_conv_backward_data cudnn_frontend::OperationBuilder_v8::build_conv_backward_filter cudnn_frontend::OperationBuilder_v8::build_conv_forward cudnn_frontend::OperationBuilder_v8::build_matmul_op cudnn_frontend::OperationBuilder_v8::build_pointwise_op cudnn_frontend::OperationBuilder_v8::build_reduction_op cudnn_frontend::Engine_v8::buildKnobs cudnn_frontend::ExecutionPlan_v8::buildTag cudnn_frontend::ExecutionPlan_v8::computeWorkSpaceSize cudnn_frontend::Engine_v8::Engine_v8 cudnn_frontend::EngineConfig_v8::EngineConfig_v8 cudnn_frontend::ExecutionPlan_v8::fetchNotes cudnn_frontend::EngineHeuristics_v8::getEngineConfig cudnn_frontend::EngineHeuristics_v8::getEngineConfigCount cudnn_frontend::OperationGraph_v8::getEngineCount cudnn_frontend::OperationBuilder_v8::setaMatDesc cudnn_frontend::OperationBuilder_v8::setbDesc cudnn_frontend::OperationBuilder_v8::setbMatDesc cudnn_frontend::OperationBuilder_v8::setcDesc cudnn_frontend::OperationBuilder_v8::setcMatDesc cudnn_frontend::EngineConfigBuilder_v8::setEngine cudnn_frontend::OperationBuilder_v8::setmatmulDesc cudnn_frontend::OperationBuilder_v8::setpwDesc cudnn_frontend::OperationBuilder_v8::setreductionDesc cudnn_frontend::OperationBuilder_v8::setwDesc

Execution Plan Caching: Goal is to auto-tune once and then save the best auto-tuned result for a problem for later use. For every unique Operation Graph (denoted by a string) we have a set of plans identified by a feature vector. The feature vector could be Tensor dimension/data_type and so on. Multiple operation Graph can share a feature vector type but may have different Execution Plan(s). The v1 cache has the following format. It is the reponsibility of the user to query the correct cache for the given device/operation graph combination.