Skip to content

GitLab

Commit b30f3cdb authored by xiabo's avatar xiabo
Browse files

添加下载的代码

parent e38ee081
Hide whitespace changes
Inline Side-by-side
Showing with 7864 additions and 0 deletions
3rdparty/core-r22.12/include/triton/core/tritonserver.h 0 → 100644
View file @ b30f3cdb
// Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
// * Neither the name of NVIDIA CORPORATION nor the names of its
// contributors may be used to endorse or promote products derived
// from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
/// \file
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
#ifdef _COMPILING_TRITONSERVER
#if defined(_MSC_VER)
#define TRITONSERVER_DECLSPEC __declspec(dllexport)
#elif defined(__GNUC__)
#define TRITONSERVER_DECLSPEC __attribute__((__visibility__("default")))
#else
#define TRITONSERVER_DECLSPEC
#endif
#else
#if defined(_MSC_VER)
#define TRITONSERVER_DECLSPEC __declspec(dllimport)
#else
#define TRITONSERVER_DECLSPEC
#endif
#endif
struct TRITONSERVER_BufferAttributes;
struct TRITONSERVER_Error;
struct TRITONSERVER_InferenceRequest;
struct TRITONSERVER_InferenceResponse;
struct TRITONSERVER_InferenceTrace;
struct TRITONSERVER_Message;
struct TRITONSERVER_Metrics;
struct TRITONSERVER_Parameter;
struct TRITONSERVER_ResponseAllocator;
struct TRITONSERVER_Server;
struct TRITONSERVER_ServerOptions;
struct TRITONSERVER_Metric;
struct TRITONSERVER_MetricFamily;
///
/// TRITONSERVER API Version
///
/// The TRITONSERVER API is versioned with major and minor version
/// numbers. Any change to the API that does not impact backwards
/// compatibility (for example, adding a non-required function)
/// increases the minor version number. Any change that breaks
/// backwards compatibility (for example, deleting or changing the
/// behavior of a function) increases the major version number. A
/// client should check that the API version used to compile the
/// client is compatible with the API version of the Triton shared
/// library that it is linking against. This is typically done by code
/// similar to the following which makes sure that the major versions
/// are equal and that the minor version of the Triton shared library
/// is >= the minor version used to build the client.
///
/// uint32_t api_version_major, api_version_minor;
/// TRITONSERVER_ApiVersion(&api_version_major, &api_version_minor);
/// if ((api_version_major != TRITONSERVER_API_VERSION_MAJOR) ||
/// (api_version_minor < TRITONSERVER_API_VERSION_MINOR)) {
/// return TRITONSERVER_ErrorNew(
/// TRITONSERVER_ERROR_UNSUPPORTED,
/// "triton server API version does not support this client");
/// }
///
#define TRITONSERVER_API_VERSION_MAJOR 1
#define TRITONSERVER_API_VERSION_MINOR 17
/// Get the TRITONBACKEND API version supported by the Triton shared
/// library. This value can be compared against the
/// TRITONSERVER_API_VERSION_MAJOR and TRITONSERVER_API_VERSION_MINOR
/// used to build the client to ensure that Triton shared library is
/// compatible with the client.
///
/// \param major Returns the TRITONSERVER API major version supported
/// by Triton.
/// \param minor Returns the TRITONSERVER API minor version supported
/// by Triton.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ApiVersion(
uint32_t* major, uint32_t* minor);
/// TRITONSERVER_DataType
///
/// Tensor data types recognized by TRITONSERVER.
///
typedef enum TRITONSERVER_datatype_enum {
TRITONSERVER_TYPE_INVALID,
TRITONSERVER_TYPE_BOOL,
TRITONSERVER_TYPE_UINT8,
TRITONSERVER_TYPE_UINT16,
TRITONSERVER_TYPE_UINT32,
TRITONSERVER_TYPE_UINT64,
TRITONSERVER_TYPE_INT8,
TRITONSERVER_TYPE_INT16,
TRITONSERVER_TYPE_INT32,
TRITONSERVER_TYPE_INT64,
TRITONSERVER_TYPE_FP16,
TRITONSERVER_TYPE_FP32,
TRITONSERVER_TYPE_FP64,
TRITONSERVER_TYPE_BYTES,
TRITONSERVER_TYPE_BF16
} TRITONSERVER_DataType;
/// Get the string representation of a data type. The returned string
/// is not owned by the caller and so should not be modified or freed.
///
/// \param datatype The data type.
/// \return The string representation of the data type.
TRITONSERVER_DECLSPEC const char* TRITONSERVER_DataTypeString(
TRITONSERVER_DataType datatype);
/// Get the Triton datatype corresponding to a string representation
/// of a datatype.
///
/// \param dtype The datatype string representation.
/// \return The Triton data type or TRITONSERVER_TYPE_INVALID if the
/// string does not represent a data type.
TRITONSERVER_DECLSPEC TRITONSERVER_DataType
TRITONSERVER_StringToDataType(const char* dtype);
/// Get the size of a Triton datatype in bytes. Zero is returned for
/// TRITONSERVER_TYPE_BYTES because it have variable size. Zero is
/// returned for TRITONSERVER_TYPE_INVALID.
///
/// \param dtype The datatype.
/// \return The size of the datatype.
TRITONSERVER_DECLSPEC uint32_t
TRITONSERVER_DataTypeByteSize(TRITONSERVER_DataType datatype);
/// TRITONSERVER_MemoryType
///
/// Types of memory recognized by TRITONSERVER.
///
typedef enum TRITONSERVER_memorytype_enum {
TRITONSERVER_MEMORY_CPU,
TRITONSERVER_MEMORY_CPU_PINNED,
TRITONSERVER_MEMORY_GPU
} TRITONSERVER_MemoryType;
/// Get the string representation of a memory type. The returned
/// string is not owned by the caller and so should not be modified or
/// freed.
///
/// \param memtype The memory type.
/// \return The string representation of the memory type.
TRITONSERVER_DECLSPEC const char* TRITONSERVER_MemoryTypeString(
TRITONSERVER_MemoryType memtype);
/// TRITONSERVER_ParameterType
///
/// Types of parameters recognized by TRITONSERVER.
///
typedef enum TRITONSERVER_parametertype_enum {
TRITONSERVER_PARAMETER_STRING,
TRITONSERVER_PARAMETER_INT,
TRITONSERVER_PARAMETER_BOOL,
TRITONSERVER_PARAMETER_BYTES
} TRITONSERVER_ParameterType;
/// Get the string representation of a parameter type. The returned
/// string is not owned by the caller and so should not be modified or
/// freed.
///
/// \param paramtype The parameter type.
/// \return The string representation of the parameter type.
TRITONSERVER_DECLSPEC const char* TRITONSERVER_ParameterTypeString(
TRITONSERVER_ParameterType paramtype);
/// Create a new parameter object. The caller takes ownership of the
/// TRITONSERVER_Parameter object and must call TRITONSERVER_ParameterDelete to
/// release the object. The object will maintain its own copy of the 'value'
///
/// \param name The parameter name.
/// \param type The parameter type.
/// \param value The pointer to the value.
/// \return A new TRITONSERVER_Parameter object. 'nullptr' will be returned if
/// 'type' is 'TRITONSERVER_PARAMETER_BYTES'. The caller should use
/// TRITONSERVER_ParameterBytesNew to create parameter with bytes type.
TRITONSERVER_DECLSPEC TRITONSERVER_Parameter* TRITONSERVER_ParameterNew(
const char* name, const TRITONSERVER_ParameterType type, const void* value);
/// Create a new parameter object with type TRITONSERVER_PARAMETER_BYTES.
/// The caller takes ownership of the TRITONSERVER_Parameter object and must
/// call TRITONSERVER_ParameterDelete to release the object. The object only
/// maintains a shallow copy of the 'byte_ptr' so the data content must be
/// valid until the parameter object is deleted.
///
/// \param name The parameter name.
/// \param byte_ptr The pointer to the data content.
/// \param size The size of the data content.
/// \return A new TRITONSERVER_Error object.
TRITONSERVER_DECLSPEC TRITONSERVER_Parameter* TRITONSERVER_ParameterBytesNew(
const char* name, const void* byte_ptr, const uint64_t size);
/// Delete an parameter object.
///
/// \param parameter The parameter object.
TRITONSERVER_DECLSPEC void TRITONSERVER_ParameterDelete(
TRITONSERVER_Parameter* parameter);
/// TRITONSERVER_InstanceGroupKind
///
/// Kinds of instance groups recognized by TRITONSERVER.
///
typedef enum TRITONSERVER_instancegroupkind_enum {
TRITONSERVER_INSTANCEGROUPKIND_AUTO,
TRITONSERVER_INSTANCEGROUPKIND_CPU,
TRITONSERVER_INSTANCEGROUPKIND_GPU,
TRITONSERVER_INSTANCEGROUPKIND_MODEL
} TRITONSERVER_InstanceGroupKind;
/// Get the string representation of an instance-group kind. The
/// returned string is not owned by the caller and so should not be
/// modified or freed.
///
/// \param kind The instance-group kind.
/// \return The string representation of the kind.
TRITONSERVER_DECLSPEC const char* TRITONSERVER_InstanceGroupKindString(
TRITONSERVER_InstanceGroupKind kind);
/// TRITONSERVER_Logging
///
/// Types/levels of logging.
///
typedef enum TRITONSERVER_loglevel_enum {
TRITONSERVER_LOG_INFO,
TRITONSERVER_LOG_WARN,
TRITONSERVER_LOG_ERROR,
TRITONSERVER_LOG_VERBOSE
} TRITONSERVER_LogLevel;
///
/// Format of logging.
///
/// TRITONSERVER_LOG_DEFAULT: the log severity (L) and timestamp will be
/// logged as "LMMDD hh:mm:ss.ssssss".
///
/// TRITONSERVER_LOG_ISO8601: the log format will be "YYYY-MM-DDThh:mm:ssZ L".
///
typedef enum TRITONSERVER_logformat_enum {
TRITONSERVER_LOG_DEFAULT,
TRITONSERVER_LOG_ISO8601
} TRITONSERVER_LogFormat;
/// Is a log level enabled?
///
/// \param level The log level.
/// \return True if the log level is enabled, false if not enabled.
TRITONSERVER_DECLSPEC bool TRITONSERVER_LogIsEnabled(
TRITONSERVER_LogLevel level);
/// Log a message at a given log level if that level is enabled.
///
/// \param level The log level.
/// \param filename The file name of the location of the log message.
/// \param line The line number of the log message.
/// \param msg The log message.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_LogMessage(
TRITONSERVER_LogLevel level, const char* filename, const int line,
const char* msg);
/// TRITONSERVER_Error
///
/// Errors are reported by a TRITONSERVER_Error object. A NULL
/// TRITONSERVER_Error indicates no error, a non-NULL TRITONSERVER_Error
/// indicates error and the code and message for the error can be
/// retrieved from the object.
///
/// The caller takes ownership of a TRITONSERVER_Error object returned by
/// the API and must call TRITONSERVER_ErrorDelete to release the object.
///
/// The TRITONSERVER_Error error codes
typedef enum TRITONSERVER_errorcode_enum {
TRITONSERVER_ERROR_UNKNOWN,
TRITONSERVER_ERROR_INTERNAL,
TRITONSERVER_ERROR_NOT_FOUND,
TRITONSERVER_ERROR_INVALID_ARG,
TRITONSERVER_ERROR_UNAVAILABLE,
TRITONSERVER_ERROR_UNSUPPORTED,
TRITONSERVER_ERROR_ALREADY_EXISTS
} TRITONSERVER_Error_Code;
/// Create a new error object. The caller takes ownership of the
/// TRITONSERVER_Error object and must call TRITONSERVER_ErrorDelete to
/// release the object.
///
/// \param code The error code.
/// \param msg The error message.
/// \return A new TRITONSERVER_Error object.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ErrorNew(
TRITONSERVER_Error_Code code, const char* msg);
/// Delete an error object.
///
/// \param error The error object.
TRITONSERVER_DECLSPEC void TRITONSERVER_ErrorDelete(TRITONSERVER_Error* error);
/// Get the error code.
///
/// \param error The error object.
/// \return The error code.
TRITONSERVER_DECLSPEC TRITONSERVER_Error_Code
TRITONSERVER_ErrorCode(TRITONSERVER_Error* error);
/// Get the string representation of an error code. The returned
/// string is not owned by the caller and so should not be modified or
/// freed. The lifetime of the returned string extends only as long as
/// 'error' and must not be accessed once 'error' is deleted.
///
/// \param error The error object.
/// \return The string representation of the error code.
TRITONSERVER_DECLSPEC const char* TRITONSERVER_ErrorCodeString(
TRITONSERVER_Error* error);
/// Get the error message. The returned string is not owned by the
/// caller and so should not be modified or freed. The lifetime of the
/// returned string extends only as long as 'error' and must not be
/// accessed once 'error' is deleted.
///
/// \param error The error object.
/// \return The error message.
TRITONSERVER_DECLSPEC const char* TRITONSERVER_ErrorMessage(
TRITONSERVER_Error* error);
/// TRITONSERVER_ResponseAllocator
///
/// Object representing a memory allocator for output tensors in an
/// inference response.
///
/// Type for allocation function that allocates a buffer to hold an
/// output tensor.
///
/// \param allocator The allocator that is provided in the call to
/// TRITONSERVER_InferenceRequestSetResponseCallback.
/// \param tensor_name The name of the output tensor to allocate for.
/// \param byte_size The size of the buffer to allocate.
/// \param memory_type The type of memory that the caller prefers for
/// the buffer allocation.
/// \param memory_type_id The ID of the memory that the caller prefers
/// for the buffer allocation.
/// \param userp The user data pointer that is provided as
/// 'response_allocator_userp' in the call to
/// TRITONSERVER_InferenceRequestSetResponseCallback.
/// \param buffer Returns a pointer to the allocated memory.
/// \param buffer_userp Returns a user-specified value to associate
/// with the buffer, or nullptr if no user-specified value should be
/// associated with the buffer. This value will be provided in the
/// call to TRITONSERVER_ResponseAllocatorReleaseFn_t when the buffer
/// is released and will also be returned by
/// TRITONSERVER_InferenceResponseOutput.
/// \param actual_memory_type Returns the type of memory where the
/// allocation resides. May be different than the type of memory
/// requested by 'memory_type'.
/// \param actual_memory_type_id Returns the ID of the memory where
/// the allocation resides. May be different than the ID of the memory
/// requested by 'memory_type_id'.
/// \return a TRITONSERVER_Error object if a failure occurs while
/// attempting an allocation. If an error is returned all other return
/// values will be ignored.
typedef TRITONSERVER_Error* (*TRITONSERVER_ResponseAllocatorAllocFn_t)(
TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name,
size_t byte_size, TRITONSERVER_MemoryType memory_type,
int64_t memory_type_id, void* userp, void** buffer, void** buffer_userp,
TRITONSERVER_MemoryType* actual_memory_type,
int64_t* actual_memory_type_id);
/// Type for allocation function that allocates a buffer to hold an
/// output tensor with buffer attributes. The callback function must fill in the
/// appropriate buffer attributes information related to this buffer. If set,
/// this function is always called after TRITONSERVER_ResponseAllocatorAllocFn_t
/// function.
///
/// \param allocator The allocator that is provided in the call to
/// TRITONSERVER_InferenceRequestSetResponseCallback.
/// \param tensor_name The name of the output tensor to allocate for.
/// \param buffer_attributes The buffer attributes associated with the buffer.
/// \param userp The user data pointer that is provided as
/// 'response_allocator_userp' in the call to
/// TRITONSERVER_InferenceRequestSetResponseCallback.
/// \param buffer_userp Returns a user-specified value to associate
/// with the buffer, or nullptr if no user-specified value should be
/// associated with the buffer. This value will be provided in the
/// call to TRITONSERVER_ResponseAllocatorReleaseFn_t when the buffer
/// is released and will also be returned by
/// TRITONSERVER_InferenceResponseOutput.
/// \return a TRITONSERVER_Error object if a failure occurs while
/// attempting an allocation. If an error is returned all other return
/// values will be ignored.
typedef TRITONSERVER_Error* (
*TRITONSERVER_ResponseAllocatorBufferAttributesFn_t)(
TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name,
TRITONSERVER_BufferAttributes* buffer_attributes, void* userp,
void* buffer_userp);
/// Type for function that is called to query the allocator's preferred memory
/// type and memory type ID. As much as possible, the allocator should attempt
/// to return the same memory_type and memory_type_id values that will be
/// returned by the subsequent call to TRITONSERVER_ResponseAllocatorAllocFn_t.
/// But the allocator is not required to do so.
///
/// \param allocator The allocator that is provided in the call to
/// TRITONSERVER_InferenceRequestSetResponseCallback.
/// \param userp The user data pointer that is provided as
/// 'response_allocator_userp' in the call to
/// TRITONSERVER_InferenceRequestSetResponseCallback.
/// \param tensor_name The name of the output tensor. This is optional
/// and it should be set to nullptr to indicate that the tensor name has
/// not determined.
/// \param byte_size The expected size of the buffer. This is optional
/// and it should be set to nullptr to indicate that the byte size has
/// not determined.
/// \param memory_type Acts as both input and output. On input gives
/// the memory type preferred by the caller. Returns memory type preferred
/// by the allocator, taken account of the caller preferred type.
/// \param memory_type_id Acts as both input and output. On input gives
/// the memory type ID preferred by the caller. Returns memory type ID preferred
/// by the allocator, taken account of the caller preferred type ID.
/// \return a TRITONSERVER_Error object if a failure occurs.
typedef TRITONSERVER_Error* (*TRITONSERVER_ResponseAllocatorQueryFn_t)(
TRITONSERVER_ResponseAllocator* allocator, void* userp,
const char* tensor_name, size_t* byte_size,
TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id);
/// Type for function that is called when the server no longer holds
/// any reference to a buffer allocated by
/// TRITONSERVER_ResponseAllocatorAllocFn_t. In practice this function
/// is typically called when the response object associated with the
/// buffer is deleted by TRITONSERVER_InferenceResponseDelete.
///
/// \param allocator The allocator that is provided in the call to
/// TRITONSERVER_InferenceRequestSetResponseCallback.
/// \param buffer Pointer to the buffer to be freed.
/// \param buffer_userp The user-specified value associated
/// with the buffer in TRITONSERVER_ResponseAllocatorAllocFn_t.
/// \param byte_size The size of the buffer.
/// \param memory_type The type of memory holding the buffer.
/// \param memory_type_id The ID of the memory holding the buffer.
/// \return a TRITONSERVER_Error object if a failure occurs while
/// attempting the release. If an error is returned Triton will not
/// attempt to release the buffer again.
typedef TRITONSERVER_Error* (*TRITONSERVER_ResponseAllocatorReleaseFn_t)(
TRITONSERVER_ResponseAllocator* allocator, void* buffer, void* buffer_userp,
size_t byte_size, TRITONSERVER_MemoryType memory_type,
int64_t memory_type_id);
/// Type for function that is called to indicate that subsequent
/// allocation requests will refer to a new response.
///
/// \param allocator The allocator that is provided in the call to
/// TRITONSERVER_InferenceRequestSetResponseCallback.
/// \param userp The user data pointer that is provided as
/// 'response_allocator_userp' in the call to
/// TRITONSERVER_InferenceRequestSetResponseCallback.
/// \return a TRITONSERVER_Error object if a failure occurs.
typedef TRITONSERVER_Error* (*TRITONSERVER_ResponseAllocatorStartFn_t)(
TRITONSERVER_ResponseAllocator* allocator, void* userp);
/// Create a new response allocator object.
///
/// The response allocator object is used by Triton to allocate
/// buffers to hold the output tensors in inference responses. Most
/// models generate a single response for each inference request
/// (TRITONSERVER_TXN_ONE_TO_ONE). For these models the order of
/// callbacks will be:
///
/// TRITONSERVER_ServerInferAsync called
/// - start_fn : optional (and typically not required)
/// - alloc_fn : called once for each output tensor in response
/// TRITONSERVER_InferenceResponseDelete called
/// - release_fn: called once for each output tensor in response
///
/// For models that generate multiple responses for each inference
/// request (TRITONSERVER_TXN_DECOUPLED), the start_fn callback can be
/// used to determine sets of alloc_fn callbacks that belong to the
/// same response:
///
/// TRITONSERVER_ServerInferAsync called
/// - start_fn
/// - alloc_fn : called once for each output tensor in response
/// - start_fn
/// - alloc_fn : called once for each output tensor in response
/// ...
/// For each response, TRITONSERVER_InferenceResponseDelete called
/// - release_fn: called once for each output tensor in the response
///
/// In all cases the start_fn, alloc_fn and release_fn callback
/// functions must be thread-safe. Typically making these functions
/// thread-safe does not require explicit locking. The recommended way
/// to implement these functions is to have each inference request
/// provide a 'response_allocator_userp' object that is unique to that
/// request with TRITONSERVER_InferenceRequestSetResponseCallback. The
/// callback functions then operate only on this unique state. Locking
/// is required only when the callback function needs to access state
/// that is shared across inference requests (for example, a common
/// allocation pool).
///
/// \param allocator Returns the new response allocator object.
/// \param alloc_fn The function to call to allocate buffers for result
/// tensors.
/// \param release_fn The function to call when the server no longer
/// holds a reference to an allocated buffer.
/// \param start_fn The function to call to indicate that the
/// subsequent 'alloc_fn' calls are for a new response. This callback
/// is optional (use nullptr to indicate that it should not be
/// invoked).
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ResponseAllocatorNew(
TRITONSERVER_ResponseAllocator** allocator,
TRITONSERVER_ResponseAllocatorAllocFn_t alloc_fn,
TRITONSERVER_ResponseAllocatorReleaseFn_t release_fn,
TRITONSERVER_ResponseAllocatorStartFn_t start_fn);
/// Set the buffer attributes function for a response allocator object.
/// The function will be called after alloc_fn to set the buffer attributes
/// associated with the output buffer.
///
/// The thread-safy requirement for buffer_attributes_fn is the same as other
/// allocator callbacks.
///
/// \param allocator The response allocator object.
/// \param buffer_attributes_fn The function to call to get the buffer
/// attributes information for an allocated buffer.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_ResponseAllocatorSetBufferAttributesFunction(
TRITONSERVER_ResponseAllocator* allocator,
TRITONSERVER_ResponseAllocatorBufferAttributesFn_t buffer_attributes_fn);
/// Set the query function to a response allocator object. Usually the
/// function will be called before alloc_fn to understand what is the
/// allocator's preferred memory type and memory type ID at the current
/// situation to make different execution decision.
///
/// The thread-safy requirement for query_fn is the same as other allocator
/// callbacks.
///
/// \param allocator The response allocator object.
/// \param query_fn The function to call to query allocator's preferred memory
/// type and memory type ID.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_ResponseAllocatorSetQueryFunction(
TRITONSERVER_ResponseAllocator* allocator,
TRITONSERVER_ResponseAllocatorQueryFn_t query_fn);
/// Delete a response allocator.
///
/// \param allocator The response allocator object.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ResponseAllocatorDelete(
TRITONSERVER_ResponseAllocator* allocator);
/// TRITONSERVER_Message
///
/// Object representing a Triton Server message.
///
/// Create a new message object from serialized JSON string.
///
/// \param message The message object.
/// \param base The base of the serialized JSON.
/// \param byte_size The size, in bytes, of the serialized message.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_MessageNewFromSerializedJson(
TRITONSERVER_Message** message, const char* base, size_t byte_size);
/// Delete a message object.
///
/// \param message The message object.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_MessageDelete(
TRITONSERVER_Message* message);
/// Get the base and size of the buffer containing the serialized
/// message in JSON format. The buffer is owned by the
/// TRITONSERVER_Message object and should not be modified or freed by
/// the caller. The lifetime of the buffer extends only as long as
/// 'message' and must not be accessed once 'message' is deleted.
///
/// \param message The message object.
/// \param base Returns the base of the serialized message.
/// \param byte_size Returns the size, in bytes, of the serialized
/// message.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_MessageSerializeToJson(
TRITONSERVER_Message* message, const char** base, size_t* byte_size);
/// TRITONSERVER_Metrics
///
/// Object representing metrics.
///
/// Metric format types
typedef enum tritonserver_metricformat_enum {
TRITONSERVER_METRIC_PROMETHEUS
} TRITONSERVER_MetricFormat;
/// Delete a metrics object.
///
/// \param metrics The metrics object.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_MetricsDelete(
TRITONSERVER_Metrics* metrics);
/// Get a buffer containing the metrics in the specified format. For
/// each format the buffer contains the following:
///
/// TRITONSERVER_METRIC_PROMETHEUS: 'base' points to a single multiline
/// string (char*) that gives a text representation of the metrics in
/// prometheus format. 'byte_size' returns the length of the string
/// in bytes.
///
/// The buffer is owned by the 'metrics' object and should not be
/// modified or freed by the caller. The lifetime of the buffer
/// extends only as long as 'metrics' and must not be accessed once
/// 'metrics' is deleted.
///
/// \param metrics The metrics object.
/// \param format The format to use for the returned metrics.
/// \param base Returns a pointer to the base of the formatted
/// metrics, as described above.
/// \param byte_size Returns the size, in bytes, of the formatted
/// metrics.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_MetricsFormatted(
TRITONSERVER_Metrics* metrics, TRITONSERVER_MetricFormat format,
const char** base, size_t* byte_size);
/// TRITONSERVER_InferenceTrace
///
/// Object that represents tracing for an inference request.
///
/// Trace levels. The trace level controls the type of trace
/// activities that are reported for an inference request.
///
/// Trace level values are power-of-2 and can be combined to trace
/// multiple types of activities. For example, use
/// (TRITONSERVER_TRACE_LEVEL_TIMESTAMPS |
/// TRITONSERVER_TRACE_LEVEL_TENSORS) to trace both timestamps and
/// tensors for an inference request.
///
/// TRITONSERVER_TRACE_LEVEL_MIN and TRITONSERVER_TRACE_LEVEL_MAX are
/// deprecated and should not be used.
typedef enum tritonserver_tracelevel_enum {
/// Tracing disabled. No trace activities are reported.
TRITONSERVER_TRACE_LEVEL_DISABLED = 0,
/// Deprecated. Use TRITONSERVER_TRACE_LEVEL_TIMESTAMPS.
TRITONSERVER_TRACE_LEVEL_MIN = 1,
/// Deprecated. Use TRITONSERVER_TRACE_LEVEL_TIMESTAMPS.
TRITONSERVER_TRACE_LEVEL_MAX = 2,
/// Record timestamps for the inference request.
TRITONSERVER_TRACE_LEVEL_TIMESTAMPS = 0x4,
/// Record input and output tensor values for the inference request.
TRITONSERVER_TRACE_LEVEL_TENSORS = 0x8
} TRITONSERVER_InferenceTraceLevel;
/// Get the string representation of a trace level. The returned
/// string is not owned by the caller and so should not be modified or
/// freed.
///
/// \param level The trace level.
/// \return The string representation of the trace level.
TRITONSERVER_DECLSPEC const char* TRITONSERVER_InferenceTraceLevelString(
TRITONSERVER_InferenceTraceLevel level);
/// Trace activities
typedef enum tritonserver_traceactivity_enum {
TRITONSERVER_TRACE_REQUEST_START = 0,
TRITONSERVER_TRACE_QUEUE_START = 1,
TRITONSERVER_TRACE_COMPUTE_START = 2,
TRITONSERVER_TRACE_COMPUTE_INPUT_END = 3,
TRITONSERVER_TRACE_COMPUTE_OUTPUT_START = 4,
TRITONSERVER_TRACE_COMPUTE_END = 5,
TRITONSERVER_TRACE_REQUEST_END = 6,
TRITONSERVER_TRACE_TENSOR_QUEUE_INPUT = 7,
TRITONSERVER_TRACE_TENSOR_BACKEND_INPUT = 8,
TRITONSERVER_TRACE_TENSOR_BACKEND_OUTPUT = 9
} TRITONSERVER_InferenceTraceActivity;
/// Get the string representation of a trace activity. The returned
/// string is not owned by the caller and so should not be modified or
/// freed.
///
/// \param activity The trace activity.
/// \return The string representation of the trace activity.
TRITONSERVER_DECLSPEC const char* TRITONSERVER_InferenceTraceActivityString(
TRITONSERVER_InferenceTraceActivity activity);
/// Type for trace timeline activity callback function. This callback function
/// is used to report activity occurring for a trace. This function
/// does not take ownership of 'trace' and so any information needed
/// from that object must be copied before returning. The 'userp' data
/// is the same as what is supplied in the call to
/// TRITONSERVER_InferenceTraceNew.
typedef void (*TRITONSERVER_InferenceTraceActivityFn_t)(
TRITONSERVER_InferenceTrace* trace,
TRITONSERVER_InferenceTraceActivity activity, uint64_t timestamp_ns,
void* userp);
/// Type for trace tensor activity callback function. This callback function
/// is used to report tensor activity occurring for a trace. This function
/// does not take ownership of 'trace' and so any information needed
/// from that object must be copied before returning. The 'userp' data
/// is the same as what is supplied in the call to
/// TRITONSERVER_InferenceTraceTensorNew.
typedef void (*TRITONSERVER_InferenceTraceTensorActivityFn_t)(
TRITONSERVER_InferenceTrace* trace,
TRITONSERVER_InferenceTraceActivity activity, const char* name,
TRITONSERVER_DataType datatype, const void* base, size_t byte_size,
const int64_t* shape, uint64_t dim_count,
TRITONSERVER_MemoryType memory_type, int64_t memory_type_id, void* userp);
/// Type for trace release callback function. This callback function
/// is called when all activity for the trace has completed. The
/// callback function takes ownership of the
/// TRITONSERVER_InferenceTrace object. The 'userp' data is the same
/// as what is supplied in the call to TRITONSERVER_InferenceTraceNew.
typedef void (*TRITONSERVER_InferenceTraceReleaseFn_t)(
TRITONSERVER_InferenceTrace* trace, void* userp);
/// Create a new inference trace object. The caller takes ownership of
/// the TRITONSERVER_InferenceTrace object and must call
/// TRITONSERVER_InferenceTraceDelete to release the object.
///
/// The activity callback function will be called to report activity
/// for 'trace' as well as for any child traces that are spawned by
/// 'trace', and so the activity callback must check the trace object
/// to determine specifically what activity is being reported.
///
/// The release callback is called for both 'trace' and for any child
/// traces spawned by 'trace'.
///
/// \param trace Returns the new inference trace object.
/// \param level The tracing level.
/// \param parent_id The parent trace id for this trace. A value of 0
/// indicates that there is not parent trace.
/// \param activity_fn The callback function where activity for the
/// trace is reported.
/// \param release_fn The callback function called when all activity
/// is complete for the trace.
/// \param trace_userp User-provided pointer that is delivered to
/// the activity and release callback functions.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceTraceNew(
TRITONSERVER_InferenceTrace** trace, TRITONSERVER_InferenceTraceLevel level,
uint64_t parent_id, TRITONSERVER_InferenceTraceActivityFn_t activity_fn,
TRITONSERVER_InferenceTraceReleaseFn_t release_fn, void* trace_userp);
/// Create a new inference trace object. The caller takes ownership of
/// the TRITONSERVER_InferenceTrace object and must call
/// TRITONSERVER_InferenceTraceDelete to release the object.
///
/// The timeline and tensor activity callback function will be called to report
/// activity for 'trace' as well as for any child traces that are spawned by
/// 'trace', and so the activity callback must check the trace object
/// to determine specifically what activity is being reported.
///
/// The release callback is called for both 'trace' and for any child
/// traces spawned by 'trace'.
///
/// \param trace Returns the new inference trace object.
/// \param level The tracing level.
/// \param parent_id The parent trace id for this trace. A value of 0
/// indicates that there is not parent trace.
/// \param activity_fn The callback function where timeline activity for the
/// trace is reported.
/// \param tensor_activity_fn The callback function where tensor activity for
/// the trace is reported.
/// \param release_fn The callback function called when all activity
/// is complete for the trace.
/// \param trace_userp User-provided pointer that is delivered to
/// the activity and release callback functions.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceTraceTensorNew(
TRITONSERVER_InferenceTrace** trace, TRITONSERVER_InferenceTraceLevel level,
uint64_t parent_id, TRITONSERVER_InferenceTraceActivityFn_t activity_fn,
TRITONSERVER_InferenceTraceTensorActivityFn_t tensor_activity_fn,
TRITONSERVER_InferenceTraceReleaseFn_t release_fn, void* trace_userp);
/// Delete a trace object.
///
/// \param trace The trace object.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceTraceDelete(
TRITONSERVER_InferenceTrace* trace);
/// Get the id associated with a trace. Every trace is assigned an id
/// that is unique across all traces created for a Triton server.
///
/// \param trace The trace.
/// \param id Returns the id associated with the trace.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceTraceId(
TRITONSERVER_InferenceTrace* trace, uint64_t* id);
/// Get the parent id associated with a trace. The parent id indicates
/// a parent-child relationship between two traces. A parent id value
/// of 0 indicates that there is no parent trace.
///
/// \param trace The trace.
/// \param id Returns the parent id associated with the trace.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceTraceParentId(
TRITONSERVER_InferenceTrace* trace, uint64_t* parent_id);
/// Get the name of the model associated with a trace. The caller does
/// not own the returned string and must not modify or delete it. The
/// lifetime of the returned string extends only as long as 'trace'.
///
/// \param trace The trace.
/// \param model_name Returns the name of the model associated with
/// the trace.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceTraceModelName(
TRITONSERVER_InferenceTrace* trace, const char** model_name);
/// Get the version of the model associated with a trace.
///
/// \param trace The trace.
/// \param model_version Returns the version of the model associated
/// with the trace.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_InferenceTraceModelVersion(
TRITONSERVER_InferenceTrace* trace, int64_t* model_version);
/// TRITONSERVER_InferenceRequest
///
/// Object representing an inference request. The inference request
/// provides the meta-data and input tensor values needed for an
/// inference and returns the inference result meta-data and output
/// tensors. An inference request object can be modified and reused
/// multiple times.
///
/// Inference request flags. The enum values must be power-of-2 values.
typedef enum tritonserver_requestflag_enum {
TRITONSERVER_REQUEST_FLAG_SEQUENCE_START = 1,
TRITONSERVER_REQUEST_FLAG_SEQUENCE_END = 2
} TRITONSERVER_RequestFlag;
/// Inference request release flags. The enum values must be
/// power-of-2 values.
typedef enum tritonserver_requestreleaseflag_enum {
TRITONSERVER_REQUEST_RELEASE_ALL = 1
} TRITONSERVER_RequestReleaseFlag;
/// Inference response complete flags. The enum values must be
/// power-of-2 values.
typedef enum tritonserver_responsecompleteflag_enum {
TRITONSERVER_RESPONSE_COMPLETE_FINAL = 1
} TRITONSERVER_ResponseCompleteFlag;
/// Type for inference request release callback function. The callback
/// indicates what type of release is being performed on the request
/// and for some of these the callback function takes ownership of the
/// TRITONSERVER_InferenceRequest object. The 'userp' data is the data
/// provided as 'request_release_userp' in the call to
/// TRITONSERVER_InferenceRequestSetReleaseCallback.
///
/// One or more flags will be specified when the callback is invoked,
/// and the callback must take the following actions:
///
/// - TRITONSERVER_REQUEST_RELEASE_ALL: The entire inference request
/// is being released and ownership is passed to the callback
/// function. Triton will not longer access the 'request' object
/// itself nor any input tensor data associated with the
/// request. The callback should free or otherwise manage the
/// 'request' object and all associated tensor data.
///
/// Note that currently TRITONSERVER_REQUEST_RELEASE_ALL should always
/// be set when the callback is invoked but in the future that may
/// change, so the callback should explicitly check for the flag
/// before taking ownership of the request object.
///
typedef void (*TRITONSERVER_InferenceRequestReleaseFn_t)(
TRITONSERVER_InferenceRequest* request, const uint32_t flags, void* userp);
/// Type for callback function indicating that an inference response
/// has completed. The callback function takes ownership of the
/// TRITONSERVER_InferenceResponse object. The 'userp' data is the
/// data provided as 'response_userp' in the call to
/// TRITONSERVER_InferenceRequestSetResponseCallback.
///
/// One or more flags may be specified when the callback is invoked:
///
/// - TRITONSERVER_RESPONSE_COMPLETE_FINAL: Indicates that no more
/// responses will be generated for a given request (more
/// specifically, that no more responses will be generated for the
/// inference request that set this callback and 'userp'). When
/// this flag is set 'response' may be a response object or may be
/// nullptr. If 'response' is not nullptr, then 'response' is the
/// last response that Triton will produce for the request. If
/// 'response' is nullptr then Triton is indicating that no more
/// responses will be produced for the request.
typedef void (*TRITONSERVER_InferenceResponseCompleteFn_t)(
TRITONSERVER_InferenceResponse* response, const uint32_t flags,
void* userp);
/// Create a new inference request object.
///
/// \param inference_request Returns the new request object.
/// \param server the inference server object.
/// \param model_name The name of the model to use for the request.
/// \param model_version The version of the model to use for the
/// request. If -1 then the server will choose a version based on the
/// model's policy.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestNew(
TRITONSERVER_InferenceRequest** inference_request,
TRITONSERVER_Server* server, const char* model_name,
const int64_t model_version);
/// Delete an inference request object.
///
/// \param inference_request The request object.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestDelete(
TRITONSERVER_InferenceRequest* inference_request);
/// Get the ID for a request. The returned ID is owned by
/// 'inference_request' and must not be modified or freed by the
/// caller.
///
/// \param inference_request The request object.
/// \param id Returns the ID.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestId(
TRITONSERVER_InferenceRequest* inference_request, const char** id);
/// Set the ID for a request.
///
/// \param inference_request The request object.
/// \param id The ID.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestSetId(
TRITONSERVER_InferenceRequest* inference_request, const char* id);
/// Get the flag(s) associated with a request. On return 'flags' holds
/// a bitwise-or of all flag values, see TRITONSERVER_RequestFlag for
/// available flags.
///
/// \param inference_request The request object.
/// \param flags Returns the flags.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestFlags(
TRITONSERVER_InferenceRequest* inference_request, uint32_t* flags);
/// Set the flag(s) associated with a request. 'flags' should hold a
/// bitwise-or of all flag values, see TRITONSERVER_RequestFlag for
/// available flags.
///
/// \param inference_request The request object.
/// \param flags The flags.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestSetFlags(
TRITONSERVER_InferenceRequest* inference_request, uint32_t flags);
/// Get the correlation ID of the inference request as an unsigned integer.
/// Default is 0, which indicates that the request has no correlation ID.
/// If the correlation id associated with the inference request is a string,
/// this function will return a failure. The correlation ID is used
/// to indicate two or more inference request are related to each other.
/// How this relationship is handled by the inference server is determined by
/// the model's scheduling policy.
///
/// \param inference_request The request object.
/// \param correlation_id Returns the correlation ID.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_InferenceRequestCorrelationId(
TRITONSERVER_InferenceRequest* inference_request, uint64_t* correlation_id);
/// Get the correlation ID of the inference request as a string.
/// Default is empty "", which indicates that the request has no correlation ID.
/// If the correlation id associated with the inference request is an unsigned
/// integer, then this function will return a failure. The correlation ID
/// is used to indicate two or more inference request are related to each other.
/// How this relationship is handled by the inference server is determined by
/// the model's scheduling policy.
///
/// \param inference_request The request object.
/// \param correlation_id Returns the correlation ID.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_InferenceRequestCorrelationIdString(
TRITONSERVER_InferenceRequest* inference_request,
const char** correlation_id);
/// Set the correlation ID of the inference request to be an unsigned integer.
/// Default is 0, which indicates that the request has no correlation ID.
/// The correlation ID is used to indicate two or more inference request
/// are related to each other. How this relationship is handled by the
/// inference server is determined by the model's scheduling policy.
///
/// \param inference_request The request object.
/// \param correlation_id The correlation ID.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_InferenceRequestSetCorrelationId(
TRITONSERVER_InferenceRequest* inference_request, uint64_t correlation_id);
/// Set the correlation ID of the inference request to be a string.
/// The correlation ID is used to indicate two or more inference
/// request are related to each other. How this relationship is
/// handled by the inference server is determined by the model's
/// scheduling policy.
///
/// \param inference_request The request object.
/// \param correlation_id The correlation ID.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_InferenceRequestSetCorrelationIdString(
TRITONSERVER_InferenceRequest* inference_request,
const char* correlation_id);
/// Get the priority for a request. The default is 0 indicating that
/// the request does not specify a priority and so will use the
/// model's default priority.
///
/// \param inference_request The request object.
/// \param priority Returns the priority level.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestPriority(
TRITONSERVER_InferenceRequest* inference_request, uint32_t* priority);
/// Set the priority for a request. The default is 0 indicating that
/// the request does not specify a priority and so will use the
/// model's default priority.
///
/// \param inference_request The request object.
/// \param priority The priority level.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_InferenceRequestSetPriority(
TRITONSERVER_InferenceRequest* inference_request, uint32_t priority);
/// Get the timeout for a request, in microseconds. The default is 0
/// which indicates that the request has no timeout.
///
/// \param inference_request The request object.
/// \param timeout_us Returns the timeout, in microseconds.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_InferenceRequestTimeoutMicroseconds(
TRITONSERVER_InferenceRequest* inference_request, uint64_t* timeout_us);
/// Set the timeout for a request, in microseconds. The default is 0
/// which indicates that the request has no timeout.
///
/// \param inference_request The request object.
/// \param timeout_us The timeout, in microseconds.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_InferenceRequestSetTimeoutMicroseconds(
TRITONSERVER_InferenceRequest* inference_request, uint64_t timeout_us);
/// Add an input to a request.
///
/// \param inference_request The request object.
/// \param name The name of the input.
/// \param datatype The type of the input. Valid type names are BOOL,
/// UINT8, UINT16, UINT32, UINT64, INT8, INT16, INT32, INT64, FP16,
/// FP32, FP64, and BYTES.
/// \param shape The shape of the input.
/// \param dim_count The number of dimensions of 'shape'.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceRequestAddInput(
TRITONSERVER_InferenceRequest* inference_request, const char* name,
const TRITONSERVER_DataType datatype, const int64_t* shape,
uint64_t dim_count);
/// Add a raw input to a request. The name recognized by the model, data type
/// and shape of the input will be deduced from model configuration.
/// This function must be called at most once on request with no other input to
/// ensure the deduction is accurate.
///
/// \param inference_request The request object.
/// \param name The name of the input. This name is only used as a reference
/// of the raw input in other Tritonserver APIs. It doesn't assoicate with the
/// name used in the model.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_InferenceRequestAddRawInput(
TRITONSERVER_InferenceRequest* inference_request, const char* name);
/// Remove an input from a request.
///
/// \param inference_request The request object.
/// \param name The name of the input.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_InferenceRequestRemoveInput(
TRITONSERVER_InferenceRequest* inference_request, const char* name);
/// Remove all inputs from a request.
///
/// \param inference_request The request object.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_InferenceRequestRemoveAllInputs(
TRITONSERVER_InferenceRequest* inference_request);
/// Assign a buffer of data to an input. The buffer will be appended
/// to any existing buffers for that input. The 'inference_request'
/// object takes ownership of the buffer and so the caller should not
/// modify or free the buffer until that ownership is released by
/// 'inference_request' being deleted or by the input being removed
/// from 'inference_request'.
///
/// \param inference_request The request object.
/// \param name The name of the input.
/// \param base The base address of the input data.
/// \param byte_size The size, in bytes, of the input data.
/// \param memory_type The memory type of the input data.
/// \param memory_type_id The memory type id of the input data.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_InferenceRequestAppendInputData(
TRITONSERVER_InferenceRequest* inference_request, const char* name,
const void* base, size_t byte_size, TRITONSERVER_MemoryType memory_type,
int64_t memory_type_id);
/// Assign a buffer of data to an input for execution on all model instances
/// with the specified host policy. The buffer will be appended to any existing
/// buffers for that input on all devices with this host policy. The
/// 'inference_request' object takes ownership of the buffer and so the caller
/// should not modify or free the buffer until that ownership is released by
/// 'inference_request' being deleted or by the input being removed from
/// 'inference_request'. If the execution is scheduled on a device that does not
/// have a input buffer specified using this function, then the input buffer
/// specified with TRITONSERVER_InferenceRequestAppendInputData will be used so
/// a non-host policy specific version of data must be added using that API.
/// \param inference_request The request object.
/// \param name The name of the input.
/// \param base The base address of the input data.
/// \param byte_size The size, in bytes, of the input data.
/// \param memory_type The memory type of the input data.
/// \param memory_type_id The memory type id of the input data.
/// \param host_policy_name All model instances executing with this host_policy
/// will use this input buffer for execution.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_InferenceRequestAppendInputDataWithHostPolicy(
TRITONSERVER_InferenceRequest* inference_request, const char* name,
const void* base, size_t byte_size, TRITONSERVER_MemoryType memory_type,
int64_t memory_type_id, const char* host_policy_name);
/// Assign a buffer of data to an input. The buffer will be appended
/// to any existing buffers for that input. The 'inference_request'
/// object takes ownership of the buffer and so the caller should not
/// modify or free the buffer until that ownership is released by
/// 'inference_request' being deleted or by the input being removed
/// from 'inference_request'.
///
/// \param inference_request The request object.
/// \param name The name of the input.
/// \param base The base address of the input data.
/// \param buffer_attributes The buffer attrubutes of the input.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_InferenceRequestAppendInputDataWithBufferAttributes(
TRITONSERVER_InferenceRequest* inference_request, const char* name,
const void* base, TRITONSERVER_BufferAttributes* buffer_attributes);
/// Clear all input data from an input, releasing ownership of the
/// buffer(s) that were appended to the input with
/// TRITONSERVER_InferenceRequestAppendInputData or
/// TRITONSERVER_InferenceRequestAppendInputDataWithHostPolicy
/// \param inference_request The request object.
/// \param name The name of the input.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_InferenceRequestRemoveAllInputData(
TRITONSERVER_InferenceRequest* inference_request, const char* name);
/// Add an output request to an inference request.
///
/// \param inference_request The request object.
/// \param name The name of the output.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_InferenceRequestAddRequestedOutput(
TRITONSERVER_InferenceRequest* inference_request, const char* name);
/// Remove an output request from an inference request.
///
/// \param inference_request The request object.
/// \param name The name of the output.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_InferenceRequestRemoveRequestedOutput(
TRITONSERVER_InferenceRequest* inference_request, const char* name);
/// Remove all output requests from an inference request.
///
/// \param inference_request The request object.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_InferenceRequestRemoveAllRequestedOutputs(
TRITONSERVER_InferenceRequest* inference_request);
/// Set the release callback for an inference request. The release
/// callback is called by Triton to return ownership of the request
/// object.
///
/// \param inference_request The request object.
/// \param request_release_fn The function called to return ownership
/// of the 'inference_request' object.
/// \param request_release_userp User-provided pointer that is
/// delivered to the 'request_release_fn' callback.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_InferenceRequestSetReleaseCallback(
TRITONSERVER_InferenceRequest* inference_request,
TRITONSERVER_InferenceRequestReleaseFn_t request_release_fn,
void* request_release_userp);
/// Set the allocator and response callback for an inference
/// request. The allocator is used to allocate buffers for any output
/// tensors included in responses that are produced for this
/// request. The response callback is called to return response
/// objects representing responses produced for this request.
///
/// \param inference_request The request object.
/// \param response_allocator The TRITONSERVER_ResponseAllocator to use
/// to allocate buffers to hold inference results.
/// \param response_allocator_userp User-provided pointer that is
/// delivered to the response allocator's start and allocation functions.
/// \param response_fn The function called to deliver an inference
/// response for this request.
/// \param response_userp User-provided pointer that is delivered to
/// the 'response_fn' callback.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_InferenceRequestSetResponseCallback(
TRITONSERVER_InferenceRequest* inference_request,
TRITONSERVER_ResponseAllocator* response_allocator,
void* response_allocator_userp,
TRITONSERVER_InferenceResponseCompleteFn_t response_fn,
void* response_userp);
/// TRITONSERVER_InferenceResponse
///
/// Object representing an inference response. The inference response
/// provides the meta-data and output tensor values calculated by the
/// inference.
///
/// Delete an inference response object.
///
/// \param inference_response The response object.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceResponseDelete(
TRITONSERVER_InferenceResponse* inference_response);
/// Return the error status of an inference response. Return a
/// TRITONSERVER_Error object on failure, return nullptr on success.
/// The returned error object is owned by 'inference_response' and so
/// should not be deleted by the caller.
///
/// \param inference_response The response object.
/// \return a TRITONSERVER_Error indicating the success or failure
/// status of the response.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceResponseError(
TRITONSERVER_InferenceResponse* inference_response);
/// Get model used to produce a response. The caller does not own the
/// returned model name value and must not modify or delete it. The
/// lifetime of all returned values extends until 'inference_response'
/// is deleted.
///
/// \param inference_response The response object.
/// \param model_name Returns the name of the model.
/// \param model_version Returns the version of the model.
/// this response.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceResponseModel(
TRITONSERVER_InferenceResponse* inference_response, const char** model_name,
int64_t* model_version);
/// Get the ID of the request corresponding to a response. The caller
/// does not own the returned ID and must not modify or delete it. The
/// lifetime of all returned values extends until 'inference_response'
/// is deleted.
///
/// \param inference_response The response object.
/// \param request_id Returns the ID of the request corresponding to
/// this response.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceResponseId(
TRITONSERVER_InferenceResponse* inference_response,
const char** request_id);
/// Get the number of parameters available in the response.
///
/// \param inference_response The response object.
/// \param count Returns the number of parameters.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_InferenceResponseParameterCount(
TRITONSERVER_InferenceResponse* inference_response, uint32_t* count);
/// Get all information about a parameter. The caller does not own any
/// of the returned values and must not modify or delete them. The
/// lifetime of all returned values extends until 'inference_response'
/// is deleted.
///
/// The 'vvalue' returns a void* pointer that must be cast
/// appropriately based on 'type'. For example:
///
/// void* vvalue;
/// TRITONSERVER_ParameterType type;
/// TRITONSERVER_InferenceResponseParameter(
/// response, index, &name, &type, &vvalue);
/// switch (type) {
/// case TRITONSERVER_PARAMETER_BOOL:
/// bool value = *(reinterpret_cast<bool*>(vvalue));
/// ...
/// case TRITONSERVER_PARAMETER_INT:
/// int64_t value = *(reinterpret_cast<int64_t*>(vvalue));
/// ...
/// case TRITONSERVER_PARAMETER_STRING:
/// const char* value = reinterpret_cast<const char*>(vvalue);
/// ...
///
/// \param inference_response The response object.
/// \param index The index of the parameter, must be 0 <= index <
/// count, where 'count' is the value returned by
/// TRITONSERVER_InferenceResponseParameterCount.
/// \param name Returns the name of the parameter.
/// \param type Returns the type of the parameter.
/// \param vvalue Returns a pointer to the parameter value.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_InferenceResponseParameter(
TRITONSERVER_InferenceResponse* inference_response, const uint32_t index,
const char** name, TRITONSERVER_ParameterType* type, const void** vvalue);
/// Get the number of outputs available in the response.
///
/// \param inference_response The response object.
/// \param count Returns the number of output tensors.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_InferenceResponseOutputCount(
TRITONSERVER_InferenceResponse* inference_response, uint32_t* count);
/// Get all information about an output tensor. The tensor data is
/// returned as the base pointer to the data and the size, in bytes,
/// of the data. The caller does not own any of the returned values
/// and must not modify or delete them. The lifetime of all returned
/// values extends until 'inference_response' is deleted.
///
/// \param inference_response The response object.
/// \param index The index of the output tensor, must be 0 <= index <
/// count, where 'count' is the value returned by
/// TRITONSERVER_InferenceResponseOutputCount.
/// \param name Returns the name of the output.
/// \param datatype Returns the type of the output.
/// \param shape Returns the shape of the output.
/// \param dim_count Returns the number of dimensions of the returned
/// shape.
/// \param base Returns the tensor data for the output.
/// \param byte_size Returns the size, in bytes, of the data.
/// \param memory_type Returns the memory type of the data.
/// \param memory_type_id Returns the memory type id of the data.
/// \param userp The user-specified value associated with the buffer
/// in TRITONSERVER_ResponseAllocatorAllocFn_t.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_InferenceResponseOutput(
TRITONSERVER_InferenceResponse* inference_response, const uint32_t index,
const char** name, TRITONSERVER_DataType* datatype, const int64_t** shape,
uint64_t* dim_count, const void** base, size_t* byte_size,
TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id,
void** userp);
/// Get a classification label associated with an output for a given
/// index. The caller does not own the returned label and must not
/// modify or delete it. The lifetime of all returned label extends
/// until 'inference_response' is deleted.
///
/// \param inference_response The response object.
/// \param index The index of the output tensor, must be 0 <= index <
/// count, where 'count' is the value returned by
/// TRITONSERVER_InferenceResponseOutputCount.
/// \param class_index The index of the class.
/// \param name Returns the label corresponding to 'class_index' or
/// nullptr if no label.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_InferenceResponseOutputClassificationLabel(
TRITONSERVER_InferenceResponse* inference_response, const uint32_t index,
const size_t class_index, const char** label);
/// TRITONSERVER_BufferAttributes
///
/// API to create, modify, or retrieve attributes associated with a buffer.
///
/// Create a new buffer attributes object. The caller takes ownership of
/// the TRITONSERVER_BufferAttributes object and must call
/// TRITONSERVER_BufferAttributesDelete to release the object.
///
/// \param buffer_attributes Returns the new buffer attributes object.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_BufferAttributesNew(
TRITONSERVER_BufferAttributes** buffer_attributes);
/// Delete a buffer attributes object.
///
/// \param buffer_attributes The buffer_attributes object.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_BufferAttributesDelete(
TRITONSERVER_BufferAttributes* buffer_attributes);
/// Set the memory type id field of the buffer attributes.
///
/// \param buffer_attributes The buffer attributes object.
/// \param memory_type_id Memory type id to assign to the buffer attributes
/// object.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_BufferAttributesSetMemoryTypeId(
TRITONSERVER_BufferAttributes* buffer_attributes, int64_t memory_type_id);
/// Set the memory type field of the buffer attributes.
///
/// \param buffer_attributes The buffer attributes object.
/// \param memory_type Memory type to assign to the buffer attributes object.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_BufferAttributesSetMemoryType(
TRITONSERVER_BufferAttributes* buffer_attributes,
TRITONSERVER_MemoryType memory_type);
/// Set the CudaIpcHandle field of the buffer attributes.
///
/// \param buffer_attributes The buffer attributes object.
/// \param cuda_ipc_handle The CudaIpcHandle to assign to the buffer attributes
/// object.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_BufferAttributesSetCudaIpcHandle(
TRITONSERVER_BufferAttributes* buffer_attributes, void* cuda_ipc_handle);
/// Set the byte size field of the buffer attributes.
///
/// \param buffer_attributes The buffer attributes object.
/// \param byte_size Byte size to assign to the buffer attributes object.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_BufferAttributesSetByteSize(
TRITONSERVER_BufferAttributes* buffer_attributes, size_t byte_size);
/// Get the memory type id field of the buffer attributes.
///
/// \param buffer_attributes The buffer attributes object.
/// \param memory_type_id Returns the memory type id associated with the buffer
/// attributes object.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_BufferAttributesMemoryTypeId(
TRITONSERVER_BufferAttributes* buffer_attributes, int64_t* memory_type_id);
/// Get the memory type field of the buffer attributes.
///
/// \param buffer_attributes The buffer attributes object.
/// \param memory_type Returns the memory type associated with the buffer
/// attributes object.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_BufferAttributesMemoryType(
TRITONSERVER_BufferAttributes* buffer_attributes,
TRITONSERVER_MemoryType* memory_type);
/// Get the CudaIpcHandle field of the buffer attributes object.
///
/// \param buffer_attributes The buffer attributes object.
/// \param cuda_ipc_handle Returns the memory type associated with the buffer
/// attributes object. If the cudaIpcHandle does not exist for the buffer,
/// nullptr will be returned.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_BufferAttributesCudaIpcHandle(
TRITONSERVER_BufferAttributes* buffer_attributes, void** cuda_ipc_handle);
/// Get the byte size field of the buffer attributes.
///
/// \param buffer_attributes The buffer attributes object.
/// \param byte_size Returns the byte size associated with the buffer attributes
/// object.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_BufferAttributesByteSize(
TRITONSERVER_BufferAttributes* buffer_attributes, size_t* byte_size);
/// TRITONSERVER_ServerOptions
///
/// Options to use when creating an inference server.
///
/// Model control modes
typedef enum tritonserver_modelcontrolmode_enum {
TRITONSERVER_MODEL_CONTROL_NONE,
TRITONSERVER_MODEL_CONTROL_POLL,
TRITONSERVER_MODEL_CONTROL_EXPLICIT
} TRITONSERVER_ModelControlMode;
/// Rate limit modes
typedef enum tritonserver_ratelimitmode_enum {
TRITONSERVER_RATE_LIMIT_OFF,
TRITONSERVER_RATE_LIMIT_EXEC_COUNT
} TRITONSERVER_RateLimitMode;
/// Create a new server options object. The caller takes ownership of
/// the TRITONSERVER_ServerOptions object and must call
/// TRITONSERVER_ServerOptionsDelete to release the object.
///
/// \param options Returns the new server options object.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsNew(
TRITONSERVER_ServerOptions** options);
/// Delete a server options object.
///
/// \param options The server options object.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsDelete(
TRITONSERVER_ServerOptions* options);
/// Set the textual ID for the server in a server options. The ID is a
/// name that identifies the server.
///
/// \param options The server options object.
/// \param server_id The server identifier.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetServerId(
TRITONSERVER_ServerOptions* options, const char* server_id);
/// Set the model repository path in a server options. The path must be
/// the full absolute path to the model repository. This function can be called
/// multiple times with different paths to set multiple model repositories.
/// Note that if a model is not unique across all model repositories
/// at any time, the model will not be available.
///
/// \param options The server options object.
/// \param model_repository_path The full path to the model repository.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_ServerOptionsSetModelRepositoryPath(
TRITONSERVER_ServerOptions* options, const char* model_repository_path);
/// Set the model control mode in a server options. For each mode the models
/// will be managed as the following:
///
/// TRITONSERVER_MODEL_CONTROL_NONE: the models in model repository will be
/// loaded on startup. After startup any changes to the model repository will
/// be ignored. Calling TRITONSERVER_ServerPollModelRepository will result in
/// an error.
///
/// TRITONSERVER_MODEL_CONTROL_POLL: the models in model repository will be
/// loaded on startup. The model repository can be polled periodically using
/// TRITONSERVER_ServerPollModelRepository and the server will load, unload,
/// and updated models according to changes in the model repository.
///
/// TRITONSERVER_MODEL_CONTROL_EXPLICIT: the models in model repository will
/// not be loaded on startup. The corresponding model control APIs must be
/// called to load / unload a model in the model repository.
///
/// \param options The server options object.
/// \param mode The mode to use for the model control.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_ServerOptionsSetModelControlMode(
TRITONSERVER_ServerOptions* options, TRITONSERVER_ModelControlMode mode);
/// Set the model to be loaded at startup in a server options. The model must be
/// present in one, and only one, of the specified model repositories.
/// This function can be called multiple times with different model name
/// to set multiple startup models.
/// Note that it only takes affect on TRITONSERVER_MODEL_CONTROL_EXPLICIT mode.
///
/// \param options The server options object.
/// \param mode_name The name of the model to load on startup.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_ServerOptionsSetStartupModel(
TRITONSERVER_ServerOptions* options, const char* model_name);
/// Enable or disable strict model configuration handling in a server
/// options.
///
/// \param options The server options object.
/// \param strict True to enable strict model configuration handling,
/// false to disable.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_ServerOptionsSetStrictModelConfig(
TRITONSERVER_ServerOptions* options, bool strict);
/// Set the rate limit mode in a server options.
///
/// TRITONSERVER_RATE_LIMIT_EXEC_COUNT: The rate limiting prioritizes the
/// inference execution using the number of times each instance has got a
/// chance to run. The execution gets to run only when its resource
/// constraints are satisfied.
///
/// TRITONSERVER_RATE_LIMIT_OFF: The rate limiting is turned off and the
/// inference gets executed whenever an instance is available.
///
/// \param options The server options object.
/// \param mode The mode to use for the rate limiting. By default, execution
/// count is used to determine the priorities.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_ServerOptionsSetRateLimiterMode(
TRITONSERVER_ServerOptions* options, TRITONSERVER_RateLimitMode mode);
/// Add resource count for rate limiting.
///
/// \param options The server options object.
/// \param name The name of the resource.
/// \param count The count of the resource.
/// \param device The device identifier for the resource. A value of -1
/// indicates that the specified number of resources are available on every
/// device. The device value is ignored for a global resource. The server
/// will use the rate limiter configuration specified for instance groups
/// in model config to determine whether resource is global. In case of
/// conflicting resource type in different model configurations, server
/// will raise an appropriate error while loading model.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_ServerOptionsAddRateLimiterResource(
TRITONSERVER_ServerOptions* options, const char* resource_name,
const size_t resource_count, const int device);
/// Set the total pinned memory byte size that the server can allocate
/// in a server options. The pinned memory pool will be shared across
/// Triton itself and the backends that use
/// TRITONBACKEND_MemoryManager to allocate memory.
///
/// \param options The server options object.
/// \param size The pinned memory pool byte size.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_ServerOptionsSetPinnedMemoryPoolByteSize(
TRITONSERVER_ServerOptions* options, uint64_t size);
/// Set the total CUDA memory byte size that the server can allocate
/// on given GPU device in a server options. The pinned memory pool
/// will be shared across Triton itself and the backends that use
/// TRITONBACKEND_MemoryManager to allocate memory.
///
/// \param options The server options object.
/// \param gpu_device The GPU device to allocate the memory pool.
/// \param size The CUDA memory pool byte size.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_ServerOptionsSetCudaMemoryPoolByteSize(
TRITONSERVER_ServerOptions* options, int gpu_device, uint64_t size);
/// Set the total response cache byte size that the server can allocate in CPU
/// memory. The response cache will be shared across all inference requests and
/// across all models.
///
/// \param options The server options object.
/// \param size The total response cache byte size.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_ServerOptionsSetResponseCacheByteSize(
TRITONSERVER_ServerOptions* options, uint64_t size);
/// Set the minimum support CUDA compute capability in a server
/// options.
///
/// \param options The server options object.
/// \param cc The minimum CUDA compute capability.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_ServerOptionsSetMinSupportedComputeCapability(
TRITONSERVER_ServerOptions* options, double cc);
/// Enable or disable exit-on-error in a server options.
///
/// \param options The server options object.
/// \param exit True to enable exiting on intialization error, false
/// to continue.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_ServerOptionsSetExitOnError(
TRITONSERVER_ServerOptions* options, bool exit);
/// Enable or disable strict readiness handling in a server options.
///
/// \param options The server options object.
/// \param strict True to enable strict readiness handling, false to
/// disable.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_ServerOptionsSetStrictReadiness(
TRITONSERVER_ServerOptions* options, bool strict);
/// Set the exit timeout, in seconds, for the server in a server
/// options.
///
/// \param options The server options object.
/// \param timeout The exit timeout, in seconds.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_ServerOptionsSetExitTimeout(
TRITONSERVER_ServerOptions* options, unsigned int timeout);
/// Set the number of threads used in buffer manager in a server options.
///
/// \param options The server options object.
/// \param thread_count The number of threads.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_ServerOptionsSetBufferManagerThreadCount(
TRITONSERVER_ServerOptions* options, unsigned int thread_count);
/// Set the number of threads to concurrently load models in a server options.
///
/// \param options The server options object.
/// \param thread_count The number of threads.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_ServerOptionsSetModelLoadThreadCount(
TRITONSERVER_ServerOptions* options, unsigned int thread_count);
/// Provide a log output file.
///
/// \param options The server options object.
/// \param file a string defining the file where the log outputs will be saved.
/// An empty string for the file name will cause triton to direct logging
/// facilities to the console
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetLogFile(
TRITONSERVER_ServerOptions* options, const char* file);
/// Enable or disable info level logging.
///
/// \param options The server options object.
/// \param log True to enable info logging, false to disable.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetLogInfo(
TRITONSERVER_ServerOptions* options, bool log);
/// Enable or disable warning level logging.
///
/// \param options The server options object.
/// \param log True to enable warning logging, false to disable.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetLogWarn(
TRITONSERVER_ServerOptions* options, bool log);
/// Enable or disable error level logging.
///
/// \param options The server options object.
/// \param log True to enable error logging, false to disable.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetLogError(
TRITONSERVER_ServerOptions* options, bool log);
/// Set the logging format.
///
/// \param options The server options object.
/// \param format The logging format.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_ServerOptionsSetLogFormat(
TRITONSERVER_ServerOptions* options, const TRITONSERVER_LogFormat format);
/// Set verbose logging level. Level zero disables verbose logging.
///
/// \param options The server options object.
/// \param level The verbose logging level.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_ServerOptionsSetLogVerbose(
TRITONSERVER_ServerOptions* options, int level);
/// Enable or disable metrics collection in a server options.
///
/// \param options The server options object.
/// \param metrics True to enable metrics, false to disable.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerOptionsSetMetrics(
TRITONSERVER_ServerOptions* options, bool metrics);
/// Enable or disable GPU metrics collection in a server options. GPU
/// metrics are collected if both this option and
/// TRITONSERVER_ServerOptionsSetMetrics are true.
///
/// \param options The server options object.
/// \param gpu_metrics True to enable GPU metrics, false to disable.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_ServerOptionsSetGpuMetrics(
TRITONSERVER_ServerOptions* options, bool gpu_metrics);
/// Enable or disable CPU metrics collection in a server options. CPU
/// metrics are collected if both this option and
/// TRITONSERVER_ServerOptionsSetMetrics are true.
///
/// \param options The server options object.
/// \param cpu_metrics True to enable CPU metrics, false to disable.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_ServerOptionsSetCpuMetrics(
TRITONSERVER_ServerOptions* options, bool cpu_metrics);
/// Set the interval for metrics collection in a server options.
/// This is 2000 milliseconds by default.
///
/// \param options The server options object.
/// \param metrics_interval_ms The time interval in ms between
/// successive metrics updates.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_ServerOptionsSetMetricsInterval(
TRITONSERVER_ServerOptions* options, uint64_t metrics_interval_ms);
/// Set the directory containing backend shared libraries. This
/// directory is searched last after the version and model directory
/// in the model repository when looking for the backend shared
/// library for a model. If the backend is named 'be' the directory
/// searched is 'backend_dir'/be/libtriton_be.so.
///
/// \param options The server options object.
/// \param backend_dir The full path of the backend directory.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_ServerOptionsSetBackendDirectory(
TRITONSERVER_ServerOptions* options, const char* backend_dir);
/// Set the directory containing repository agent shared libraries. This
/// directory is searched when looking for the repository agent shared
/// library for a model. If the backend is named 'ra' the directory
/// searched is 'repoagent_dir'/ra/libtritonrepoagent_ra.so.
///
/// \param options The server options object.
/// \param repoagent_dir The full path of the repository agent directory.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_ServerOptionsSetRepoAgentDirectory(
TRITONSERVER_ServerOptions* options, const char* repoagent_dir);
/// Specify the limit on memory usage as a fraction on the device identified by
/// 'kind' and 'device_id'. If model loading on the device is requested and the
/// current memory usage exceeds the limit, the load will be rejected. If not
/// specified, the limit will not be set.
///
/// Currently support TRITONSERVER_INSTANCEGROUPKIND_GPU
///
/// \param options The server options object.
/// \param kind The kind of the device.
/// \param device_id The id of the device.
/// \param fraction The limit on memory usage as a fraction
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_ServerOptionsSetModelLoadDeviceLimit(
TRITONSERVER_ServerOptions* options,
const TRITONSERVER_InstanceGroupKind kind, const int device_id,
const double fraction);
/// Set a configuration setting for a named backend in a server
/// options.
///
/// \param options The server options object.
/// \param backend_name The name of the backend.
/// \param setting The name of the setting.
/// \param value The setting value.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_ServerOptionsSetBackendConfig(
TRITONSERVER_ServerOptions* options, const char* backend_name,
const char* setting, const char* value);
/// Set a host policy setting for a given policy name in a server options.
///
/// \param options The server options object.
/// \param policy_name The name of the policy.
/// \param setting The name of the setting.
/// \param value The setting value.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_ServerOptionsSetHostPolicy(
TRITONSERVER_ServerOptions* options, const char* policy_name,
const char* setting, const char* value);
/// TRITONSERVER_Server
///
/// An inference server.
///
/// Model batch flags. The enum values must be power-of-2 values.
typedef enum tritonserver_batchflag_enum {
TRITONSERVER_BATCH_UNKNOWN = 1,
TRITONSERVER_BATCH_FIRST_DIM = 2
} TRITONSERVER_ModelBatchFlag;
/// Model index flags. The enum values must be power-of-2 values.
typedef enum tritonserver_modelindexflag_enum {
TRITONSERVER_INDEX_FLAG_READY = 1
} TRITONSERVER_ModelIndexFlag;
/// Model transaction policy flags. The enum values must be
/// power-of-2 values.
typedef enum tritonserver_txn_property_flag_enum {
TRITONSERVER_TXN_ONE_TO_ONE = 1,
TRITONSERVER_TXN_DECOUPLED = 2
} TRITONSERVER_ModelTxnPropertyFlag;
/// Create a new server object. The caller takes ownership of the
/// TRITONSERVER_Server object and must call TRITONSERVER_ServerDelete
/// to release the object.
///
/// \param server Returns the new inference server object.
/// \param options The inference server options object.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerNew(
TRITONSERVER_Server** server, TRITONSERVER_ServerOptions* options);
/// Delete a server object. If server is not already stopped it is
/// stopped before being deleted.
///
/// \param server The inference server object.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerDelete(
TRITONSERVER_Server* server);
/// Stop a server object. A server can't be restarted once it is
/// stopped.
///
/// \param server The inference server object.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerStop(
TRITONSERVER_Server* server);
/// Register a new model repository. Not available in polling mode.
///
/// \param server The inference server object.
/// \param repository_path The full path to the model repository.
/// \param name_mapping List of name_mapping parameters. Each mapping has
/// the model directory name as its key, overriden model name as its value.
/// \param model_count Number of mappings provided.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_ServerRegisterModelRepository(
TRITONSERVER_Server* server, const char* repository_path,
const TRITONSERVER_Parameter** name_mapping, const uint32_t mapping_count);
/// Unregister a model repository. Not available in polling mode.
///
/// \param server The inference server object.
/// \param repository_path The full path to the model repository.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_ServerUnregisterModelRepository(
TRITONSERVER_Server* server, const char* repository_path);
/// Check the model repository for changes and update server state
/// based on those changes.
///
/// \param server The inference server object.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_ServerPollModelRepository(TRITONSERVER_Server* server);
/// Is the server live?
///
/// \param server The inference server object.
/// \param live Returns true if server is live, false otherwise.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerIsLive(
TRITONSERVER_Server* server, bool* live);
/// Is the server ready?
///
/// \param server The inference server object.
/// \param ready Returns true if server is ready, false otherwise.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerIsReady(
TRITONSERVER_Server* server, bool* ready);
/// Is the model ready?
///
/// \param server The inference server object.
/// \param model_name The name of the model to get readiness for.
/// \param model_version The version of the model to get readiness
/// for. If -1 then the server will choose a version based on the
/// model's policy.
/// \param ready Returns true if server is ready, false otherwise.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerModelIsReady(
TRITONSERVER_Server* server, const char* model_name,
const int64_t model_version, bool* ready);
/// Get the batch properties of the model. The properties are
/// communicated by a flags value and an (optional) object returned by
/// 'voidp'.
///
/// - TRITONSERVER_BATCH_UNKNOWN: Triton cannot determine the
/// batching properties of the model. This means that the model
/// does not support batching in any way that is useable by
/// Triton. The returned 'voidp' value is nullptr.
///
/// - TRITONSERVER_BATCH_FIRST_DIM: The model supports batching
/// along the first dimension of every input and output
/// tensor. Triton schedulers that perform batching can
/// automatically batch inference requests along this dimension.
/// The returned 'voidp' value is nullptr.
///
/// \param server The inference server object.
/// \param model_name The name of the model.
/// \param model_version The version of the model. If -1 then the
/// server will choose a version based on the model's policy.
/// \param flags Returns flags indicating the batch properties of the
/// model.
/// \param voidp If non-nullptr, returns a point specific to the
/// 'flags' value.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_ServerModelBatchProperties(
TRITONSERVER_Server* server, const char* model_name,
const int64_t model_version, uint32_t* flags, void** voidp);
/// Get the transaction policy of the model. The policy is
/// communicated by a flags value.
///
/// - TRITONSERVER_TXN_ONE_TO_ONE: The model generates exactly
/// one response per request.
///
/// - TRITONSERVER_TXN_DECOUPLED: The model may generate zero
/// to many responses per request.
///
/// \param server The inference server object.
/// \param model_name The name of the model.
/// \param model_version The version of the model. If -1 then the
/// server will choose a version based on the model's policy.
/// \param txn_flags Returns flags indicating the transaction policy of the
/// model.
/// \param voidp If non-nullptr, returns a point specific to the 'flags' value.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_ServerModelTransactionProperties(
TRITONSERVER_Server* server, const char* model_name,
const int64_t model_version, uint32_t* txn_flags, void** voidp);
/// Get the metadata of the server as a TRITONSERVER_Message object.
/// The caller takes ownership of the message object and must call
/// TRITONSERVER_MessageDelete to release the object.
///
/// \param server The inference server object.
/// \param server_metadata Returns the server metadata message.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerMetadata(
TRITONSERVER_Server* server, TRITONSERVER_Message** server_metadata);
/// Get the metadata of a model as a TRITONSERVER_Message
/// object. The caller takes ownership of the message object and must
/// call TRITONSERVER_MessageDelete to release the object.
///
/// \param server The inference server object.
/// \param model_name The name of the model.
/// \param model_version The version of the model.
/// If -1 then the server will choose a version based on the model's
/// policy.
/// \param model_metadata Returns the model metadata message.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerModelMetadata(
TRITONSERVER_Server* server, const char* model_name,
const int64_t model_version, TRITONSERVER_Message** model_metadata);
/// Get the statistics of a model as a TRITONSERVER_Message
/// object. The caller takes ownership of the object and must call
/// TRITONSERVER_MessageDelete to release the object.
///
/// \param server The inference server object.
/// \param model_name The name of the model.
/// If empty, then statistics for all available models will be returned,
/// and the server will choose a version based on those models' policies.
/// \param model_version The version of the model. If -1 then the
/// server will choose a version based on the model's policy.
/// \param model_stats Returns the model statistics message.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerModelStatistics(
TRITONSERVER_Server* server, const char* model_name,
const int64_t model_version, TRITONSERVER_Message** model_stats);
/// Get the configuration of a model as a TRITONSERVER_Message object.
/// The caller takes ownership of the message object and must call
/// TRITONSERVER_MessageDelete to release the object.
///
/// \param server The inference server object.
/// \param model_name The name of the model.
/// \param model_version The version of the model. If -1 then the
/// server will choose a version based on the model's policy.
/// \param config_version The model configuration will be returned in
/// a format matching this version. If the configuration cannot be
/// represented in the requested version's format then an error will
/// be returned. Currently only version 1 is supported.
/// \param model_config Returns the model config message.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerModelConfig(
TRITONSERVER_Server* server, const char* model_name,
const int64_t model_version, const uint32_t config_version,
TRITONSERVER_Message** model_config);
/// Get the index of all unique models in the model repositories as a
/// TRITONSERVER_Message object. The caller takes ownership of the
/// message object and must call TRITONSERVER_MessageDelete to release
/// the object.
///
/// If TRITONSERVER_INDEX_FLAG_READY is set in 'flags' only the models
/// that are loaded into the server and ready for inferencing are
/// returned.
///
/// \param server The inference server object.
/// \param flags TRITONSERVER_ModelIndexFlag flags that control how to
/// collect the index.
/// \param model_index Return the model index message that holds the
/// index of all models contained in the server's model repository(s).
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerModelIndex(
TRITONSERVER_Server* server, uint32_t flags,
TRITONSERVER_Message** model_index);
/// Load the requested model or reload the model if it is already
/// loaded. The function does not return until the model is loaded or
/// fails to load. Returned error indicates if model loaded
/// successfully or not.
///
/// \param server The inference server object.
/// \param model_name The name of the model.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerLoadModel(
TRITONSERVER_Server* server, const char* model_name);
/// Load the requested model or reload the model if it is already
/// loaded, with load parameters provided. The function does not return until
/// the model is loaded or fails to load. Returned error indicates if model
/// loaded successfully or not.
/// Currently the below parameter names are recognized:
/// - "config" : string parameter that contains a JSON representation of the
/// model configuration. This config will be used for loading the model instead
/// of the one in the model directory.
///
/// \param server The inference server object.
/// \param model_name The name of the model.
/// \param parameters The array of load parameters.
/// \param parameter_count The number of parameters.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_ServerLoadModelWithParameters(
TRITONSERVER_Server* server, const char* model_name,
const TRITONSERVER_Parameter** parameters, const uint64_t parameter_count);
/// Unload the requested model. Unloading a model that is not loaded
/// on server has no affect and success code will be returned.
/// The function does not wait for the requested model to be fully unload
/// and success code will be returned.
/// Returned error indicates if model unloaded successfully or not.
///
/// \param server The inference server object.
/// \param model_name The name of the model.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerUnloadModel(
TRITONSERVER_Server* server, const char* model_name);
/// Unload the requested model, and also unload any dependent model that
/// was loaded along with the requested model (for example, the models composing
/// an ensemble). Unloading a model that is not loaded
/// on server has no affect and success code will be returned.
/// The function does not wait for the requested model and all dependent
/// models to be fully unload and success code will be returned.
/// Returned error indicates if model unloaded successfully or not.
///
/// \param server The inference server object.
/// \param model_name The name of the model.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error*
TRITONSERVER_ServerUnloadModelAndDependents(
TRITONSERVER_Server* server, const char* model_name);
/// Get the current metrics for the server. The caller takes ownership
/// of the metrics object and must call TRITONSERVER_MetricsDelete to
/// release the object.
///
/// \param server The inference server object.
/// \param metrics Returns the metrics.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerMetrics(
TRITONSERVER_Server* server, TRITONSERVER_Metrics** metrics);
/// Perform inference using the meta-data and inputs supplied by the
/// 'inference_request'. If the function returns success, then the
/// caller releases ownership of 'inference_request' and must not
/// access it in any way after this call, until ownership is returned
/// via the 'request_release_fn' callback registered in the request
/// object with TRITONSERVER_InferenceRequestSetReleaseCallback.
///
/// The function unconditionally takes ownership of 'trace' and so the
/// caller must not access it in any way after this call (except in
/// the trace activity callbacks) until ownership is returned via the
/// trace's release_fn callback.
///
/// Responses produced for this request are returned using the
/// allocator and callback registered with the request by
/// TRITONSERVER_InferenceRequestSetResponseCallback.
///
/// \param server The inference server object.
/// \param inference_request The request object.
/// \param trace The trace object for this request, or nullptr if no
/// tracing.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_ServerInferAsync(
TRITONSERVER_Server* server,
TRITONSERVER_InferenceRequest* inference_request,
TRITONSERVER_InferenceTrace* trace);
/// TRITONSERVER_MetricKind
///
/// Types of metrics recognized by TRITONSERVER.
///
typedef enum TRITONSERVER_metrickind_enum {
TRITONSERVER_METRIC_KIND_COUNTER,
TRITONSERVER_METRIC_KIND_GAUGE
} TRITONSERVER_MetricKind;
/// Create a new metric family object. The caller takes ownership of the
/// TRITONSERVER_MetricFamily object and must call
/// TRITONSERVER_MetricFamilyDelete to release the object.
///
/// \param family Returns the new metric family object.
/// \param kind The type of metric family to create.
/// \param name The name of the metric family seen when calling the metrics
/// endpoint.
/// \param description The description of the metric family seen when
/// calling the metrics endpoint.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_MetricFamilyNew(
TRITONSERVER_MetricFamily** family, const TRITONSERVER_MetricKind kind,
const char* name, const char* description);
/// Delete a metric family object.
/// A TRITONSERVER_MetricFamily* object should be deleted AFTER its
/// corresponding TRITONSERVER_Metric* objects have been deleted.
/// Attempting to delete a family before its metrics will return an error.
///
/// \param family The metric family object to delete.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_MetricFamilyDelete(
TRITONSERVER_MetricFamily* family);
/// Create a new metric object. The caller takes ownership of the
/// TRITONSERVER_Metric object and must call
/// TRITONSERVER_MetricDelete to release the object. The caller is also
/// responsible for ownership of the labels passed in. Each label can be deleted
/// immediately after creating the metric with TRITONSERVER_ParameterDelete
/// if not re-using the labels.
///
/// \param metric Returns the new metric object.
/// \param family The metric family to add this new metric to.
/// \param labels The array of labels to associate with this new metric.
/// \param label_count The number of labels.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_MetricNew(
TRITONSERVER_Metric** metric, TRITONSERVER_MetricFamily* family,
const TRITONSERVER_Parameter** labels, const uint64_t label_count);
/// Delete a metric object.
/// All TRITONSERVER_Metric* objects should be deleted BEFORE their
/// corresponding TRITONSERVER_MetricFamily* objects have been deleted.
/// If a family is deleted before its metrics, an error will be returned.
///
/// \param metric The metric object to delete.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_MetricDelete(
TRITONSERVER_Metric* metric);
/// Get the current value of a metric object.
/// Supports metrics of kind TRITONSERVER_METRIC_KIND_COUNTER
/// and TRITONSERVER_METRIC_KIND_GAUGE, and returns
/// TRITONSERVER_ERROR_UNSUPPORTED for unsupported TRITONSERVER_MetricKind.
///
/// \param metric The metric object to query.
/// \param value Returns the current value of the metric object.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_MetricValue(
TRITONSERVER_Metric* metric, double* value);
/// Increment the current value of metric by value.
/// Supports metrics of kind TRITONSERVER_METRIC_KIND_GAUGE for any value,
/// and TRITONSERVER_METRIC_KIND_COUNTER for non-negative values. Returns
/// TRITONSERVER_ERROR_UNSUPPORTED for unsupported TRITONSERVER_MetricKind
/// and TRITONSERVER_ERROR_INVALID_ARG for negative values on a
/// TRITONSERVER_METRIC_KIND_COUNTER metric.
///
/// \param metric The metric object to update.
/// \param value The amount to increment the metric's value by.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_MetricIncrement(
TRITONSERVER_Metric* metric, double value);
/// Set the current value of metric to value.
/// Supports metrics of kind TRITONSERVER_METRIC_KIND_GAUGE and returns
/// TRITONSERVER_ERROR_UNSUPPORTED for unsupported TRITONSERVER_MetricKind.
///
/// \param metric The metric object to update.
/// \param value The amount to set metric's value to.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_MetricSet(
TRITONSERVER_Metric* metric, double value);
/// Get the TRITONSERVER_MetricKind of metric and its corresponding family.
///
/// \param metric The metric object to query.
/// \param kind Returns the TRITONSERVER_MetricKind of metric.
/// \return a TRITONSERVER_Error indicating success or failure.
TRITONSERVER_DECLSPEC TRITONSERVER_Error* TRITONSERVER_GetMetricKind(
TRITONSERVER_Metric* metric, TRITONSERVER_MetricKind* kind);
#ifdef __cplusplus
}
#endif
3rdparty/core-r22.12/src/backend_config.cc 0 → 100644
View file @ b30f3cdb
// Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
// * Neither the name of NVIDIA CORPORATION nor the names of its
// contributors may be used to endorse or promote products derived
// from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "backend_config.h"
#include "status.h"
#include "triton/common/logging.h"
#include "triton/common/model_config.h"
namespace triton { namespace core {
namespace {
Status
GetTFSpecializedBackendName(
const triton::common::BackendCmdlineConfigMap& config_map,
std::string* specialized_name)
{
std::string tf_version_str = "2";
const auto& itr = config_map.find("tensorflow");
if (itr != config_map.end()) {
if (BackendConfiguration(itr->second, "version", &tf_version_str).IsOk()) {
if ((tf_version_str != "1") && (tf_version_str != "2")) {
return Status(
Status::Code::INVALID_ARG,
"unexpected TensorFlow library version '" + tf_version_str +
"', expects 1 or 2.");
}
}
}
*specialized_name += tf_version_str;
return Status::Success;
}
} // namespace
Status
BackendConfiguration(
const triton::common::BackendCmdlineConfig& config, const std::string& key,
std::string* val)
{
for (const auto& pr : config) {
if (pr.first == key) {
*val = pr.second;
return Status::Success;
}
}
return Status(
Status::Code::INTERNAL,
std::string("unable to find common backend configuration for '") + key +
"'");
}
Status
BackendConfigurationParseStringToDouble(const std::string& str, double* val)
{
try {
*val = std::stod(str);
}
catch (...) {
return Status(
Status::Code::INTERNAL,
"unable to parse common backend configuration as double");
}
return Status::Success;
}
Status
BackendConfigurationParseStringToBool(const std::string& str, bool* val)
{
try {
std::string lowercase_str{str};
std::transform(
lowercase_str.begin(), lowercase_str.end(), lowercase_str.begin(),
[](unsigned char c) { return std::tolower(c); });
*val = (lowercase_str == "true");
}
catch (...) {
return Status(
Status::Code::INTERNAL,
"unable to parse common backend configuration as bool");
}
return Status::Success;
}
Status
BackendConfigurationGlobalBackendsDirectory(
const triton::common::BackendCmdlineConfigMap& config_map, std::string* dir)
{
const auto& itr = config_map.find(std::string());
if (itr == config_map.end()) {
return Status(
Status::Code::INTERNAL,
"unable to find global backends directory configuration");
}
RETURN_IF_ERROR(BackendConfiguration(itr->second, "backend-directory", dir));
return Status::Success;
}
Status
BackendConfigurationMinComputeCapability(
const triton::common::BackendCmdlineConfigMap& config_map, double* mcc)
{
#ifdef TRITON_ENABLE_GPU
*mcc = TRITON_MIN_COMPUTE_CAPABILITY;
#else
*mcc = 0;
#endif // TRITON_ENABLE_GPU
const auto& itr = config_map.find(std::string());
if (itr == config_map.end()) {
return Status(
Status::Code::INTERNAL, "unable to find common backend configuration");
}
std::string min_compute_capability_str;
RETURN_IF_ERROR(BackendConfiguration(
itr->second, "min-compute-capability", &min_compute_capability_str));
RETURN_IF_ERROR(
BackendConfigurationParseStringToDouble(min_compute_capability_str, mcc));
return Status::Success;
}
Status
BackendConfigurationAutoCompleteConfig(
const triton::common::BackendCmdlineConfigMap& config_map, bool* acc)
{
const auto& itr = config_map.find(std::string());
if (itr == config_map.end()) {
return Status(
Status::Code::INTERNAL, "unable to find auto-complete configuration");
}
std::string auto_complete_config_str;
RETURN_IF_ERROR(BackendConfiguration(
itr->second, "auto-complete-config", &auto_complete_config_str));
RETURN_IF_ERROR(
BackendConfigurationParseStringToBool(auto_complete_config_str, acc));
return Status::Success;
}
Status
BackendConfigurationSpecializeBackendName(
const triton::common::BackendCmdlineConfigMap& config_map,
const std::string& backend_name, std::string* specialized_name)
{
*specialized_name = backend_name;
if (backend_name == "tensorflow") {
RETURN_IF_ERROR(GetTFSpecializedBackendName(config_map, specialized_name));
}
return Status::Success;
}
Status
BackendConfigurationBackendLibraryName(
const std::string& backend_name, std::string* libname)
{
#ifdef _WIN32
*libname = "triton_" + backend_name + ".dll";
#else
*libname = "libtriton_" + backend_name + ".so";
#endif
return Status::Success;
}
Status
BackendConfigurationModelLoadGpuFraction(
const triton::common::BackendCmdlineConfigMap& config_map,
const int device_id, double* memory_limit)
{
*memory_limit = 1.0;
const auto& itr = config_map.find(std::string());
if (itr == config_map.end()) {
return Status(
Status::Code::INTERNAL,
"unable to find global backends directory configuration");
}
static std::string key_prefix = "model-load-gpu-limit-device-";
std::string memory_limit_str;
auto status = BackendConfiguration(
itr->second, key_prefix + std::to_string(device_id), &memory_limit_str);
// Allow missing key, default to 1.0 (no limit) if the limit is not specified
if (status.IsOk()) {
RETURN_IF_ERROR(BackendConfigurationParseStringToDouble(
memory_limit_str, memory_limit));
}
return Status::Success;
}
}} // namespace triton::core
3rdparty/core-r22.12/src/backend_config.h 0 → 100644
View file @ b30f3cdb
// Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
// * Neither the name of NVIDIA CORPORATION nor the names of its
// contributors may be used to endorse or promote products derived
// from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include "status.h"
#include "triton/common/model_config.h"
namespace triton { namespace core {
/// Get a key's string value from a backend configuration.
Status BackendConfiguration(
const triton::common::BackendCmdlineConfig& config, const std::string& key,
std::string* val);
/// Convert a backend configuration string value into a double.
Status BackendConfigurationParseStringToDouble(
const std::string& str, double* val);
/// Convert a backend configuration string value into a bool.
Status BackendConfigurationParseStringToBool(const std::string& str, bool* val);
/// Get the global backends directory from the backend configuration.
Status BackendConfigurationGlobalBackendsDirectory(
const triton::common::BackendCmdlineConfigMap& config_map,
std::string* dir);
/// Get the minimum compute capability from the backend configuration.
Status BackendConfigurationMinComputeCapability(
const triton::common::BackendCmdlineConfigMap& config_map, double* mcc);
/// Get the model configuration auto-complete setting from the backend
/// configuration.
Status BackendConfigurationAutoCompleteConfig(
const triton::common::BackendCmdlineConfigMap& config_map, bool* acc);
/// Convert a backend name to the specialized version of that name
/// based on the backend configuration. For example, "tensorflow" will
/// convert to either "tensorflow1" or "tensorflow2" depending on how
/// tritonserver is run.
Status BackendConfigurationSpecializeBackendName(
const triton::common::BackendCmdlineConfigMap& config_map,
const std::string& backend_name, std::string* specialized_name);
/// Return the shared library name for a backend.
Status BackendConfigurationBackendLibraryName(
const std::string& backend_name, std::string* libname);
/// Get GPU memory limit fraction for model loading
/// from the backend configuration.
Status BackendConfigurationModelLoadGpuFraction(
const triton::common::BackendCmdlineConfigMap& config_map,
const int device_id, double* memory_limit);
}} // namespace triton::core
3rdparty/core-r22.12/src/backend_manager.cc 0 → 100644
View file @ b30f3cdb
// Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
// * Neither the name of NVIDIA CORPORATION nor the names of its
// contributors may be used to endorse or promote products derived
// from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "backend_manager.h"
#include "backend_memory_manager.h"
#include "server_message.h"
#include "shared_library.h"
#include "triton/common/logging.h"
// For unknown reason, windows will not export the TRITONBACKEND_*
// functions declared with dllexport in tritonbackend.h. To get those
// functions exported it is (also?) necessary to mark the definitions
// in this file with dllexport as well.
#if defined(_MSC_VER)
#define TRITONAPI_DECLSPEC __declspec(dllexport)
#elif defined(__GNUC__)
#define TRITONAPI_DECLSPEC __attribute__((__visibility__("default")))
#else
#define TRITONAPI_DECLSPEC
#endif
namespace triton { namespace core {
//
// TritonBackend
//
Status
TritonBackend::Create(
const std::string& name, const std::string& dir, const std::string& libpath,
const triton::common::BackendCmdlineConfig& backend_cmdline_config,
std::shared_ptr<TritonBackend>* backend)
{
// Create the JSON representation of the backend configuration.
triton::common::TritonJson::Value backend_config_json(
triton::common::TritonJson::ValueType::OBJECT);
if (!backend_cmdline_config.empty()) {
triton::common::TritonJson::Value cmdline_json(
backend_config_json, triton::common::TritonJson::ValueType::OBJECT);
for (const auto& pr : backend_cmdline_config) {
RETURN_IF_ERROR(cmdline_json.AddString(pr.first.c_str(), pr.second));
}
RETURN_IF_ERROR(
backend_config_json.Add("cmdline", std::move(cmdline_json)));
}
TritonServerMessage backend_config(backend_config_json);
auto local_backend = std::shared_ptr<TritonBackend>(
new TritonBackend(name, dir, libpath, backend_config));
// Load the library and initialize all the entrypoints
RETURN_IF_ERROR(local_backend->LoadBackendLibrary());
// Backend initialization is optional... The TRITONBACKEND_Backend
// object is this TritonBackend object. We must set set shared
// library path to point to the backend directory in case the
// backend library attempts to load additional shared libaries.
if (local_backend->backend_init_fn_ != nullptr) {
std::unique_ptr<SharedLibrary> slib;
RETURN_IF_ERROR(SharedLibrary::Acquire(&slib));
RETURN_IF_ERROR(slib->SetLibraryDirectory(local_backend->dir_));
TRITONSERVER_Error* err = local_backend->backend_init_fn_(
reinterpret_cast<TRITONBACKEND_Backend*>(local_backend.get()));
RETURN_IF_ERROR(slib->ResetLibraryDirectory());
RETURN_IF_TRITONSERVER_ERROR(err);
}
local_backend->UpdateAttributes();
*backend = std::move(local_backend);
return Status::Success;
}
Status
TritonBackend::UpdateAttributes()
{
if (backend_attri_fn_ == nullptr) {
return Status::Success;
}
// Create an Attribute object for the backend to fill, note that it copies
// some fields from 'attributes_' while the others use default value. This
// is an ad hoc way to determine whether the attribute is set by the backend
// and keep / update current value.
Attribute latest;
latest.exec_policy_ = attributes_.exec_policy_;
RETURN_IF_TRITONSERVER_ERROR(backend_attri_fn_(
reinterpret_cast<TRITONBACKEND_Backend*>(this),
reinterpret_cast<TRITONBACKEND_BackendAttribute*>(&latest)));
// Update attributes that were set
attributes_.exec_policy_ = latest.exec_policy_;
if (!latest.preferred_groups_.empty()) {
attributes_.preferred_groups_ = latest.preferred_groups_;
}
return Status::Success;
}
TritonBackend::TritonBackend(
const std::string& name, const std::string& dir, const std::string& libpath,
const TritonServerMessage& backend_config)
: name_(name), dir_(dir), libpath_(libpath),
backend_config_(backend_config), state_(nullptr)
{
ClearHandles();
}
TritonBackend::~TritonBackend()
{
LOG_VERBOSE(1) << "unloading backend '" << name_ << "'";
// Backend finalization is optional... The TRITONBACKEND_Backend
// object is this TritonBackend object.
if (backend_fini_fn_ != nullptr) {
LOG_TRITONSERVER_ERROR(
backend_fini_fn_(reinterpret_cast<TRITONBACKEND_Backend*>(this)),
"failed finalizing backend");
}
ClearHandles();
}
void
TritonBackend::ClearHandles()
{
dlhandle_ = nullptr;
backend_init_fn_ = nullptr;
backend_fini_fn_ = nullptr;
backend_attri_fn_ = nullptr;
model_init_fn_ = nullptr;
model_fini_fn_ = nullptr;
inst_init_fn_ = nullptr;
inst_fini_fn_ = nullptr;
inst_exec_fn_ = nullptr;
}
Status
TritonBackend::LoadBackendLibrary()
{
TritonBackendInitFn_t bifn;
TritonBackendFiniFn_t bffn;
TritonBackendAttriFn_t bafn;
TritonModelInitFn_t mifn;
TritonModelFiniFn_t mffn;
TritonModelInstanceInitFn_t iifn;
TritonModelInstanceFiniFn_t iffn;
TritonModelInstanceExecFn_t iefn;
{
std::unique_ptr<SharedLibrary> slib;
RETURN_IF_ERROR(SharedLibrary::Acquire(&slib));
RETURN_IF_ERROR(slib->OpenLibraryHandle(libpath_, &dlhandle_));
// Backend initialize and finalize functions, optional
RETURN_IF_ERROR(slib->GetEntrypoint(
dlhandle_, "TRITONBACKEND_Initialize", true /* optional */,
reinterpret_cast<void**>(&bifn)));
RETURN_IF_ERROR(slib->GetEntrypoint(
dlhandle_, "TRITONBACKEND_Finalize", true /* optional */,
reinterpret_cast<void**>(&bffn)));
// Backend attribute function, optional
RETURN_IF_ERROR(slib->GetEntrypoint(
dlhandle_, "TRITONBACKEND_GetBackendAttribute", true /* optional */,
reinterpret_cast<void**>(&bafn)));
// Model initialize and finalize functions, optional
RETURN_IF_ERROR(slib->GetEntrypoint(
dlhandle_, "TRITONBACKEND_ModelInitialize", true /* optional */,
reinterpret_cast<void**>(&mifn)));
RETURN_IF_ERROR(slib->GetEntrypoint(
dlhandle_, "TRITONBACKEND_ModelFinalize", true /* optional */,
reinterpret_cast<void**>(&mffn)));
// Model instance initialize and finalize functions, optional
RETURN_IF_ERROR(slib->GetEntrypoint(
dlhandle_, "TRITONBACKEND_ModelInstanceInitialize", true /* optional */,
reinterpret_cast<void**>(&iifn)));
RETURN_IF_ERROR(slib->GetEntrypoint(
dlhandle_, "TRITONBACKEND_ModelInstanceFinalize", true /* optional */,
reinterpret_cast<void**>(&iffn)));
// Model instance execute function, required
RETURN_IF_ERROR(slib->GetEntrypoint(
dlhandle_, "TRITONBACKEND_ModelInstanceExecute", false /* optional */,
reinterpret_cast<void**>(&iefn)));
}
backend_init_fn_ = bifn;
backend_fini_fn_ = bffn;
backend_attri_fn_ = bafn;
model_init_fn_ = mifn;
model_fini_fn_ = mffn;
inst_init_fn_ = iifn;
inst_fini_fn_ = iffn;
inst_exec_fn_ = iefn;
return Status::Success;
}
extern "C" {
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_ApiVersion(uint32_t* major, uint32_t* minor)
{
*major = TRITONBACKEND_API_VERSION_MAJOR;
*minor = TRITONBACKEND_API_VERSION_MINOR;
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_BackendName(TRITONBACKEND_Backend* backend, const char** name)
{
TritonBackend* tb = reinterpret_cast<TritonBackend*>(backend);
*name = tb->Name().c_str();
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_BackendConfig(
TRITONBACKEND_Backend* backend, TRITONSERVER_Message** backend_config)
{
TritonBackend* tb = reinterpret_cast<TritonBackend*>(backend);
*backend_config = const_cast<TRITONSERVER_Message*>(
reinterpret_cast<const TRITONSERVER_Message*>(&tb->BackendConfig()));
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_BackendExecutionPolicy(
TRITONBACKEND_Backend* backend, TRITONBACKEND_ExecutionPolicy* policy)
{
TritonBackend* tb = reinterpret_cast<TritonBackend*>(backend);
*policy = tb->ExecutionPolicy();
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_BackendSetExecutionPolicy(
TRITONBACKEND_Backend* backend, TRITONBACKEND_ExecutionPolicy policy)
{
TritonBackend* tb = reinterpret_cast<TritonBackend*>(backend);
tb->SetExecutionPolicy(policy);
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_BackendArtifacts(
TRITONBACKEND_Backend* backend, TRITONBACKEND_ArtifactType* artifact_type,
const char** location)
{
TritonBackend* tb = reinterpret_cast<TritonBackend*>(backend);
*artifact_type = TRITONBACKEND_ARTIFACT_FILESYSTEM;
*location = tb->Directory().c_str();
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_BackendMemoryManager(
TRITONBACKEND_Backend* backend, TRITONBACKEND_MemoryManager** manager)
{
static TritonMemoryManager gMemoryManager;
*manager = reinterpret_cast<TRITONBACKEND_MemoryManager*>(&gMemoryManager);
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_BackendState(TRITONBACKEND_Backend* backend, void** state)
{
TritonBackend* tb = reinterpret_cast<TritonBackend*>(backend);
*state = tb->State();
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_BackendSetState(TRITONBACKEND_Backend* backend, void* state)
{
TritonBackend* tb = reinterpret_cast<TritonBackend*>(backend);
tb->SetState(state);
return nullptr; // success
}
} // extern C
//
// TritonBackendManager
//
static std::weak_ptr<TritonBackendManager> backend_manager_;
static std::mutex mu_;
Status
TritonBackendManager::Create(std::shared_ptr<TritonBackendManager>* manager)
{
std::lock_guard<std::mutex> lock(mu_);
// If there is already a manager then we just use it...
*manager = backend_manager_.lock();
if (*manager != nullptr) {
return Status::Success;
}
manager->reset(new TritonBackendManager());
backend_manager_ = *manager;
return Status::Success;
}
Status
TritonBackendManager::CreateBackend(
const std::string& name, const std::string& dir, const std::string& libpath,
const triton::common::BackendCmdlineConfig& backend_cmdline_config,
std::shared_ptr<TritonBackend>* backend)
{
std::lock_guard<std::mutex> lock(mu_);
const auto& itr = backend_map_.find(libpath);
if (itr != backend_map_.end()) {
*backend = itr->second;
return Status::Success;
}
RETURN_IF_ERROR(TritonBackend::Create(
name, dir, libpath, backend_cmdline_config, backend));
backend_map_.insert({libpath, *backend});
return Status::Success;
}
Status
TritonBackendManager::BackendState(
std::unique_ptr<std::unordered_map<std::string, std::vector<std::string>>>*
backend_state)
{
std::lock_guard<std::mutex> lock(mu_);
std::unique_ptr<std::unordered_map<std::string, std::vector<std::string>>>
backend_state_map(
new std::unordered_map<std::string, std::vector<std::string>>);
for (const auto& backend_pair : backend_map_) {
auto& libpath = backend_pair.first;
auto backend = backend_pair.second;
const char* backend_config;
size_t backend_config_size;
backend->BackendConfig().Serialize(&backend_config, &backend_config_size);
backend_state_map->insert(
{backend->Name(), std::vector<std::string>{libpath, backend_config}});
}
*backend_state = std::move(backend_state_map);
return Status::Success;
}
}} // namespace triton::core
3rdparty/core-r22.12/src/backend_manager.h 0 → 100644
View file @ b30f3cdb
// Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
// * Neither the name of NVIDIA CORPORATION nor the names of its
// contributors may be used to endorse or promote products derived
// from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include <memory>
#include <mutex>
#include <string>
#include <unordered_map>
#include "constants.h"
#include "server_message.h"
#include "status.h"
#include "triton/common/model_config.h"
#include "tritonserver_apis.h"
namespace triton { namespace core {
//
// Proxy to a backend shared library.
//
class TritonBackend {
public:
struct Attribute {
Attribute() : exec_policy_(TRITONBACKEND_EXECUTION_BLOCKING) {}
TRITONBACKEND_ExecutionPolicy exec_policy_;
std::vector<inference::ModelInstanceGroup> preferred_groups_;
};
typedef TRITONSERVER_Error* (*TritonModelInitFn_t)(
TRITONBACKEND_Model* model);
typedef TRITONSERVER_Error* (*TritonModelFiniFn_t)(
TRITONBACKEND_Model* model);
typedef TRITONSERVER_Error* (*TritonModelInstanceInitFn_t)(
TRITONBACKEND_ModelInstance* instance);
typedef TRITONSERVER_Error* (*TritonModelInstanceFiniFn_t)(
TRITONBACKEND_ModelInstance* instance);
typedef TRITONSERVER_Error* (*TritonModelInstanceExecFn_t)(
TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Request** requests,
const uint32_t request_cnt);
static Status Create(
const std::string& name, const std::string& dir,
const std::string& libpath,
const triton::common::BackendCmdlineConfig& backend_cmdline_config,
std::shared_ptr<TritonBackend>* backend);
~TritonBackend();
const std::string& Name() const { return name_; }
const std::string& Directory() const { return dir_; }
const TritonServerMessage& BackendConfig() const { return backend_config_; }
const Attribute& BackendAttributes() const { return attributes_; }
TRITONBACKEND_ExecutionPolicy ExecutionPolicy() const
{
return attributes_.exec_policy_;
}
void SetExecutionPolicy(const TRITONBACKEND_ExecutionPolicy policy)
{
attributes_.exec_policy_ = policy;
}
void* State() { return state_; }
void SetState(void* state) { state_ = state; }
TritonModelInitFn_t ModelInitFn() const { return model_init_fn_; }
TritonModelFiniFn_t ModelFiniFn() const { return model_fini_fn_; }
TritonModelInstanceInitFn_t ModelInstanceInitFn() const
{
return inst_init_fn_;
}
TritonModelInstanceFiniFn_t ModelInstanceFiniFn() const
{
return inst_fini_fn_;
}
TritonModelInstanceExecFn_t ModelInstanceExecFn() const
{
return inst_exec_fn_;
}
private:
typedef TRITONSERVER_Error* (*TritonBackendInitFn_t)(
TRITONBACKEND_Backend* backend);
typedef TRITONSERVER_Error* (*TritonBackendFiniFn_t)(
TRITONBACKEND_Backend* backend);
typedef TRITONSERVER_Error* (*TritonBackendAttriFn_t)(
TRITONBACKEND_Backend* backend,
TRITONBACKEND_BackendAttribute* backend_attributes);
TritonBackend(
const std::string& name, const std::string& dir,
const std::string& libpath, const TritonServerMessage& backend_config);
void ClearHandles();
Status LoadBackendLibrary();
Status UpdateAttributes();
// The name of the backend.
const std::string name_;
// Full path to the directory holding backend shared library and
// other artifacts.
const std::string dir_;
// Full path to the backend shared library.
const std::string libpath_;
// Backend configuration as JSON
TritonServerMessage backend_config_;
// backend attributes
Attribute attributes_;
// dlopen / dlsym handles
void* dlhandle_;
TritonBackendInitFn_t backend_init_fn_;
TritonBackendFiniFn_t backend_fini_fn_;
TritonBackendAttriFn_t backend_attri_fn_;
TritonModelInitFn_t model_init_fn_;
TritonModelFiniFn_t model_fini_fn_;
TritonModelInstanceInitFn_t inst_init_fn_;
TritonModelInstanceFiniFn_t inst_fini_fn_;
TritonModelInstanceExecFn_t inst_exec_fn_;
// Opaque state associated with the backend.
void* state_;
};
//
// Manage communication with Triton backends and their lifecycle.
//
class TritonBackendManager {
public:
static Status Create(std::shared_ptr<TritonBackendManager>* manager);
Status CreateBackend(
const std::string& name, const std::string& dir,
const std::string& libpath,
const triton::common::BackendCmdlineConfig& backend_cmdline_config,
std::shared_ptr<TritonBackend>* backend);
Status BackendState(
std::unique_ptr<
std::unordered_map<std::string, std::vector<std::string>>>*
backend_state);
private:
DISALLOW_COPY_AND_ASSIGN(TritonBackendManager);
TritonBackendManager() = default;
std::unordered_map<std::string, std::shared_ptr<TritonBackend>> backend_map_;
};
}} // namespace triton::core
3rdparty/core-r22.12/src/backend_memory_manager.cc 0 → 100644
View file @ b30f3cdb
// Copyright 2020-2022, NVIDIA CORPORATION. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
// * Neither the name of NVIDIA CORPORATION nor the names of its
// contributors may be used to endorse or promote products derived
// from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "backend_memory_manager.h"
#include "pinned_memory_manager.h"
#include "status.h"
#include "tritonserver_apis.h"
#ifdef TRITON_ENABLE_GPU
#include <cuda_runtime_api.h>
#include "cuda_memory_manager.h"
#endif // TRITON_ENABLE_GPU
// For unknown reason, windows will not export the TRITONBACKEND_*
// functions declared with dllexport in tritonbackend.h. To get those
// functions exported it is (also?) necessary to mark the definitions
// in this file with dllexport as well.
#if defined(_MSC_VER)
#define TRITONAPI_DECLSPEC __declspec(dllexport)
#elif defined(__GNUC__)
#define TRITONAPI_DECLSPEC __attribute__((__visibility__("default")))
#else
#define TRITONAPI_DECLSPEC
#endif
namespace triton { namespace core {
extern "C" {
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_MemoryManagerAllocate(
TRITONBACKEND_MemoryManager* manager, void** buffer,
const TRITONSERVER_MemoryType memory_type, const int64_t memory_type_id,
const uint64_t byte_size)
{
switch (memory_type) {
case TRITONSERVER_MEMORY_GPU:
#ifdef TRITON_ENABLE_GPU
{
auto status = CudaMemoryManager::Alloc(buffer, byte_size, memory_type_id);
if (!status.IsOk()) {
return TRITONSERVER_ErrorNew(
StatusCodeToTritonCode(status.ErrorCode()),
status.Message().c_str());
}
break;
}
#else
return TRITONSERVER_ErrorNew(
TRITONSERVER_ERROR_UNSUPPORTED,
"GPU memory allocation not supported");
#endif // TRITON_ENABLE_GPU
case TRITONSERVER_MEMORY_CPU_PINNED:
#ifdef TRITON_ENABLE_GPU
{
TRITONSERVER_MemoryType mt = memory_type;
auto status = PinnedMemoryManager::Alloc(buffer, byte_size, &mt, false);
if (!status.IsOk()) {
return TRITONSERVER_ErrorNew(
StatusCodeToTritonCode(status.ErrorCode()),
status.Message().c_str());
}
break;
}
#else
return TRITONSERVER_ErrorNew(
TRITONSERVER_ERROR_UNSUPPORTED,
"Pinned memory allocation not supported");
#endif // TRITON_ENABLE_GPU
case TRITONSERVER_MEMORY_CPU: {
*buffer = malloc(byte_size);
if (*buffer == nullptr) {
return TRITONSERVER_ErrorNew(
TRITONSERVER_ERROR_UNAVAILABLE, "CPU memory allocation failed");
}
break;
}
}
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_MemoryManagerFree(
TRITONBACKEND_MemoryManager* manager, void* buffer,
const TRITONSERVER_MemoryType memory_type, const int64_t memory_type_id)
{
switch (memory_type) {
case TRITONSERVER_MEMORY_GPU: {
#ifdef TRITON_ENABLE_GPU
auto status = CudaMemoryManager::Free(buffer, memory_type_id);
if (!status.IsOk()) {
return TRITONSERVER_ErrorNew(
StatusCodeToTritonCode(status.StatusCode()),
status.Message().c_str());
}
#endif // TRITON_ENABLE_GPU
break;
}
case TRITONSERVER_MEMORY_CPU_PINNED: {
#ifdef TRITON_ENABLE_GPU
auto status = PinnedMemoryManager::Free(buffer);
if (!status.IsOk()) {
return TRITONSERVER_ErrorNew(
StatusCodeToTritonCode(status.StatusCode()),
status.Message().c_str());
}
#endif // TRITON_ENABLE_GPU
break;
}
case TRITONSERVER_MEMORY_CPU:
free(buffer);
break;
}
return nullptr; // success
}
} // extern C
}} // namespace triton::core
3rdparty/core-r22.12/src/backend_memory_manager.h 0 → 100644
View file @ b30f3cdb
// Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
// * Neither the name of NVIDIA CORPORATION nor the names of its
// contributors may be used to endorse or promote products derived
// from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
namespace triton { namespace core {
// Currently there is just a global memory manager that is used for
// all backends and which simply forwards requests on to the core
// memory manager.
struct TritonMemoryManager {
};
}} // namespace triton::core
3rdparty/core-r22.12/src/backend_model.cc 0 → 100644
View file @ b30f3cdb
// Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
// * Neither the name of NVIDIA CORPORATION nor the names of its
// contributors may be used to endorse or promote products derived
// from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "backend_model.h"
#include <vector>
#include "backend_config.h"
#include "backend_model_instance.h"
#include "dynamic_batch_scheduler.h"
#include "filesystem.h"
#include "model_config_utils.h"
#include "numa_utils.h"
#include "sequence_batch_scheduler.h"
#include "sequence_state.h"
#include "server.h"
#include "server_message.h"
#include "shared_library.h"
#include "triton/common/logging.h"
#include "tritonserver_apis.h"
// For unknown reason, windows will not export the TRITONBACKEND_*
// functions declared with dllexport in tritonbackend.h. To get those
// functions exported it is (also?) necessary to mark the definitions
// in this file with dllexport as well.
#if defined(_MSC_VER)
#define TRITONAPI_DECLSPEC __declspec(dllexport)
#elif defined(__GNUC__)
#define TRITONAPI_DECLSPEC __attribute__((__visibility__("default")))
#else
#define TRITONAPI_DECLSPEC
#endif
namespace triton { namespace core {
Status
TritonModel::Create(
InferenceServer* server, const std::string& model_path,
const triton::common::BackendCmdlineConfigMap& backend_cmdline_config_map,
const triton::common::HostPolicyCmdlineConfigMap& host_policy_map,
const std::string& model_name, const int64_t version,
inference::ModelConfig model_config, const bool is_config_provided,
std::unique_ptr<TritonModel>* model)
{
model->reset();
// The model configuration must specify a backend. The name of the
// corresponding shared library must be libtriton_<backend>.so.
if (model_config.backend().empty()) {
return Status(
Status::Code::INVALID_ARG,
"must specify 'backend' for '" + model_config.name() + "'");
}
// Localize the content of the model repository corresponding to
// 'model_name'. This model holds a handle to the localized content
// so that it persists as long as the model is loaded.
std::shared_ptr<LocalizedPath> localized_model_dir;
RETURN_IF_ERROR(LocalizePath(model_path, &localized_model_dir));
// Localize paths in backend model config
// [FIXME] Remove once a more permanent solution is implemented (DLIS-4211)
RETURN_IF_ERROR(LocalizePythonBackendExecutionEnvironmentPath(
model_path, &model_config, &localized_model_dir));
// Get some internal configuration values needed for initialization.
std::string backend_dir;
RETURN_IF_ERROR(BackendConfigurationGlobalBackendsDirectory(
backend_cmdline_config_map, &backend_dir));
bool auto_complete_config = false;
RETURN_IF_ERROR(BackendConfigurationAutoCompleteConfig(
backend_cmdline_config_map, &auto_complete_config));
double min_compute_capability = 0;
RETURN_IF_ERROR(BackendConfigurationMinComputeCapability(
backend_cmdline_config_map, &min_compute_capability));
std::string specialized_backend_name;
RETURN_IF_ERROR(BackendConfigurationSpecializeBackendName(
backend_cmdline_config_map, model_config.backend(),
&specialized_backend_name));
std::string backend_libname;
RETURN_IF_ERROR(BackendConfigurationBackendLibraryName(
specialized_backend_name, &backend_libname));
// Get the path to the backend shared library. Search path is
// version directory, model directory, global backend directory.
const auto localized_model_path = localized_model_dir->Path();
const auto version_path =
JoinPath({localized_model_path, std::to_string(version)});
const std::string global_path =
JoinPath({backend_dir, specialized_backend_name});
const std::vector<std::string> search_paths = {
version_path, localized_model_path, global_path};
std::string backend_libdir;
std::string backend_libpath;
for (const auto& path : search_paths) {
const auto full_path = JoinPath({path, backend_libname});
bool exists = false;
RETURN_IF_ERROR(FileExists(full_path, &exists));
if (exists) {
backend_libdir = path;
backend_libpath = full_path;
break;
}
}
if (backend_libpath.empty()) {
return Status(
Status::Code::INVALID_ARG, "unable to find '" + backend_libname +
"' for model '" + model_config.name() +
"', searched: " + version_path + ", " +
model_path + ", " + global_path);
}
// Resolve the global backend configuration with the specific backend
// configuration
triton::common::BackendCmdlineConfig config;
RETURN_IF_ERROR(ResolveBackendConfigs(
backend_cmdline_config_map, model_config.backend(), config));
RETURN_IF_ERROR(SetBackendConfigDefaults(config));
std::shared_ptr<TritonBackend> backend;
RETURN_IF_ERROR(server->BackendManager()->CreateBackend(
model_config.backend(), backend_libdir, backend_libpath, config,
&backend));
// Normalize backend-dependent config
{
const auto& attributes = backend->BackendAttributes();
// [WIP] formalize config normalization / validation
RETURN_IF_ERROR(NormalizeInstanceGroup(
min_compute_capability, attributes.preferred_groups_, &model_config));
RETURN_IF_ERROR(
ValidateInstanceGroup(model_config, min_compute_capability));
}
// Create and initialize the model.
std::unique_ptr<TritonModel> local_model(new TritonModel(
server, localized_model_dir, backend, min_compute_capability, version,
model_config, auto_complete_config));
TritonModel* raw_local_model = local_model.get();
// Model initialization is optional... The TRITONBACKEND_Model
// object is this TritonModel object. We must set set shared library
// path to point to the backend directory in case the backend
// library attempts to load additional shared libaries.
if (backend->ModelInitFn() != nullptr) {
std::unique_ptr<SharedLibrary> slib;
RETURN_IF_ERROR(SharedLibrary::Acquire(&slib));
RETURN_IF_ERROR(slib->SetLibraryDirectory(backend->Directory()));
TRITONSERVER_Error* err = backend->ModelInitFn()(
reinterpret_cast<TRITONBACKEND_Model*>(raw_local_model));
RETURN_IF_ERROR(slib->ResetLibraryDirectory());
RETURN_IF_TRITONSERVER_ERROR(err);
}
// Initialize the model for Triton core usage
RETURN_IF_ERROR(local_model->Init(is_config_provided));
bool device_blocking = false;
if (local_model->backend_->ExecutionPolicy() ==
TRITONBACKEND_EXECUTION_DEVICE_BLOCKING) {
if (model_config.has_sequence_batching()) {
LOG_INFO << "Overriding execution policy to "
"\"TRITONBACKEND_EXECUTION_BLOCKING\" for sequence model \""
<< model_config.name() << "\"";
} else {
device_blocking = true;
}
}
// Create and initialize the model instances for this model.
RETURN_IF_ERROR(TritonModelInstance::CreateInstances(
raw_local_model, backend_cmdline_config_map, host_policy_map,
model_config, device_blocking));
RETURN_IF_ERROR(local_model->SetConfiguredScheduler());
*model = std::move(local_model);
return Status::Success;
}
Status
TritonModel::ResolveBackendConfigs(
const triton::common::BackendCmdlineConfigMap& backend_cmdline_config_map,
const std::string& backend_name,
triton::common::BackendCmdlineConfig& config)
{
const auto& global_itr = backend_cmdline_config_map.find(std::string());
const auto& specific_itr = backend_cmdline_config_map.find(backend_name);
if (specific_itr == backend_cmdline_config_map.end() &&
global_itr != backend_cmdline_config_map.end()) {
for (auto setting : global_itr->second) {
config.push_back(setting);
}
} else if (
specific_itr != backend_cmdline_config_map.end() &&
global_itr == backend_cmdline_config_map.end()) {
for (auto setting : specific_itr->second) {
config.push_back(setting);
}
} else if (
specific_itr != backend_cmdline_config_map.end() &&
global_itr != backend_cmdline_config_map.end()) {
triton::common::BackendCmdlineConfig global_backend_config =
global_itr->second;
triton::common::BackendCmdlineConfig specific_backend_config =
specific_itr->second;
std::sort(global_backend_config.begin(), global_backend_config.end());
std::sort(specific_backend_config.begin(), specific_backend_config.end());
size_t global_index = 0;
size_t specific_index = 0;
while (global_index < global_backend_config.size() &&
specific_index < specific_backend_config.size()) {
auto& current_global_setting = global_backend_config.at(global_index);
auto& current_specific_setting =
specific_backend_config.at(specific_index);
if (current_specific_setting.first.compare(
current_global_setting.first) == 0) {
// specific setting overrides global setting
config.push_back(current_specific_setting);
++global_index;
++specific_index;
} else if (
current_specific_setting.first.compare(current_global_setting.first) <
0) {
config.push_back(current_specific_setting);
++specific_index;
} else {
config.push_back(current_global_setting);
++global_index;
}
}
// add the rest of the global configs
if (global_index < global_backend_config.size()) {
auto& current_global_setting = global_backend_config.at(global_index);
config.push_back(current_global_setting);
}
// add the rest of the specific settings
if (specific_index < specific_backend_config.size()) {
auto& current_specific_setting =
specific_backend_config.at(specific_index);
config.push_back(current_specific_setting);
}
} // else empty config
return Status::Success;
}
const std::unordered_map<std::string, std::string> backend_config_defaults(
{{"default-max-batch-size", "4"}});
Status
TritonModel::SetBackendConfigDefaults(
triton::common::BackendCmdlineConfig& config)
{
auto backend_config_defaults_copy = backend_config_defaults;
for (auto& setting : config) {
if (setting.first.compare("default-max-batch-size") == 0) {
LOG_VERBOSE(1) << "Found overwritten default setting: " << setting.first
<< "," << setting.second;
backend_config_defaults_copy.erase(setting.first);
}
if (backend_config_defaults_copy.empty()) {
break;
}
}
// Anything left should be added to the config
for (const auto& default_setting : backend_config_defaults_copy) {
LOG_VERBOSE(1) << "Adding default backend config setting: "
<< default_setting.first << "," << default_setting.second;
config.push_back(
std::make_pair(default_setting.first, default_setting.second));
}
return Status::Success;
}
Status
TritonModel::AddInstance(
std::unique_ptr<TritonModelInstance>&& instance, const bool passive)
{
if (passive) {
passive_instances_.emplace_back(std::move(instance));
} else {
instances_.emplace_back(std::move(instance));
}
return Status::Success;
}
Status
TritonModel::UpdateModelConfig(
const uint32_t config_version, TRITONSERVER_Message* updated_config_message)
{
const char* buffer;
size_t byte_size;
RETURN_IF_TRITONSERVER_ERROR(TRITONSERVER_MessageSerializeToJson(
updated_config_message, &buffer, &byte_size));
inference::ModelConfig updated_config;
RETURN_IF_ERROR(
JsonToModelConfig({buffer, byte_size}, config_version, &updated_config));
auto config = Config();
config.set_max_batch_size(updated_config.max_batch_size());
auto inputs_config = config.mutable_input();
*inputs_config = updated_config.input();
auto outputs_config = config.mutable_output();
*outputs_config = updated_config.output();
if (!config.scheduling_choice_case()) {
if (updated_config.has_dynamic_batching()) {
auto dynamic_batching_config = config.mutable_dynamic_batching();
*dynamic_batching_config = updated_config.dynamic_batching();
} else if (updated_config.has_sequence_batching()) {
auto sequence_batching_config = config.mutable_sequence_batching();
*sequence_batching_config = updated_config.sequence_batching();
} else if (updated_config.has_ensemble_scheduling()) {
auto ensemble_scheduling_config = config.mutable_ensemble_scheduling();
*ensemble_scheduling_config = updated_config.ensemble_scheduling();
} // else do nothing
} else if (
config.scheduling_choice_case() !=
updated_config.scheduling_choice_case()) {
return Status(
triton::common::Error::Code::INTERNAL,
(std::string("Cannot update scheduling choice from ") +
std::to_string(config.scheduling_choice_case()) + std::string(" to ") +
std::to_string(config.scheduling_choice_case()) +
std::string(" when auto-completing."))
.c_str());
} // else do nothing
// Need to normalize the model configuration for
// populating missing fields.
RETURN_IF_ERROR(NormalizeModelConfig(min_compute_capability_, &config));
RETURN_IF_ERROR(SetModelConfig(config));
return Status::Success;
}
Status
TritonModel::SetConfiguredScheduler()
{
std::unique_ptr<Scheduler> scheduler;
// Need to enforce equal shape batches (i.e. non-ragged batches) if
// the model 1) allows one or more variable-size input tensors that
// are not marked as 'allow_ragged_batch' or 2) has one or more
// shape-tensor inputs. This is not needed if all input shapes are
// non-variable and if there are no shape tensors... so we don't
// enable it in that case for efficiency reasons.
std::unordered_map<std::string, bool> enforce_equal_shape_tensors;
for (const auto input : config_.input()) {
if (input.is_shape_tensor()) {
enforce_equal_shape_tensors.insert({input.name(), true});
} else if (
!input.allow_ragged_batch() &&
(triton::common::GetElementCount(input) == -1)) {
enforce_equal_shape_tensors.insert({input.name(), false});
}
}
// If 'sequence_batching' is configured, then use the SequenceBatchScheduler,
// otherwise use the default DynamicBatchScheduler.
if (config_.has_sequence_batching()) {
// Sequence batcher
RETURN_IF_ERROR(SequenceBatchScheduler::Create(
this, enforce_equal_shape_tensors, &scheduler));
} else if (config_.has_dynamic_batching()) {
// Dynamic batcher
RETURN_IF_ERROR(DynamicBatchScheduler::Create(
this, nullptr, 0 /*nice*/, true /* dynamic_batching_enabled */,
config_.max_batch_size(), enforce_equal_shape_tensors,
config_.dynamic_batching(),
config_.response_cache().enable() /* response_cache_enable */,
&scheduler));
} else {
// Default scheduler. Use dynamic batch scheduler (with batching
// disabled) as the default scheduler.
RETURN_IF_ERROR(DynamicBatchScheduler::Create(
this, nullptr, 0 /*nice*/, false /* dynamic_batching_enabled */,
1 /* max_batch_size */,
std::unordered_map<
std::string, bool>() /* enforce_equal_shape_tensors */,
false /* preserve_ordering */,
config_.response_cache().enable() /* response_cache_enable */,
std::set<int32_t>() /* preferred_batch_sizes */,
0 /* max_queue_delay_microseconds */, &scheduler));
}
return SetScheduler(std::move(scheduler));
}
Status
TritonModel::Initialize()
{
for (const auto& instance : instances_) {
RETURN_IF_ERROR(instance->Initialize());
}
return Status::Success;
}
Status
TritonModel::WarmUp()
{
for (const auto& instance : instances_) {
RETURN_IF_ERROR(instance->WarmUp());
}
return Status::Success;
}
TritonModel::TritonModel(
InferenceServer* server,
const std::shared_ptr<LocalizedPath>& localized_model_dir,
const std::shared_ptr<TritonBackend>& backend,
const double min_compute_capability, const int64_t version,
const inference::ModelConfig& config, const bool auto_complete_config)
: Model(
min_compute_capability, localized_model_dir->Path(), version, config),
server_(server), min_compute_capability_(min_compute_capability),
auto_complete_config_(auto_complete_config),
localized_model_dir_(localized_model_dir), backend_(backend),
state_(nullptr)
{
}
TritonModel::~TritonModel()
{
// Explicitly delete/finalize all model instances before finalizing
// the model itself.
instances_.clear();
passive_instances_.clear();
// Unregister itself from the rate limiter. Note this should happen
// after all instances are destructed. Destrucing instances ensures
// there are no instance threads waiting on rate limiter for
// receiving their payloads.
server_->GetRateLimiter()->UnregisterModel(this);
// Model finalization is optional... The TRITONBACKEND_Model
// object is this TritonModel object.
if (backend_->ModelFiniFn() != nullptr) {
LOG_TRITONSERVER_ERROR(
backend_->ModelFiniFn()(reinterpret_cast<TRITONBACKEND_Model*>(this)),
"failed finalizing model");
}
}
extern "C" {
//
// TRITONBACKEND_Model
//
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_ModelName(TRITONBACKEND_Model* model, const char** name)
{
TritonModel* tm = reinterpret_cast<TritonModel*>(model);
*name = tm->Name().c_str();
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_ModelVersion(TRITONBACKEND_Model* model, uint64_t* version)
{
TritonModel* tm = reinterpret_cast<TritonModel*>(model);
*version = tm->Version();
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_ModelRepository(
TRITONBACKEND_Model* model, TRITONBACKEND_ArtifactType* artifact_type,
const char** location)
{
TritonModel* tm = reinterpret_cast<TritonModel*>(model);
*artifact_type = TRITONBACKEND_ARTIFACT_FILESYSTEM;
*location = tm->LocalizedModelPath().c_str();
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_ModelConfig(
TRITONBACKEND_Model* model, const uint32_t config_version,
TRITONSERVER_Message** model_config)
{
TritonModel* tm = reinterpret_cast<TritonModel*>(model);
std::string model_config_json;
Status status =
ModelConfigToJson(tm->Config(), config_version, &model_config_json);
if (!status.IsOk()) {
return TRITONSERVER_ErrorNew(
StatusCodeToTritonCode(status.StatusCode()), status.Message().c_str());
}
*model_config = reinterpret_cast<TRITONSERVER_Message*>(
new TritonServerMessage(std::move(model_config_json)));
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_ModelAutoCompleteConfig(
TRITONBACKEND_Model* model, bool* auto_complete_config)
{
TritonModel* tm = reinterpret_cast<TritonModel*>(model);
*auto_complete_config = tm->AutoCompleteConfig();
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_ModelSetConfig(
TRITONBACKEND_Model* model, const uint32_t config_version,
TRITONSERVER_Message* model_config)
{
TritonModel* tm = reinterpret_cast<TritonModel*>(model);
Status status = tm->UpdateModelConfig(config_version, model_config);
if (!status.IsOk()) {
return TRITONSERVER_ErrorNew(
StatusCodeToTritonCode(status.StatusCode()), status.Message().c_str());
}
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_ModelServer(
TRITONBACKEND_Model* model, TRITONSERVER_Server** server)
{
TritonModel* tm = reinterpret_cast<TritonModel*>(model);
*server = reinterpret_cast<TRITONSERVER_Server*>(tm->Server());
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_ModelBackend(
TRITONBACKEND_Model* model, TRITONBACKEND_Backend** backend)
{
TritonModel* tm = reinterpret_cast<TritonModel*>(model);
*backend = reinterpret_cast<TRITONBACKEND_Backend*>(tm->Backend().get());
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_ModelState(TRITONBACKEND_Model* model, void** state)
{
TritonModel* tm = reinterpret_cast<TritonModel*>(model);
*state = tm->State();
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_ModelSetState(TRITONBACKEND_Model* model, void* state)
{
TritonModel* tm = reinterpret_cast<TritonModel*>(model);
tm->SetState(state);
return nullptr; // success
}
///
/// TRITONBACKEND_Request
///
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_RequestId(TRITONBACKEND_Request* request, const char** id)
{
InferenceRequest* tr = reinterpret_cast<InferenceRequest*>(request);
*id = tr->Id().c_str();
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_RequestCorrelationId(TRITONBACKEND_Request* request, uint64_t* id)
{
InferenceRequest* tr = reinterpret_cast<InferenceRequest*>(request);
const InferenceRequest::SequenceId& correlation_id = tr->CorrelationId();
if (correlation_id.Type() != InferenceRequest::SequenceId::DataType::UINT64) {
return TRITONSERVER_ErrorNew(
TRITONSERVER_ERROR_INVALID_ARG,
(tr->LogRequest() + "correlation ID in request is not an unsigned int")
.c_str());
}
*id = correlation_id.UnsignedIntValue();
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_RequestFlags(TRITONBACKEND_Request* request, uint32_t* flags)
{
InferenceRequest* tr = reinterpret_cast<InferenceRequest*>(request);
*flags = tr->Flags();
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_RequestCorrelationIdString(
TRITONBACKEND_Request* request, const char** id)
{
InferenceRequest* tr = reinterpret_cast<InferenceRequest*>(request);
const InferenceRequest::SequenceId& correlation_id = tr->CorrelationId();
if (correlation_id.Type() != InferenceRequest::SequenceId::DataType::STRING) {
return TRITONSERVER_ErrorNew(
TRITONSERVER_ERROR_INVALID_ARG,
(tr->LogRequest() + "correlation ID in request is not a string")
.c_str());
}
*id = correlation_id.StringValue().c_str();
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_RequestInputCount(TRITONBACKEND_Request* request, uint32_t* count)
{
InferenceRequest* tr = reinterpret_cast<InferenceRequest*>(request);
*count = tr->ImmutableInputs().size();
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_RequestInputName(
TRITONBACKEND_Request* request, const uint32_t index,
const char** input_name)
{
*input_name = nullptr;
InferenceRequest* tr = reinterpret_cast<InferenceRequest*>(request);
const auto& inputs = tr->ImmutableInputs();
if (index >= inputs.size()) {
return TRITONSERVER_ErrorNew(
TRITONSERVER_ERROR_INVALID_ARG,
(tr->LogRequest() + "out of bounds index " + std::to_string(index) +
": request has " + std::to_string(inputs.size()) + " inputs")
.c_str());
}
// The request inputs are not allowed to change once the request
// makes it to the backend, so it is ok to just iterate through the
// map. This linear search is the best we can do given the need for
// the inputs to be in a map and given the typical small number of
// inputs is better than having every request maintain the inputs as
// both map and vector.
uint32_t cnt = 0;
for (const auto& pr : inputs) {
if (cnt++ == index) {
InferenceRequest::Input* in = pr.second;
*input_name = in->Name().c_str();
break;
}
}
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_RequestInput(
TRITONBACKEND_Request* request, const char* name,
TRITONBACKEND_Input** input)
{
InferenceRequest* tr = reinterpret_cast<InferenceRequest*>(request);
const auto& inputs = tr->ImmutableInputs();
const auto& itr = inputs.find(name);
if (itr == inputs.end()) {
*input = nullptr;
return TRITONSERVER_ErrorNew(
TRITONSERVER_ERROR_INVALID_ARG,
(tr->LogRequest() + "unknown request input name " + name).c_str());
}
InferenceRequest::Input* in = itr->second;
*input = reinterpret_cast<TRITONBACKEND_Input*>(in);
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_RequestInputByIndex(
TRITONBACKEND_Request* request, const uint32_t index,
TRITONBACKEND_Input** input)
{
InferenceRequest* tr = reinterpret_cast<InferenceRequest*>(request);
const auto& inputs = tr->ImmutableInputs();
if (index >= inputs.size()) {
return TRITONSERVER_ErrorNew(
TRITONSERVER_ERROR_INVALID_ARG,
(tr->LogRequest() + "out of bounds index " + std::to_string(index) +
": request has " + std::to_string(inputs.size()) + " inputs")
.c_str());
}
// The request inputs are not allowed to change once the request
// makes it to the backend, so it is ok to just iterate through the
// map. This linear search is the best we can do given the need for
// the inputs to be in a map and given the typical small number of
// inputs is better than having every request maintain the inputs as
// both map and vector.
uint32_t cnt = 0;
for (const auto& pr : inputs) {
if (cnt++ == index) {
InferenceRequest::Input* in = pr.second;
*input = reinterpret_cast<TRITONBACKEND_Input*>(in);
break;
}
}
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_RequestOutputCount(
TRITONBACKEND_Request* request, uint32_t* count)
{
InferenceRequest* tr = reinterpret_cast<InferenceRequest*>(request);
*count = tr->ImmutableRequestedOutputs().size();
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_RequestOutputName(
TRITONBACKEND_Request* request, const uint32_t index,
const char** output_name)
{
*output_name = nullptr;
InferenceRequest* tr = reinterpret_cast<InferenceRequest*>(request);
const auto& routputs = tr->ImmutableRequestedOutputs();
if (index >= routputs.size()) {
return TRITONSERVER_ErrorNew(
TRITONSERVER_ERROR_INVALID_ARG,
(tr->LogRequest() + "out of bounds index " + std::to_string(index) +
": request has " + std::to_string(routputs.size()) +
" requested outputs")
.c_str());
}
// The requested outputs are not allowed to change once the request
// makes it to the backend, so it is ok to just iterate through the
// set. This linear search is the best we can do given the requested
// outputs being in a set and given the typical small number of
// requested outputs it should not be a performance issue.
uint32_t cnt = 0;
for (const auto& rout : routputs) {
if (cnt++ == index) {
*output_name = rout.c_str();
break;
}
}
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_RequestOutputBufferProperties(
TRITONBACKEND_Request* request, const char* name, size_t* byte_size,
TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id)
{
InferenceRequest* tr = reinterpret_cast<InferenceRequest*>(request);
auto status =
tr->OutputBufferProperties(name, byte_size, memory_type, memory_type_id);
if (!status.IsOk()) {
return TRITONSERVER_ErrorNew(
StatusCodeToTritonCode(status.StatusCode()), status.Message().c_str());
}
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_RequestRelease(
TRITONBACKEND_Request* request, uint32_t release_flags)
{
InferenceRequest* tr = reinterpret_cast<InferenceRequest*>(request);
std::unique_ptr<InferenceRequest> ur(tr);
InferenceRequest::Release(std::move(ur), release_flags);
return nullptr; // success
}
///
/// TRITONBACKEND_State
///
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_StateUpdate(TRITONBACKEND_State* state)
{
SequenceState* ts = reinterpret_cast<SequenceState*>(state);
auto status = ts->Update();
if (!status.IsOk()) {
return TRITONSERVER_ErrorNew(
StatusCodeToTritonCode(status.StatusCode()), status.Message().c_str());
}
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_StateNew(
TRITONBACKEND_State** state, TRITONBACKEND_Request* request,
const char* name, const TRITONSERVER_DataType datatype,
const int64_t* shape, const uint32_t dims_count)
{
InferenceRequest* tr = reinterpret_cast<InferenceRequest*>(request);
SequenceState* lstate;
std::vector<int64_t> lshape(shape, shape + dims_count);
auto& sequence_state = tr->GetSequenceStates();
if (sequence_state == nullptr) {
return TRITONSERVER_ErrorNew(
TRITONSERVER_ERROR_INVALID_ARG,
(std::string("unable to add state '") + name +
"'. State configuration is missing for model '" + tr->ModelName() +
"'.")
.c_str());
}
Status status = sequence_state->OutputState(
name, TritonToDataType(datatype), lshape, &lstate);
if (!status.IsOk()) {
*state = nullptr;
return TRITONSERVER_ErrorNew(
StatusCodeToTritonCode(status.StatusCode()), status.Message().c_str());
}
*state = reinterpret_cast<TRITONBACKEND_State*>(lstate);
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_StateBuffer(
TRITONBACKEND_State* state, void** buffer, const uint64_t buffer_byte_size,
TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id)
{
SequenceState* to = reinterpret_cast<SequenceState*>(state);
Status status = Status::Success;
// If the buffer size exactly matches the buffer available, reuse the
// currently allocated buffer.
if (to->Data()->TotalByteSize() == buffer_byte_size) {
const std::shared_ptr<AllocatedMemory>& memory =
reinterpret_cast<const std::shared_ptr<AllocatedMemory>&>(to->Data());
TRITONSERVER_MemoryType current_memory_type;
int64_t current_memory_type_id;
void* lbuffer =
memory->MutableBuffer(&current_memory_type, &current_memory_type_id);
// If the requested memory type doesn't match the current buffer, allocate a
// new buffer with the requested memory type and memory type id.
if (current_memory_type == *memory_type &&
current_memory_type_id == *memory_type_id) {
*buffer = lbuffer;
} else {
std::shared_ptr<AllocatedMemory> memory =
std::make_shared<AllocatedMemory>(
buffer_byte_size, *memory_type, *memory_type_id);
*buffer = memory->MutableBuffer(memory_type, memory_type_id);
to->RemoveAllData();
status = to->SetData(memory);
}
} else {
std::shared_ptr<AllocatedMemory> memory = std::make_shared<AllocatedMemory>(
buffer_byte_size, *memory_type, *memory_type_id);
*buffer = memory->MutableBuffer(memory_type, memory_type_id);
to->RemoveAllData();
status = to->SetData(memory);
}
if (!status.IsOk()) {
*buffer = nullptr;
return TRITONSERVER_ErrorNew(
StatusCodeToTritonCode(status.StatusCode()), status.Message().c_str());
}
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_StateBufferAttributes(
TRITONBACKEND_State* state,
TRITONSERVER_BufferAttributes** buffer_attributes)
{
SequenceState* to = reinterpret_cast<SequenceState*>(state);
to->Data()->BufferAt(
0, reinterpret_cast<BufferAttributes**>(buffer_attributes));
return nullptr; // success
}
//
// TRITONBACKEND_ResponseFactory
//
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_ResponseFactoryNew(
TRITONBACKEND_ResponseFactory** factory, TRITONBACKEND_Request* request)
{
InferenceRequest* tr = reinterpret_cast<InferenceRequest*>(request);
std::shared_ptr<InferenceResponseFactory>* response_factory =
new std::shared_ptr<InferenceResponseFactory>(tr->ResponseFactory());
*factory = reinterpret_cast<TRITONBACKEND_ResponseFactory*>(response_factory);
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_ResponseFactoryDelete(TRITONBACKEND_ResponseFactory* factory)
{
std::shared_ptr<InferenceResponseFactory>* response_factory =
reinterpret_cast<std::shared_ptr<InferenceResponseFactory>*>(factory);
delete response_factory;
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_ResponseFactorySendFlags(
TRITONBACKEND_ResponseFactory* factory, const uint32_t send_flags)
{
std::shared_ptr<InferenceResponseFactory>* response_factory =
reinterpret_cast<std::shared_ptr<InferenceResponseFactory>*>(factory);
Status status = (*response_factory)->SendFlags(send_flags);
if (!status.IsOk()) {
return TRITONSERVER_ErrorNew(
StatusCodeToTritonCode(status.StatusCode()), status.Message().c_str());
}
return nullptr; // success
}
///
/// TRITONBACKEND_Response
///
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_ResponseNew(
TRITONBACKEND_Response** response, TRITONBACKEND_Request* request)
{
InferenceRequest* tr = reinterpret_cast<InferenceRequest*>(request);
std::unique_ptr<InferenceResponse> tresp;
Status status = tr->ResponseFactory()->CreateResponse(&tresp);
if (!status.IsOk()) {
*response = nullptr;
return TRITONSERVER_ErrorNew(
StatusCodeToTritonCode(status.StatusCode()), status.Message().c_str());
}
*response = reinterpret_cast<TRITONBACKEND_Response*>(tresp.release());
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_ResponseNewFromFactory(
TRITONBACKEND_Response** response, TRITONBACKEND_ResponseFactory* factory)
{
std::shared_ptr<InferenceResponseFactory>* response_factory =
reinterpret_cast<std::shared_ptr<InferenceResponseFactory>*>(factory);
std::unique_ptr<InferenceResponse> tr;
Status status = (*response_factory)->CreateResponse(&tr);
if (!status.IsOk()) {
*response = nullptr;
return TRITONSERVER_ErrorNew(
StatusCodeToTritonCode(status.StatusCode()), status.Message().c_str());
}
*response = reinterpret_cast<TRITONBACKEND_Response*>(tr.release());
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_ResponseDelete(TRITONBACKEND_Response* response)
{
InferenceResponse* tr = reinterpret_cast<InferenceResponse*>(response);
delete tr;
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_ResponseSetStringParameter(
TRITONBACKEND_Response* response, const char* name, const char* value)
{
InferenceResponse* tr = reinterpret_cast<InferenceResponse*>(response);
Status status = tr->AddParameter(name, value);
if (!status.IsOk()) {
return TRITONSERVER_ErrorNew(
StatusCodeToTritonCode(status.StatusCode()), status.Message().c_str());
}
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_ResponseSetIntParameter(
TRITONBACKEND_Response* response, const char* name, const int64_t value)
{
InferenceResponse* tr = reinterpret_cast<InferenceResponse*>(response);
Status status = tr->AddParameter(name, value);
if (!status.IsOk()) {
return TRITONSERVER_ErrorNew(
StatusCodeToTritonCode(status.StatusCode()), status.Message().c_str());
}
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_ResponseSetBoolParameter(
TRITONBACKEND_Response* response, const char* name, const bool value)
{
InferenceResponse* tr = reinterpret_cast<InferenceResponse*>(response);
Status status = tr->AddParameter(name, value);
if (!status.IsOk()) {
return TRITONSERVER_ErrorNew(
StatusCodeToTritonCode(status.StatusCode()), status.Message().c_str());
}
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_ResponseOutput(
TRITONBACKEND_Response* response, TRITONBACKEND_Output** output,
const char* name, const TRITONSERVER_DataType datatype,
const int64_t* shape, const uint32_t dims_count)
{
InferenceResponse* tr = reinterpret_cast<InferenceResponse*>(response);
std::vector<int64_t> lshape(shape, shape + dims_count);
InferenceResponse::Output* loutput;
Status status = tr->AddOutput(
name, TritonToDataType(datatype), std::move(lshape), &loutput);
if (!status.IsOk()) {
*output = nullptr;
return TRITONSERVER_ErrorNew(
StatusCodeToTritonCode(status.StatusCode()), status.Message().c_str());
}
*output = reinterpret_cast<TRITONBACKEND_Output*>(loutput);
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_ResponseSend(
TRITONBACKEND_Response* response, const uint32_t send_flags,
TRITONSERVER_Error* error)
{
InferenceResponse* tr = reinterpret_cast<InferenceResponse*>(response);
Status status;
std::unique_ptr<InferenceResponse> utr(tr);
if (error == nullptr) {
status = InferenceResponse::Send(std::move(utr), send_flags);
} else {
status = InferenceResponse::SendWithStatus(
std::move(utr), send_flags,
Status(
TritonCodeToStatusCode(TRITONSERVER_ErrorCode(error)),
TRITONSERVER_ErrorMessage(error)));
}
if (!status.IsOk()) {
return TRITONSERVER_ErrorNew(
StatusCodeToTritonCode(status.StatusCode()), status.Message().c_str());
}
return nullptr; // success
}
///
/// TRITONBACKEND_Input
///
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_InputProperties(
TRITONBACKEND_Input* input, const char** name,
TRITONSERVER_DataType* datatype, const int64_t** shape,
uint32_t* dims_count, uint64_t* byte_size, uint32_t* buffer_count)
{
InferenceRequest::Input* ti =
reinterpret_cast<InferenceRequest::Input*>(input);
if (name != nullptr) {
*name = ti->Name().c_str();
}
if (datatype != nullptr) {
*datatype = DataTypeToTriton(ti->DType());
}
if (shape != nullptr) {
*shape = ti->ShapeWithBatchDim().data();
}
if (dims_count != nullptr) {
*dims_count = ti->ShapeWithBatchDim().size();
}
if (byte_size != nullptr) {
*byte_size = ti->Data()->TotalByteSize();
}
if (buffer_count != nullptr) {
*buffer_count = ti->DataBufferCount();
}
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_InputPropertiesForHostPolicy(
TRITONBACKEND_Input* input, const char* host_policy_name, const char** name,
TRITONSERVER_DataType* datatype, const int64_t** shape,
uint32_t* dims_count, uint64_t* byte_size, uint32_t* buffer_count)
{
InferenceRequest::Input* ti =
reinterpret_cast<InferenceRequest::Input*>(input);
if (name != nullptr) {
*name = ti->Name().c_str();
}
if (datatype != nullptr) {
*datatype = DataTypeToTriton(ti->DType());
}
if (shape != nullptr) {
*shape = ti->ShapeWithBatchDim().data();
}
if (dims_count != nullptr) {
*dims_count = ti->ShapeWithBatchDim().size();
}
if (host_policy_name != nullptr) {
if (byte_size != nullptr) {
*byte_size = ti->Data(host_policy_name)->TotalByteSize();
}
if (buffer_count != nullptr) {
*buffer_count = ti->DataBufferCountForHostPolicy(host_policy_name);
}
} else {
if (byte_size != nullptr) {
*byte_size = ti->Data()->TotalByteSize();
}
if (buffer_count != nullptr) {
*buffer_count = ti->DataBufferCount();
}
}
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_InputBuffer(
TRITONBACKEND_Input* input, const uint32_t index, const void** buffer,
uint64_t* buffer_byte_size, TRITONSERVER_MemoryType* memory_type,
int64_t* memory_type_id)
{
InferenceRequest::Input* ti =
reinterpret_cast<InferenceRequest::Input*>(input);
Status status = ti->DataBuffer(
index, buffer, buffer_byte_size, memory_type, memory_type_id);
if (!status.IsOk()) {
*buffer = nullptr;
*buffer_byte_size = 0;
return TRITONSERVER_ErrorNew(
StatusCodeToTritonCode(status.StatusCode()), status.Message().c_str());
}
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_InputBufferAttributes(
TRITONBACKEND_Input* input, const uint32_t index, const void** buffer,
TRITONSERVER_BufferAttributes** buffer_attributes)
{
InferenceRequest::Input* ti =
reinterpret_cast<InferenceRequest::Input*>(input);
Status status = ti->DataBufferAttributes(
index, buffer, reinterpret_cast<BufferAttributes**>(buffer_attributes));
if (!status.IsOk()) {
*buffer = nullptr;
*buffer_attributes = nullptr;
return TRITONSERVER_ErrorNew(
StatusCodeToTritonCode(status.StatusCode()), status.Message().c_str());
}
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_InputBufferForHostPolicy(
TRITONBACKEND_Input* input, const char* host_policy_name,
const uint32_t index, const void** buffer, uint64_t* buffer_byte_size,
TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id)
{
InferenceRequest::Input* ti =
reinterpret_cast<InferenceRequest::Input*>(input);
Status status =
(host_policy_name == nullptr)
? ti->DataBuffer(
index, buffer, buffer_byte_size, memory_type, memory_type_id)
: ti->DataBufferForHostPolicy(
index, buffer, buffer_byte_size, memory_type, memory_type_id,
host_policy_name);
if (!status.IsOk()) {
*buffer = nullptr;
*buffer_byte_size = 0;
return TRITONSERVER_ErrorNew(
StatusCodeToTritonCode(status.StatusCode()), status.Message().c_str());
}
return nullptr; // success
}
///
/// TRITONBACKEND_Output
///
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_OutputBuffer(
TRITONBACKEND_Output* output, void** buffer,
const uint64_t buffer_byte_size, TRITONSERVER_MemoryType* memory_type,
int64_t* memory_type_id)
{
InferenceResponse::Output* to =
reinterpret_cast<InferenceResponse::Output*>(output);
Status status = to->AllocateDataBuffer(
buffer, buffer_byte_size, memory_type, memory_type_id);
if (!status.IsOk()) {
*buffer = nullptr;
return TRITONSERVER_ErrorNew(
StatusCodeToTritonCode(status.StatusCode()), status.Message().c_str());
}
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_OutputBufferAttributes(
TRITONBACKEND_Output* output,
TRITONSERVER_BufferAttributes** buffer_attributes)
{
InferenceResponse::Output* to =
reinterpret_cast<InferenceResponse::Output*>(output);
*buffer_attributes = reinterpret_cast<TRITONSERVER_BufferAttributes*>(
to->GetBufferAttributes());
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_BackendAttributeAddPreferredInstanceGroup(
TRITONBACKEND_BackendAttribute* backend_attributes,
const TRITONSERVER_InstanceGroupKind kind, const uint64_t count,
const uint64_t* device_ids, const uint64_t id_count)
{
auto ba = reinterpret_cast<TritonBackend::Attribute*>(backend_attributes);
ba->preferred_groups_.emplace_back();
auto& pg = ba->preferred_groups_.back();
switch (kind) {
case TRITONSERVER_INSTANCEGROUPKIND_AUTO:
pg.set_kind(inference::ModelInstanceGroup::KIND_AUTO);
break;
case TRITONSERVER_INSTANCEGROUPKIND_CPU:
pg.set_kind(inference::ModelInstanceGroup::KIND_CPU);
break;
case TRITONSERVER_INSTANCEGROUPKIND_GPU:
pg.set_kind(inference::ModelInstanceGroup::KIND_GPU);
break;
case TRITONSERVER_INSTANCEGROUPKIND_MODEL:
pg.set_kind(inference::ModelInstanceGroup::KIND_MODEL);
break;
}
pg.set_count(count);
if (device_ids != nullptr) {
for (size_t i = 0; i < id_count; ++i) {
pg.add_gpus(device_ids[i]);
}
}
return nullptr;
}
} // extern C
}} // namespace triton::core
3rdparty/core-r22.12/src/backend_model.h 0 → 100644
View file @ b30f3cdb
// Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
// * Neither the name of NVIDIA CORPORATION nor the names of its
// contributors may be used to endorse or promote products derived
// from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include <memory>
#include <string>
#include "backend_manager.h"
#include "filesystem.h"
#include "infer_request.h"
#include "model.h"
#include "model_config.pb.h"
#include "status.h"
namespace triton { namespace core {
class InferenceServer;
class TritonModelInstance;
//
// Represents a model.
//
// Inheriting from Model to implement backend APIs
//
class TritonModel : public Model {
public:
static Status Create(
InferenceServer* server, const std::string& model_path,
const triton::common::BackendCmdlineConfigMap& backend_cmdline_config_map,
const triton::common::HostPolicyCmdlineConfigMap& host_policy_map,
const std::string& model_name, const int64_t version,
inference::ModelConfig model_config, const bool is_config_provided,
std::unique_ptr<TritonModel>* model);
~TritonModel();
const std::string& LocalizedModelPath() const
{
return localized_model_dir_->Path();
}
InferenceServer* Server() { return server_; }
bool AutoCompleteConfig() const { return auto_complete_config_; }
Status UpdateModelConfig(
const uint32_t config_version,
TRITONSERVER_Message* updated_config_message);
const std::shared_ptr<TritonBackend>& Backend() const { return backend_; }
const std::vector<std::unique_ptr<TritonModelInstance>>& Instances() const
{
return instances_;
}
void* State() { return state_; }
void SetState(void* state) { state_ = state; }
Status AddInstance(
std::unique_ptr<TritonModelInstance>&& instance, const bool passive);
private:
DISALLOW_COPY_AND_ASSIGN(TritonModel);
TritonModel(
InferenceServer* server,
const std::shared_ptr<LocalizedPath>& localized_model_dir,
const std::shared_ptr<TritonBackend>& backend,
const double min_compute_capability, const int64_t version,
const inference::ModelConfig& config, const bool auto_complete_config);
// Set the scheduler based on the model configuration. The scheduler
// can only be set once for a backend.
Status SetConfiguredScheduler();
// Merges the global backend configs with the specific
// backend configs.
static Status ResolveBackendConfigs(
const triton::common::BackendCmdlineConfigMap& backend_cmdline_config_map,
const std::string& backend_name,
triton::common::BackendCmdlineConfig& config);
// Sets defaults for some backend configurations when none are specified on
// the command line.
static Status SetBackendConfigDefaults(
triton::common::BackendCmdlineConfig& config);
Status Initialize();
Status WarmUp();
// The server object that owns this model. The model holds this as a
// raw pointer because the lifetime of the server is guaranteed to
// be longer than the lifetime of a model owned by the server.
InferenceServer* server_;
// The minimum supported compute capability on device.
const double min_compute_capability_;
// Whether the backend should attempt to auto-complete the model config.
const bool auto_complete_config_;
// The localized repo directory holding the model. If localization
// required creation of a temporary local copy then that copy will
// persist as along as this object is retained by this model.
std::shared_ptr<LocalizedPath> localized_model_dir_;
// Backend used by this model.
std::shared_ptr<TritonBackend> backend_;
// The model instances for this model.
std::vector<std::unique_ptr<TritonModelInstance>> instances_;
std::vector<std::unique_ptr<TritonModelInstance>> passive_instances_;
// Opaque state associated with this model.
void* state_;
};
}} // namespace triton::core
3rdparty/core-r22.12/src/backend_model_instance.cc 0 → 100644
View file @ b30f3cdb
// Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
// * Neither the name of NVIDIA CORPORATION nor the names of its
// contributors may be used to endorse or promote products derived
// from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "backend_model_instance.h"
#ifndef _WIN32
#include <sys/resource.h>
#include <sys/syscall.h>
#include <unistd.h>
#endif
#include "backend_config.h"
#include "backend_model.h"
#include "cuda_utils.h"
#include "metrics.h"
#include "model_config.pb.h"
#include "numa_utils.h"
#include "server.h"
#include "shared_library.h"
#include "triton/common/logging.h"
#include "triton/common/nvtx.h"
#include "tritonserver_apis.h"
// For unknown reason, windows will not export the TRITONBACKEND_*
// functions declared with dllexport in tritonbackend.h. To get those
// functions exported it is (also?) necessary to mark the definitions
// in this file with dllexport as well.
#if defined(_MSC_VER)
#define TRITONAPI_DECLSPEC __declspec(dllexport)
#elif defined(__GNUC__)
#define TRITONAPI_DECLSPEC __attribute__((__visibility__("default")))
#else
#define TRITONAPI_DECLSPEC
#endif
namespace triton { namespace core {
namespace {
// Utilities for warmup feature
TRITONSERVER_Error*
WarmupResponseAlloc(
TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name,
size_t byte_size, TRITONSERVER_MemoryType preferred_memory_type,
int64_t preferred_memory_type_id, void* userp, void** buffer,
void** buffer_userp, TRITONSERVER_MemoryType* actual_memory_type,
int64_t* actual_memory_type_id)
{
*buffer = malloc(byte_size);
if (*buffer != nullptr) {
*actual_memory_type = TRITONSERVER_MEMORY_CPU;
*actual_memory_type_id = 0;
return nullptr;
}
return TRITONSERVER_ErrorNew(
TRITONSERVER_ERROR_INTERNAL,
"failed to allocate output buffer for warmup.");
}
TRITONSERVER_Error*
WarmupResponseRelease(
TRITONSERVER_ResponseAllocator* allocator, void* buffer, void* buffer_userp,
size_t byte_size, TRITONSERVER_MemoryType memory_type,
int64_t memory_type_id)
{
free(buffer);
return nullptr;
}
ResponseAllocator warmup_allocator = ResponseAllocator(
WarmupResponseAlloc, WarmupResponseRelease, nullptr /* start_fn */);
void
WarmupResponseComplete(
TRITONSERVER_InferenceResponse* iresponse, const uint32_t flags,
void* userp)
{
auto res_pair = reinterpret_cast<
std::pair<std::promise<void>, std::vector<std::string>*>*>(userp);
if (iresponse != nullptr) {
auto err = TRITONSERVER_InferenceResponseError(iresponse);
if (err != nullptr) {
// The error vector is shared by all requests in the batch for now
static std::mutex res_mtx;
{
std::lock_guard<std::mutex> lk(res_mtx);
res_pair->second->emplace_back(TRITONSERVER_ErrorMessage(err));
}
TRITONSERVER_ErrorDelete(err);
}
// Just delete the response, warmup doesn't check for correctness
LOG_TRITONSERVER_ERROR(
TRITONSERVER_InferenceResponseDelete(iresponse),
"deleting warmup response");
}
// Last response
if ((flags & TRITONSERVER_RESPONSE_COMPLETE_FINAL) != 0) {
res_pair->first.set_value();
}
}
void
WarmupRequestComplete(
TRITONSERVER_InferenceRequest* request, const uint32_t flags, void* userp)
{
if ((flags & TRITONSERVER_REQUEST_RELEASE_ALL) != 0) {
// Don't need to release request here, it is managed in WarmupData
if (userp != nullptr) {
auto warmup_promise = reinterpret_cast<std::promise<void>*>(userp);
warmup_promise->set_value();
}
}
}
} // namespace
TritonModelInstance::TritonModelInstance(
TritonModel* model, const std::string& name, const size_t index,
const TRITONSERVER_InstanceGroupKind kind, const int32_t device_id,
const std::vector<std::string>& profile_names, const bool passive,
const triton::common::HostPolicyCmdlineConfig& host_policy,
const TritonServerMessage& host_policy_message,
const std::vector<SecondaryDevice>& secondary_devices)
: model_(model), name_(name), index_(index), kind_(kind),
device_id_(device_id), host_policy_(host_policy),
host_policy_message_(host_policy_message), profile_names_(profile_names),
passive_(passive), secondary_devices_(secondary_devices), state_(nullptr)
{
#ifdef TRITON_ENABLE_METRICS
if (Metrics::Enabled()) {
// Use an ID in the metric only for GPU instances. Otherwise use
// METRIC_REPORTER_ID_CPU to indicate no device should be reported in the
// metric.
const int id = (kind_ == TRITONSERVER_INSTANCEGROUPKIND_GPU)
? device_id_
: METRIC_REPORTER_ID_CPU;
MetricModelReporter::Create(
model_->Name(), model_->Version(), id, model_->Config().metric_tags(),
&reporter_);
}
#endif // TRITON_ENABLE_METRICS
}
TritonModelInstance::~TritonModelInstance()
{
if (triton_backend_thread_.get() != nullptr) {
triton_backend_thread_->StopBackendThread();
}
// Model finalization is optional...
if (model_->Backend()->ModelInstanceFiniFn() != nullptr) {
LOG_TRITONSERVER_ERROR(
model_->Backend()->ModelInstanceFiniFn()(
reinterpret_cast<TRITONBACKEND_ModelInstance*>(this)),
"failed finalizing model instance");
}
}
Status
TritonModelInstance::CreateInstances(
TritonModel* model,
const triton::common::BackendCmdlineConfigMap& backend_cmdline_config_map,
const triton::common::HostPolicyCmdlineConfigMap& host_policy_map,
const inference::ModelConfig& model_config, const bool device_blocking)
{
static triton::common::HostPolicyCmdlineConfig empty_host_policy;
// This structure is used to allocate TritonBackendThread to instances on same
// device for device blocking execution policy.
std::map<uint32_t, std::shared_ptr<TritonBackendThread>> device_to_thread_map;
for (const auto& group : model_config.instance_group()) {
std::vector<std::string> profile_names;
for (const auto& profile_name : group.profile()) {
profile_names.push_back(profile_name);
}
std::vector<SecondaryDevice> secondary_devices;
for (const auto& secondary_device : group.secondary_devices()) {
secondary_devices.emplace_back(
inference::
ModelInstanceGroup_SecondaryDevice_SecondaryDeviceKind_Name(
secondary_device.kind()),
secondary_device.device_id());
}
for (int32_t c = 0; c < group.count(); ++c) {
std::string instance_name{group.count() > 1
? group.name() + "_" + std::to_string(c)
: group.name()};
const bool passive = group.passive();
std::vector<std::tuple<
std::string, TRITONSERVER_InstanceGroupKind, int32_t,
const inference::ModelRateLimiter*>>
instance_setting;
if (group.kind() == inference::ModelInstanceGroup::KIND_CPU) {
instance_setting.emplace_back(
group.host_policy().empty() ? "cpu" : group.host_policy(),
TRITONSERVER_INSTANCEGROUPKIND_CPU, 0 /* device_id */,
&group.rate_limiter());
} else if (group.kind() == inference::ModelInstanceGroup::KIND_GPU) {
for (const int32_t device_id : group.gpus()) {
instance_setting.emplace_back(
group.host_policy().empty() ? ("gpu_" + std::to_string(device_id))
: group.host_policy(),
TRITONSERVER_INSTANCEGROUPKIND_GPU, device_id,
&group.rate_limiter());
}
} else if (group.kind() == inference::ModelInstanceGroup::KIND_MODEL) {
instance_setting.emplace_back(
group.host_policy().empty() ? "model" : group.host_policy(),
TRITONSERVER_INSTANCEGROUPKIND_MODEL, 0 /* device_id */,
&group.rate_limiter());
} else {
return Status(
Status::Code::INVALID_ARG,
std::string("instance_group kind ") +
ModelInstanceGroup_Kind_Name(group.kind()) + " not supported");
}
for (const auto is : instance_setting) {
const auto& kind = std::get<1>(is);
const auto& id = std::get<2>(is);
const std::string& policy_name = std::get<0>(is);
const triton::common::HostPolicyCmdlineConfig* host_policy;
const auto policy_it = host_policy_map.find(policy_name);
if (policy_it != host_policy_map.end()) {
host_policy = &policy_it->second;
} else {
host_policy = &empty_host_policy;
}
RETURN_IF_ERROR(SetNumaConfigOnThread(*host_policy));
auto err = CreateInstance(
model, instance_name, c, kind, id, profile_names, passive,
policy_name, *host_policy, *(std::get<3>(is)), device_blocking,
&device_to_thread_map, secondary_devices);
RETURN_IF_ERROR(ResetNumaMemoryPolicy());
RETURN_IF_ERROR(err);
// When deploying on GPU, we want to make sure the GPU memory usage
// is within allowed range, otherwise, stop the creation to ensure
// there is sufficient GPU memory for other use.
// We check the usage after loading the instance to better enforcing
// the limit. If we check before loading, we may create instance
// that occupies the rest of available memory which against the purpose
if (kind == TRITONSERVER_INSTANCEGROUPKIND_GPU) {
size_t free, total;
double memory_limit;
RETURN_IF_ERROR(GetDeviceMemoryInfo(id, &free, &total));
RETURN_IF_ERROR(BackendConfigurationModelLoadGpuFraction(
backend_cmdline_config_map, id, &memory_limit));
const size_t allow = total * memory_limit;
const size_t used = total - free;
if (used > allow) {
return Status(
Status::Code::UNAVAILABLE,
std::string("can not create model '") + instance_name +
"': memory limit set for " +
TRITONSERVER_InstanceGroupKindString(kind) + " " +
std::to_string(id) +
" has exceeded, model loading is rejected.");
}
}
}
}
}
return Status::Success;
}
Status
TritonModelInstance::CreateInstance(
TritonModel* model, const std::string& name, const size_t index,
const TRITONSERVER_InstanceGroupKind kind, const int32_t device_id,
const std::vector<std::string>& profile_names, const bool passive,
const std::string& host_policy_name,
const triton::common::HostPolicyCmdlineConfig& host_policy,
const inference::ModelRateLimiter& rate_limiter_config,
const bool device_blocking,
std::map<uint32_t, std::shared_ptr<TritonBackendThread>>*
device_to_thread_map,
const std::vector<SecondaryDevice>& secondary_devices)
{
// Create the JSON representation of the backend configuration.
triton::common::TritonJson::Value host_policy_json(
triton::common::TritonJson::ValueType::OBJECT);
triton::common::TritonJson::Value policy_setting_json(
host_policy_json, triton::common::TritonJson::ValueType::OBJECT);
for (const auto& pr : host_policy) {
RETURN_IF_ERROR(policy_setting_json.AddString(pr.first.c_str(), pr.second));
}
RETURN_IF_ERROR(host_policy_json.Add(
host_policy_name.c_str(), std::move(policy_setting_json)));
TritonServerMessage host_policy_message(host_policy_json);
std::unique_ptr<TritonModelInstance> local_instance(new TritonModelInstance(
model, name, index, kind, device_id, profile_names, passive, host_policy,
host_policy_message, secondary_devices));
TRITONBACKEND_ModelInstance* triton_instance =
reinterpret_cast<TRITONBACKEND_ModelInstance*>(local_instance.get());
// Instance initialization is optional... We must set set shared
// library path to point to the backend directory in case the
// backend library attempts to load additional shared libaries.
if (model->Backend()->ModelInstanceInitFn() != nullptr) {
std::unique_ptr<SharedLibrary> slib;
RETURN_IF_ERROR(SharedLibrary::Acquire(&slib));
RETURN_IF_ERROR(slib->SetLibraryDirectory(model->Backend()->Directory()));
TRITONSERVER_Error* err =
model->Backend()->ModelInstanceInitFn()(triton_instance);
RETURN_IF_ERROR(slib->ResetLibraryDirectory());
RETURN_IF_TRITONSERVER_ERROR(err);
}
if (!passive) {
RETURN_IF_ERROR(local_instance->GenerateWarmupData());
RETURN_IF_ERROR(model->Server()->GetRateLimiter()->RegisterModelInstance(
local_instance.get(), rate_limiter_config));
RETURN_IF_ERROR(local_instance->SetBackendThread(
kind, device_id, device_blocking, device_to_thread_map));
}
RETURN_IF_ERROR(model->AddInstance(std::move(local_instance), passive));
return Status::Success;
}
Status
TritonModelInstance::SetBackendThread(
const TRITONSERVER_InstanceGroupKind kind, const int32_t device_id,
const bool device_blocking,
std::map<uint32_t, std::shared_ptr<TritonBackendThread>>*
device_to_thread_map)
{
if (device_blocking && (kind == TRITONSERVER_INSTANCEGROUPKIND_GPU)) {
auto thread_it = device_to_thread_map->find(device_id);
if (thread_it != device_to_thread_map->end()) {
LOG_VERBOSE(1) << "Using already started backend thread for " << Name()
<< " on device " << device_id;
triton_backend_thread_ = thread_it->second;
}
}
if (triton_backend_thread_.get() == nullptr) {
std::unique_ptr<TritonBackendThread> local_backend_thread;
RETURN_IF_ERROR(TritonBackendThread::CreateBackendThread(
Name(), this, 0 /* nice */, device_id, &local_backend_thread));
triton_backend_thread_ = std::move(local_backend_thread);
device_to_thread_map->insert({device_id, triton_backend_thread_});
} else {
triton_backend_thread_->AddModelInstance(this);
}
RETURN_IF_ERROR(triton_backend_thread_->InitAndWarmUpModelInstance(this));
return Status::Success;
}
Status
TritonModelInstance::GenerateWarmupData()
{
warmup_samples_.clear();
for (const auto& warmup_setting : model_->Config().model_warmup()) {
if (warmup_setting.batch_size() == 0) {
LOG_VERBOSE(1) << "Skipping batch 0 warmup sample '"
<< warmup_setting.name() << "'";
continue;
}
LOG_VERBOSE(1) << "Generating warmup sample data for '"
<< warmup_setting.name() << "'";
// Two passes. First pass to get max byte size for synthetic
// data. Second pass to add original inputs and override inputs
// for control inputs.
int64_t max_zero_byte_size = 0;
int64_t max_random_byte_size = 0;
for (const auto& input_meta : warmup_setting.inputs()) {
auto element_count =
triton::common::GetElementCount(input_meta.second.dims());
if (element_count == -1) {
return Status(
Status::Code::INVALID_ARG,
"warmup setting expects all variable-size dimensions are specified "
"for input '" +
input_meta.first + "'");
}
int64_t batch_byte_size =
element_count *
triton::common::GetDataTypeByteSize(input_meta.second.data_type());
if (batch_byte_size == 0) {
batch_byte_size = element_count * sizeof(int32_t);
}
switch (input_meta.second.input_data_type_case()) {
case inference::ModelWarmup_Input::InputDataTypeCase::kZeroData:
max_zero_byte_size = std::max(batch_byte_size, max_zero_byte_size);
break;
case inference::ModelWarmup_Input::InputDataTypeCase::kRandomData: {
// Because Triton expects STRING type to be in special format
// (prepend 4 bytes to specify string length), so using zero data
// for simplicity (4 bytes * element count of zeros).
if (input_meta.second.data_type() ==
inference::DataType::TYPE_STRING) {
max_zero_byte_size = std::max(batch_byte_size, max_zero_byte_size);
} else {
max_random_byte_size =
std::max(batch_byte_size, max_random_byte_size);
}
break;
}
default:
break;
}
}
warmup_samples_.emplace_back(warmup_setting.name(), warmup_setting.count());
auto& warmup_data = warmup_samples_.back();
// Create buffers for synthetic data
TRITONSERVER_MemoryType type;
int64_t type_id;
warmup_data.zero_data_.reset(new AllocatedMemory(
max_zero_byte_size, TRITONSERVER_MEMORY_CPU_PINNED /* memory_type */,
0 /* memory_type_id */));
char* zero_buffer = warmup_data.zero_data_->MutableBuffer(&type, &type_id);
memset(zero_buffer, 0, max_zero_byte_size);
warmup_data.random_data_.reset(new AllocatedMemory(
max_random_byte_size, TRITONSERVER_MEMORY_CPU_PINNED /* memory_type */,
0 /* memory_type_id */));
char* random_buffer =
warmup_data.random_data_->MutableBuffer(&type, &type_id);
for (int64_t offset = 0; offset < max_random_byte_size; offset++) {
random_buffer[offset] = rand();
}
// Prepare the inference request for the specified sample, not using
// in-process C API because the request doesn't go through the same pipeline
// (i.e. no normalization / scheduler) so we need to prepare the request to
// the state just before calling instance execute function.
for (size_t cnt = 0; cnt < warmup_setting.batch_size(); cnt++) {
warmup_data.requests_.emplace_back(
new InferenceRequest(model_, model_->Version()));
auto& lrequest = warmup_data.requests_.back();
// Second pass to prepare original inputs.
std::vector<std::shared_ptr<InferenceRequest::Input>> input_sps;
for (const auto& input_meta : warmup_setting.inputs()) {
auto batch1_element_count =
triton::common::GetElementCount(input_meta.second.dims());
auto batch_byte_size =
batch1_element_count *
triton::common::GetDataTypeByteSize(input_meta.second.data_type());
if (batch_byte_size == 0) {
batch_byte_size = batch1_element_count * sizeof(int32_t);
}
const char* allocated_ptr;
switch (input_meta.second.input_data_type_case()) {
case inference::ModelWarmup_Input::InputDataTypeCase::kZeroData:
allocated_ptr = zero_buffer;
break;
case inference::ModelWarmup_Input::InputDataTypeCase::kRandomData: {
if (input_meta.second.data_type() ==
inference::DataType::TYPE_STRING) {
allocated_ptr = zero_buffer;
} else {
allocated_ptr = random_buffer;
}
break;
}
case inference::ModelWarmup_Input::InputDataTypeCase::
kInputDataFile: {
// For data provided from file, we can set buffer in first pass
warmup_data.provided_data_.emplace_back(new std::string());
auto input_data = warmup_data.provided_data_.back().get();
RETURN_IF_ERROR(ReadTextFile(
JoinPath({model_->LocalizedModelPath(), kWarmupDataFolder,
input_meta.second.input_data_file()}),
input_data));
if (input_meta.second.data_type() ==
inference::DataType::TYPE_STRING) {
batch_byte_size = input_data->size();
} else if (((size_t)batch_byte_size) > input_data->size()) {
return Status(
Status::Code::INVALID_ARG,
lrequest->LogRequest() + "warmup setting expects " +
std::to_string(batch_byte_size) +
" bytes, but the data "
"provided from " +
input_meta.second.input_data_file() + "only has " +
std::to_string(input_data->size()) + " bytes");
}
allocated_ptr = input_data->data();
break;
}
default:
return Status(
Status::Code::INVALID_ARG,
lrequest->LogRequest() + "warmup setting expects input '" +
input_meta.first + "' to have input_data_type set");
}
const inference::ModelInput* input_config;
bool is_original_input =
model_->GetInput(input_meta.first, &input_config).IsOk();
InferenceRequest::Input* input = nullptr;
std::vector<int64_t> input_meta_shape;
// Append batch size only if the model supports batching
// and not control inpt.
if ((model_->Config().max_batch_size() != 0) && is_original_input) {
input_meta_shape.push_back(1);
}
for (auto d : input_meta.second.dims()) {
input_meta_shape.push_back(d);
}
if (is_original_input) {
RETURN_IF_ERROR(lrequest->AddOriginalInput(
input_meta.first, input_meta.second.data_type(), input_meta_shape,
&input));
} else {
input_sps.emplace_back();
RETURN_IF_ERROR(lrequest->AddOverrideInput(
input_meta.first, input_meta.second.data_type(),
(model_->Config().max_batch_size() != 0 ? 1 : 0),
input_meta_shape, &input_sps.back()));
input = input_sps.back().get();
}
RETURN_IF_ERROR(input->AppendData(
allocated_ptr, batch_byte_size,
TRITONSERVER_MEMORY_CPU /* memory_type */, 0 /* memory_type_id */));
}
RETURN_IF_ERROR(lrequest->PrepareForInference());
// Override inputs must be added after PrepareForInference() is called
for (const auto& sp : input_sps) {
RETURN_IF_ERROR(lrequest->AddOverrideInput(sp));
}
}
}
return Status::Success;
}
void
TritonModelInstance::Schedule(
std::vector<std::unique_ptr<InferenceRequest>>&& requests,
const std::function<void()>& OnCompletion)
{
// Use a thread local vector to avoid needing to malloc each
// time an inference is run.
thread_local std::vector<TRITONBACKEND_Request*> triton_requests(1024);
triton_requests.clear();
for (auto& r : requests) {
// Load the input states for the inference request.
r->LoadInputStates();
triton_requests.push_back(
reinterpret_cast<TRITONBACKEND_Request*>(r.release()));
}
Execute(triton_requests);
OnCompletion();
}
Status
TritonModelInstance::Initialize()
{
RETURN_IF_ERROR(SetNumaConfigOnThread(HostPolicy()));
return Status::Success;
}
Status
TritonModelInstance::WarmUp()
{
// move samples to local variable for scoped cleanup
std::vector<triton::core::TritonModelInstance::WarmupData> lwarmup_samples;
lwarmup_samples.swap(warmup_samples_);
for (auto& sample : lwarmup_samples) {
for (size_t iteration = 1; iteration <= sample.count_; ++iteration) {
LOG_VERBOSE(1) << "model '" << sample.requests_.back()->ModelName()
<< "' instance " << Name() << " is running warmup sample '"
<< sample.sample_name_ << "' for iteration " << iteration;
// request/response complete is asynchronous so use promise to wait for
// completion. Also collects error message from the responses in a vector.
std::vector<std::promise<void>> request_complete(sample.requests_.size());
std::vector<std::string> response_errors;
std::vector<std::pair<std::promise<void>, std::vector<std::string>*>>
response_complete(sample.requests_.size());
std::vector<TRITONBACKEND_Request*> triton_requests;
for (size_t i = 0; i < sample.requests_.size(); ++i) {
auto& request = sample.requests_[i];
request->SetReleaseCallback(
WarmupRequestComplete, &request_complete[i]);
response_complete[i].second = &response_errors;
request->SetResponseCallback(
&warmup_allocator, nullptr, WarmupResponseComplete,
&response_complete[i]);
// Capture timestamp before run to avoid incorrect accumulation from
// sequential warmup runs
#ifdef TRITON_ENABLE_STATS
request->CaptureRequestStartNs();
#endif // TRITON_ENABLE_STATS
request->CaptureQueueStartNs();
triton_requests.push_back(
reinterpret_cast<TRITONBACKEND_Request*>(request.get()));
}
Execute(triton_requests);
// Wait for warmup sample to complete and check error
for (size_t i = 0; i < sample.requests_.size(); ++i) {
request_complete[i].get_future().get();
response_complete[i].first.get_future().get();
}
if (response_errors.size() != 0) {
std::string err_str =
"failed to run warmup sample '" + sample.sample_name_ + "': ";
for (const auto& error : response_errors) {
err_str += (error + "; ");
}
// End warmup as soon as there is failing sample
LOG_VERBOSE(1) << "model '" << sample.requests_.back()->ModelName()
<< "' instance " << Name()
<< " failed to run warmup sample '"
<< sample.sample_name_ << "'";
return Status(Status::Code::INVALID_ARG, err_str);
}
}
}
return Status::Success;
}
void
TritonModelInstance::Execute(
std::vector<TRITONBACKEND_Request*>& triton_requests)
{
TRITONBACKEND_ModelInstance* triton_model_instance =
reinterpret_cast<TRITONBACKEND_ModelInstance*>(this);
TritonBackend::TritonModelInstanceExecFn_t inst_exec_fn =
model_->Backend()->ModelInstanceExecFn();
// If there is an error then we retain ownership of 'requests'
// and must send error responses.
TRITONSERVER_Error* err = inst_exec_fn(
triton_model_instance, &triton_requests[0], triton_requests.size());
if (err != nullptr) {
Status status = Status(
TritonCodeToStatusCode(TRITONSERVER_ErrorCode(err)),
TRITONSERVER_ErrorMessage(err));
for (TRITONBACKEND_Request* tr : triton_requests) {
std::unique_ptr<InferenceRequest> ur(
reinterpret_cast<InferenceRequest*>(tr));
InferenceRequest::RespondIfError(ur, status, true /* release_requests */);
}
TRITONSERVER_ErrorDelete(err);
}
}
Status
TritonModelInstance::TritonBackendThread::CreateBackendThread(
const std::string name, TritonModelInstance* model_instance, const int nice,
const int32_t device_id,
std::unique_ptr<TritonBackendThread>* triton_backend_thread)
{
TritonBackendThread* raw_triton_backend_thread =
new TritonBackendThread(name, model_instance->Model());
std::unique_ptr<TritonBackendThread> runner(raw_triton_backend_thread);
runner->AddModelInstance(model_instance);
runner->backend_thread_ =
std::thread([raw_triton_backend_thread, nice, device_id]() {
raw_triton_backend_thread->BackendThread(nice, device_id);
});
triton_backend_thread->reset(runner.release());
return Status::Success;
}
void
TritonModelInstance::TritonBackendThread::AddModelInstance(
TritonModelInstance* model_instance)
{
model_instances_.push_back(model_instance);
}
Status
TritonModelInstance::TritonBackendThread::InitAndWarmUpModelInstance(
TritonModelInstance* model_instance)
{
// Initialize the instance on the backend thread
auto init_payload = model_->Server()->GetRateLimiter()->GetPayload(
Payload::Operation::INIT, model_instance);
RETURN_IF_ERROR(
model_->Server()->GetRateLimiter()->EnqueuePayload(model_, init_payload));
RETURN_IF_ERROR(init_payload->Wait());
// Warm-up the instance on the backend thread
auto warmup_payload = model_->Server()->GetRateLimiter()->GetPayload(
Payload::Operation::WARM_UP, model_instance);
RETURN_IF_ERROR(model_->Server()->GetRateLimiter()->EnqueuePayload(
model_, warmup_payload));
RETURN_IF_ERROR(warmup_payload->Wait());
return Status::Success;
}
TritonModelInstance::TritonBackendThread::TritonBackendThread(
const std::string& name, TritonModel* model)
: name_(name), model_(model)
{
}
TritonModelInstance::TritonBackendThread::~TritonBackendThread()
{
StopBackendThread();
}
void
TritonModelInstance::TritonBackendThread::StopBackendThread()
{
if (backend_thread_.joinable()) {
// Signal the backend thread to exit and then wait for it...
auto exit_payload = model_->Server()->GetRateLimiter()->GetPayload(
Payload::Operation::EXIT, model_instances_.back());
model_->Server()->GetRateLimiter()->EnqueuePayload(model_, exit_payload);
backend_thread_.join();
}
}
void
TritonModelInstance::TritonBackendThread::BackendThread(
const int nice, const int32_t device_id)
{
#ifndef _WIN32
if (setpriority(PRIO_PROCESS, syscall(SYS_gettid), nice) == 0) {
LOG_VERBOSE(1) << "Starting backend thread for " << name_ << " at nice "
<< nice << " on device " << device_id << "...";
} else {
LOG_VERBOSE(1) << "Starting backend thread for " << name_
<< " at default nice (requested nice " << nice << " failed)"
<< " on device " << device_id << "...";
}
#else
LOG_VERBOSE(1) << "Starting backend thread for " << name_
<< " at default nice on device " << device_id << "...";
#endif
bool should_exit = false;
while (!should_exit) {
std::shared_ptr<Payload> payload;
model_->Server()->GetRateLimiter()->DequeuePayload(
model_instances_, &payload);
NVTX_RANGE(nvtx_, "BackendThread " + name_);
payload->Execute(&should_exit);
model_instances_.push_back(payload->GetInstance());
// Release the payload to the RateLimiter
model_->Server()->GetRateLimiter()->PayloadRelease(payload);
}
LOG_VERBOSE(1) << "Stopping backend thread for " << name_ << "...";
}
extern "C" {
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_ModelInstanceName(
TRITONBACKEND_ModelInstance* instance, const char** name)
{
TritonModelInstance* ti = reinterpret_cast<TritonModelInstance*>(instance);
*name = ti->Name().c_str();
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_ModelInstanceKind(
TRITONBACKEND_ModelInstance* instance, TRITONSERVER_InstanceGroupKind* kind)
{
TritonModelInstance* ti = reinterpret_cast<TritonModelInstance*>(instance);
*kind = ti->Kind();
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_ModelInstanceDeviceId(
TRITONBACKEND_ModelInstance* instance, int32_t* device_id)
{
TritonModelInstance* ti = reinterpret_cast<TritonModelInstance*>(instance);
*device_id = ti->DeviceId();
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_ModelInstanceHostPolicy(
TRITONBACKEND_ModelInstance* instance, TRITONSERVER_Message** host_policy)
{
TritonModelInstance* ti = reinterpret_cast<TritonModelInstance*>(instance);
*host_policy = const_cast<TRITONSERVER_Message*>(
reinterpret_cast<const TRITONSERVER_Message*>(&ti->HostPolicyMessage()));
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_ModelInstanceProfileCount(
TRITONBACKEND_ModelInstance* instance, uint32_t* count)
{
TritonModelInstance* ti = reinterpret_cast<TritonModelInstance*>(instance);
*count = ti->Profiles().size();
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_ModelInstanceProfileName(
TRITONBACKEND_ModelInstance* instance, const uint32_t index,
const char** profile_name)
{
*profile_name = nullptr;
TritonModelInstance* ti = reinterpret_cast<TritonModelInstance*>(instance);
const auto& rprofiles = ti->Profiles();
if (index >= rprofiles.size()) {
return TRITONSERVER_ErrorNew(
TRITONSERVER_ERROR_INVALID_ARG,
(std::string("out of bounds index ") + std::to_string(index) +
": instance is configured with " + std::to_string(rprofiles.size()) +
" profiles")
.c_str());
}
*profile_name = rprofiles[index].c_str();
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_ModelInstanceSecondaryDeviceCount(
TRITONBACKEND_ModelInstance* instance, uint32_t* count)
{
TritonModelInstance* ti = reinterpret_cast<TritonModelInstance*>(instance);
*count = ti->SecondaryDevices().size();
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_ModelInstanceSecondaryDeviceProperties(
TRITONBACKEND_ModelInstance* instance, uint32_t index, const char** kind,
int64_t* id)
{
TritonModelInstance* ti = reinterpret_cast<TritonModelInstance*>(instance);
const auto& rsecondarydevices = ti->SecondaryDevices();
if (index >= rsecondarydevices.size()) {
return TRITONSERVER_ErrorNew(
TRITONSERVER_ERROR_INVALID_ARG,
(std::string("out of bounds index ") + std::to_string(index) +
": instance is configured with " +
std::to_string(rsecondarydevices.size()) + " secondary devices")
.c_str());
}
*kind = rsecondarydevices[index].kind_.c_str();
*id = rsecondarydevices[index].id_;
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_ModelInstanceIsPassive(
TRITONBACKEND_ModelInstance* instance, bool* is_passive)
{
TritonModelInstance* ti = reinterpret_cast<TritonModelInstance*>(instance);
*is_passive = ti->IsPassive();
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_ModelInstanceModel(
TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Model** model)
{
TritonModelInstance* ti = reinterpret_cast<TritonModelInstance*>(instance);
*model = reinterpret_cast<TRITONBACKEND_Model*>(ti->Model());
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_ModelInstanceState(
TRITONBACKEND_ModelInstance* instance, void** state)
{
TritonModelInstance* ti = reinterpret_cast<TritonModelInstance*>(instance);
*state = ti->State();
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_ModelInstanceSetState(
TRITONBACKEND_ModelInstance* instance, void* state)
{
TritonModelInstance* ti = reinterpret_cast<TritonModelInstance*>(instance);
ti->SetState(state);
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_ModelInstanceReportStatistics(
TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Request* request,
const bool success, const uint64_t exec_start_ns,
const uint64_t compute_start_ns, const uint64_t compute_end_ns,
const uint64_t exec_end_ns)
{
#ifdef TRITON_ENABLE_STATS
TritonModelInstance* ti = reinterpret_cast<TritonModelInstance*>(instance);
InferenceRequest* tr = reinterpret_cast<InferenceRequest*>(request);
tr->ReportStatistics(
ti->MetricReporter(), success, exec_start_ns, compute_start_ns,
compute_end_ns, exec_end_ns);
#endif // TRITON_ENABLE_STATS
return nullptr; // success
}
TRITONAPI_DECLSPEC TRITONSERVER_Error*
TRITONBACKEND_ModelInstanceReportBatchStatistics(
TRITONBACKEND_ModelInstance* instance, const uint64_t batch_size,
const uint64_t exec_start_ns, const uint64_t compute_start_ns,
const uint64_t compute_end_ns, const uint64_t exec_end_ns)
{
#ifdef TRITON_ENABLE_STATS
TritonModelInstance* ti = reinterpret_cast<TritonModelInstance*>(instance);
ti->Model()->MutableStatsAggregator()->UpdateInferBatchStats(
ti->MetricReporter(), batch_size, exec_start_ns, compute_start_ns,
compute_end_ns, exec_end_ns);
#endif // TRITON_ENABLE_STATS
return nullptr; // success
}
} // extern C
}} // namespace triton::core
3rdparty/core-r22.12/src/backend_model_instance.h 0 → 100644
View file @ b30f3cdb
// Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
// * Neither the name of NVIDIA CORPORATION nor the names of its
// contributors may be used to endorse or promote products derived
// from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include <functional>
#include <future>
#include <memory>
#include <string>
#include <thread>
#include "constants.h"
#include "memory.h"
#include "metric_model_reporter.h"
#include "model_config.pb.h"
#include "server_message.h"
#include "status.h"
#include "triton/common/sync_queue.h"
namespace triton { namespace core {
class TritonModel;
class InferenceRequest;
//
// Represents a model instance.
//
class TritonModelInstance {
public:
static Status CreateInstances(
TritonModel* model,
const triton::common::BackendCmdlineConfigMap& backend_cmdline_config_map,
const triton::common::HostPolicyCmdlineConfigMap& host_policy_map,
const inference::ModelConfig& model_config, const bool device_blocking);
~TritonModelInstance();
const std::string& Name() const { return name_; }
size_t Index() const { return index_; }
TRITONSERVER_InstanceGroupKind Kind() const { return kind_; }
int32_t DeviceId() const { return device_id_; }
const triton::common::HostPolicyCmdlineConfig& HostPolicy() const
{
return host_policy_;
}
const TritonServerMessage& HostPolicyMessage() const
{
return host_policy_message_;
}
bool IsPassive() const { return passive_; }
const std::vector<std::string>& Profiles() const { return profile_names_; }
struct SecondaryDevice {
SecondaryDevice(const std::string kind, const int64_t id)
: kind_(kind), id_(id)
{
}
const std::string kind_;
const int64_t id_;
};
const std::vector<SecondaryDevice>& SecondaryDevices() const
{
return secondary_devices_;
}
Status Initialize();
Status WarmUp();
void Schedule(
std::vector<std::unique_ptr<InferenceRequest>>&& requests,
const std::function<void()>& OnCompletion);
TritonModel* Model() const { return model_; }
void* State() { return state_; }
void SetState(void* state) { state_ = state; }
MetricModelReporter* MetricReporter() const { return reporter_.get(); }
private:
DISALLOW_COPY_AND_ASSIGN(TritonModelInstance);
class TritonBackendThread;
TritonModelInstance(
TritonModel* model, const std::string& name, const size_t index,
const TRITONSERVER_InstanceGroupKind kind, const int32_t device_id,
const std::vector<std::string>& profile_names, const bool passive,
const triton::common::HostPolicyCmdlineConfig& host_policy,
const TritonServerMessage& host_policy_message,
const std::vector<SecondaryDevice>& secondary_devices);
static Status CreateInstance(
TritonModel* model, const std::string& name, const size_t index,
const TRITONSERVER_InstanceGroupKind kind, const int32_t device_id,
const std::vector<std::string>& profile_names, const bool passive,
const std::string& host_policy_name,
const triton::common::HostPolicyCmdlineConfig& host_policy,
const inference::ModelRateLimiter& rate_limiter_config,
const bool device_blocking,
std::map<uint32_t, std::shared_ptr<TritonBackendThread>>*
device_to_thread_map,
const std::vector<SecondaryDevice>& secondary_devices);
Status SetBackendThread(
const TRITONSERVER_InstanceGroupKind kind, const int32_t device_id,
const bool device_blocking,
std::map<uint32_t, std::shared_ptr<TritonBackendThread>>*
device_to_thread_map);
Status GenerateWarmupData();
void Execute(std::vector<TRITONBACKEND_Request*>& triton_requests);
class TritonBackendThread {
public:
static Status CreateBackendThread(
const std::string name, TritonModelInstance* model, const int nice,
const int32_t device_id,
std::unique_ptr<TritonBackendThread>* triton_backend_thread);
void AddModelInstance(TritonModelInstance* model_instance);
Status InitAndWarmUpModelInstance(TritonModelInstance* model_instance);
void StopBackendThread();
~TritonBackendThread();
private:
TritonBackendThread(const std::string& name, TritonModel* model);
void BackendThread(const int nice, const int32_t device_id);
std::string name_;
TritonModel* model_;
std::deque<TritonModelInstance*> model_instances_;
std::thread backend_thread_;
std::atomic<bool> backend_thread_exit_;
};
std::shared_ptr<TritonBackendThread> triton_backend_thread_;
struct WarmupData {
WarmupData(const std::string& sample_name, const size_t count)
: sample_name_(sample_name), count_(std::max(count, size_t{1}))
{
}
std::string sample_name_;
size_t count_;
// Using a batch of requests to satisfy batch size, this provides better
// alignment on the batch expected by the model, especially for sequence
// model.
std::vector<std::unique_ptr<InferenceRequest>> requests_;
// Placeholder for input data
std::unique_ptr<AllocatedMemory> zero_data_;
std::unique_ptr<AllocatedMemory> random_data_;
std::vector<std::unique_ptr<std::string>> provided_data_;
};
std::vector<WarmupData> warmup_samples_;
// The TritonModel object that owns this instance. The instance
// holds this as a raw pointer because the lifetime of the model is
// guaranteed to be longer than the lifetime of an instance owned by the
// model.
TritonModel* model_;
std::string name_;
size_t index_;
// For CPU device_id_ is always 0. For GPU device_id_ indicates the
// GPU device to be used by the instance.
TRITONSERVER_InstanceGroupKind kind_;
int32_t device_id_;
const triton::common::HostPolicyCmdlineConfig host_policy_;
TritonServerMessage host_policy_message_;
std::vector<std::string> profile_names_;
bool passive_;
std::vector<SecondaryDevice> secondary_devices_;
// Reporter for metrics, or nullptr if no metrics should be reported
std::shared_ptr<MetricModelReporter> reporter_;
// Opaque state associated with this model instance.
void* state_;
};
}} // namespace triton::core
3rdparty/core-r22.12/src/buffer_attributes.cc 0 → 100644
View file @ b30f3cdb
// Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
// * Neither the name of NVIDIA CORPORATION nor the names of its
// contributors may be used to endorse or promote products derived
// from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "buffer_attributes.h"
#include <cstring>
#include "constants.h"
namespace triton { namespace core {
void
BufferAttributes::SetByteSize(const size_t& byte_size)
{
byte_size_ = byte_size;
}
void
BufferAttributes::SetMemoryType(const TRITONSERVER_MemoryType& memory_type)
{
memory_type_ = memory_type;
}
void
BufferAttributes::SetMemoryTypeId(const int64_t& memory_type_id)
{
memory_type_id_ = memory_type_id;
}
void
BufferAttributes::SetCudaIpcHandle(void* cuda_ipc_handle)
{
char* lcuda_ipc_handle = reinterpret_cast<char*>(cuda_ipc_handle);
cuda_ipc_handle_.clear();
std::copy(
lcuda_ipc_handle, lcuda_ipc_handle + CUDA_IPC_STRUCT_SIZE,
std::back_inserter(cuda_ipc_handle_));
}
void*
BufferAttributes::CudaIpcHandle()
{
if (cuda_ipc_handle_.empty()) {
return nullptr;
} else {
return reinterpret_cast<void*>(cuda_ipc_handle_.data());
}
}
size_t
BufferAttributes::ByteSize() const
{
return byte_size_;
}
TRITONSERVER_MemoryType
BufferAttributes::MemoryType() const
{
return memory_type_;
}
int64_t
BufferAttributes::MemoryTypeId() const
{
return memory_type_id_;
}
BufferAttributes::BufferAttributes(
size_t byte_size, TRITONSERVER_MemoryType memory_type,
int64_t memory_type_id, char* cuda_ipc_handle)
: byte_size_(byte_size), memory_type_(memory_type),
memory_type_id_(memory_type_id)
{
// cuda ipc handle size
cuda_ipc_handle_.reserve(CUDA_IPC_STRUCT_SIZE);
if (cuda_ipc_handle != nullptr) {
std::copy(
cuda_ipc_handle, cuda_ipc_handle + CUDA_IPC_STRUCT_SIZE,
std::back_inserter(cuda_ipc_handle_));
}
}
}} // namespace triton::core
3rdparty/core-r22.12/src/buffer_attributes.h 0 → 100644
View file @ b30f3cdb
// Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
// * Neither the name of NVIDIA CORPORATION nor the names of its
// contributors may be used to endorse or promote products derived
// from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <iterator>
#include <vector>
#include "tritonserver_apis.h"
#pragma once
namespace triton { namespace core {
//
// A class to hold information about the buffer allocation.
//
class BufferAttributes {
public:
BufferAttributes(
size_t byte_size, TRITONSERVER_MemoryType memory_type,
int64_t memory_type_id, char cuda_ipc_handle[64]);
BufferAttributes()
{
memory_type_ = TRITONSERVER_MEMORY_CPU;
memory_type_id_ = 0;
cuda_ipc_handle_.reserve(64);
}
// Set the buffer byte size
void SetByteSize(const size_t& byte_size);
// Set the buffer memory_type
void SetMemoryType(const TRITONSERVER_MemoryType& memory_type);
// Set the buffer memory type id
void SetMemoryTypeId(const int64_t& memory_type_id);
// Set the cuda ipc handle
void SetCudaIpcHandle(void* cuda_ipc_handle);
// Get the cuda ipc handle
void* CudaIpcHandle();
// Get the byte size
size_t ByteSize() const;
// Get the memory type
TRITONSERVER_MemoryType MemoryType() const;
// Get the memory type id
int64_t MemoryTypeId() const;
private:
size_t byte_size_;
TRITONSERVER_MemoryType memory_type_;
int64_t memory_type_id_;
std::vector<char> cuda_ipc_handle_;
};
}} // namespace triton::core
3rdparty/core-r22.12/src/constants.h 0 → 100644
View file @ b30f3cdb
// Copyright 2018-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
// * Neither the name of NVIDIA CORPORATION nor the names of its
// contributors may be used to endorse or promote products derived
// from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include <stdint.h>
namespace triton { namespace core {
constexpr char kInferHeaderContentLengthHTTPHeader[] =
"Inference-Header-Content-Length";
constexpr char kAcceptEncodingHTTPHeader[] = "Accept-Encoding";
constexpr char kContentEncodingHTTPHeader[] = "Content-Encoding";
constexpr char kContentTypeHeader[] = "Content-Type";
constexpr char kContentLengthHeader[] = "Content-Length";
constexpr char kTensorFlowGraphDefPlatform[] = "tensorflow_graphdef";
constexpr char kTensorFlowSavedModelPlatform[] = "tensorflow_savedmodel";
constexpr char kTensorFlowGraphDefFilename[] = "model.graphdef";
constexpr char kTensorFlowSavedModelFilename[] = "model.savedmodel";
constexpr char kTensorFlowBackend[] = "tensorflow";
constexpr char kTensorRTPlanPlatform[] = "tensorrt_plan";
constexpr char kTensorRTPlanFilename[] = "model.plan";
constexpr char kTensorRTBackend[] = "tensorrt";
constexpr char kOnnxRuntimeOnnxPlatform[] = "onnxruntime_onnx";
constexpr char kOnnxRuntimeOnnxFilename[] = "model.onnx";
constexpr char kOnnxRuntimeBackend[] = "onnxruntime";
constexpr char kOpenVINORuntimeOpenVINOFilename[] = "model.xml";
constexpr char kOpenVINORuntimeBackend[] = "openvino";
constexpr char kPyTorchLibTorchPlatform[] = "pytorch_libtorch";
constexpr char kPyTorchLibTorchFilename[] = "model.pt";
constexpr char kPyTorchBackend[] = "pytorch";
constexpr char kPythonFilename[] = "model.py";
constexpr char kPythonBackend[] = "python";
#ifdef TRITON_ENABLE_ENSEMBLE
constexpr char kEnsemblePlatform[] = "ensemble";
#endif // TRITON_ENABLE_ENSEMBLE
constexpr char kTensorRTExecutionAccelerator[] = "tensorrt";
constexpr char kOpenVINOExecutionAccelerator[] = "openvino";
constexpr char kGPUIOExecutionAccelerator[] = "gpu_io";
constexpr char kAutoMixedPrecisionExecutionAccelerator[] =
"auto_mixed_precision";
constexpr char kModelConfigPbTxt[] = "config.pbtxt";
constexpr char kMetricsLabelModelName[] = "model";
constexpr char kMetricsLabelModelVersion[] = "version";
constexpr char kMetricsLabelGpuUuid[] = "gpu_uuid";
constexpr char kWarmupDataFolder[] = "warmup";
constexpr char kInitialStateFolder[] = "initial_state";
constexpr uint64_t NANOS_PER_SECOND = 1000000000;
constexpr uint64_t NANOS_PER_MILLIS = 1000000;
constexpr int MAX_GRPC_MESSAGE_SIZE = INT32_MAX;
constexpr uint64_t SEQUENCE_IDLE_DEFAULT_MICROSECONDS = 1000 * 1000;
constexpr size_t STRING_CORRELATION_ID_MAX_LENGTH_BYTES = 128;
constexpr size_t CUDA_IPC_STRUCT_SIZE = 64;
#ifdef TRITON_ENABLE_METRICS
// MetricModelReporter expects a device ID for GPUs, but we reuse this device
// ID for other metrics as well such as for CPU and Response Cache metrics
constexpr int METRIC_REPORTER_ID_CPU = -1;
constexpr int METRIC_REPORTER_ID_RESPONSE_CACHE = -2;
#endif
#define TIMESPEC_TO_NANOS(TS) \
((TS).tv_sec * triton::core::NANOS_PER_SECOND + (TS).tv_nsec)
#define TIMESPEC_TO_MILLIS(TS) \
(TIMESPEC_TO_NANOS(TS) / triton::core::NANOS_PER_MILLIS)
#define DISALLOW_MOVE(TypeName) TypeName(Context&& o) = delete;
#define DISALLOW_COPY(TypeName) TypeName(const TypeName&) = delete;
#define DISALLOW_ASSIGN(TypeName) void operator=(const TypeName&) = delete;
#define DISALLOW_COPY_AND_ASSIGN(TypeName) \
DISALLOW_COPY(TypeName) \
DISALLOW_ASSIGN(TypeName)
}} // namespace triton::core
3rdparty/core-r22.12/src/cuda_memory_manager.cc 0 → 100644
View file @ b30f3cdb
// Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
// * Neither the name of NVIDIA CORPORATION nor the names of its
// contributors may be used to endorse or promote products derived
// from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
#include "cuda_memory_manager.h"
#include <cnmem.h>
#include <string.h>
#include <set>
#include "cuda_utils.h"
#include "triton/common/logging.h"
namespace {
#define RETURN_IF_CNMEM_ERROR(S, MSG) \
do { \
auto status__ = (S); \
if (status__ != CNMEM_STATUS_SUCCESS) { \
return Status( \
Status::Code::INTERNAL, \
(MSG) + ": " + cnmemGetErrorString(status__)); \
} \
} while (false)
std::string
PointerToString(void* ptr)
{
std::stringstream ss;
ss << ptr;
return ss.str();
}
} // namespace
namespace triton { namespace core {
std::unique_ptr<CudaMemoryManager> CudaMemoryManager::instance_;
std::mutex CudaMemoryManager::instance_mu_;
CudaMemoryManager::~CudaMemoryManager()
{
if (has_allocation_) {
auto status = cnmemFinalize();
if (status != CNMEM_STATUS_SUCCESS) {
LOG_ERROR << "Failed to finalize CUDA memory manager: [" << status << "] "
<< cnmemGetErrorString(status);
}
}
}
void
CudaMemoryManager::Reset()
{
std::lock_guard<std::mutex> lock(instance_mu_);
instance_.reset();
}
Status
CudaMemoryManager::Create(const CudaMemoryManager::Options& options)
{
// Ensure thread-safe creation of CUDA memory pool
std::lock_guard<std::mutex> lock(instance_mu_);
if (instance_ != nullptr) {
LOG_WARNING << "New CUDA memory pools could not be created since they "
"already exists";
return Status::Success;
}
std::set<int> supported_gpus;
auto status = GetSupportedGPUs(
&supported_gpus, options.min_supported_compute_capability_);
if (status.IsOk()) {
std::vector<cnmemDevice_t> devices;
for (auto gpu : supported_gpus) {
const auto it = options.memory_pool_byte_size_.find(gpu);
if ((it != options.memory_pool_byte_size_.end()) && (it->second != 0)) {
devices.emplace_back();
auto& device = devices.back();
memset(&device, 0, sizeof(device));
device.device = gpu;
device.size = it->second;
LOG_INFO << "CUDA memory pool is created on device " << device.device
<< " with size " << device.size;
}
}
if (!devices.empty()) {
RETURN_IF_CNMEM_ERROR(
cnmemInit(devices.size(), devices.data(), CNMEM_FLAGS_CANNOT_GROW),
std::string("Failed to finalize CUDA memory manager"));
} else {
LOG_INFO << "CUDA memory pool disabled";
}
// Use to finalize CNMeM properly when out of scope
instance_.reset(new CudaMemoryManager(!devices.empty()));
} else {
return Status(
status.ErrorCode(),
"Failed to initialize CUDA memory manager: " + status.Message());
}
return Status::Success;
}
Status
CudaMemoryManager::Alloc(void** ptr, uint64_t size, int64_t device_id)
{
if (instance_ == nullptr) {
return Status(
Status::Code::UNAVAILABLE, "CudaMemoryManager has not been created");
} else if (!instance_->has_allocation_) {
return Status(
Status::Code::UNAVAILABLE,
"CudaMemoryManager has no preallocated CUDA memory");
}
int current_device;
RETURN_IF_CUDA_ERR(
cudaGetDevice(&current_device), std::string("Failed to get device"));
bool overridden = (current_device != device_id);
if (overridden) {
RETURN_IF_CUDA_ERR(
cudaSetDevice(device_id), std::string("Failed to set device"));
}
// Defer returning error to make sure the device is recovered
auto err = cnmemMalloc(ptr, size, nullptr);
if (overridden) {
cudaSetDevice(current_device);
}
RETURN_IF_CNMEM_ERROR(
err, std::string("Failed to allocate CUDA memory with byte size ") +
std::to_string(size) + " on GPU " + std::to_string(device_id));
return Status::Success;
}
Status
CudaMemoryManager::Free(void* ptr, int64_t device_id)
{
if (instance_ == nullptr) {
return Status(
Status::Code::UNAVAILABLE, "CudaMemoryManager has not been created");
} else if (!instance_->has_allocation_) {
return Status(
Status::Code::UNAVAILABLE,
"CudaMemoryManager has no preallocated CUDA memory");
}
int current_device;
RETURN_IF_CUDA_ERR(
cudaGetDevice(&current_device), std::string("Failed to get device"));
bool overridden = (current_device != device_id);
if (overridden) {
RETURN_IF_CUDA_ERR(
cudaSetDevice(device_id), std::string("Failed to set device"));
}
// Defer returning error to make sure the device is recovered
auto err = cnmemFree(ptr, nullptr);
if (overridden) {
cudaSetDevice(current_device);
}
RETURN_IF_CNMEM_ERROR(
err, std::string("Failed to deallocate CUDA memory at address ") +
PointerToString(ptr) + " on GPU " + std::to_string(device_id));
return Status::Success;
}
}} // namespace triton::core
3rdparty/core-r22.12/src/cuda_memory_manager.h 0 → 100644
View file @ b30f3cdb
// Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
// * Neither the name of NVIDIA CORPORATION nor the names of its
// contributors may be used to endorse or promote products derived
// from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
#pragma once
#include <map>
#include <memory>
#include <mutex>
#include "status.h"
namespace triton { namespace core {
// This is a singleton class responsible for maintaining CUDA memory pool
// used by the inference server. CUDA memory allocations and deallocations
// must be requested via functions provided by this class.
class CudaMemoryManager {
public:
// Options to configure CUDA memory manager.
struct Options {
Options(double cc = 6.0, const std::map<int, uint64_t>& s = {})
: min_supported_compute_capability_(cc), memory_pool_byte_size_(s)
{
}
// The minimum compute capability of the supported devices.
double min_supported_compute_capability_;
// The size of CUDA memory reserved for the specified devices.
// The memory size will be rounded up to align with
// the default granularity (512 bytes).
// No memory will be reserved for devices that is not listed.
std::map<int, uint64_t> memory_pool_byte_size_;
};
~CudaMemoryManager();
// Create the memory manager based on 'options' specified.
// Return Status object indicating success or failure.
static Status Create(const Options& options);
// Allocate CUDA memory on GPU 'device_id' with
// the requested 'size' and return the pointer in 'ptr'.
// Return Status object indicating success or failure.
static Status Alloc(void** ptr, uint64_t size, int64_t device_id);
// Free the memory allocated by the memory manager on 'device_id'.
// Return Status object indicating success or failure.
static Status Free(void* ptr, int64_t device_id);
protected:
// Provide explicit control on the lifecycle of the CUDA memory manager,
// for testing only.
static void Reset();
private:
CudaMemoryManager(bool has_allocation) : has_allocation_(has_allocation) {}
bool has_allocation_;
static std::unique_ptr<CudaMemoryManager> instance_;
static std::mutex instance_mu_;
};
}} // namespace triton::core
3rdparty/core-r22.12/src/cuda_utils.cc 0 → 100644
View file @ b30f3cdb
// Copyright 2019-2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
// * Neither the name of NVIDIA CORPORATION nor the names of its
// contributors may be used to endorse or promote products derived
// from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "cuda_utils.h"
#include "model_config_utils.h"
#include "triton/common/nvtx.h"
namespace triton { namespace core {
#ifdef TRITON_ENABLE_GPU
void CUDART_CB
MemcpyHost(void* args)
{
auto* copy_params = reinterpret_cast<CopyParams*>(args);
memcpy(copy_params->dst_, copy_params->src_, copy_params->byte_size_);
delete copy_params;
}
#endif // TRITON_ENABLE_GPU
Status
GetDeviceMemoryInfo(const int device_id, size_t* free, size_t* total)
{
*free = 0;
*total = 0;
#ifdef TRITON_ENABLE_GPU
// Make sure that correct device is set before creating stream and
// then restore the device to what was set by the caller.
int current_device;
auto cuerr = cudaGetDevice(&current_device);
bool overridden = false;
if (cuerr == cudaSuccess) {
overridden = (current_device != device_id);
if (overridden) {
cuerr = cudaSetDevice(device_id);
}
}
if (cuerr == cudaSuccess) {
cuerr = cudaMemGetInfo(free, total);
}
if (overridden) {
cudaSetDevice(current_device);
}
if (cuerr != cudaSuccess) {
return Status(
Status::Code::INTERNAL,
(std::string("unable to get memory info for device ") +
std::to_string(device_id) + ": " + cudaGetErrorString(cuerr)));
}
#endif // TRITON_ENABLE_GPU
return Status::Success;
}
Status
EnablePeerAccess(const double min_compute_capability)
{
#ifdef TRITON_ENABLE_GPU
// If we can't enable peer access for one device pair, the best we can
// do is skipping it...
std::set<int> supported_gpus;
bool all_enabled = false;
if (GetSupportedGPUs(&supported_gpus, min_compute_capability).IsOk()) {
all_enabled = true;
int can_access_peer = false;
for (const auto& host : supported_gpus) {
auto cuerr = cudaSetDevice(host);
if (cuerr == cudaSuccess) {
for (const auto& peer : supported_gpus) {
if (host == peer) {
continue;
}
cuerr = cudaDeviceCanAccessPeer(&can_access_peer, host, peer);
if ((cuerr == cudaSuccess) && (can_access_peer == 1)) {
cuerr = cudaDeviceEnablePeerAccess(peer, 0);
}
all_enabled &= ((cuerr == cudaSuccess) && (can_access_peer == 1));
}
}
}
}
if (!all_enabled) {
return Status(
Status::Code::UNSUPPORTED,
"failed to enable peer access for some device pairs");
}
#endif // TRITON_ENABLE_GPU
return Status::Success;
}
Status
CopyBuffer(
const std::string& msg, const TRITONSERVER_MemoryType src_memory_type,
const int64_t src_memory_type_id,
const TRITONSERVER_MemoryType dst_memory_type,
const int64_t dst_memory_type_id, const size_t byte_size, const void* src,
void* dst, cudaStream_t cuda_stream, bool* cuda_used, bool copy_on_stream)
{
NVTX_RANGE(nvtx_, "CopyBuffer");
*cuda_used = false;
// For CUDA memcpy, all host to host copy will be blocked in respect to the
// host, so use memcpy() directly. In this case, need to be careful on whether
// the src buffer is valid.
if ((src_memory_type != TRITONSERVER_MEMORY_GPU) &&
(dst_memory_type != TRITONSERVER_MEMORY_GPU)) {
#ifdef TRITON_ENABLE_GPU
if (copy_on_stream) {
auto params = new CopyParams(dst, src, byte_size);
cudaLaunchHostFunc(
cuda_stream, MemcpyHost, reinterpret_cast<void*>(params));
*cuda_used = true;
} else {
memcpy(dst, src, byte_size);
}
#else
memcpy(dst, src, byte_size);
#endif // TRITON_ENABLE_GPU
} else {
#ifdef TRITON_ENABLE_GPU
RETURN_IF_CUDA_ERR(
cudaMemcpyAsync(dst, src, byte_size, cudaMemcpyDefault, cuda_stream),
msg + ": failed to perform CUDA copy");
*cuda_used = true;
#else
return Status(
Status::Code::INTERNAL,
msg + ": try to use CUDA copy while GPU is not supported");
#endif // TRITON_ENABLE_GPU
}
return Status::Success;
}
void
CopyBufferHandler(
const std::string& msg, const TRITONSERVER_MemoryType src_memory_type,
const int64_t src_memory_type_id,
const TRITONSERVER_MemoryType dst_memory_type,
const int64_t dst_memory_type_id, const size_t byte_size, const void* src,
void* dst, cudaStream_t cuda_stream, void* response_ptr,
triton::common::SyncQueue<std::tuple<Status, bool, void*>>*
completion_queue)
{
bool cuda_used = false;
Status status = CopyBuffer(
msg, src_memory_type, src_memory_type_id, dst_memory_type,
dst_memory_type_id, byte_size, src, dst, cuda_stream, &cuda_used);
completion_queue->Put(std::make_tuple(status, cuda_used, response_ptr));
}
#ifdef TRITON_ENABLE_GPU
Status
CheckGPUCompatibility(const int gpu_id, const double min_compute_capability)
{
// Query the compute capability from the device
cudaDeviceProp cuprops;
cudaError_t cuerr = cudaGetDeviceProperties(&cuprops, gpu_id);
if (cuerr != cudaSuccess) {
return Status(
Status::Code::INTERNAL,
"unable to get CUDA device properties for GPU ID" +
std::to_string(gpu_id) + ": " + cudaGetErrorString(cuerr));
}
double compute_compability = cuprops.major + (cuprops.minor / 10.0);
if ((compute_compability > min_compute_capability) ||
(abs(compute_compability - min_compute_capability) < 0.01)) {
return Status::Success;
} else {
return Status(
Status::Code::UNSUPPORTED,
"gpu " + std::to_string(gpu_id) + " has compute capability '" +
std::to_string(cuprops.major) + "." +
std::to_string(cuprops.minor) +
"' which is less than the minimum supported of '" +
std::to_string(min_compute_capability) + "'");
}
}
Status
GetSupportedGPUs(
std::set<int>* supported_gpus, const double min_compute_capability)
{
// Make sure set is empty before starting
supported_gpus->clear();
int device_cnt;
cudaError_t cuerr = cudaGetDeviceCount(&device_cnt);
if ((cuerr == cudaErrorNoDevice) || (cuerr == cudaErrorInsufficientDriver)) {
device_cnt = 0;
} else if (cuerr != cudaSuccess) {
return Status(
Status::Code::INTERNAL, "unable to get number of CUDA devices: " +
std::string(cudaGetErrorString(cuerr)));
}
// populates supported_gpus
for (int gpu_id = 0; gpu_id < device_cnt; gpu_id++) {
Status status = CheckGPUCompatibility(gpu_id, min_compute_capability);
if (status.IsOk()) {
supported_gpus->insert(gpu_id);
}
}
return Status::Success;
}
Status
SupportsIntegratedZeroCopy(const int gpu_id, bool* zero_copy_support)
{
// Query the device to check if integrated
cudaDeviceProp cuprops;
cudaError_t cuerr = cudaGetDeviceProperties(&cuprops, gpu_id);
if (cuerr != cudaSuccess) {
return Status(
Status::Code::INTERNAL,
"unable to get CUDA device properties for GPU ID" +
std::to_string(gpu_id) + ": " + cudaGetErrorString(cuerr));
}
// Zero-copy supported only on integrated GPU when it can map host memory
if (cuprops.integrated && cuprops.canMapHostMemory) {
*zero_copy_support = true;
} else {
*zero_copy_support = false;
}
return Status::Success;
}
#endif
}} // namespace triton::core
3rdparty/core-r22.12/src/cuda_utils.h 0 → 100644
View file @ b30f3cdb
// Copyright 2019-2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
// * Neither the name of NVIDIA CORPORATION nor the names of its
// contributors may be used to endorse or promote products derived
// from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include <set>
#include "status.h"
#include "triton/common/sync_queue.h"
#ifdef TRITON_ENABLE_GPU
#include <cuda_runtime_api.h>
#endif // TRITON_ENABLE_GPU
namespace triton { namespace core {
#ifdef TRITON_ENABLE_GPU
#define RETURN_IF_CUDA_ERR(X, MSG) \
do { \
cudaError_t err__ = (X); \
if (err__ != cudaSuccess) { \
return Status( \
Status::Code::INTERNAL, (MSG) + ": " + cudaGetErrorString(err__)); \
} \
} while (false)
#endif // TRITON_ENABLE_GPU
#ifndef TRITON_ENABLE_GPU
using cudaStream_t = void*;
#endif // !TRITON_ENABLE_GPU
/// Get the memory info for the specified device.
/// \param device_id The device ID.
/// \param free Return free memory in bytes.
/// \param total Return total memory in bytes.
/// \return The error status. A non-OK status means failure to get memory info.
Status GetDeviceMemoryInfo(const int device_id, size_t* free, size_t* total);
/// Enable peer access for all GPU device pairs
/// \param min_compute_capability The minimum support CUDA compute
/// capability.
/// \return The error status. A non-OK status means not all pairs are enabled
Status EnablePeerAccess(const double min_compute_capability);
/// Copy buffer from 'src' to 'dst' for given 'byte_size'. The buffer location
/// is identified by the memory type and id, and the corresponding copy will be
/// initiated.
/// \param msg The message to be prepended in error message.
/// \param src_memory_type The memory type CPU/GPU of the source.
/// \param src_memory_type_id The device id of the source.
/// \param dst_memory_type The memory type CPU/GPU of the destination.
/// \param dst_memory_type_id The device id of the destination.
/// \param byte_size The size in bytes to me copied from source to destination.
/// \param src The buffer start address of the source.
/// \param dst The buffer start address of the destination.
/// \param cuda_stream The stream to be associated with, and 0 can be
/// passed for default stream.
/// \param cuda_used returns whether a CUDA memory copy is initiated. If true,
/// the caller should synchronize on the given 'cuda_stream' to ensure data copy
/// is completed.
/// \param copy_on_stream whether the memory copies should be performed in cuda
/// host functions on the 'cuda_stream'.
/// \return The error status. A non-ok status indicates failure to copy the
/// buffer.
Status CopyBuffer(
const std::string& msg, const TRITONSERVER_MemoryType src_memory_type,
const int64_t src_memory_type_id,
const TRITONSERVER_MemoryType dst_memory_type,
const int64_t dst_memory_type_id, const size_t byte_size, const void* src,
void* dst, cudaStream_t cuda_stream, bool* cuda_used,
bool copy_on_stream = false);
#ifdef TRITON_ENABLE_GPU
/// Validates the compute capability of the GPU indexed
/// \param gpu_id The index of the target GPU.
/// \param min_compute_capability The minimum support CUDA compute
/// capability.
/// \return The error status. A non-OK status means the target GPU is
/// not supported.
Status CheckGPUCompatibility(
const int gpu_id, const double min_compute_capability);
/// Obtains a set of gpu ids that is supported by triton.
/// \param supported_gpus Returns the set of integers which is
/// populated by ids of supported GPUS
/// \param min_compute_capability The minimum support CUDA compute
/// capability.
/// \return The error status. A non-ok status means there were
/// errors encountered while querying GPU devices.
Status GetSupportedGPUs(
std::set<int>* supported_gpus, const double min_compute_capability);
/// Checks if the GPU specified is an integrated GPU and supports Zero-copy.
/// \param gpu_id The index of the target GPU.
/// \param zero_copy_support If true, Zero-copy is supported by this GPU.
/// \return The error status. A non-OK status means the target GPU is
/// not supported.
Status SupportsIntegratedZeroCopy(const int gpu_id, bool* zero_copy_support);
#endif
// Helper around CopyBuffer that updates the completion queue with the returned
// status and cuda_used flag.
void CopyBufferHandler(
const std::string& msg, const TRITONSERVER_MemoryType src_memory_type,
const int64_t src_memory_type_id,
const TRITONSERVER_MemoryType dst_memory_type,
const int64_t dst_memory_type_id, const size_t byte_size, const void* src,
void* dst, cudaStream_t cuda_stream, void* response_ptr,
triton::common::SyncQueue<std::tuple<Status, bool, void*>>*
completion_queue);
struct CopyParams {
CopyParams(void* dst, const void* src, const size_t byte_size)
: dst_(dst), src_(src), byte_size_(byte_size)
{
}
void* dst_;
const void* src_;
const size_t byte_size_;
};
}} // namespace triton::core
3rdparty/core-r22.12/src/dynamic_batch_scheduler.cc 0 → 100644
View file @ b30f3cdb
// Copyright 2018-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
// * Neither the name of NVIDIA CORPORATION nor the names of its
// contributors may be used to endorse or promote products derived
// from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "dynamic_batch_scheduler.h"
#ifndef _WIN32
#include <sys/resource.h>
#include <sys/syscall.h>
#include <unistd.h>
#endif
#include "constants.h"
#include "server.h"
#include "triton/common/logging.h"
#include "triton/common/model_config.h"
#include "triton/common/nvtx.h"
namespace triton { namespace core {
bool
IsStaleState(Payload::State payload_state)
{
return (
(payload_state == Payload::State::EXECUTING) ||
(payload_state == Payload::State::RELEASED));
}
DynamicBatchScheduler::DynamicBatchScheduler(
TritonModel* model, TritonModelInstance* model_instance,
const bool dynamic_batching_enabled, const int32_t max_batch_size,
const std::unordered_map<std::string, bool>& enforce_equal_shape_tensors,
const bool preserve_ordering, const bool response_cache_enable,
const std::set<int32_t>& preferred_batch_sizes,
const uint64_t max_queue_delay_microseconds,
const inference::ModelQueuePolicy& default_queue_policy,
const uint32_t priority_levels, const ModelQueuePolicyMap& queue_policy_map)
: model_(model), model_instance_(model_instance),
model_name_(model->Name()),
dynamic_batching_enabled_(dynamic_batching_enabled),
queue_(default_queue_policy, priority_levels, queue_policy_map),
stop_(false), max_batch_size_((size_t)std::max(1, max_batch_size)),
preferred_batch_sizes_(preferred_batch_sizes),
pending_batch_delay_ns_(max_queue_delay_microseconds * 1000),
pending_batch_size_(0), queued_batch_size_(0),
next_preferred_batch_size_(0),
enforce_equal_shape_tensors_(enforce_equal_shape_tensors),
has_optional_input_(false), preserve_ordering_(preserve_ordering)
{
rate_limiter_ = model_->Server()->GetRateLimiter();
// Both the server and model config should specify
// caching enabled for model to utilize response cache.
response_cache_enabled_ =
(model_->Server()->ResponseCacheEnabled() && response_cache_enable);
#ifdef TRITON_ENABLE_METRICS
// Initialize metric reporter for cache statistics if cache enabled
if (response_cache_enabled_) {
MetricModelReporter::Create(
model_name_, model_->Version(), METRIC_REPORTER_ID_RESPONSE_CACHE,
model_->Config().metric_tags(), &reporter_);
}
#endif // TRITON_ENABLE_METRICS
max_preferred_batch_size_ = 0;
for (const auto size : preferred_batch_sizes_) {
max_preferred_batch_size_ =
std::max(max_preferred_batch_size_, (size_t)size);
}
for (const auto& input : model_->Config().input()) {
if (input.optional()) {
has_optional_input_ = true;
break;
}
}
}
Status
DynamicBatchScheduler::Create(
TritonModel* model, TritonModelInstance* model_instance, const int nice,
const bool dynamic_batching_enabled, const int32_t max_batch_size,
const std::unordered_map<std::string, bool>& enforce_equal_shape_tensors,
const bool preserve_ordering, const bool response_cache_enable,
const std::set<int32_t>& preferred_batch_sizes,
const uint64_t max_queue_delay_microseconds,
std::unique_ptr<Scheduler>* scheduler)
{
inference::ModelDynamicBatching batcher_config;
batcher_config.set_preserve_ordering(preserve_ordering);
for (const auto& bs : preferred_batch_sizes) {
batcher_config.add_preferred_batch_size(bs);
}
batcher_config.set_max_queue_delay_microseconds(max_queue_delay_microseconds);
return Create(
model, model_instance, nice, dynamic_batching_enabled, max_batch_size,
enforce_equal_shape_tensors, batcher_config, response_cache_enable,
scheduler);
}
Status
DynamicBatchScheduler::Create(
TritonModel* model, TritonModelInstance* model_instance, const int nice,
const bool dynamic_batching_enabled, const int32_t max_batch_size,
const std::unordered_map<std::string, bool>& enforce_equal_shape_tensors,
const inference::ModelDynamicBatching& batcher_config,
const bool response_cache_enable, std::unique_ptr<Scheduler>* scheduler)
{
std::set<int32_t> preferred_batch_sizes;
for (const auto size : batcher_config.preferred_batch_size()) {
preferred_batch_sizes.insert(size);
}
DynamicBatchScheduler* dyna_sched = new DynamicBatchScheduler(
model, model_instance, dynamic_batching_enabled, max_batch_size,
enforce_equal_shape_tensors, batcher_config.preserve_ordering(),
response_cache_enable, preferred_batch_sizes,
batcher_config.max_queue_delay_microseconds(),
batcher_config.default_queue_policy(), batcher_config.priority_levels(),
batcher_config.priority_queue_policy());
std::unique_ptr<DynamicBatchScheduler> sched(dyna_sched);
sched->scheduler_thread_exit_.store(false);
if (dynamic_batching_enabled) {
sched->NewPayload();
sched->scheduler_thread_ =
std::thread([dyna_sched, nice]() { dyna_sched->BatcherThread(nice); });
}
scheduler->reset(sched.release());
return Status::Success;
}
DynamicBatchScheduler::~DynamicBatchScheduler()
{
// Signal the scheduler thread to exit and then wait for it..
scheduler_thread_exit_.store(true);
cv_.notify_one();
if (scheduler_thread_.joinable()) {
scheduler_thread_.join();
}
}
Status
DynamicBatchScheduler::Enqueue(std::unique_ptr<InferenceRequest>& request)
{
if (stop_) {
return Status(
Status::Code::UNAVAILABLE,
request->LogRequest() +
"Server is stopping, scheduler for model has stopped accepting new "
"inference requests");
}
// If queue start timestamp hasn't been set, queue timer starts at
// the beginning of the queueing and scheduling process. Otherwise,
// dynamic batcher is used as component of another batcher and should not
// overwrite the queue start timestamp.
if (request->QueueStartNs() == 0) {
request->CaptureQueueStartNs();
INFER_TRACE_ACTIVITY(
request->Trace(), TRITONSERVER_TRACE_QUEUE_START,
request->QueueStartNs());
#ifdef TRITON_ENABLE_TRACING
request->TraceInputTensors(
TRITONSERVER_TRACE_TENSOR_QUEUE_INPUT, "DynamicBatchScheduler Enqueue");
#endif // TRITON_ENABLE_TRACING
}
// Record time at the beginning of the batcher queueing. In the case of
// oldest sequence batcher, this will overwrite the value that was previously
// set by sequence batcher, which is okay as by this point, the previous
// batcher won't be needing this value and it can be safely reused by
// the dynamic batcher.
request->CaptureBatcherStartNs();
std::unique_ptr<InferenceResponse> cached_response;
if (response_cache_enabled_) {
CacheLookUp(request, cached_response);
}
if (cached_response != nullptr) {
// If there was a cache hit then try sending the cached response
// and release the request.
if (preserve_ordering_) {
// In order to preserve the order, the response send must be
// delegated.
DelegateResponse(request);
}
// Send cached response and release request
InferenceResponse::Send(
std::move(cached_response), TRITONSERVER_RESPONSE_COMPLETE_FINAL);
InferenceRequest::Release(
std::move(request), TRITONSERVER_REQUEST_RELEASE_ALL);
return Status::Success;
}
if (!dynamic_batching_enabled_) {
if (preserve_ordering_ || response_cache_enabled_) {
DelegateResponse(request);
}
// If not using dynamic batching, directly enqueue the
// request to model for execution
auto payload = model_->Server()->GetRateLimiter()->GetPayload(
Payload::Operation::INFER_RUN, nullptr /* TritonModelInstance*/);
payload->AddRequest(std::move(request));
RETURN_IF_ERROR(
model_->Server()->GetRateLimiter()->EnqueuePayload(model_, payload));
} else {
bool wake_batcher = true;
{
std::lock_guard<std::mutex> lock(mu_);
queued_batch_size_ += std::max(1U, request->BatchSize());
// Assuming no error is returned, this call takes ownership of
// 'request' and so we can't use it after this point.
RETURN_IF_ERROR(queue_.Enqueue(request->Priority(), request));
// If there are any idle runners and the queued batch size is greater or
// equal to next preferred batch size, then wake batcher up to service
// this request. We do the actual wake outside of the lock to avoid
// having the woken thread immediately block on the lock
wake_batcher =
model_->Server()->GetRateLimiter()->PayloadSlotAvailable(model_);
// We may wake up runner less often if we don't enforce equal shape
// within a batch, otherwise must always wake up runner to check it
if (enforce_equal_shape_tensors_.empty()) {
std::lock_guard<std::mutex> exec_lock(*(curr_payload_->GetExecMutex()));
auto payload_state = curr_payload_->GetState();
wake_batcher &=
(payload_saturated_ || IsStaleState(payload_state) ||
(queued_batch_size_ >= next_preferred_batch_size_));
}
}
if (wake_batcher) {
cv_.notify_one();
}
}
return Status::Success;
}
void
DynamicBatchScheduler::NewPayload()
{
curr_payload_ = model_->Server()->GetRateLimiter()->GetPayload(
Payload::Operation::INFER_RUN, model_instance_);
payload_saturated_ = false;
}
void
DynamicBatchScheduler::BatcherThread(const int nice)
{
#ifndef _WIN32
if (setpriority(PRIO_PROCESS, syscall(SYS_gettid), nice) == 0) {
LOG_VERBOSE(1) << "Starting dynamic-batcher thread for " << model_name_
<< " at nice " << nice << "...";
} else {
LOG_VERBOSE(1) << "Starting dynamic-batcher thread for " << model_name_
<< " at default nice (requested nice " << nice
<< " failed)...";
}
#else
LOG_VERBOSE(1) << "Starting dynamic-batcher thread for " << model_name_
<< " at default nice...";
#endif
// For debugging/testing, delay start of threads until the queue
// contains the specified number of entries.
size_t delay_cnt = 0;
{
const char* dstr = getenv("TRITONSERVER_DELAY_SCHEDULER");
if (dstr != nullptr) {
delay_cnt = atoi(dstr);
LOG_VERBOSE(1) << "Delaying batcher thread for " << model_name_
<< " until " << delay_cnt << " queued requests...";
}
}
auto wait_for_slots = [this]() {
return model_->Server()->GetRateLimiter()->PayloadSlotAvailable(model_);
};
const uint64_t default_wait_microseconds = 500 * 1000;
while (!scheduler_thread_exit_.load()) {
NVTX_RANGE(nvtx_, "DynamicBatcher " + model_name_);
std::shared_ptr<std::vector<std::deque<std::unique_ptr<InferenceRequest>>>>
rejected_requests;
uint64_t wait_microseconds = 0;
// Hold the lock for as short a time as possible.
{
std::unique_lock<std::mutex> lock(mu_);
{
std::lock_guard<std::mutex> exec_lock(*(curr_payload_->GetExecMutex()));
auto payload_state = curr_payload_->GetState();
if (payload_saturated_ || IsStaleState(payload_state)) {
NewPayload();
next_preferred_batch_size_ = 0;
}
}
if (delay_cnt > 0) {
// Debugging/testing... wait until queue contains 'delay_cnt'
// items...
wait_microseconds = 10 * 1000;
if (queue_.Size() >= delay_cnt) {
delay_cnt = 0;
}
LOG_VERBOSE(1) << "Delaying batcher thread " << model_name_ << " until "
<< delay_cnt
<< " queued requests, current total = " << queue_.Size();
} else if (queue_.Empty()) {
wait_microseconds = default_wait_microseconds;
} else {
if (payload_saturated_) {
continue;
}
cv_.wait(lock, wait_for_slots);
{
std::lock_guard<std::mutex> exec_lock(
*(curr_payload_->GetExecMutex()));
auto payload_state = curr_payload_->GetState();
if (IsStaleState(payload_state)) {
continue;
}
// Use dynamic batching to get request(s) to execute.
wait_microseconds = GetDynamicBatch();
// Get requests that are rejected from searching dynamic batch.
queue_.ReleaseRejectedRequests(&rejected_requests);
// Extract batch only if there is pending batch
auto pending_batch_queue_cnt = queue_.PendingBatchCount();
if ((wait_microseconds == 0) && (pending_batch_queue_cnt != 0)) {
curr_payload_->ReserveRequests(pending_batch_queue_cnt);
for (size_t idx = 0; idx < pending_batch_queue_cnt; ++idx) {
std::unique_ptr<InferenceRequest> request;
auto status = queue_.Dequeue(&request);
if (status.IsOk()) {
if (preserve_ordering_ || response_cache_enabled_) {
DelegateResponse(request);
}
curr_payload_->AddRequest(std::move(request));
} else {
// The queue is empty which conflicts with pending batch
// count. Send the current batch if any and reset related
// variables.
LOG_ERROR << request->LogRequest()
<< "Failed to retrieve request from scheduler queue: "
<< status.Message();
queue_.ResetCursor();
queued_batch_size_ = 0;
pending_batch_size_ = 0;
break;
}
}
if (curr_payload_->GetState() == Payload::State::UNINITIALIZED) {
curr_payload_->SetState(Payload::State::READY);
}
queued_batch_size_ -= pending_batch_size_;
pending_batch_size_ = 0;
}
}
}
// If no requests are to be handled, wait for notification or
// for the specified timeout before checking the queue again.
if (wait_microseconds > 0) {
std::chrono::microseconds wait_timeout(wait_microseconds);
cv_.wait_for(lock, wait_timeout);
}
}
if (curr_payload_->GetState() == Payload::State::READY) {
auto callback = [this]() { cv_.notify_one(); };
curr_payload_->SetCallback(callback);
model_->Server()->GetRateLimiter()->EnqueuePayload(model_, curr_payload_);
}
// Finish rejected requests if any
if (rejected_requests != nullptr) {
static Status rejected_status =
Status(Status::Code::UNAVAILABLE, "Request timeout expired");
for (auto& rejected_queue : *rejected_requests) {
for (auto& rejected_request : rejected_queue) {
InferenceRequest::RespondIfError(
rejected_request, rejected_status, true);
}
}
}
} // end runner loop
LOG_VERBOSE(1) << "Stopping dynamic-batcher thread for " << model_name_
<< "...";
}
uint64_t
DynamicBatchScheduler::GetDynamicBatch()
{
// 'mu_' mutex must be held when this function is called. queue_
// must not be empty.
// Examine the new requests. If adding these new requests to the
// pending batch allows a preferred batch size then execute it
// immediately. Stop examining requests if the maximum preferred
// batch size would be exceeded or if the shape of the next request
// does not match the shape of the pending batch.
bool send_now = false;
if (!queue_.IsCursorValid()) {
queue_.ResetCursor();
pending_batch_size_ = 0;
}
size_t best_preferred_batch_size = 0;
queued_batch_size_ -= queue_.ApplyPolicyAtCursor();
// When there is optional input or input shape must be enforced,
// the inputs in the requests must be examined for forming a batch
const bool check_input =
!enforce_equal_shape_tensors_.empty() || has_optional_input_;
auto payload_batch_size = curr_payload_->BatchSize();
while (!queue_.CursorEnd()) {
const auto batch_size = std::max(1U, queue_.RequestAtCursor()->BatchSize());
// If there is no pending batch, then this request is starting a
// new batch.
if ((payload_batch_size + queue_.PendingBatchCount()) == 0) {
// Get the shape of the new batch that is being started...
if (check_input) {
if (!curr_payload_->MutableRequiredEqualInputs()
->Initialize(
queue_.RequestAtCursor(), enforce_equal_shape_tensors_,
has_optional_input_)
.IsOk()) {
send_now = true;
break;
}
}
} else {
// There is a pending batch and adding this request would make
// the batch size larger than all of the preferred batch sizes,
// so mark the cursor at this point. Not sending the pending batch so
// that we can examine the queue delay of requests that fits in a batch.
if (((payload_batch_size + pending_batch_size_ + batch_size) >
max_preferred_batch_size_) &&
(best_preferred_batch_size == 0)) {
best_preferred_batch_size = pending_batch_size_;
queue_.MarkCursor();
payload_saturated_ = true;
}
if ((payload_batch_size + pending_batch_size_ + batch_size) >
max_batch_size_) {
send_now = true;
break;
}
// There is a pending batch and it has a different shape then
// this request, so send the pending batch as it is.
if (check_input &&
!curr_payload_->MutableRequiredEqualInputs()->HasEqualInputs(
queue_.RequestAtCursor())) {
curr_payload_->MarkSaturated();
send_now = true;
break;
}
}
pending_batch_size_ += batch_size;
queue_.AdvanceCursor();
queued_batch_size_ -= queue_.ApplyPolicyAtCursor();
if (preferred_batch_sizes_.find(pending_batch_size_ + payload_batch_size) !=
preferred_batch_sizes_.end()) {
best_preferred_batch_size = pending_batch_size_;
queue_.MarkCursor();
}
}
// Obatin the age of the oldest pending request to compare with the maximum
// batch queuing delay
uint64_t now_ns = std::chrono::duration_cast<std::chrono::nanoseconds>(
std::chrono::steady_clock::now().time_since_epoch())
.count();
uint64_t delay_ns = now_ns - queue_.OldestEnqueueTime();
bool delay_is_exceeded =
(pending_batch_delay_ns_ != 0) && (delay_ns >= pending_batch_delay_ns_);
// If we found a preferred batch size and the queue delay hasn't been
// exceeded, then execute that.
if ((best_preferred_batch_size != 0) && !delay_is_exceeded) {
if (pending_batch_delay_ns_ == 0) {
payload_saturated_ = true;
}
pending_batch_size_ = best_preferred_batch_size;
queue_.SetCursorToMark();
return 0;
}
// No request in pending batch happens when all queued requests have expired
// timeout and the policies are REJECT
if (queue_.PendingBatchCount() == 0) {
return 0;
}
// If the delay has been exceeded, or if the current batch can't grow
// any larger then just immediately execute whatever is pending.
if (send_now || ((payload_batch_size + pending_batch_size_) >=
max_preferred_batch_size_)) {
payload_saturated_ = true;
return 0;
}
if (delay_is_exceeded || (pending_batch_delay_ns_ == 0)) {
return 0;
}
// Set the next preferred batch size given the pending batch size
auto next_preferred_batch_size_it = preferred_batch_sizes_.upper_bound(
pending_batch_size_ + payload_batch_size);
if (next_preferred_batch_size_it != preferred_batch_sizes_.end()) {
next_preferred_batch_size_ = *next_preferred_batch_size_it;
} else {
next_preferred_batch_size_ =
preferred_batch_sizes_.empty() ? 0 : *preferred_batch_sizes_.begin();
}
if (next_preferred_batch_size_ != 0) {
next_preferred_batch_size_ -= payload_batch_size;
}
// By this point, we have not seen the pending batch that should be executed
// immediately. However, if we have scheduled a payload that can be grown and
// not yet in preferred batch size, we should move the pending batch over to
// ensure the model instance will pick up largest available batch even if it
// is not the preferred batch.
if (!payload_saturated_ && (payload_batch_size != 0) &&
(preferred_batch_sizes_.find(payload_batch_size) ==
preferred_batch_sizes_.end())) {
return 0;
}
uint64_t wait_ns = pending_batch_delay_ns_ - delay_ns;
// Note that taking request timeout into consideration allows us to reset
// pending batch as soon as it is invalidated. But the cost is that in edge
// case where the timeout will be expired one by one, the thread will be
// waken frequently.
if (queue_.ClosestTimeout() != 0) {
if (now_ns <= queue_.ClosestTimeout()) {
wait_ns = std::min(queue_.ClosestTimeout() - now_ns, wait_ns);
} else {
// A request in pending batch is timed-out, wait for 1 us to force the
// thread to reset the pending batch right the way.
wait_ns = 1000;
}
}
// Return non-zero wait microseconds to cause this thread to wait
// until the queue delay or the closest timeout has expired.
// Another thread may be awaken due to incoming request to handle the
// pending batch before this thread wakes and that is ok. But if no other
// request comes in then this thread will wake and revisit the pending batch
// (and at that time will then see the delay has been exceeded and will send
// the batch).
return wait_ns / 1000;
}
void
DynamicBatchScheduler::DelegateResponse(
std::unique_ptr<InferenceRequest>& request)
{
std::lock_guard<std::mutex> lock(completion_queue_mtx_);
completion_queue_.emplace_back();
auto queue_slot = &completion_queue_.back();
// Pass raw ptr to lambda for tracking stats from cache and updating
// metric reporter on cache miss stats after insertion
InferenceRequest* raw_request_ptr = request.get();
request->SetResponseDelegator(
[this, queue_slot, raw_request_ptr](
std::unique_ptr<InferenceResponse>&& response, const uint32_t flags) {
if (response_cache_enabled_ && raw_request_ptr->CacheKeyIsSet()) {
// Cache insertion happens here because we need the backend to have
// computed the inference response first in the case of cache miss
auto cache = model_->Server()->GetResponseCache();
auto status = cache->Insert(*response, raw_request_ptr);
bool cache_miss =
(status.StatusCode() != Status::Code::ALREADY_EXISTS);
if (cache_miss) {
#ifdef TRITON_ENABLE_STATS
// Update cache miss statistics even on failure to insert
// as we still spend time on lookup and attempting to insert
raw_request_ptr->ReportStatisticsCacheMiss(reporter_.get());
#endif // TRITON_ENABLE_STATS
if (!status.IsOk()) {
LOG_ERROR << raw_request_ptr->LogRequest()
<< "Failed to insert request_hash ["
<< raw_request_ptr->CacheKey()
<< "] into response cache: " << status.Message();
}
} // Otherwise do nothing; we update cache hit statistics on Lookup
}
if (preserve_ordering_) {
{
std::lock_guard<std::mutex> lock(completion_queue_mtx_);
queue_slot->emplace_back(std::move(response), flags);
}
FinalizeResponses();
} else {
InferenceResponse::Send(std::move(response), flags);
}
});
}
void
DynamicBatchScheduler::CacheLookUp(
std::unique_ptr<InferenceRequest>& request,
std::unique_ptr<InferenceResponse>& cached_response)
{
auto cache = model_->Server()->GetResponseCache();
// Lookup request in cache
std::unique_ptr<InferenceResponse> local_response;
request->ResponseFactory()->CreateResponse(&local_response);
auto status = cache->Lookup(local_response.get(), request.get());
if (status.IsOk() && (local_response != nullptr)) {
cached_response = std::move(local_response);
#ifdef TRITON_ENABLE_STATS
// Update model metrics/stats on cache hits
// Backends will update metrics as normal on cache misses
request->ReportStatisticsCacheHit(reporter_.get());
#endif // TRITON_ENABLE_STATS
}
}
void
DynamicBatchScheduler::FinalizeResponses()
{
// Need exclusive access of the function to ensure responses are sent
// in order
std::lock_guard<std::mutex> lock(finalize_mtx_);
// Finalize the completed payloads in-order as far as possible
std::deque<std::pair<std::unique_ptr<InferenceResponse>, const uint32_t>>
responses;
{
std::lock_guard<std::mutex> queue_lock(completion_queue_mtx_);
while (!completion_queue_.empty() && !completion_queue_.front().empty()) {
bool response_complete = false;
for (auto& response_pair : completion_queue_.front()) {
// Assuming FINAL flag is set only in the last response of the request
response_complete =
((response_pair.second & TRITONSERVER_RESPONSE_COMPLETE_FINAL) !=
0);
responses.emplace_back(std::move(response_pair));
}
if (response_complete) {
completion_queue_.pop_front();
} else {
completion_queue_.front().clear();
}
}
}
for (auto& response : responses) {
InferenceResponse::Send(std::move(response.first), response.second);
}
}
}} // namespace triton::core
3rdparty/core-r22.12/src/dynamic_batch_scheduler.h 0 → 100644
View file @ b30f3cdb
// Copyright 2018-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
// * Neither the name of NVIDIA CORPORATION nor the names of its
// contributors may be used to endorse or promote products derived
// from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include <atomic>
#include <condition_variable>
#include <deque>
#include <future>
#include <map>
#include <mutex>
#include <queue>
#include <set>
#include <thread>
#include "backend_model.h"
#include "backend_model_instance.h"
#include "model_config.pb.h"
#include "rate_limiter.h"
#include "scheduler.h"
#include "scheduler_utils.h"
#include "status.h"
#include "triton/common/model_config.h"
namespace triton { namespace core {
// Scheduler that implements dynamic batching.
class DynamicBatchScheduler : public Scheduler {
public:
// Create a scheduler to support a given number of runners and a run
// function to call when a request is scheduled.
static Status Create(
TritonModel* model, TritonModelInstance* model_instance, const int nice,
const bool dynamic_batching_enabled, const int32_t max_batch_size,
const std::unordered_map<std::string, bool>& enforce_equal_shape_tensors,
const bool preserve_ordering, const bool response_cache_enable,
const std::set<int32_t>& preferred_batch_sizes,
const uint64_t max_queue_delay_microseconds,
std::unique_ptr<Scheduler>* scheduler);
// Create a scheduler to support a given number of runners and a run
// function to call when a request is scheduled. And the scheduler also
// supports different queue policies for different priority levels.
static Status Create(
TritonModel* model, TritonModelInstance* model_instance, const int nice,
const bool dynamic_batching_enabled, const int32_t max_batch_size,
const std::unordered_map<std::string, bool>& enforce_equal_shape_tensors,
const inference::ModelDynamicBatching& batcher_config,
const bool response_cache_enable, std::unique_ptr<Scheduler>* scheduler);
~DynamicBatchScheduler();
// \see Scheduler::Enqueue()
Status Enqueue(std::unique_ptr<InferenceRequest>& request) override;
// \see Scheduler::InflightInferenceCount()
size_t InflightInferenceCount() override
{
std::unique_lock<std::mutex> lock(mu_);
if (curr_payload_ != nullptr) {
return queue_.Size() + curr_payload_->RequestCount();
}
return queue_.Size();
}
// \see Scheduler::Stop()
void Stop() override { stop_ = true; }
MetricModelReporter* MetricReporter() const { return reporter_.get(); }
private:
DynamicBatchScheduler(
TritonModel* model, TritonModelInstance* model_instance,
const bool dynamic_batching_enabled, const int32_t max_batch_size,
const std::unordered_map<std::string, bool>& enforce_equal_shape_tensors,
const bool preserve_ordering, const bool response_cache_enable,
const std::set<int32_t>& preferred_batch_sizes,
const uint64_t max_queue_delay_microseconds,
const inference::ModelQueuePolicy& default_queue_policy,
const uint32_t priority_levels,
const ModelQueuePolicyMap& queue_policy_map);
void BatcherThread(const int nice);
void NewPayload();
uint64_t GetDynamicBatch();
void DelegateResponse(std::unique_ptr<InferenceRequest>& request);
void CacheLookUp(
std::unique_ptr<InferenceRequest>& request,
std::unique_ptr<InferenceResponse>& cached_response);
void FinalizeResponses();
TritonModel* model_;
TritonModelInstance* model_instance_;
// Name of the model.
std::string model_name_;
// True if dynamic batching is enabled.
const bool dynamic_batching_enabled_;
// Map from priority level to queue holding inference requests for the model
// represented by this scheduler. If priority queues are not supported by the
// scheduler, then priority zero entry is used as the single queue.
PriorityQueue queue_;
bool stop_;
std::thread scheduler_thread_;
std::atomic<bool> scheduler_thread_exit_;
// Mutex and condvar for signaling scheduler thread
std::mutex mu_;
std::condition_variable cv_;
std::shared_ptr<RateLimiter> rate_limiter_;
std::shared_ptr<Payload> curr_payload_;
bool payload_saturated_;
size_t max_batch_size_;
size_t max_preferred_batch_size_;
std::set<int32_t> preferred_batch_sizes_;
uint64_t pending_batch_delay_ns_;
size_t pending_batch_size_;
size_t queued_batch_size_;
size_t next_preferred_batch_size_;
// The input tensors that require shape checking before being
// allowed in a batch. As a map from the tensor name to a bool. If
// tensor is in map then its shape must match shape of same tensor
// in requests already in the batch. If value is "true" then
// additional tensor is treated as a shape tensor and the values
// contained in the shape tensor must match same tensor already in
// the batch.
const std::unordered_map<std::string, bool> enforce_equal_shape_tensors_;
// Store information on whether the model contains optional inputs.
bool has_optional_input_;
// If true the ordering of responses matches the order of requests
// even when there are multiple scheduler threads.
const bool preserve_ordering_;
// If true, the scheduler will try to retrieve responses from cache.
bool response_cache_enabled_;
// Per completion-id queues to store the ready responses
std::deque<
std::vector<std::pair<std::unique_ptr<InferenceResponse>, uint32_t>>>
completion_queue_;
// Lock to protect the completion_queues_
std::mutex completion_queue_mtx_;
// Preserves the order in which responses are finalized
std::mutex finalize_mtx_;
// Reporter for metrics, or nullptr if no metrics should be reported
std::shared_ptr<MetricModelReporter> reporter_;
};
}} // namespace triton::core
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment