添加下载的代码

3rdparty/common-r22.12/include/triton/common/table_printer.h

+// Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+//  * Redistributions of source code must retain the above copyright
+//    notice, this list of conditions and the following disclaimer.
+//  * Redistributions in binary form must reproduce the above copyright
+//    notice, this list of conditions and the following disclaimer in the
+//    documentation and/or other materials provided with the distribution.
+//  * Neither the name of NVIDIA CORPORATION nor the names of its
+//    contributors may be used to endorse or promote products derived
+//    from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#pragma once
+
+#include <memory>
+#include <sstream>
+#include <string>
+#include <vector>
+
+namespace triton { namespace common {
+
+//
+// An ASCII table printer.
+//
+class TablePrinter {
+ public:
+  // Insert a row at the end of the table
+  void InsertRow(const std::vector<std::string>& row);
+
+  // Print the table
+  std::string PrintTable();
+
+  // TablePrinter will take the ownership of `headers`.
+  TablePrinter(const std::vector<std::string>& headers);
+
+ private:
+  // Update the `shares_` such that all the excess
+  // amount of space not used a column is fairly allocated
+  // to the other columns
+  void FairShare();
+
+  // Append a row to `table`. This function handles the cases where a wrapping
+  // occurs.
+  void AddRow(std::stringstream& table, size_t row_index);
+
+  // Add a row divider
+  void AddRowDivider(std::stringstream& table);
+
+  // Max row width
+  std::vector<size_t> max_widths_;
+
+  // Max row height
+  std::vector<size_t> max_heights_;
+
+  // A vector of vectors of vectors containing data items for every column
+  // The record is stored in a vector of string, where each of the vector items
+  // contains a single line from the record. For example, ["Item 1", "Item 2",
+  // "Item 3\n Item 3 line 2"] will be stored as [["Item 1"], ["Item 2"], ["Item
+  // 3", "Item 3 line 2"]]
+  std::vector<std::vector<std::vector<std::string>>> data_;
+
+  // Fair share of every column
+  std::vector<float> shares_;
+};
+
+}}  // namespace triton::common

3rdparty/common-r22.12/include/triton/common/thread_pool.h

+// Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+//  * Redistributions of source code must retain the above copyright
+//    notice, this list of conditions and the following disclaimer.
+//  * Redistributions in binary form must reproduce the above copyright
+//    notice, this list of conditions and the following disclaimer in the
+//    documentation and/or other materials provided with the distribution.
+//  * Neither the name of NVIDIA CORPORATION nor the names of its
+//    contributors may be used to endorse or promote products derived
+//    from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#pragma once
+
+#include <condition_variable>
+#include <functional>
+#include <queue>
+#include <thread>
+
+namespace triton { namespace common {
+
+// Generic fixed-size Thread Pool to execute tasks asynchronously
+
+class ThreadPool {
+ public:
+  explicit ThreadPool(std::size_t thread_count);
+  ~ThreadPool();
+  ThreadPool(const ThreadPool&) = delete;
+  ThreadPool& operator=(const ThreadPool&) = delete;
+
+  using Task = std::function<void(void)>;
+  // Assigns "task" to the task queue for a worker thread to execute when
+  // available. This will not track the return value of the task.
+  void Enqueue(Task&& task);
+  // Returns the number of threads in thread pool
+  size_t Size() { return workers_.size(); }
+
+ private:
+  std::queue<Task> task_queue_;
+  std::mutex queue_mtx_;
+  std::condition_variable cv_;
+  std::vector<std::thread> workers_;
+  // If true, tells pool to stop accepting work and tells awake worker threads
+  // to exit when no tasks are left on the queue.
+  bool stop_ = false;
+};
+
+}}  // namespace triton::common

3rdparty/common-r22.12/include/triton/common/triton_json.h

+// Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+//  * Redistributions of source code must retain the above copyright
+//    notice, this list of conditions and the following disclaimer.
+//  * Redistributions in binary form must reproduce the above copyright
+//    notice, this list of conditions and the following disclaimer in the
+//    documentation and/or other materials provided with the distribution.
+//  * Neither the name of NVIDIA CORPORATION nor the names of its
+//    contributors may be used to endorse or promote products derived
+//    from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#pragma once
+
+#ifdef _WIN32
+// Remove GetObject definition from windows.h, which prevents calls to
+// RapidJSON's GetObject.
+// https://github.com/Tencent/rapidjson/issues/1448
+#undef GetObject
+#include <rapidjson/document.h>
+#else
+// Disable class-memaccess warning to facilitate compilation with gcc>7
+// https://github.com/Tencent/rapidjson/issues/1700
+#pragma GCC diagnostic push
+#if defined(__GNUC__) && __GNUC__ >= 8
+#pragma GCC diagnostic ignored "-Wclass-memaccess"
+#endif
+#include <rapidjson/document.h>
+#pragma GCC diagnostic pop
+#endif  // _WIN32
+
+#include <rapidjson/allocators.h>  // CrtAllocator (default) for Writer instantiation
+#include <rapidjson/encodings.h>  // UTF8 (default) for Writer instantiation
+#include <rapidjson/error/en.h>
+#include <rapidjson/prettywriter.h>
+#include <rapidjson/rapidjson.h>
+#include <rapidjson/stringbuffer.h>
+#include <rapidjson/writer.h>
+#include <string>
+#include <vector>
+
+// This header can be used both within Triton server and externally
+// (i.e. in source that interacts only via TRITONSERVER or
+// TRITONBACKEND API). Status is handled differently in these cases so
+// the following macros must be defined before including this
+// header. As an example the defines are shown here as returned by the
+// TRITONSERVER API.
+//
+//   #define TRITONJSON_STATUSTYPE TRITONSERVER_Error*
+//   #define TRITONJSON_STATUSRETURN(M)
+//        return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, (M).c_str())
+//   #define TRITONJSON_STATUSSUCCESS nullptr
+
+namespace triton { namespace common {
+
+//
+// A JSON parser/writer. Currently based on rapidjson but the intent
+// is to provide an abstraction for JSON functions that make it easy
+// to substitute a different JSON parser. Specifically for rapidjson
+// the class is also designed to provide safe access and error
+// reporting to avoid the cases where rapidjson would just abort the
+// entire application (!).
+//
+class TritonJson {
+ public:
+  class Value;
+  enum class ValueType {
+    OBJECT = rapidjson::kObjectType,
+    ARRAY = rapidjson::kArrayType,
+  };
+
+  //
+  // Buffer used when writing JSON representation.
+  //
+  class WriteBuffer {
+   public:
+    // Get buffer base address.
+    const char* Base() const { return buffer_.c_str(); }
+
+    // Get a reference to the buffer itself. Useful to efficiently
+    // move the contents out of the buffer.
+    std::string& MutableContents() { return buffer_; }
+
+    // Immutable contents.
+    const std::string& Contents() const { return buffer_; }
+
+    // Interface required by rapidjson::Writer
+    typedef char Ch;
+    void Put(char c) { buffer_.push_back(c); }
+    void Clear() { buffer_.clear(); }
+    void Flush() { return; }
+    size_t Size() const { return buffer_.size(); }
+
+   private:
+    std::string buffer_;
+  };
+
+  //
+  // Value representing the entire document or an element within a
+  // document.
+  //
+  class Value {
+   public:
+    // Empty value. Will become a top-level Document value if
+    // initialized by parsing or a non-top-level value if initialized
+    // any other way.
+    explicit Value() : value_(nullptr), allocator_(nullptr) {}
+
+    // Construct a top-level JSON document.
+    explicit Value(const ValueType type)
+        : document_(static_cast<rapidjson::Type>(type)), value_(nullptr),
+          allocator_(&document_.GetAllocator())
+    {
+    }
+
+    // Construct a non-top-level JSON value in a 'document'.
+    explicit Value(TritonJson::Value& document, const ValueType type)
+    {
+      allocator_ = &document.document_.GetAllocator();
+      value_ = new (allocator_->Malloc(sizeof(rapidjson::Value)))
+          rapidjson::Value(static_cast<rapidjson::Type>(type));
+    }
+
+    // Move constructor.
+    explicit Value(Value&& other) { *this = std::move(other); }
+
+    // Move assignment operator.
+    Value& operator=(Value&& other)
+    {
+      document_ = std::move(other.document_);
+      value_ = other.value_;
+      allocator_ = other.allocator_;
+      other.value_ = nullptr;
+      other.allocator_ = nullptr;
+      return *this;
+    }
+
+    // Parse JSON into document. Can only be called on top-level
+    // document value, otherwise error is returned.
+    TRITONJSON_STATUSTYPE Parse(const char* base, const size_t size)
+    {
+      if (value_ != nullptr) {
+        TRITONJSON_STATUSRETURN(
+            std::string("JSON parsing only available for top-level document"));
+      }
+      const unsigned int parseFlags = rapidjson::kParseNanAndInfFlag;
+      document_.Parse<parseFlags>(base, size);
+      if (document_.HasParseError()) {
+        TRITONJSON_STATUSRETURN(std::string(
+            "failed to parse the request JSON buffer: " +
+            std::string(GetParseError_En(document_.GetParseError())) + " at " +
+            std::to_string(document_.GetErrorOffset())));
+      }
+      allocator_ = &document_.GetAllocator();
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // \see Parse(const char* base, const size_t size)
+    TRITONJSON_STATUSTYPE Parse(const std::string& json)
+    {
+      return Parse(json.data(), json.size());
+    }
+
+    // Write JSON representation into a 'buffer' in a compact
+    // format. Can only be called for a top-level document value,
+    // otherwise error is returned.
+    TRITONJSON_STATUSTYPE Write(WriteBuffer* buffer) const
+    {
+      if (value_ != nullptr) {
+        TRITONJSON_STATUSRETURN(
+            std::string("JSON writing only available for top-level document"));
+      }
+      const unsigned int writeFlags = rapidjson::kWriteNanAndInfFlag;
+      // Provide default template arguments to pass writeFlags
+      rapidjson::Writer<
+          WriteBuffer, rapidjson::UTF8<>, rapidjson::UTF8<>,
+          rapidjson::CrtAllocator, writeFlags>
+          writer(*buffer);
+      if (!document_.Accept(writer)) {
+        TRITONJSON_STATUSRETURN(
+            std::string("Failed to accept document, invalid JSON."));
+      }
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Write JSON representation into a 'buffer' in an easy-to-read
+    // format. Can only be called for a top-level document value,
+    // otherwise error is returned.
+    TRITONJSON_STATUSTYPE PrettyWrite(WriteBuffer* buffer) const
+    {
+      if (value_ != nullptr) {
+        TRITONJSON_STATUSRETURN(
+            std::string("JSON writing only available for top-level document"));
+      }
+
+      // Can't pass writeFlags with latest release v1.1.0 of rapidjson-dev.
+      // We would need to build rapidjson from source to capture latest fixes.
+      // See this issue:
+      // https://github.com/Tencent/rapidjson/issues/905#issuecomment-370981353
+      // PrettyWrite is only used for displaying model configs currently, so
+      // this should not be an issue.
+      rapidjson::PrettyWriter<WriteBuffer> writer(*buffer);
+      if (!document_.Accept(writer)) {
+        TRITONJSON_STATUSRETURN(
+            std::string("Failed to accept document, invalid JSON."));
+      }
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Swap a value with another.
+    TRITONJSON_STATUSTYPE Swap(TritonJson::Value& other)
+    {
+      rapidjson::Value& value = AsMutableValue();
+      value.Swap(other.AsMutableValue());
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // FIXME Should have Set* for all types.
+
+    // Set/overwrite a signed integer in a value. This changes the
+    // type of the value to signed int.
+    TRITONJSON_STATUSTYPE SetInt(const int64_t value)
+    {
+      rapidjson::Value& v = AsMutableValue();
+      v.SetInt64(value);
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Set/overwrite a string in a value. This changes the
+    // type of the value to string
+    TRITONJSON_STATUSTYPE SetString(const std::string& value)
+    {
+      rapidjson::Value& v = AsMutableValue();
+      v.SetString(value.c_str(), value.length(), *allocator_);
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Set/overwrite a string member with provided name and value in this object
+    TRITONJSON_STATUSTYPE SetStringObject(
+        const char* name, const std::string& value)
+    {
+      rapidjson::Value& object = AsMutableValue();
+      if (!object.IsObject()) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to add/replace JSON member '") + name +
+            "' to non-object");
+      }
+      auto itr = object.FindMember(name);
+      if (itr == object.MemberEnd()) {
+        AddString(name, value);
+      } else {
+        object.RemoveMember(itr);
+        object.AddMember(
+            rapidjson::Value(rapidjson::StringRef(name)).Move(),
+            rapidjson::Value(value.c_str(), value.size(), *allocator_),
+            *allocator_);
+      }
+
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Add an array or object as a new member to this value. 'value'
+    // is moved into this value and so on return 'value' should not be
+    // used. It is assumed that 'name' can be used by reference, it is
+    // the caller's responsibility to make sure the lifetime of 'name'
+    // extends at least as long as the object.
+    TRITONJSON_STATUSTYPE Add(const char* name, TritonJson::Value&& value)
+    {
+      rapidjson::Value& object = AsMutableValue();
+      if (!object.IsObject()) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to add JSON member '") + name +
+            "' to non-object");
+      }
+      if (value.value_ == nullptr) {
+        rapidjson::Value v2;
+        v2.CopyFrom(value.document_, *allocator_);
+        object.AddMember(
+            rapidjson::Value(rapidjson::StringRef(name)).Move(), v2.Move(),
+            *allocator_);
+      } else {
+        object.AddMember(
+            rapidjson::Value(rapidjson::StringRef(name)).Move(),
+            value.value_->Move(), *allocator_);
+      }
+      value.Release();
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Add a copy of a string as a new member to this value. It is
+    // assumed that 'name' can be used by reference, it is the
+    // caller's responsibility to make sure the lifetime of 'name'
+    // extends at least as long as the object.
+    TRITONJSON_STATUSTYPE AddString(const char* name, const std::string& value)
+    {
+      rapidjson::Value& object = AsMutableValue();
+      if (!object.IsObject()) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to add JSON member '") + name +
+            "' to non-object");
+      }
+      object.AddMember(
+          rapidjson::Value(rapidjson::StringRef(name)).Move(),
+          rapidjson::Value(value.c_str(), value.size(), *allocator_).Move(),
+          *allocator_);
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Add a copy of a explicit-length string as a new member to this
+    // value. It is assumed that 'name' can be used by reference, it
+    // is the caller's responsibility to make sure the lifetime of
+    // 'name' extends at least as long as the object.
+    TRITONJSON_STATUSTYPE AddString(
+        const char* name, const char* value, const size_t len)
+    {
+      rapidjson::Value& object = AsMutableValue();
+      if (!object.IsObject()) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to add JSON member '") + name +
+            "' to non-object");
+      }
+      object.AddMember(
+          rapidjson::Value(rapidjson::StringRef(name)).Move(),
+          rapidjson::Value(value, len, *allocator_).Move(), *allocator_);
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Add a reference to a string as a new member to this value. It
+    // is assumed that 'name' and 'value' can be used by reference, it
+    // is the caller's responsibility to make sure the lifetime of
+    // 'name' and 'value' extend at least as long as the object.
+    TRITONJSON_STATUSTYPE AddStringRef(const char* name, const char* value)
+    {
+      rapidjson::Value& object = AsMutableValue();
+      if (!object.IsObject()) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to add JSON member '") + name +
+            "' to non-object");
+      }
+      object.AddMember(
+          rapidjson::Value(rapidjson::StringRef(name)).Move(),
+          rapidjson::StringRef(value), *allocator_);
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Add a reference to a expicit-length string as a new member to
+    // this value. It is assumed that 'name' and 'value' can be used
+    // by reference, it is the caller's responsibility to make sure
+    // the lifetime of 'name' and 'value' extend at least as long as
+    // the object.
+    TRITONJSON_STATUSTYPE AddStringRef(
+        const char* name, const char* value, const size_t len)
+    {
+      rapidjson::Value& object = AsMutableValue();
+      if (!object.IsObject()) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to add JSON member '") + name +
+            "' to non-object");
+      }
+      object.AddMember(
+          rapidjson::Value(rapidjson::StringRef(name)).Move(),
+          rapidjson::StringRef(value, len), *allocator_);
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Add a boolean new member to this value. It is assumed that
+    // 'name' can be used by reference, it is the caller's
+    // responsibility to make sure the lifetime of 'name' extends at
+    // least as long as the object.
+    TRITONJSON_STATUSTYPE AddBool(const char* name, const bool value)
+    {
+      rapidjson::Value& object = AsMutableValue();
+      if (!object.IsObject()) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to add JSON member '") + name +
+            "' to non-object");
+      }
+      object.AddMember(
+          rapidjson::Value(rapidjson::StringRef(name)).Move(),
+          rapidjson::Value(value).Move(), *allocator_);
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Add a signed integer as a new member to this value. It is
+    // assumed that 'name' can be used by reference, it is the
+    // caller's responsibility to make sure the lifetime of 'name'
+    // extends at least as long as the object.
+    TRITONJSON_STATUSTYPE AddInt(const char* name, const int64_t value)
+    {
+      rapidjson::Value& object = AsMutableValue();
+      if (!object.IsObject()) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to add JSON member '") + name +
+            "' to non-object");
+      }
+      object.AddMember(
+          rapidjson::Value(rapidjson::StringRef(name)).Move(),
+          rapidjson::Value(value).Move(), *allocator_);
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Add an unsigned integer as a new member to this value. It is
+    // assumed that 'name' can be used by reference, it is the
+    // caller's responsibility to make sure the lifetime of 'name'
+    // extends at least as long as the object.
+    TRITONJSON_STATUSTYPE AddUInt(const char* name, const uint64_t value)
+    {
+      rapidjson::Value& object = AsMutableValue();
+      if (!object.IsObject()) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to add JSON member '") + name +
+            "' to non-object");
+      }
+      object.AddMember(
+          rapidjson::Value(rapidjson::StringRef(name)).Move(),
+          rapidjson::Value(value).Move(), *allocator_);
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Add a double as a new member to this value. It is assumed that
+    // 'name' can be used by reference, it is the caller's
+    // responsibility to make sure the lifetime of 'name' extends at
+    // least as long as the object.
+    TRITONJSON_STATUSTYPE AddDouble(const char* name, const double value)
+    {
+      rapidjson::Value& object = AsMutableValue();
+      if (!object.IsObject()) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to add JSON member '") + name +
+            "' to non-object");
+      }
+      object.AddMember(
+          rapidjson::Value(rapidjson::StringRef(name)).Move(),
+          rapidjson::Value(value).Move(), *allocator_);
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Append an array or object to this value, which must be an
+    // array. 'value' is moved into this value and so on return
+    // 'value' should not be used.
+    TRITONJSON_STATUSTYPE Append(TritonJson::Value&& value)
+    {
+      rapidjson::Value& array = AsMutableValue();
+      if (!array.IsArray()) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to append JSON member to non-array"));
+      }
+      if (value.value_ == nullptr) {
+        rapidjson::Value v2;
+        v2.CopyFrom(value.document_, *allocator_);
+        array.PushBack(v2.Move(), *allocator_);
+      } else {
+        array.PushBack(value.value_->Move(), *allocator_);
+      }
+
+      value.Release();
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Append a copy of a string to this value, which must be an
+    // array.
+    TRITONJSON_STATUSTYPE AppendString(const std::string& value)
+    {
+      rapidjson::Value& array = AsMutableValue();
+      if (!array.IsArray()) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to append JSON member to non-array"));
+      }
+      array.PushBack(
+          rapidjson::Value(value.c_str(), value.size(), *allocator_).Move(),
+          *allocator_);
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Append a copy of an explicit-length string to this value, which
+    // must be an array.
+    TRITONJSON_STATUSTYPE AppendString(const char* value, const size_t len)
+    {
+      rapidjson::Value& array = AsMutableValue();
+      if (!array.IsArray()) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to append JSON member to non-array"));
+      }
+      array.PushBack(
+          rapidjson::Value(value, len, *allocator_).Move(), *allocator_);
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Append a reference to a string to this value, which must be an
+    // array. It is assumed that 'value' can be used by reference, it
+    // is the caller's responsibility to make sure the lifetime of
+    // 'value' extends at least as long as the object.
+    TRITONJSON_STATUSTYPE AppendStringRef(const char* value)
+    {
+      rapidjson::Value& array = AsMutableValue();
+      if (!array.IsArray()) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to append JSON member to non-array"));
+      }
+      array.PushBack(rapidjson::StringRef(value), *allocator_);
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Append a reference to a expicit-length string to this value,
+    // which must be an array. It is assumed that 'value' can be used
+    // by reference, it is the caller's responsibility to make sure
+    // the lifetime of 'value' extends at least as long as the object.
+    TRITONJSON_STATUSTYPE AppendStringRef(const char* value, const size_t len)
+    {
+      rapidjson::Value& array = AsMutableValue();
+      if (!array.IsArray()) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to append JSON member to non-array"));
+      }
+      array.PushBack(rapidjson::StringRef(value, len), *allocator_);
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Append a boolean to this value, which must be an array.
+    TRITONJSON_STATUSTYPE AppendBool(const bool value)
+    {
+      rapidjson::Value& array = AsMutableValue();
+      if (!array.IsArray()) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to append JSON member to non-array"));
+      }
+
+      array.PushBack(rapidjson::Value(value).Move(), *allocator_);
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Append a signed integer to this value, which must be an array.
+    TRITONJSON_STATUSTYPE AppendInt(const int64_t value)
+    {
+      rapidjson::Value& array = AsMutableValue();
+      if (!array.IsArray()) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to append JSON member to non-array"));
+      }
+
+      array.PushBack(rapidjson::Value(value).Move(), *allocator_);
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Append an unsigned integer to this value, which must be an
+    // array.
+    TRITONJSON_STATUSTYPE AppendUInt(const uint64_t value)
+    {
+      rapidjson::Value& array = AsMutableValue();
+      if (!array.IsArray()) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to append JSON member to non-array"));
+      }
+
+      array.PushBack(rapidjson::Value(value).Move(), *allocator_);
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Append a double to this value, which must be an array.
+    TRITONJSON_STATUSTYPE AppendDouble(const double value)
+    {
+      rapidjson::Value& array = AsMutableValue();
+      if (!array.IsArray()) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to append JSON member to non-array"));
+      }
+
+      array.PushBack(rapidjson::Value(value).Move(), *allocator_);
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Remove member from this object
+    TRITONJSON_STATUSTYPE Remove(const char* name)
+    {
+      rapidjson::Value& object = AsMutableValue();
+      if (!object.IsObject()) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to remove JSON member '") + name +
+            "' to non-object");
+      }
+      auto itr = object.FindMember(name);
+      if (itr != object.MemberEnd()) {
+        object.RemoveMember(itr);
+      }  // else report success
+
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Check if this value is of the specified type. Return appropriate
+    // error if not.
+    TRITONJSON_STATUSTYPE AssertType(TritonJson::ValueType type) const
+    {
+      if (static_cast<rapidjson::Type>(type) != AsValue().GetType()) {
+        TRITONJSON_STATUSRETURN(std::string("unexpected type"));
+      }
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Get the size of an array. If called on non-array returns zero.
+    size_t ArraySize() const
+    {
+      const rapidjson::Value& array = AsValue();
+      if (!array.IsArray()) {
+        return 0;
+      }
+      return array.GetArray().Size();
+    }
+
+    // Return the specified index contained in this array.
+    TRITONJSON_STATUSTYPE At(
+        const size_t idx, TritonJson::Value* value = nullptr)
+    {
+      rapidjson::Value& array = AsMutableValue();
+      if (!array.IsArray() || (idx >= array.GetArray().Size())) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to access non-existing array index '") +
+            std::to_string(idx) + "'");
+      }
+      *value = TritonJson::Value(array[idx], allocator_);
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Get the names of all members in an object.  Error if value is
+    // not an object.
+    TRITONJSON_STATUSTYPE Members(std::vector<std::string>* names) const
+    {
+      const rapidjson::Value& object = AsValue();
+      if (!object.IsObject()) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to get members for non-object"));
+      }
+      for (const auto& m : object.GetObject()) {
+        names->push_back(m.name.GetString());
+      }
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Return true if this value is an object and the named member is
+    // contained in this object.
+    bool Find(const char* name) const
+    {
+      const rapidjson::Value& object = AsValue();
+      return object.IsObject() && object.HasMember(name);
+    }
+
+    // Return true if this value is an object and the named member is
+    // contained in this object. Return the member in 'value'.
+    bool Find(const char* name, TritonJson::Value* value)
+    {
+      rapidjson::Value& object = AsMutableValue();
+      if (object.IsObject() && object.HasMember(name)) {
+        if (value != nullptr) {
+          *value = TritonJson::Value(object[name], allocator_);
+        }
+        return true;
+      }
+
+      return false;
+    }
+
+    // Whether the object is null value. Note that false will also be retuned
+    // if the object is not a JSON value.
+    bool IsNull() const { return ((value_ != nullptr) && value_->IsNull()); }
+
+    // Return true if the object is an object and it has no members;
+    // false otherwise.
+    bool IsEmpty() const
+    {
+      const rapidjson::Value& object = AsValue();
+      if (object.IsObject() && object.MemberCount() == 0) {
+        return true;
+      }
+      return false;
+    }
+
+    // Get value as a string. The string may contain null or other
+    // special characters and so 'len' must be used to determine length.
+    // Error if value is not a string.
+    TRITONJSON_STATUSTYPE AsString(const char** value, size_t* len) const
+    {
+      if ((value_ == nullptr) || !value_->IsString()) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to access JSON non-string as string"));
+      }
+      *value = value_->GetString();
+      *len = value_->GetStringLength();
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Get value as a string. The string may contain null or other
+    // special characters.  Error if value is not a string.
+    TRITONJSON_STATUSTYPE AsString(std::string* str) const
+    {
+      if ((value_ == nullptr) || !value_->IsString()) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to access JSON non-string as string"));
+      }
+      str->assign(value_->GetString(), value_->GetStringLength());
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Get value as a boolean. Error if value is not a boolean.
+    TRITONJSON_STATUSTYPE AsBool(bool* value) const
+    {
+      if ((value_ == nullptr) || !value_->IsBool()) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to access JSON non-boolean as boolean"));
+      }
+      *value = value_->GetBool();
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Get value as a signed integer. Error if value is not a signed
+    // integer.
+    TRITONJSON_STATUSTYPE AsInt(int64_t* value) const
+    {
+      if ((value_ == nullptr) || !value_->IsInt64()) {
+        TRITONJSON_STATUSRETURN(std::string(
+            "attempt to access JSON non-signed-integer as signed-integer"));
+      }
+      *value = value_->GetInt64();
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Get value as an unsigned integer. Error if value is not an
+    // unsigned integer.
+    TRITONJSON_STATUSTYPE AsUInt(uint64_t* value) const
+    {
+      if ((value_ == nullptr) || !value_->IsUint64()) {
+        TRITONJSON_STATUSRETURN(std::string(
+            "attempt to access JSON non-unsigned-integer as unsigned-integer"));
+      }
+      *value = value_->GetUint64();
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Get value as a double. Error if value is not a double.
+    TRITONJSON_STATUSTYPE AsDouble(double* value) const
+    {
+      if ((value_ == nullptr) || !value_->IsNumber()) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to access JSON non-number as double"));
+      }
+      *value = value_->GetDouble();
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Get named array member contained in this object.
+    TRITONJSON_STATUSTYPE MemberAsArray(
+        const char* name, TritonJson::Value* value)
+    {
+      rapidjson::Value& object = AsMutableValue();
+      if (!object.IsObject() || !object.HasMember(name)) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to access non-existing object member '") +
+            name + "'");
+      }
+      auto& v = object[name];
+      if (!v.IsArray()) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to access JSON non-array as array"));
+      }
+      *value = TritonJson::Value(v, allocator_);
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Get named object member contained in this object.
+    TRITONJSON_STATUSTYPE MemberAsObject(
+        const char* name, TritonJson::Value* value)
+    {
+      rapidjson::Value& object = AsMutableValue();
+      if (!object.IsObject() || !object.HasMember(name)) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to access non-existing object member '") +
+            name + "'");
+      }
+      auto& v = object[name];
+      if (!v.IsObject()) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to access JSON non-object as object"));
+      }
+      *value = TritonJson::Value(v, allocator_);
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Get object member as a string. The string may contain null or other
+    // special characters and so 'len' must be used to determine length.
+    // Error if this is not an object or if the member is not a string.
+    TRITONJSON_STATUSTYPE MemberAsString(
+        const char* name, const char** value, size_t* len) const
+    {
+      const rapidjson::Value& object = AsValue();
+      if (!object.IsObject() || !object.HasMember(name)) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to access non-existing object member '") +
+            name + "'");
+      }
+      const auto& v = object[name];
+      if (!v.IsString()) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to access JSON non-string as string"));
+      }
+      *value = v.GetString();
+      *len = v.GetStringLength();
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Get object member as a string. The string may contain null or
+    // other special characters.  Error if this is not an object or if
+    // the member is not a string.
+    TRITONJSON_STATUSTYPE MemberAsString(
+        const char* name, std::string* str) const
+    {
+      const rapidjson::Value& object = AsValue();
+      if (!object.IsObject() || !object.HasMember(name)) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to access non-existing object member '") +
+            name + "'");
+      }
+      const auto& v = object[name];
+      if (!v.IsString()) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to access JSON non-string as string"));
+      }
+      str->assign(v.GetString(), v.GetStringLength());
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Get object member as a boolean.  Error if this is not an object
+    // or if the member is not a boolean.
+    TRITONJSON_STATUSTYPE MemberAsBool(const char* name, bool* value) const
+    {
+      const rapidjson::Value& object = AsValue();
+      if (!object.IsObject() || !object.HasMember(name)) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to access non-existing object member '") +
+            name + "'");
+      }
+      const auto& v = object[name];
+      if (!v.IsBool()) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to access JSON non-boolean as boolean"));
+      }
+      *value = v.GetBool();
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Get object member as a signed integer.  Error if this is not an object
+    // or if the member is not a signed integer.
+    TRITONJSON_STATUSTYPE MemberAsInt(const char* name, int64_t* value) const
+    {
+      const rapidjson::Value& object = AsValue();
+      if (!object.IsObject() || !object.HasMember(name)) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to access non-existing object member '") +
+            name + "'");
+      }
+      const auto& v = object[name];
+      if (!v.IsInt64()) {
+        TRITONJSON_STATUSRETURN(std::string(
+            "attempt to access JSON non-signed-integer as signed-integer"));
+      }
+      *value = v.GetInt64();
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Get object member as an unsigned integer.  Error if this is not an object
+    // or if the member is not an unsigned integer.
+    TRITONJSON_STATUSTYPE MemberAsUInt(const char* name, uint64_t* value) const
+    {
+      const rapidjson::Value& object = AsValue();
+      if (!object.IsObject() || !object.HasMember(name)) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to access non-existing object member '") +
+            name + "'");
+      }
+      const auto& v = object[name];
+      if (!v.IsUint64()) {
+        TRITONJSON_STATUSRETURN(std::string(
+            "attempt to access JSON non-unsigned-integer as unsigned-integer"));
+      }
+      *value = v.GetUint64();
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Get object member as a double.  Error if this is not an object
+    // or if the member is not a double.
+    TRITONJSON_STATUSTYPE MemberAsDouble(const char* name, double* value) const
+    {
+      const rapidjson::Value& object = AsValue();
+      if (!object.IsObject() || !object.HasMember(name)) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to access non-existing object member '") +
+            name + "'");
+      }
+      const auto& v = object[name];
+      if (!v.IsNumber()) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to access JSON non-number as double"));
+      }
+      *value = v.GetDouble();
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Get array element at a given index within this array.
+    TRITONJSON_STATUSTYPE IndexAsArray(
+        const size_t idx, TritonJson::Value* value)
+    {
+      rapidjson::Value& array = AsMutableValue();
+      if (!array.IsArray() || (idx >= array.GetArray().Size())) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to access non-existing array index '") +
+            std::to_string(idx) + "'");
+      }
+      auto& v = array[idx];
+      if (!v.IsArray()) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to access JSON non-array as array"));
+      }
+      *value = TritonJson::Value(v, allocator_);
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Get object element at a given index within this array.
+    TRITONJSON_STATUSTYPE IndexAsObject(
+        const size_t idx, TritonJson::Value* value)
+    {
+      rapidjson::Value& array = AsMutableValue();
+      if (!array.IsArray() || (idx >= array.GetArray().Size())) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to access non-existing array index '") +
+            std::to_string(idx) + "'");
+      }
+      auto& v = array[idx];
+      if (!v.IsObject()) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to access JSON non-object as object"));
+      }
+      *value = TritonJson::Value(v, allocator_);
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Get array index as a string. The string may contain null or
+    // other special characters and so 'len' must be used to determine
+    // length.  Error if this is not an array or if the index element
+    // is not a string.
+    TRITONJSON_STATUSTYPE IndexAsString(
+        const size_t idx, const char** value, size_t* len) const
+    {
+      const rapidjson::Value& array = AsValue();
+      if (!array.IsArray() || (idx >= array.GetArray().Size())) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to access non-existing array index '") +
+            std::to_string(idx) + "'");
+      }
+      const auto& v = array[idx];
+      if (!v.IsString()) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to access JSON non-string as string"));
+      }
+      *value = v.GetString();
+      *len = v.GetStringLength();
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Get array index as a string. The string may contain null or
+    // other special characters.  Error if this is not an array or if
+    // the index element is not a string.
+    TRITONJSON_STATUSTYPE IndexAsString(
+        const size_t idx, std::string* str) const
+    {
+      const rapidjson::Value& array = AsValue();
+      if (!array.IsArray() || (idx >= array.GetArray().Size())) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to access non-existing array index '") +
+            std::to_string(idx) + "'");
+      }
+      const auto& v = array[idx];
+      if (!v.IsString()) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to access JSON non-string as string"));
+      }
+      str->assign(v.GetString(), v.GetStringLength());
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Get array index as a boolean.  Error if this is not an array or
+    // if the index element is not a boolean.
+    TRITONJSON_STATUSTYPE IndexAsBool(const size_t idx, bool* value) const
+    {
+      const rapidjson::Value& array = AsValue();
+      if (!array.IsArray() || (idx >= array.GetArray().Size())) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to access non-existing array index '") +
+            std::to_string(idx) + "'");
+      }
+      const auto& v = array[idx];
+      if (!v.IsBool()) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to access JSON non-boolean as boolean"));
+      }
+      *value = v.GetBool();
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Get array index as a signed integer.  Error if this is not an array or
+    // if the index element is not a signed integer.
+    TRITONJSON_STATUSTYPE IndexAsInt(const size_t idx, int64_t* value) const
+    {
+      const rapidjson::Value& array = AsValue();
+      if (!array.IsArray() || (idx >= array.GetArray().Size())) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to access non-existing array index '") +
+            std::to_string(idx) + "'");
+      }
+      const auto& v = array[idx];
+      if (!v.IsInt64()) {
+        TRITONJSON_STATUSRETURN(std::string(
+            "attempt to access JSON non-signed-integer as signed-integer"));
+      }
+      *value = v.GetInt64();
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Get array index as an unsigned integer.  Error if this is not an array or
+    // if the index element is not an unsigned integer.
+    TRITONJSON_STATUSTYPE IndexAsUInt(const size_t idx, uint64_t* value) const
+    {
+      const rapidjson::Value& array = AsValue();
+      if (!array.IsArray() || (idx >= array.GetArray().Size())) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to access non-existing array index '") +
+            std::to_string(idx) + "'");
+      }
+      const auto& v = array[idx];
+      if (!v.IsUint64()) {
+        TRITONJSON_STATUSRETURN(std::string(
+            "attempt to access JSON non-unsigned-integer as unsigned-integer"));
+      }
+      *value = v.GetUint64();
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Get array index as a double.  Error if this is not an array or
+    // if the index element is not a double.
+    TRITONJSON_STATUSTYPE IndexAsDouble(const size_t idx, double* value) const
+    {
+      const rapidjson::Value& array = AsValue();
+      if (!array.IsArray() || (idx >= array.GetArray().Size())) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to access non-existing array index '") +
+            std::to_string(idx) + "'");
+      }
+      const auto& v = array[idx];
+      if (!v.IsNumber()) {
+        TRITONJSON_STATUSRETURN(
+            std::string("attempt to access JSON non-number as double"));
+      }
+      *value = v.GetDouble();
+      return TRITONJSON_STATUSSUCCESS;
+    }
+
+    // Release/clear a value.
+    void Release()
+    {
+      if (value_ != nullptr) {
+        allocator_->Free(value_);
+      }
+    }
+
+   private:
+    // Construct a non-top-level JSON value that references an
+    // existing element in a document.
+    explicit Value(
+        rapidjson::Value& v, rapidjson::Document::AllocatorType* allocator)
+        : value_(&v), allocator_(allocator)
+    {
+    }
+
+    // Return a value object that can be used for both a top-level
+    // document as well as an element within a document.
+    const rapidjson::Value& AsValue() const
+    {
+      if (value_ == nullptr) {
+        return document_;
+      }
+      return *value_;
+    }
+
+    rapidjson::Value& AsMutableValue()
+    {
+      if (value_ == nullptr) {
+        return document_;
+      }
+      return *value_;
+    }
+
+    // If this object a document or value. Based on this only one or
+    // document_ or value_ is valid.
+    rapidjson::Document document_;
+    rapidjson::Value* value_;
+    rapidjson::Document::AllocatorType* allocator_;
+  };
+};
+
+}}  // namespace triton::common

3rdparty/common-r22.12/protobuf/grpc_service.proto

+// Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+//  * Redistributions of source code must retain the above copyright
+//    notice, this list of conditions and the following disclaimer.
+//  * Redistributions in binary form must reproduce the above copyright
+//    notice, this list of conditions and the following disclaimer in the
+//    documentation and/or other materials provided with the distribution.
+//  * Neither the name of NVIDIA CORPORATION nor the names of its
+//    contributors may be used to endorse or promote products derived
+//    from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+syntax = "proto3";
+
+package inference;
+
+//@@.. cpp:namespace:: inference
+
+import "model_config.proto";
+
+//@@
+//@@.. cpp:var:: service InferenceService
+//@@
+//@@   Inference Server GRPC endpoints.
+//@@
+service GRPCInferenceService
+{
+  //@@  .. cpp:var:: rpc ServerLive(ServerLiveRequest) returns
+  //@@       (ServerLiveResponse)
+  //@@
+  //@@     Check liveness of the inference server.
+  //@@
+  rpc ServerLive(ServerLiveRequest) returns (ServerLiveResponse) {}
+
+  //@@  .. cpp:var:: rpc ServerReady(ServerReadyRequest) returns
+  //@@       (ServerReadyResponse)
+  //@@
+  //@@     Check readiness of the inference server.
+  //@@
+  rpc ServerReady(ServerReadyRequest) returns (ServerReadyResponse) {}
+
+  //@@  .. cpp:var:: rpc ModelReady(ModelReadyRequest) returns
+  //@@       (ModelReadyResponse)
+  //@@
+  //@@     Check readiness of a model in the inference server.
+  //@@
+  rpc ModelReady(ModelReadyRequest) returns (ModelReadyResponse) {}
+
+  //@@  .. cpp:var:: rpc ServerMetadata(ServerMetadataRequest) returns
+  //@@       (ServerMetadataResponse)
+  //@@
+  //@@     Get server metadata.
+  //@@
+  rpc ServerMetadata(ServerMetadataRequest) returns (ServerMetadataResponse) {}
+
+  //@@  .. cpp:var:: rpc ModelMetadata(ModelMetadataRequest) returns
+  //@@       (ModelMetadataResponse)
+  //@@
+  //@@     Get model metadata.
+  //@@
+  rpc ModelMetadata(ModelMetadataRequest) returns (ModelMetadataResponse) {}
+
+  //@@  .. cpp:var:: rpc ModelInfer(ModelInferRequest) returns
+  //@@       (ModelInferResponse)
+  //@@
+  //@@     Perform inference using a specific model.
+  //@@
+  rpc ModelInfer(ModelInferRequest) returns (ModelInferResponse) {}
+
+  //@@  .. cpp:var:: rpc ModelStreamInfer(stream ModelInferRequest) returns
+  //@@       (stream ModelStreamInferResponse)
+  //@@
+  //@@     Perform streaming inference.
+  //@@
+  rpc ModelStreamInfer(stream ModelInferRequest)
+      returns (stream ModelStreamInferResponse)
+  {
+  }
+
+  //@@  .. cpp:var:: rpc ModelConfig(ModelConfigRequest) returns
+  //@@       (ModelConfigResponse)
+  //@@
+  //@@     Get model configuration.
+  //@@
+  rpc ModelConfig(ModelConfigRequest) returns (ModelConfigResponse) {}
+
+  //@@  .. cpp:var:: rpc ModelStatistics(
+  //@@                     ModelStatisticsRequest)
+  //@@                   returns (ModelStatisticsResponse)
+  //@@
+  //@@     Get the cumulative inference statistics for a model.
+  //@@
+  rpc ModelStatistics(ModelStatisticsRequest) returns (ModelStatisticsResponse)
+  {
+  }
+
+  //@@  .. cpp:var:: rpc RepositoryIndex(RepositoryIndexRequest) returns
+  //@@       (RepositoryIndexResponse)
+  //@@
+  //@@     Get the index of model repository contents.
+  //@@
+  rpc RepositoryIndex(RepositoryIndexRequest) returns (RepositoryIndexResponse)
+  {
+  }
+
+  //@@  .. cpp:var:: rpc RepositoryModelLoad(RepositoryModelLoadRequest) returns
+  //@@       (RepositoryModelLoadResponse)
+  //@@
+  //@@     Load or reload a model from a repository.
+  //@@
+  rpc RepositoryModelLoad(RepositoryModelLoadRequest)
+      returns (RepositoryModelLoadResponse)
+  {
+  }
+
+  //@@  .. cpp:var:: rpc RepositoryModelUnload(RepositoryModelUnloadRequest)
+  //@@       returns (RepositoryModelUnloadResponse)
+  //@@
+  //@@     Unload a model.
+  //@@
+  rpc RepositoryModelUnload(RepositoryModelUnloadRequest)
+      returns (RepositoryModelUnloadResponse)
+  {
+  }
+
+  //@@  .. cpp:var:: rpc SystemSharedMemoryStatus(
+  //@@                     SystemSharedMemoryStatusRequest)
+  //@@                   returns (SystemSharedMemoryStatusRespose)
+  //@@
+  //@@     Get the status of all registered system-shared-memory regions.
+  //@@
+  rpc SystemSharedMemoryStatus(SystemSharedMemoryStatusRequest)
+      returns (SystemSharedMemoryStatusResponse)
+  {
+  }
+
+  //@@  .. cpp:var:: rpc SystemSharedMemoryRegister(
+  //@@                     SystemSharedMemoryRegisterRequest)
+  //@@                   returns (SystemSharedMemoryRegisterResponse)
+  //@@
+  //@@     Register a system-shared-memory region.
+  //@@
+  rpc SystemSharedMemoryRegister(SystemSharedMemoryRegisterRequest)
+      returns (SystemSharedMemoryRegisterResponse)
+  {
+  }
+
+  //@@  .. cpp:var:: rpc SystemSharedMemoryUnregister(
+  //@@                     SystemSharedMemoryUnregisterRequest)
+  //@@                   returns (SystemSharedMemoryUnregisterResponse)
+  //@@
+  //@@     Unregister a system-shared-memory region.
+  //@@
+  rpc SystemSharedMemoryUnregister(SystemSharedMemoryUnregisterRequest)
+      returns (SystemSharedMemoryUnregisterResponse)
+  {
+  }
+
+  //@@  .. cpp:var:: rpc CudaSharedMemoryStatus(
+  //@@                     CudaSharedMemoryStatusRequest)
+  //@@                   returns (CudaSharedMemoryStatusRespose)
+  //@@
+  //@@     Get the status of all registered CUDA-shared-memory regions.
+  //@@
+  rpc CudaSharedMemoryStatus(CudaSharedMemoryStatusRequest)
+      returns (CudaSharedMemoryStatusResponse)
+  {
+  }
+
+  //@@  .. cpp:var:: rpc CudaSharedMemoryRegister(
+  //@@                     CudaSharedMemoryRegisterRequest)
+  //@@                   returns (CudaSharedMemoryRegisterResponse)
+  //@@
+  //@@     Register a CUDA-shared-memory region.
+  //@@
+  rpc CudaSharedMemoryRegister(CudaSharedMemoryRegisterRequest)
+      returns (CudaSharedMemoryRegisterResponse)
+  {
+  }
+
+  //@@  .. cpp:var:: rpc CudaSharedMemoryUnregister(
+  //@@                     CudaSharedMemoryUnregisterRequest)
+  //@@                   returns (CudaSharedMemoryUnregisterResponse)
+  //@@
+  //@@     Unregister a CUDA-shared-memory region.
+  //@@
+  rpc CudaSharedMemoryUnregister(CudaSharedMemoryUnregisterRequest)
+      returns (CudaSharedMemoryUnregisterResponse)
+  {
+  }
+
+  //@@  .. cpp:var:: rpc TraceSetting(TraceSettingRequest)
+  //@@                   returns (TraceSettingResponse)
+  //@@
+  //@@     Update and get the trace setting of the Triton server.
+  //@@
+  rpc TraceSetting(TraceSettingRequest) returns (TraceSettingResponse)
+  {
+  }
+
+  //@@  .. cpp:var:: rpc LogSettings(LogSettingsRequest)
+  //@@                   returns (LogSettingsResponse)
+  //@@
+  //@@     Update and get the log settings of the Triton server.
+  //@@
+  rpc LogSettings(LogSettingsRequest) returns (LogSettingsResponse)
+  {
+  }
+}
+
+//@@
+//@@.. cpp:var:: message ServerLiveRequest
+//@@
+//@@   Request message for ServerLive.
+//@@
+message ServerLiveRequest {}
+
+//@@
+//@@.. cpp:var:: message ServerLiveResponse
+//@@
+//@@   Response message for ServerLive.
+//@@
+message ServerLiveResponse
+{
+  //@@
+  //@@  .. cpp:var:: bool live
+  //@@
+  //@@     True if the inference server is live, false it not live.
+  //@@
+  bool live = 1;
+}
+
+//@@
+//@@.. cpp:var:: message ServerReadyRequest
+//@@
+//@@   Request message for ServerReady.
+//@@
+message ServerReadyRequest {}
+
+//@@
+//@@.. cpp:var:: message ServerReadyResponse
+//@@
+//@@   Response message for ServerReady.
+//@@
+message ServerReadyResponse
+{
+  //@@
+  //@@  .. cpp:var:: bool ready
+  //@@
+  //@@     True if the inference server is ready, false it not ready.
+  //@@
+  bool ready = 1;
+}
+
+//@@
+//@@.. cpp:var:: message ModelReadyRequest
+//@@
+//@@   Request message for ModelReady.
+//@@
+message ModelReadyRequest
+{
+  //@@
+  //@@  .. cpp:var:: string name
+  //@@
+  //@@     The name of the model to check for readiness.
+  //@@
+  string name = 1;
+
+  //@@  .. cpp:var:: string version
+  //@@
+  //@@     The version of the model to check for readiness. If not given the
+  //@@     server will choose a version based on the model and internal policy.
+  //@@
+  string version = 2;
+}
+
+//@@
+//@@.. cpp:var:: message ModelReadyResponse
+//@@
+//@@   Response message for ModelReady.
+//@@
+message ModelReadyResponse
+{
+  //@@
+  //@@  .. cpp:var:: bool ready
+  //@@
+  //@@     True if the model is ready, false it not ready.
+  //@@
+  bool ready = 1;
+}
+
+//@@
+//@@.. cpp:var:: message ServerMetadataRequest
+//@@
+//@@   Request message for ServerMetadata.
+//@@
+message ServerMetadataRequest {}
+
+//@@
+//@@.. cpp:var:: message ServerMetadataResponse
+//@@
+//@@   Response message for ServerMetadata.
+//@@
+message ServerMetadataResponse
+{
+  //@@
+  //@@  .. cpp:var:: string name
+  //@@
+  //@@     The server name.
+  //@@
+  string name = 1;
+
+  //@@
+  //@@  .. cpp:var:: string version
+  //@@
+  //@@     The server version.
+  //@@
+  string version = 2;
+
+  //@@
+  //@@  .. cpp:var:: string extensions (repeated)
+  //@@
+  //@@     The extensions supported by the server.
+  //@@
+  repeated string extensions = 3;
+}
+
+//@@
+//@@.. cpp:var:: message ModelMetadataRequest
+//@@
+//@@   Request message for ModelMetadata.
+//@@
+message ModelMetadataRequest
+{
+  //@@
+  //@@  .. cpp:var:: string name
+  //@@
+  //@@     The name of the model.
+  //@@
+  string name = 1;
+
+  //@@  .. cpp:var:: string version
+  //@@
+  //@@     The version of the model to check for readiness. If not
+  //@@     given the server will choose a version based on the
+  //@@     model and internal policy.
+  //@@
+  string version = 2;
+}
+
+//@@
+//@@.. cpp:var:: message ModelMetadataResponse
+//@@
+//@@   Response message for ModelMetadata.
+//@@
+message ModelMetadataResponse
+{
+  //@@
+  //@@  .. cpp:var:: message TensorMetadata
+  //@@
+  //@@     Metadata for a tensor.
+  //@@
+  message TensorMetadata
+  {
+    //@@
+    //@@    .. cpp:var:: string name
+    //@@
+    //@@       The tensor name.
+    //@@
+    string name = 1;
+
+    //@@
+    //@@    .. cpp:var:: string datatype
+    //@@
+    //@@       The tensor data type.
+    //@@
+    string datatype = 2;
+
+    //@@
+    //@@    .. cpp:var:: int64 shape (repeated)
+    //@@
+    //@@       The tensor shape. A variable-size dimension is represented
+    //@@       by a -1 value.
+    //@@
+    repeated int64 shape = 3;
+  }
+
+  //@@
+  //@@  .. cpp:var:: string name
+  //@@
+  //@@     The model name.
+  //@@
+  string name = 1;
+
+  //@@
+  //@@  .. cpp:var:: string versions (repeated)
+  //@@
+  //@@     The versions of the model.
+  //@@
+  repeated string versions = 2;
+
+  //@@
+  //@@  .. cpp:var:: string platform
+  //@@
+  //@@     The model's platform.
+  //@@
+  string platform = 3;
+
+  //@@
+  //@@  .. cpp:var:: TensorMetadata inputs (repeated)
+  //@@
+  //@@     The model's inputs.
+  //@@
+  repeated TensorMetadata inputs = 4;
+
+  //@@
+  //@@  .. cpp:var:: TensorMetadata outputs (repeated)
+  //@@
+  //@@     The model's outputs.
+  //@@
+  repeated TensorMetadata outputs = 5;
+}
+
+//@@
+//@@.. cpp:var:: message InferParameter
+//@@
+//@@   An inference parameter value.
+//@@
+message InferParameter
+{
+  //@@  .. cpp:var:: oneof parameter_choice
+  //@@
+  //@@     The parameter value can be a string, an int64 or
+  //@@     a boolean
+  //@@
+  oneof parameter_choice
+  {
+    //@@    .. cpp:var:: bool bool_param
+    //@@
+    //@@       A boolean parameter value.
+    //@@
+    bool bool_param = 1;
+
+    //@@    .. cpp:var:: int64 int64_param
+    //@@
+    //@@       An int64 parameter value.
+    //@@
+    int64 int64_param = 2;
+
+    //@@    .. cpp:var:: string string_param
+    //@@
+    //@@       A string parameter value.
+    //@@
+    string string_param = 3;
+  }
+}
+
+//@@
+//@@.. cpp:var:: message InferTensorContents
+//@@
+//@@   The data contained in a tensor represented by the repeated type
+//@@   that matches the tensor's data type. Protobuf oneof is not used
+//@@   because oneofs cannot contain repeated fields.
+//@@
+message InferTensorContents
+{
+  //@@
+  //@@  .. cpp:var:: bool bool_contents (repeated)
+  //@@
+  //@@     Representation for BOOL data type. The size must match what is
+  //@@     expected by the tensor's shape. The contents must be the flattened,
+  //@@     one-dimensional, row-major order of the tensor elements.
+  //@@
+  repeated bool bool_contents = 1;
+
+  //@@
+  //@@  .. cpp:var:: int32 int_contents (repeated)
+  //@@
+  //@@     Representation for INT8, INT16, and INT32 data types. The size
+  //@@     must match what is expected by the tensor's shape. The contents
+  //@@     must be the flattened, one-dimensional, row-major order of the
+  //@@     tensor elements.
+  //@@
+  repeated int32 int_contents = 2;
+
+  //@@
+  //@@  .. cpp:var:: int64 int64_contents (repeated)
+  //@@
+  //@@     Representation for INT64 data types. The size must match what
+  //@@     is expected by the tensor's shape. The contents must be the
+  //@@     flattened, one-dimensional, row-major order of the tensor elements.
+  //@@
+  repeated int64 int64_contents = 3;
+
+  //@@
+  //@@  .. cpp:var:: uint32 uint_contents (repeated)
+  //@@
+  //@@     Representation for UINT8, UINT16, and UINT32 data types. The size
+  //@@     must match what is expected by the tensor's shape. The contents
+  //@@     must be the flattened, one-dimensional, row-major order of the
+  //@@     tensor elements.
+  //@@
+  repeated uint32 uint_contents = 4;
+
+  //@@
+  //@@  .. cpp:var:: uint64 uint64_contents (repeated)
+  //@@
+  //@@     Representation for UINT64 data types. The size must match what
+  //@@     is expected by the tensor's shape. The contents must be the
+  //@@     flattened, one-dimensional, row-major order of the tensor elements.
+  //@@
+  repeated uint64 uint64_contents = 5;
+
+  //@@
+  //@@  .. cpp:var:: float fp32_contents (repeated)
+  //@@
+  //@@     Representation for FP32 data type. The size must match what is
+  //@@     expected by the tensor's shape. The contents must be the flattened,
+  //@@     one-dimensional, row-major order of the tensor elements.
+  //@@
+  repeated float fp32_contents = 6;
+
+  //@@
+  //@@  .. cpp:var:: double fp64_contents (repeated)
+  //@@
+  //@@     Representation for FP64 data type. The size must match what is
+  //@@     expected by the tensor's shape. The contents must be the flattened,
+  //@@     one-dimensional, row-major order of the tensor elements.
+  //@@
+  repeated double fp64_contents = 7;
+
+  //@@
+  //@@  .. cpp:var:: bytes bytes_contents (repeated)
+  //@@
+  //@@     Representation for BYTES data type. The size must match what is
+  //@@     expected by the tensor's shape. The contents must be the flattened,
+  //@@     one-dimensional, row-major order of the tensor elements.
+  //@@
+  repeated bytes bytes_contents = 8;
+}
+
+//@@
+//@@.. cpp:var:: message ModelInferRequest
+//@@
+//@@   Request message for ModelInfer.
+//@@
+message ModelInferRequest
+{
+  //@@
+  //@@  .. cpp:var:: message InferInputTensor
+  //@@
+  //@@     An input tensor for an inference request.
+  //@@
+  message InferInputTensor
+  {
+    //@@
+    //@@    .. cpp:var:: string name
+    //@@
+    //@@       The tensor name.
+    //@@
+    string name = 1;
+
+    //@@
+    //@@    .. cpp:var:: string datatype
+    //@@
+    //@@       The tensor data type.
+    //@@
+    string datatype = 2;
+
+    //@@
+    //@@    .. cpp:var:: int64 shape (repeated)
+    //@@
+    //@@       The tensor shape.
+    //@@
+    repeated int64 shape = 3;
+
+    //@@    .. cpp:var:: map<string,InferParameter> parameters
+    //@@
+    //@@       Optional inference input tensor parameters.
+    //@@
+    map<string, InferParameter> parameters = 4;
+
+    //@@    .. cpp:var:: InferTensorContents contents
+    //@@
+    //@@       The tensor contents using a data-type format. This field
+    //@@       must not be specified if tensor contents are being specified
+    //@@       in ModelInferRequest.raw_input_contents.
+    //@@
+    InferTensorContents contents = 5;
+  }
+
+  //@@
+  //@@  .. cpp:var:: message InferRequestedOutputTensor
+  //@@
+  //@@     An output tensor requested for an inference request.
+  //@@
+  message InferRequestedOutputTensor
+  {
+    //@@
+    //@@    .. cpp:var:: string name
+    //@@
+    //@@       The tensor name.
+    //@@
+    string name = 1;
+
+    //@@    .. cpp:var:: map<string,InferParameter> parameters
+    //@@
+    //@@       Optional requested output tensor parameters.
+    //@@
+    map<string, InferParameter> parameters = 2;
+  }
+
+  //@@  .. cpp:var:: string model_name
+  //@@
+  //@@     The name of the model to use for inferencing.
+  //@@
+  string model_name = 1;
+
+  //@@  .. cpp:var:: string model_version
+  //@@
+  //@@     The version of the model to use for inference. If not
+  //@@     given the latest/most-recent version of the model is used.
+  //@@
+  string model_version = 2;
+
+  //@@  .. cpp:var:: string id
+  //@@
+  //@@     Optional identifier for the request. If specified will be
+  //@@     returned in the response.
+  //@@
+  string id = 3;
+
+  //@@  .. cpp:var:: map<string,InferParameter> parameters
+  //@@
+  //@@     Optional inference parameters.
+  //@@
+  map<string, InferParameter> parameters = 4;
+
+  //@@
+  //@@  .. cpp:var:: InferInputTensor inputs (repeated)
+  //@@
+  //@@     The input tensors for the inference.
+  //@@
+  repeated InferInputTensor inputs = 5;
+
+  //@@
+  //@@  .. cpp:var:: InferRequestedOutputTensor outputs (repeated)
+  //@@
+  //@@     The requested output tensors for the inference. Optional, if not
+  //@@     specified all outputs specified in the model config will be
+  //@@     returned.
+  //@@
+  repeated InferRequestedOutputTensor outputs = 6;
+
+  //@@
+  //@@  .. cpp:var:: bytes raw_input_contents
+  //@@
+  //@@     The data contained in an input tensor can be represented in
+  //@@     "raw" bytes form or in the repeated type that matches the
+  //@@     tensor's data type. Using the "raw" bytes form will
+  //@@     typically allow higher performance due to the way protobuf
+  //@@     allocation and reuse interacts with GRPC. For example, see
+  //@@     https://github.com/grpc/grpc/issues/23231.
+  //@@
+  //@@     To use the raw representation 'raw_input_contents' must be
+  //@@     initialized with data for each tensor in the same order as
+  //@@     'inputs'. For each tensor, the size of this content must
+  //@@     match what is expected by the tensor's shape and data
+  //@@     type. The raw data must be the flattened, one-dimensional,
+  //@@     row-major order of the tensor elements without any stride
+  //@@     or padding between the elements. Note that the FP16 and BF16 data
+  //@@     types must be represented as raw content as there is no
+  //@@     specific data type for a 16-bit float type.
+  //@@
+  //@@     If this field is specified then InferInputTensor::contents
+  //@@     must not be specified for any input tensor.
+  //@@
+  repeated bytes raw_input_contents = 7;
+}
+
+//@@
+//@@.. cpp:var:: message ModelInferResponse
+//@@
+//@@   Response message for ModelInfer.
+//@@
+message ModelInferResponse
+{
+  //@@
+  //@@  .. cpp:var:: message InferOutputTensor
+  //@@
+  //@@     An output tensor returned for an inference request.
+  //@@
+  message InferOutputTensor
+  {
+    //@@
+    //@@    .. cpp:var:: string name
+    //@@
+    //@@       The tensor name.
+    //@@
+    string name = 1;
+
+    //@@
+    //@@    .. cpp:var:: string datatype
+    //@@
+    //@@       The tensor data type.
+    //@@
+    string datatype = 2;
+
+    //@@
+    //@@    .. cpp:var:: int64 shape (repeated)
+    //@@
+    //@@       The tensor shape.
+    //@@
+    repeated int64 shape = 3;
+
+    //@@    .. cpp:var:: map<string,InferParameter> parameters
+    //@@
+    //@@       Optional output tensor parameters.
+    //@@
+    map<string, InferParameter> parameters = 4;
+
+    //@@    .. cpp:var:: InferTensorContents contents
+    //@@
+    //@@       The tensor contents using a data-type format. This field
+    //@@       must not be specified if tensor contents are being specified
+    //@@       in ModelInferResponse.raw_output_contents.
+    //@@
+    InferTensorContents contents = 5;
+  }
+
+  //@@  .. cpp:var:: string model_name
+  //@@
+  //@@     The name of the model used for inference.
+  //@@
+  string model_name = 1;
+
+  //@@  .. cpp:var:: string model_version
+  //@@
+  //@@     The version of the model used for inference.
+  //@@
+  string model_version = 2;
+
+  //@@  .. cpp:var:: string id
+  //@@
+  //@@     The id of the inference request if one was specified.
+  //@@
+  string id = 3;
+
+  //@@  .. cpp:var:: map<string,InferParameter> parameters
+  //@@
+  //@@     Optional inference response parameters.
+  //@@
+  map<string, InferParameter> parameters = 4;
+
+  //@@
+  //@@  .. cpp:var:: InferOutputTensor outputs (repeated)
+  //@@
+  //@@     The output tensors holding inference results.
+  //@@
+  repeated InferOutputTensor outputs = 5;
+
+  //@@
+  //@@  .. cpp:var:: bytes raw_output_contents
+  //@@
+  //@@     The data contained in an output tensor can be represented in
+  //@@     "raw" bytes form or in the repeated type that matches the
+  //@@     tensor's data type. Using the "raw" bytes form will
+  //@@     typically allow higher performance due to the way protobuf
+  //@@     allocation and reuse interacts with GRPC. For example, see
+  //@@     https://github.com/grpc/grpc/issues/23231.
+  //@@
+  //@@     To use the raw representation 'raw_output_contents' must be
+  //@@     initialized with data for each tensor in the same order as
+  //@@     'outputs'. For each tensor, the size of this content must
+  //@@     match what is expected by the tensor's shape and data
+  //@@     type. The raw data must be the flattened, one-dimensional,
+  //@@     row-major order of the tensor elements without any stride
+  //@@     or padding between the elements. Note that the FP16 and BF16 data
+  //@@     types must be represented as raw content as there is no
+  //@@     specific data type for a 16-bit float type.
+  //@@
+  //@@     If this field is specified then InferOutputTensor::contents
+  //@@     must not be specified for any output tensor.
+  //@@
+  repeated bytes raw_output_contents = 6;
+}
+
+//@@
+//@@.. cpp:var:: message ModelStreamInferResponse
+//@@
+//@@   Response message for ModelStreamInfer.
+//@@
+message ModelStreamInferResponse
+{
+  //@@
+  //@@  .. cpp:var:: string error_message
+  //@@
+  //@@     The message describing the error. The empty message
+  //@@     indicates the inference was successful without errors.
+  //@@
+  string error_message = 1;
+
+  //@@
+  //@@  .. cpp:var:: ModelInferResponse infer_response
+  //@@
+  //@@     Holds the results of the request.
+  //@@
+  ModelInferResponse infer_response = 2;
+}
+
+//@@
+//@@.. cpp:var:: message ModelConfigRequest
+//@@
+//@@   Request message for ModelConfig.
+//@@
+message ModelConfigRequest
+{
+  //@@
+  //@@  .. cpp:var:: string name
+  //@@
+  //@@     The name of the model.
+  //@@
+  string name = 1;
+
+  //@@  .. cpp:var:: string version
+  //@@
+  //@@     The version of the model. If not given the model version
+  //@@     is selected automatically based on the version policy.
+  //@@
+  string version = 2;
+}
+
+//@@
+//@@.. cpp:var:: message ModelConfigResponse
+//@@
+//@@   Response message for ModelConfig.
+//@@
+message ModelConfigResponse
+{
+  //@@
+  //@@  .. cpp:var:: ModelConfig config
+  //@@
+  //@@     The model configuration.
+  //@@
+  ModelConfig config = 1;
+}
+
+//@@
+//@@.. cpp:var:: message ModelStatisticsRequest
+//@@
+//@@   Request message for ModelStatistics.
+//@@
+message ModelStatisticsRequest
+{
+  //@@  .. cpp:var:: string name
+  //@@
+  //@@     The name of the model. If not given returns statistics for
+  //@@     all models.
+  //@@
+  string name = 1;
+
+  //@@  .. cpp:var:: string version
+  //@@
+  //@@     The version of the model. If not given returns statistics for
+  //@@     all model versions.
+  //@@
+  string version = 2;
+}
+
+
+//@@
+//@@.. cpp:var:: message StatisticDuration
+//@@
+//@@   Statistic recording a cumulative duration metric.
+//@@
+message StatisticDuration
+{
+  //@@  .. cpp:var:: uint64 count
+  //@@
+  //@@     Cumulative number of times this metric occurred.
+  //@@
+  uint64 count = 1;
+
+  //@@  .. cpp:var:: uint64 total_time_ns
+  //@@
+  //@@     Total collected duration of this metric in nanoseconds.
+  //@@
+  uint64 ns = 2;
+}
+
+//@@
+//@@.. cpp:var:: message InferStatistics
+//@@
+//@@   Inference statistics.
+//@@
+message InferStatistics
+{
+  //@@  .. cpp:var:: StatisticDuration success
+  //@@
+  //@@     Cumulative count and duration for successful inference
+  //@@     request. The "success" count and cumulative duration includes
+  //@@     cache hits.
+  //@@
+  StatisticDuration success = 1;
+
+  //@@  .. cpp:var:: StatisticDuration fail
+  //@@
+  //@@     Cumulative count and duration for failed inference
+  //@@     request.
+  //@@
+  StatisticDuration fail = 2;
+
+  //@@  .. cpp:var:: StatisticDuration queue
+  //@@
+  //@@     The count and cumulative duration that inference requests wait in
+  //@@     scheduling or other queues. The "queue" count and cumulative 
+  //@@     duration includes cache hits.
+  //@@
+  StatisticDuration queue = 3;
+
+  //@@  .. cpp:var:: StatisticDuration compute_input
+  //@@
+  //@@     The count and cumulative duration to prepare input tensor data as
+  //@@     required by the model framework / backend. For example, this duration
+  //@@     should include the time to copy input tensor data to the GPU.
+  //@@     The "compute_input" count and cumulative duration do not account for
+  //@@     requests that were a cache hit. See the "cache_hit" field for more
+  //@@     info.
+  //@@
+  StatisticDuration compute_input = 4;
+
+  //@@  .. cpp:var:: StatisticDuration compute_infer
+  //@@
+  //@@     The count and cumulative duration to execute the model.
+  //@@     The "compute_infer" count and cumulative duration do not account for
+  //@@     requests that were a cache hit. See the "cache_hit" field for more
+  //@@     info.
+  //@@
+  StatisticDuration compute_infer = 5;
+
+  //@@  .. cpp:var:: StatisticDuration compute_output
+  //@@
+  //@@     The count and cumulative duration to extract output tensor data
+  //@@     produced by the model framework / backend. For example, this duration
+  //@@     should include the time to copy output tensor data from the GPU.
+  //@@     The "compute_output" count and cumulative duration do not account for
+  //@@     requests that were a cache hit. See the "cache_hit" field for more
+  //@@     info.
+  //@@
+  StatisticDuration compute_output = 6;
+
+  //@@  .. cpp:var:: StatisticDuration cache_hit
+  //@@
+  //@@     The count of response cache hits and cumulative duration to lookup
+  //@@     and extract output tensor data from the Response Cache on a cache
+  //@@     hit. For example, this duration should include the time to copy
+  //@@     output tensor data from the Response Cache to the response object.
+  //@@     On cache hits, triton does not need to go to the model/backend 
+  //@@     for the output tensor data, so the "compute_input", "compute_infer",
+  //@@     and "compute_output" fields are not updated. Assuming the response
+  //@@     cache is enabled for a given model, a cache hit occurs for a
+  //@@     request to that model when the request metadata (model name,
+  //@@     model version, model inputs) hashes to an existing entry in the
+  //@@     cache. On a cache miss, the request hash and response output tensor
+  //@@     data is added to the cache. See response cache docs for more info:
+  //@@     https://github.com/triton-inference-server/server/blob/main/docs/response_cache.md
+  //@@
+  StatisticDuration cache_hit = 7;
+
+  //@@  .. cpp:var:: StatisticDuration cache_miss
+  //@@
+  //@@     The count of response cache misses and cumulative duration to lookup
+  //@@     and insert output tensor data from the computed response to the cache.
+  //@@     For example, this duration should include the time to copy
+  //@@     output tensor data from the response object to the Response Cache.
+  //@@     Assuming the response cache is enabled for a given model, a cache
+  //@@     miss occurs for a request to that model when the request metadata
+  //@@     does NOT hash to an existing entry in the cache. See the response
+  //@@     cache docs for more info:
+  //@@     https://github.com/triton-inference-server/server/blob/main/docs/response_cache.md
+  //@@
+  StatisticDuration cache_miss = 8;
+}
+
+//@@
+//@@.. cpp:var:: message InferBatchStatistics
+//@@
+//@@   Inference batch statistics.
+//@@
+message InferBatchStatistics
+{
+  //@@  .. cpp:var:: uint64 batch_size
+  //@@
+  //@@     The size of the batch.
+  //@@
+  uint64 batch_size = 1;
+
+  //@@  .. cpp:var:: StatisticDuration compute_input
+  //@@
+  //@@     The count and cumulative duration to prepare input tensor data as
+  //@@     required by the model framework / backend with the given batch size.
+  //@@     For example, this duration should include the time to copy input
+  //@@     tensor data to the GPU.
+  //@@
+  StatisticDuration compute_input = 2;
+
+  //@@  .. cpp:var:: StatisticDuration compute_infer
+  //@@
+  //@@     The count and cumulative duration to execute the model with the given
+  //@@     batch size.
+  //@@
+  StatisticDuration compute_infer = 3;
+
+  //@@  .. cpp:var:: StatisticDuration compute_output
+  //@@
+  //@@     The count and cumulative duration to extract output tensor data
+  //@@     produced by the model framework / backend with the given batch size.
+  //@@     For example, this duration should include the time to copy output
+  //@@     tensor data from the GPU.
+  //@@
+  StatisticDuration compute_output = 4;
+}
+
+//@@
+//@@.. cpp:var:: message ModelStatistics
+//@@
+//@@   Statistics for a specific model and version.
+//@@
+message ModelStatistics
+{
+  //@@  .. cpp:var:: string name
+  //@@
+  //@@     The name of the model. If not given returns statistics for all
+  //@@
+  string name = 1;
+
+  //@@  .. cpp:var:: string version
+  //@@
+  //@@     The version of the model.
+  //@@
+  string version = 2;
+
+  //@@  .. cpp:var:: uint64 last_inference
+  //@@
+  //@@     The timestamp of the last inference request made for this model,
+  //@@     as milliseconds since the epoch.
+  //@@
+  uint64 last_inference = 3;
+
+  //@@  .. cpp:var:: uint64 last_inference
+  //@@
+  //@@     The cumulative count of successful inference requests made for this
+  //@@     model. Each inference in a batched request is counted as an
+  //@@     individual inference. For example, if a client sends a single
+  //@@     inference request with batch size 64, "inference_count" will be
+  //@@     incremented by 64. Similarly, if a clients sends 64 individual
+  //@@     requests each with batch size 1, "inference_count" will be
+  //@@     incremented by 64. The "inference_count" value DOES NOT include
+  //@@     cache hits.
+  //@@
+  uint64 inference_count = 4;
+
+  //@@  .. cpp:var:: uint64 last_inference
+  //@@
+  //@@     The cumulative count of the number of successful inference executions
+  //@@     performed for the model. When dynamic batching is enabled, a single
+  //@@     model execution can perform inferencing for more than one inference
+  //@@     request. For example, if a clients sends 64 individual requests each
+  //@@     with batch size 1 and the dynamic batcher batches them into a single
+  //@@     large batch for model execution then "execution_count" will be
+  //@@     incremented by 1. If, on the other hand, the dynamic batcher is not
+  //@@     enabled for that each of the 64 individual requests is executed
+  //@@     independently, then "execution_count" will be incremented by 64.
+  //@@     The "execution_count" value DOES NOT include cache hits.
+  //@@
+  uint64 execution_count = 5;
+
+  //@@  .. cpp:var:: InferStatistics inference_stats
+  //@@
+  //@@     The aggregate statistics for the model/version.
+  //@@
+  InferStatistics inference_stats = 6;
+
+  //@@  .. cpp:var:: InferBatchStatistics batch_stats (repeated)
+  //@@
+  //@@     The aggregate statistics for each different batch size that is
+  //@@     executed in the model. The batch statistics indicate how many actual
+  //@@     model executions were performed and show differences due to different
+  //@@     batch size (for example, larger batches typically take longer to
+  //@@     compute).
+  //@@
+  repeated InferBatchStatistics batch_stats = 7;
+}
+
+//@@
+//@@.. cpp:var:: message ModelStatisticsResponse
+//@@
+//@@   Response message for ModelStatistics.
+//@@
+message ModelStatisticsResponse
+{
+  //@@  .. cpp:var:: ModelStatistics model_stats (repeated)
+  //@@
+  //@@     Statistics for each requested model.
+  //@@
+  repeated ModelStatistics model_stats = 1;
+}
+
+//@@
+//@@.. cpp:var:: message ModelRepositoryParameter
+//@@
+//@@   An model repository parameter value.
+//@@
+message ModelRepositoryParameter
+{
+  //@@  .. cpp:var:: oneof parameter_choice
+  //@@
+  //@@     The parameter value can be a string, an int64 or
+  //@@     a boolean
+  //@@
+  oneof parameter_choice
+  {
+    //@@    .. cpp:var:: bool bool_param
+    //@@
+    //@@       A boolean parameter value.
+    //@@
+    bool bool_param = 1;
+
+    //@@    .. cpp:var:: int64 int64_param
+    //@@
+    //@@       An int64 parameter value.
+    //@@
+    int64 int64_param = 2;
+
+    //@@    .. cpp:var:: string string_param
+    //@@
+    //@@       A string parameter value.
+    //@@
+    string string_param = 3;
+
+    //@@    .. cpp:var:: bytes bytes_param
+    //@@
+    //@@       A bytes parameter value.
+    //@@
+    bytes bytes_param = 4;
+  }
+}
+
+//@@
+//@@.. cpp:var:: message RepositoryIndexRequest
+//@@
+//@@   Request message for RepositoryIndex.
+//@@
+message RepositoryIndexRequest
+{
+  //@@  .. cpp:var:: string repository_name
+  //@@
+  //@@     The name of the repository. If empty the index is returned
+  //@@     for all repositories.
+  //@@
+  string repository_name = 1;
+
+  //@@  .. cpp:var:: bool ready
+  //@@
+  //@@     If true returned only models currently ready for inferencing.
+  //@@
+  bool ready = 2;
+}
+
+//@@
+//@@.. cpp:var:: message RepositoryIndexResponse
+//@@
+//@@   Response message for RepositoryIndex.
+//@@
+message RepositoryIndexResponse
+{
+  //@@
+  //@@  .. cpp:var:: message ModelIndex
+  //@@
+  //@@     Index entry for a model.
+  //@@
+  message ModelIndex
+  {
+    //@@
+    //@@    .. cpp:var:: string name
+    //@@
+    //@@       The name of the model.
+    //@@
+    string name = 1;
+
+    //@@    .. cpp:var:: string version
+    //@@
+    //@@       The version of the model.
+    //@@
+    string version = 2;
+
+    //@@
+    //@@    .. cpp:var:: string state
+    //@@
+    //@@       The state of the model.
+    //@@
+    string state = 3;
+
+    //@@
+    //@@    .. cpp:var:: string reason
+    //@@
+    //@@       The reason, if any, that the model is in the given state.
+    //@@
+    string reason = 4;
+  }
+
+  //@@
+  //@@  .. cpp:var:: ModelIndex models (repeated)
+  //@@
+  //@@     An index entry for each model.
+  //@@
+  repeated ModelIndex models = 1;
+}
+
+//@@
+//@@.. cpp:var:: message RepositoryModelLoadRequest
+//@@
+//@@   Request message for RepositoryModelLoad.
+//@@
+message RepositoryModelLoadRequest
+{
+  //@@  .. cpp:var:: string repository_name
+  //@@
+  //@@     The name of the repository to load from. If empty the model
+  //@@     is loaded from any repository.
+  //@@
+  string repository_name = 1;
+
+  //@@  .. cpp:var:: string repository_name
+  //@@
+  //@@     The name of the model to load, or reload.
+  //@@
+  string model_name = 2;
+
+  //@@  .. cpp:var:: map<string,ModelRepositoryParameter> parameters
+  //@@
+  //@@     Optional model repository request parameters.
+  //@@
+  map<string, ModelRepositoryParameter> parameters = 3;
+}
+
+//@@
+//@@.. cpp:var:: message RepositoryModelLoadResponse
+//@@
+//@@   Response message for RepositoryModelLoad.
+//@@
+message RepositoryModelLoadResponse {}
+
+//@@
+//@@.. cpp:var:: message RepositoryModelUnloadRequest
+//@@
+//@@   Request message for RepositoryModelUnload.
+//@@
+message RepositoryModelUnloadRequest
+{
+  //@@  .. cpp:var:: string repository_name
+  //@@
+  //@@     The name of the repository from which the model was originally
+  //@@     loaded. If empty the repository is not considered.
+  //@@
+  string repository_name = 1;
+
+  //@@  .. cpp:var:: string repository_name
+  //@@
+  //@@     The name of the model to unload.
+  //@@
+  string model_name = 2;
+
+  //@@  .. cpp:var:: map<string,ModelRepositoryParameter> parameters
+  //@@
+  //@@     Optional model repository request parameters.
+  //@@
+  map<string, ModelRepositoryParameter> parameters = 3;
+}
+
+//@@
+//@@.. cpp:var:: message RepositoryModelUnloadResponse
+//@@
+//@@   Response message for RepositoryModelUnload.
+//@@
+message RepositoryModelUnloadResponse {}
+
+//@@
+//@@.. cpp:var:: message SystemSharedMemoryStatusRequest
+//@@
+//@@   Request message for SystemSharedMemoryStatus.
+//@@
+message SystemSharedMemoryStatusRequest
+{
+  //@@
+  //@@  .. cpp:var:: string name
+  //@@
+  //@@     The name of the region to get status for. If empty the
+  //@@     status is returned for all registered regions.
+  //@@
+  string name = 1;
+}
+
+//@@
+//@@.. cpp:var:: message SystemSharedMemoryStatusResponse
+//@@
+//@@   Response message for SystemSharedMemoryStatus.
+//@@
+message SystemSharedMemoryStatusResponse
+{
+  //@@
+  //@@  .. cpp:var:: message RegionStatus
+  //@@
+  //@@     Status for a shared memory region.
+  //@@
+  message RegionStatus
+  {
+    //@@
+    //@@    .. cpp:var:: string name
+    //@@
+    //@@       The name for the shared memory region.
+    //@@
+    string name = 1;
+
+    //@@    .. cpp:var:: string shared_memory_key
+    //@@
+    //@@       The key of the underlying memory object that contains the
+    //@@       shared memory region.
+    //@@
+    string key = 2;
+
+    //@@    .. cpp:var:: uint64 offset
+    //@@
+    //@@       Offset, in bytes, within the underlying memory object to
+    //@@       the start of the shared memory region.
+    //@@
+    uint64 offset = 3;
+
+    //@@    .. cpp:var:: uint64 byte_size
+    //@@
+    //@@       Size of the shared memory region, in bytes.
+    //@@
+    uint64 byte_size = 4;
+  }
+
+  //@@
+  //@@  .. cpp:var:: map<string,RegionStatus> regions
+  //@@
+  //@@     Status for each of the registered regions, indexed by
+  //@@     region name.
+  //@@
+  map<string, RegionStatus> regions = 1;
+}
+
+//@@
+//@@.. cpp:var:: message SystemSharedMemoryRegisterRequest
+//@@
+//@@   Request message for SystemSharedMemoryRegister.
+//@@
+message SystemSharedMemoryRegisterRequest
+{
+  //@@
+  //@@  .. cpp:var:: string name
+  //@@
+  //@@     The name of the region to register.
+  //@@
+  string name = 1;
+
+  //@@  .. cpp:var:: string shared_memory_key
+  //@@
+  //@@     The key of the underlying memory object that contains the
+  //@@     shared memory region.
+  //@@
+  string key = 2;
+
+  //@@  .. cpp:var:: uint64 offset
+  //@@
+  //@@     Offset, in bytes, within the underlying memory object to
+  //@@     the start of the shared memory region.
+  //@@
+  uint64 offset = 3;
+
+  //@@  .. cpp:var:: uint64 byte_size
+  //@@
+  //@@     Size of the shared memory region, in bytes.
+  //@@
+  uint64 byte_size = 4;
+}
+
+//@@
+//@@.. cpp:var:: message SystemSharedMemoryRegisterResponse
+//@@
+//@@   Response message for SystemSharedMemoryRegister.
+//@@
+message SystemSharedMemoryRegisterResponse {}
+
+//@@
+//@@.. cpp:var:: message SystemSharedMemoryUnregisterRequest
+//@@
+//@@   Request message for SystemSharedMemoryUnregister.
+//@@
+message SystemSharedMemoryUnregisterRequest
+{
+  //@@
+  //@@  .. cpp:var:: string name
+  //@@
+  //@@     The name of the system region to unregister. If empty
+  //@@     all system shared-memory regions are unregistered.
+  //@@
+  string name = 1;
+}
+
+//@@
+//@@.. cpp:var:: message SystemSharedMemoryUnregisterResponse
+//@@
+//@@   Response message for SystemSharedMemoryUnregister.
+//@@
+message SystemSharedMemoryUnregisterResponse {}
+
+//@@
+//@@.. cpp:var:: message CudaSharedMemoryStatusRequest
+//@@
+//@@   Request message for CudaSharedMemoryStatus.
+//@@
+message CudaSharedMemoryStatusRequest
+{
+  //@@
+  //@@  .. cpp:var:: string name
+  //@@
+  //@@     The name of the region to get status for. If empty the
+  //@@     status is returned for all registered regions.
+  //@@
+  string name = 1;
+}
+
+//@@
+//@@.. cpp:var:: message CudaSharedMemoryStatusResponse
+//@@
+//@@   Response message for CudaSharedMemoryStatus.
+//@@
+message CudaSharedMemoryStatusResponse
+{
+  //@@
+  //@@  .. cpp:var:: message RegionStatus
+  //@@
+  //@@     Status for a shared memory region.
+  //@@
+  message RegionStatus
+  {
+    //@@
+    //@@    .. cpp:var:: string name
+    //@@
+    //@@       The name for the shared memory region.
+    //@@
+    string name = 1;
+
+    //@@    .. cpp:var:: uin64 device_id
+    //@@
+    //@@       The GPU device ID where the cudaIPC handle was created.
+    //@@
+    uint64 device_id = 2;
+
+    //@@    .. cpp:var:: uint64 byte_size
+    //@@
+    //@@       Size of the shared memory region, in bytes.
+    //@@
+    uint64 byte_size = 3;
+  }
+
+  //@@
+  //@@  .. cpp:var:: map<string,RegionStatus> regions
+  //@@
+  //@@     Status for each of the registered regions, indexed by
+  //@@     region name.
+  //@@
+  map<string, RegionStatus> regions = 1;
+}
+
+//@@
+//@@.. cpp:var:: message CudaSharedMemoryRegisterRequest
+//@@
+//@@   Request message for CudaSharedMemoryRegister.
+//@@
+message CudaSharedMemoryRegisterRequest
+{
+  //@@
+  //@@  .. cpp:var:: string name
+  //@@
+  //@@     The name of the region to register.
+  //@@
+  string name = 1;
+
+  //@@  .. cpp:var:: bytes raw_handle
+  //@@
+  //@@     The raw serialized cudaIPC handle.
+  //@@
+  bytes raw_handle = 2;
+
+  //@@  .. cpp:var:: int64 device_id
+  //@@
+  //@@     The GPU device ID on which the cudaIPC handle was created.
+  //@@
+  int64 device_id = 3;
+
+  //@@  .. cpp:var:: uint64 byte_size
+  //@@
+  //@@     Size of the shared memory block, in bytes.
+  //@@
+  uint64 byte_size = 4;
+}
+
+//@@
+//@@.. cpp:var:: message CudaSharedMemoryRegisterResponse
+//@@
+//@@   Response message for CudaSharedMemoryRegister.
+//@@
+message CudaSharedMemoryRegisterResponse {}
+
+//@@
+//@@.. cpp:var:: message CudaSharedMemoryUnregisterRequest
+//@@
+//@@   Request message for CudaSharedMemoryUnregister.
+//@@
+message CudaSharedMemoryUnregisterRequest
+{
+  //@@
+  //@@  .. cpp:var:: string name
+  //@@
+  //@@     The name of the cuda region to unregister. If empty
+  //@@     all cuda shared-memory regions are unregistered.
+  //@@
+  string name = 1;
+}
+
+//@@
+//@@.. cpp:var:: message CudaSharedMemoryUnregisterResponse
+//@@
+//@@   Response message for CudaSharedMemoryUnregister.
+//@@
+message CudaSharedMemoryUnregisterResponse {}
+
+//@@
+//@@.. cpp:var:: message TraceSettingRequest
+//@@
+//@@   Request message for TraceSetting.
+//@@
+message TraceSettingRequest
+{
+  //@@
+  //@@  .. cpp:var:: message SettingValue
+  //@@
+  //@@     The values to be associated with a trace setting.
+  //@@     If no value is provided, the setting will be clear and
+  //@@     the global setting value will be used.
+  //@@
+  message SettingValue
+  {
+    //@@
+    //@@    .. cpp:var:: string value (repeated)
+    //@@
+    //@@       The value.
+    //@@
+    repeated string value = 1;
+  }
+
+  //@@  .. cpp:var:: map<string,SettingValue> settings
+  //@@
+  //@@     The new setting values to be updated,
+  //@@     settings that are not specified will remain unchanged.
+  //@@
+  map<string, SettingValue> settings = 1;
+
+  //@@
+  //@@  .. cpp:var:: string model_name
+  //@@
+  //@@     The name of the model to apply the new trace settings.
+  //@@     If not given, the new settings will be applied globally.
+  //@@
+  string model_name = 2;
+}
+
+//@@
+//@@.. cpp:var:: message TraceSettingResponse
+//@@
+//@@   Response message for TraceSetting.
+//@@
+message TraceSettingResponse
+{
+  //@@
+  //@@  .. cpp:var:: message SettingValue
+  //@@
+  //@@     The values to be associated with a trace setting.
+  //@@
+  message SettingValue
+  {
+    //@@
+    //@@    .. cpp:var:: string value (repeated)
+    //@@
+    //@@       The value.
+    //@@
+    repeated string value = 1;
+  }
+
+  //@@  .. cpp:var:: map<string,SettingValue> settings
+  //@@
+  //@@     The current trace settings, including any changes specified
+  //@@     by TraceSettingRequest.
+  //@@
+  map<string, SettingValue> settings = 1;
+}
+
+//@@
+//@@.. cpp:var:: message LogSettingsRequest
+//@@
+//@@   Request message for LogSettings.
+//@@
+message LogSettingsRequest
+{
+  message SettingValue
+  {
+    oneof parameter_choice
+    {
+      //@@    .. cpp:var:: bool bool_param
+      //@@
+      //@@       A boolean parameter value.
+      //@@
+      bool bool_param = 1;
+
+      //@@    .. cpp:var:: uint32 uint32_param
+      //@@
+      //@@       An uint32 parameter value.
+      //@@
+      uint32 uint32_param = 2;
+
+      //@@    .. cpp:var:: string string_param
+      //@@
+      //@@       A string parameter value.
+      //@@
+      string string_param = 3;
+    }
+  }
+  //@@  .. cpp:var:: map<string,SettingValue> settings
+  //@@
+  //@@     The current log settings.
+  //@@
+  map<string, SettingValue> settings = 1;
+}
+
+//@@
+//@@.. cpp:var:: message LogSettingsResponse
+//@@
+//@@   Response message for LogSettings.
+//@@
+message LogSettingsResponse
+{
+  message SettingValue
+  {
+    oneof parameter_choice
+    {
+      //@@    .. cpp:var:: bool bool_param
+      //@@
+      //@@       A boolean parameter value.
+      //@@
+      bool bool_param = 1;
+
+      //@@    .. cpp:var:: uint32 uint32_param
+      //@@
+      //@@       An int32 parameter value.
+      //@@
+      uint32 uint32_param = 2;
+
+      //@@    .. cpp:var:: string string_param
+      //@@
+      //@@       A string parameter value.
+      //@@
+      string string_param = 3;
+    }
+  }
+  //@@  .. cpp:var:: map<string,SettingValue> settings
+  //@@
+  //@@     The current log settings.
+  //@@
+  map<string, SettingValue> settings = 1;
+}
+

3rdparty/common-r22.12/protobuf/model_config.proto

+// Copyright 2018-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+//  * Redistributions of source code must retain the above copyright
+//    notice, this list of conditions and the following disclaimer.
+//  * Redistributions in binary form must reproduce the above copyright
+//    notice, this list of conditions and the following disclaimer in the
+//    documentation and/or other materials provided with the distribution.
+//  * Neither the name of NVIDIA CORPORATION nor the names of its
+//    contributors may be used to endorse or promote products derived
+//    from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Copyright (c) 2018, TensorFlow Authors. All rights reserved.
+
+syntax = "proto3";
+
+package inference;
+
+//@@.. cpp:namespace:: inference
+
+//@@
+//@@.. cpp:enum:: DataType
+//@@
+//@@   Data types supported for input and output tensors.
+//@@
+enum DataType {
+  //@@  .. cpp:enumerator:: DataType::INVALID = 0
+  TYPE_INVALID = 0;
+
+  //@@  .. cpp:enumerator:: DataType::BOOL = 1
+  TYPE_BOOL = 1;
+
+  //@@  .. cpp:enumerator:: DataType::UINT8 = 2
+  TYPE_UINT8 = 2;
+  //@@  .. cpp:enumerator:: DataType::UINT16 = 3
+  TYPE_UINT16 = 3;
+  //@@  .. cpp:enumerator:: DataType::UINT32 = 4
+  TYPE_UINT32 = 4;
+  //@@  .. cpp:enumerator:: DataType::UINT64 = 5
+  TYPE_UINT64 = 5;
+
+  //@@  .. cpp:enumerator:: DataType::INT8 = 6
+  TYPE_INT8 = 6;
+  //@@  .. cpp:enumerator:: DataType::INT16 = 7
+  TYPE_INT16 = 7;
+  //@@  .. cpp:enumerator:: DataType::INT32 = 8
+  TYPE_INT32 = 8;
+  //@@  .. cpp:enumerator:: DataType::INT64 = 9
+  TYPE_INT64 = 9;
+
+  //@@  .. cpp:enumerator:: DataType::FP16 = 10
+  TYPE_FP16 = 10;
+  //@@  .. cpp:enumerator:: DataType::FP32 = 11
+  TYPE_FP32 = 11;
+  //@@  .. cpp:enumerator:: DataType::FP64 = 12
+  TYPE_FP64 = 12;
+
+  //@@  .. cpp:enumerator:: DataType::STRING = 13
+  TYPE_STRING = 13;
+
+  //@@  .. cpp:enumerator:: DataType::BF16 = 14
+  TYPE_BF16 = 14;
+}
+
+//@@
+//@@  .. cpp:var:: message ModelRateLimiter
+//@@
+//@@     The specifications required by the rate limiter to properly
+//@@     schedule the inference requests across the different models
+//@@     and their instances.
+//@@
+message ModelRateLimiter
+{
+  //@@  .. cpp:var:: message Resource
+  //@@
+  //@@     The resource property.
+  //@@
+  message Resource
+  {
+    //@@  .. cpp:var:: string name
+    //@@
+    //@@     The name associated with the resource.
+    //@@
+    string name = 1;
+
+    //@@  .. cpp:var:: bool global
+    //@@
+    //@@     Whether or not the resource is global. If true then the resource
+    //@@     is assumed to be shared among the devices otherwise specified
+    //@@     count of the resource is assumed for each device associated
+    //@@     with the instance.
+    //@@
+    bool global = 2;
+
+    //@@  .. cpp:var:: uint32 count
+    //@@
+    //@@     The number of resources required for the execution of the model
+    //@@     instance.
+    //@@
+    uint32 count = 3;
+  }
+
+  //@@  .. cpp:var:: Resource resources (repeated)
+  //@@
+  //@@     The resources required to execute the request on a model instance.
+  //@@     Resources are just names with a corresponding count. The execution
+  //@@     of the instance will be blocked until the specificied resources are
+  //@@     available. By default an instance uses no rate-limiter resources.
+  //@@
+  repeated Resource resources = 1;
+
+  //@@  .. cpp:var:: uint32 priority
+  //@@
+  //@@     The optional weighting value to be used for prioritizing across
+  //@@     instances. An instance with priority 2 will be given 1/2 the
+  //@@     number of scheduling chances as an instance_group with priority
+  //@@     1. The default priority is 1. The priority of value 0 will be
+  //@@     treated as priority 1.
+  //@@
+  uint32 priority = 2;
+}
+
+//@@
+//@@.. cpp:var:: message ModelInstanceGroup
+//@@
+//@@   A group of one or more instances of a model and resources made
+//@@   available for those instances.
+//@@
+message ModelInstanceGroup
+{
+  //@@
+  //@@  .. cpp:enum:: Kind
+  //@@
+  //@@     Kind of this instance group.
+  //@@
+  enum Kind {
+    //@@    .. cpp:enumerator:: Kind::KIND_AUTO = 0
+    //@@
+    //@@       This instance group represents instances that can run on either
+    //@@       CPU or GPU. If all GPUs listed in 'gpus' are available then
+    //@@       instances will be created on GPU(s), otherwise instances will
+    //@@       be created on CPU.
+    //@@
+    KIND_AUTO = 0;
+
+    //@@    .. cpp:enumerator:: Kind::KIND_GPU = 1
+    //@@
+    //@@       This instance group represents instances that must run on the
+    //@@       GPU.
+    //@@
+    KIND_GPU = 1;
+
+    //@@    .. cpp:enumerator:: Kind::KIND_CPU = 2
+    //@@
+    //@@       This instance group represents instances that must run on the
+    //@@       CPU.
+    //@@
+    KIND_CPU = 2;
+
+    //@@    .. cpp:enumerator:: Kind::KIND_MODEL = 3
+    //@@
+    //@@       This instance group represents instances that should run on the
+    //@@       CPU and/or GPU(s) as specified by the model or backend itself.
+    //@@       The inference server will not override the model/backend
+    //@@       settings.
+    //@@
+    KIND_MODEL = 3;
+  }
+
+  //@@
+  //@@  .. cpp:var:: message SecondaryDevice
+  //@@
+  //@@     A secondary device required for a model instance.
+  //@@
+  message SecondaryDevice
+  {
+    //@@
+    //@@  .. cpp:enum:: SecondaryDeviceKind
+    //@@
+    //@@     The kind of the secondary device.
+    //@@
+    enum SecondaryDeviceKind {
+      //@@    .. cpp:enumerator:: SecondaryDeviceKind::KIND_NVDLA = 0
+      //@@
+      //@@       An NVDLA core. http://nvdla.org
+      //@@       Currently KIND_NVDLA is only supported by the TensorRT backend.
+      //@@
+      KIND_NVDLA = 0;
+    }
+
+    //@@  .. cpp:var:: SecondaryDeviceKind kind
+    //@@
+    //@@     The secondary device kind.
+    //@@
+    SecondaryDeviceKind kind = 1;
+
+    //@@  .. cpp:var:: int64 device_id
+    //@@
+    //@@     Identifier for the secondary device.
+    //@@
+    int64 device_id = 2;
+  }
+
+  //@@  .. cpp:var:: string name
+  //@@
+  //@@     Optional name of this group of instances. If not specified the
+  //@@     name will be formed as <model name>_<group number>. The name of
+  //@@     individual instances will be further formed by a unique instance
+  //@@     number and GPU index:
+  //@@
+  string name = 1;
+
+  //@@  .. cpp:var:: Kind kind
+  //@@
+  //@@     The kind of this instance group. Default is KIND_AUTO. If
+  //@@     KIND_AUTO or KIND_GPU then both 'count' and 'gpu' are valid and
+  //@@     may be specified. If KIND_CPU or KIND_MODEL only 'count' is valid
+  //@@     and 'gpu' cannot be specified.
+  //@@
+  Kind kind = 4;
+
+  //@@  .. cpp:var:: int32 count
+  //@@
+  //@@     For a group assigned to GPU, the number of instances created for
+  //@@     each GPU listed in 'gpus'. For a group assigned to CPU the number
+  //@@     of instances created. Default is 1.
+  int32 count = 2;
+
+  //@@  .. cpp:var:: ModelRateLimiter rate_limiter
+  //@@
+  //@@     The rate limiter specific settings to be associated with this
+  //@@     instance group. Optional, if not specified no rate limiting
+  //@@     will be applied to this instance group.
+  //@@
+  ModelRateLimiter rate_limiter = 6;
+
+  //@@  .. cpp:var:: int32 gpus (repeated)
+  //@@
+  //@@     GPU(s) where instances should be available. For each GPU listed,
+  //@@     'count' instances of the model will be available. Setting 'gpus'
+  //@@     to empty (or not specifying at all) is eqivalent to listing all
+  //@@     available GPUs.
+  //@@
+  repeated int32 gpus = 3;
+
+  //@@  .. cpp:var:: SecondaryDevice secondary_devices (repeated)
+  //@@
+  //@@     Secondary devices that are required by instances specified by this
+  //@@     instance group. Optional.
+  //@@
+  repeated SecondaryDevice secondary_devices = 8;
+
+  //@@  .. cpp:var:: string profile (repeated)
+  //@@
+  //@@     For TensorRT models containing multiple optimization profile, this
+  //@@     parameter specifies a set of optimization profiles available to this
+  //@@     instance group. The inference server will choose the optimal profile
+  //@@     based on the shapes of the input tensors. This field should lie
+  //@@     between 0 and <TotalNumberOfOptimizationProfilesInPlanModel> - 1
+  //@@     and be specified only for TensorRT backend, otherwise an error will
+  //@@     be generated. If not specified, the server will select the first
+  //@@     optimization profile by default.
+  //@@
+  repeated string profile = 5;
+
+  //@@  .. cpp:var:: bool passive
+  //@@
+  //@@     Whether the instances within this instance group will be accepting
+  //@@     inference requests from the scheduler. If true, the instances will
+  //@@     not be added to the scheduler. Default value is false.
+  //@@
+  bool passive = 7;
+
+  //@@  .. cpp:var:: string host_policy
+  //@@
+  //@@     The host policy name that the instance to be associated with.
+  //@@     The default value is set to reflect the device kind of the instance,
+  //@@     for instance, KIND_CPU is "cpu", KIND_MODEL is "model" and
+  //@@     KIND_GPU is "gpu_<gpu_id>".
+  //@@
+  string host_policy = 9;
+}
+
+//@@
+//@@.. cpp:var:: message ModelTensorReshape
+//@@
+//@@   Reshape specification for input and output tensors.
+//@@
+message ModelTensorReshape
+{
+  //@@  .. cpp:var:: int64 shape (repeated)
+  //@@
+  //@@     The shape to use for reshaping.
+  //@@
+  repeated int64 shape = 1;
+}
+
+//@@
+//@@.. cpp:var:: message ModelInput
+//@@
+//@@   An input required by the model.
+//@@
+message ModelInput
+{
+  //@@
+  //@@  .. cpp:enum:: Format
+  //@@
+  //@@     The format for the input.
+  //@@
+  enum Format {
+    //@@    .. cpp:enumerator:: Format::FORMAT_NONE = 0
+    //@@
+    //@@       The input has no specific format. This is the default.
+    //@@
+    FORMAT_NONE = 0;
+
+    //@@    .. cpp:enumerator:: Format::FORMAT_NHWC = 1
+    //@@
+    //@@       HWC image format. Tensors with this format require 3 dimensions
+    //@@       if the model does not support batching (max_batch_size = 0) or 4
+    //@@       dimensions if the model does support batching (max_batch_size
+    //@@       >= 1). In either case the 'dims' below should only specify the
+    //@@       3 non-batch dimensions (i.e. HWC or CHW).
+    //@@
+    FORMAT_NHWC = 1;
+
+    //@@    .. cpp:enumerator:: Format::FORMAT_NCHW = 2
+    //@@
+    //@@       CHW image format. Tensors with this format require 3 dimensions
+    //@@       if the model does not support batching (max_batch_size = 0) or 4
+    //@@       dimensions if the model does support batching (max_batch_size
+    //@@       >= 1). In either case the 'dims' below should only specify the
+    //@@       3 non-batch dimensions (i.e. HWC or CHW).
+    //@@
+    FORMAT_NCHW = 2;
+  }
+
+  //@@  .. cpp:var:: string name
+  //@@
+  //@@     The name of the input.
+  //@@
+  string name = 1;
+
+  //@@  .. cpp:var:: DataType data_type
+  //@@
+  //@@     The data-type of the input.
+  //@@
+  DataType data_type = 2;
+
+  //@@  .. cpp:var:: Format format
+  //@@
+  //@@     The format of the input. Optional.
+  //@@
+  Format format = 3;
+
+  //@@  .. cpp:var:: int64 dims (repeated)
+  //@@
+  //@@     The dimensions/shape of the input tensor that must be provided
+  //@@     when invoking the inference API for this model.
+  //@@
+  repeated int64 dims = 4;
+
+  //@@  .. cpp:var:: ModelTensorReshape reshape
+  //@@
+  //@@     The shape expected for this input by the backend. The input will
+  //@@     be reshaped to this before being presented to the backend. The
+  //@@     reshape must have the same number of elements as the input shape
+  //@@     specified by 'dims'. Optional.
+  //@@
+  ModelTensorReshape reshape = 5;
+
+  //@@  .. cpp:var:: bool is_shape_tensor
+  //@@
+  //@@     Whether or not the input is a shape tensor to the model. This field
+  //@@     is currently supported only for the TensorRT model. An error will be
+  //@@     generated if this specification does not comply with underlying
+  //@@     model.
+  //@@
+  bool is_shape_tensor = 6;
+
+  //@@  .. cpp:var:: bool allow_ragged_batch
+  //@@
+  //@@     Whether or not the input is allowed to be "ragged" in a dynamically
+  //@@     created batch. Default is false indicating that two requests will
+  //@@     only be batched if this tensor has the same shape in both requests.
+  //@@     True indicates that two requests can be batched even if this tensor
+  //@@     has a different shape in each request.
+  //@@
+  bool allow_ragged_batch = 7;
+
+  //@@  .. cpp:var:: bool optional
+  //@@
+  //@@     Whether or not the input is optional for the model execution.
+  //@@     If true, the input is not required in the inference request.
+  //@@     Default value is false.
+  //@@
+  bool optional = 8;
+}
+
+//@@
+//@@.. cpp:var:: message ModelOutput
+//@@
+//@@   An output produced by the model.
+//@@
+message ModelOutput
+{
+  //@@  .. cpp:var:: string name
+  //@@
+  //@@     The name of the output.
+  //@@
+  string name = 1;
+
+  //@@  .. cpp:var:: DataType data_type
+  //@@
+  //@@     The data-type of the output.
+  //@@
+  DataType data_type = 2;
+
+  //@@  .. cpp:var:: int64 dims (repeated)
+  //@@
+  //@@     The dimensions/shape of the output tensor.
+  //@@
+  repeated int64 dims = 3;
+
+  //@@  .. cpp:var:: ModelTensorReshape reshape
+  //@@
+  //@@     The shape produced for this output by the backend. The output will
+  //@@     be reshaped from this to the shape specifed in 'dims' before being
+  //@@     returned in the inference response. The reshape must have the same
+  //@@     number of elements as the output shape specified by 'dims'. Optional.
+  //@@
+  ModelTensorReshape reshape = 5;
+
+  //@@  .. cpp:var:: string label_filename
+  //@@
+  //@@     The label file associated with this output. Should be specified only
+  //@@     for outputs that represent classifications. Optional.
+  //@@
+  string label_filename = 4;
+
+
+  //@@  .. cpp:var:: bool is_shape_tensor
+  //@@
+  //@@     Whether or not the output is a shape tensor to the model. This field
+  //@@     is currently supported only for the TensorRT model. An error will be
+  //@@     generated if this specification does not comply with underlying
+  //@@     model.
+  //@@
+  bool is_shape_tensor = 6;
+}
+
+//@@  .. cpp:var:: message BatchInput
+//@@
+//@@     A batch input is an additional input that must be added by
+//@@     the backend based on all the requests in a batch.
+//@@
+message BatchInput
+{
+  //@@
+  //@@    .. cpp:enum:: Kind
+  //@@
+  //@@       The kind of the batch input.
+  //@@
+  enum Kind {
+    //@@      .. cpp:enumerator:: Kind::BATCH_ELEMENT_COUNT = 0
+    //@@
+    //@@         The element count of the 'source_input' will be added as
+    //@@         input with shape [1].
+    //@@
+    BATCH_ELEMENT_COUNT = 0;
+
+    //@@      .. cpp:enumerator:: Kind::BATCH_ACCUMULATED_ELEMENT_COUNT = 1
+    //@@
+    //@@         The accumulated element count of the 'source_input' will be
+    //@@         added as input with shape [1]. For example, if there is a
+    //@@         batch of two request, each with 2 elements, an input of value
+    //@@         2 will be added to the first request, and an input of value
+    //@@         4 will be added to the second request.
+    //@@
+    BATCH_ACCUMULATED_ELEMENT_COUNT = 1;
+
+    //@@      .. cpp:enumerator::
+    //@@         Kind::BATCH_ACCUMULATED_ELEMENT_COUNT_WITH_ZERO = 2
+    //@@
+    //@@         The accumulated element count of the 'source_input' will be
+    //@@         added as input with shape [1], except for the first request
+    //@@         in the batch. For the first request in the batch, the input
+    //@@         will have shape [2] where the first element is value 0.
+    //@@
+    BATCH_ACCUMULATED_ELEMENT_COUNT_WITH_ZERO = 2;
+
+    //@@      .. cpp:enumerator:: Kind::BATCH_MAX_ELEMENT_COUNT_AS_SHAPE = 3
+    //@@
+    //@@         Among the requests in the batch, the max element count of the
+    //@@         'source_input' will be added as input with shape
+    //@@         [max_element_count] for the first request in the batch.
+    //@@         For other requests, such input will be with shape [0].
+    //@@         The data of the tensor will be uninitialized.
+    //@@
+    BATCH_MAX_ELEMENT_COUNT_AS_SHAPE = 3;
+
+    //@@      .. cpp:enumerator:: Kind::BATCH_ITEM_SHAPE = 4
+    //@@
+    //@@         Among the requests in the batch, the shape of the
+    //@@         'source_input' will be added as input with shape
+    //@@         [batch_size, len(input_dim)]. For example, if one
+    //@@         batch-2 input with shape [3, 1] and batch-1 input
+    //@@         with shape [2, 2] are batched, the batch input will
+    //@@         have shape [3, 2] and value [ [3, 1], [3, 1], [2, 2]].
+    //@@
+    BATCH_ITEM_SHAPE = 4;
+
+    //@@      .. cpp:enumerator:: Kind::BATCH_ITEM_SHAPE_FLATTEN = 5
+    //@@
+    //@@         Among the requests in the batch, the shape of the
+    //@@         'source_input' will be added as input with single dimensional
+    //@@         shape [batch_size * len(input_dim)]. For example, if one
+    //@@         batch-2 input with shape [3, 1] and batch-1 input
+    //@@         with shape [2, 2] are batched, the batch input will
+    //@@         have shape [6] and value [3, 1, 3, 1, 2, 2].
+    //@@
+    BATCH_ITEM_SHAPE_FLATTEN = 5;
+  }
+
+  //@@    .. cpp:var:: Kind kind
+  //@@
+  //@@       The kind of this batch input.
+  //@@
+  Kind kind = 1;
+
+  //@@    .. cpp:var:: string target_name (repeated)
+  //@@
+  //@@       The name of the model inputs that the backend will create
+  //@@       for this batch input.
+  //@@
+  repeated string target_name = 2;
+
+  //@@    .. cpp:var:: DataType data_type
+  //@@
+  //@@       The input's datatype. The data type can be TYPE_INT32 or
+  //@@       TYPE_FP32.
+  //@@
+  DataType data_type = 3;
+
+  //@@    .. cpp:var:: string source_input (repeated)
+  //@@
+  //@@       The backend derives the value for each batch input from one or
+  //@@       more other inputs. 'source_input' gives the names of those
+  //@@       inputs.
+  //@@
+  repeated string source_input = 4;
+}
+
+//@@.. cpp:var:: message BatchOutput
+//@@
+//@@   A batch output is an output produced by the model that must be handled
+//@@   differently by the backend based on all the requests in a batch.
+//@@
+message BatchOutput
+{
+  //@@
+  //@@  .. cpp:enum:: Kind
+  //@@
+  //@@     The kind of the batch output.
+  //@@
+  enum Kind {
+    //@@    .. cpp:enumerator:: Kind::BATCH_SCATTER_WITH_INPUT_SHAPE = 0
+    //@@
+    //@@       The output should be scattered according to the shape of
+    //@@       'source_input'. The dynamic dimension of the output will
+    //@@       be set to the value of the same dimension in the input.
+    //@@
+    BATCH_SCATTER_WITH_INPUT_SHAPE = 0;
+  }
+
+  //@@  .. cpp:var:: string target_name (repeated)
+  //@@
+  //@@     The name of the outputs to be produced by this batch output
+  //@@     specification.
+  //@@
+  repeated string target_name = 1;
+
+  //@@  .. cpp:var:: Kind kind
+  //@@
+  //@@     The kind of this batch output.
+  //@@
+  Kind kind = 2;
+
+  //@@  .. cpp:var:: string source_input (repeated)
+  //@@
+  //@@     The backend derives each batch output from one or more inputs.
+  //@@     'source_input' gives the names of those inputs.
+  //@@
+  repeated string source_input = 3;
+}
+
+//@@
+//@@.. cpp:var:: message ModelVersionPolicy
+//@@
+//@@   Policy indicating which versions of a model should be made
+//@@   available by the inference server.
+//@@
+message ModelVersionPolicy
+{
+  //@@  .. cpp:var:: message Latest
+  //@@
+  //@@     Serve only the latest version(s) of a model. This is
+  //@@     the default policy.
+  //@@
+  message Latest
+  {
+    //@@    .. cpp:var:: uint32 num_versions
+    //@@
+    //@@       Serve only the 'num_versions' highest-numbered versions. T
+    //@@       The default value of 'num_versions' is 1, indicating that by
+    //@@       default only the single highest-number version of a
+    //@@       model will be served.
+    //@@
+    uint32 num_versions = 1;
+  }
+
+  //@@  .. cpp:var:: message All
+  //@@
+  //@@     Serve all versions of the model.
+  //@@
+  message All {}
+
+  //@@  .. cpp:var:: message Specific
+  //@@
+  //@@     Serve only specific versions of the model.
+  //@@
+  message Specific
+  {
+    //@@    .. cpp:var:: int64 versions (repeated)
+    //@@
+    //@@       The specific versions of the model that will be served.
+    //@@
+    repeated int64 versions = 1;
+  }
+
+  //@@  .. cpp:var:: oneof policy_choice
+  //@@
+  //@@     Each model must implement only a single version policy. The
+  //@@     default policy is 'Latest'.
+  //@@
+  oneof policy_choice
+  {
+    //@@    .. cpp:var:: Latest latest
+    //@@
+    //@@       Serve only latest version(s) of the model.
+    //@@
+    Latest latest = 1;
+
+    //@@    .. cpp:var:: All all
+    //@@
+    //@@       Serve all versions of the model.
+    //@@
+    All all = 2;
+
+    //@@    .. cpp:var:: Specific specific
+    //@@
+    //@@       Serve only specific version(s) of the model.
+    //@@
+    Specific specific = 3;
+  }
+}
+
+//@@
+//@@.. cpp:var:: message ModelOptimizationPolicy
+//@@
+//@@   Optimization settings for a model. These settings control if/how a
+//@@   model is optimized and prioritized by the backend framework when
+//@@   it is loaded.
+//@@
+message ModelOptimizationPolicy
+{
+  //@@
+  //@@  .. cpp:var:: message Graph
+  //@@
+  //@@     Enable generic graph optimization of the model. If not specified
+  //@@     the framework's default level of optimization is used. Supports
+  //@@     TensorFlow graphdef and savedmodel and Onnx models. For TensorFlow
+  //@@     causes XLA to be enabled/disabled for the model. For Onnx defaults
+  //@@     to enabling all optimizations, -1 enables only basic optimizations,
+  //@@     +1 enables only basic and extended optimizations.
+  //@@
+  message Graph
+  {
+    //@@    .. cpp:var:: int32 level
+    //@@
+    //@@       The optimization level. Defaults to 0 (zero) if not specified.
+    //@@
+    //@@         - -1: Disabled
+    //@@         -  0: Framework default
+    //@@         -  1+: Enable optimization level (greater values indicate
+    //@@            higher optimization levels)
+    //@@
+    int32 level = 1;
+  }
+
+  //@@
+  //@@  .. cpp:enum:: ModelPriority
+  //@@
+  //@@     Model priorities. A model will be given scheduling and execution
+  //@@     preference over models at lower priorities. Current model
+  //@@     priorities only work for TensorRT models.
+  //@@
+  enum ModelPriority {
+    //@@    .. cpp:enumerator:: ModelPriority::PRIORITY_DEFAULT = 0
+    //@@
+    //@@       The default model priority.
+    //@@
+    PRIORITY_DEFAULT = 0;
+
+    //@@    .. cpp:enumerator:: ModelPriority::PRIORITY_MAX = 1
+    //@@
+    //@@       The maximum model priority.
+    //@@
+    PRIORITY_MAX = 1;
+
+    //@@    .. cpp:enumerator:: ModelPriority::PRIORITY_MIN = 2
+    //@@
+    //@@       The minimum model priority.
+    //@@
+    PRIORITY_MIN = 2;
+  }
+
+  //@@
+  //@@  .. cpp:var:: message Cuda
+  //@@
+  //@@     CUDA-specific optimization settings.
+  //@@
+  message Cuda
+  {
+    //@@    .. cpp:var:: message GraphSpec
+    //@@
+    //@@       Specification of the CUDA graph to be captured.
+    //@@
+    message GraphSpec
+    {
+      //@@      .. cpp:var:: message Dims
+      //@@
+      //@@         Specification of tensor dimension.
+      //@@
+      message Shape
+      {
+        //@@        .. cpp:var:: int64 dim (repeated)
+        //@@
+        //@@           The dimension.
+        //@@
+        repeated int64 dim = 1;
+      }
+
+      message LowerBound
+      {
+        //@@      .. cpp:var:: int32 batch_size
+        //@@
+        //@@         The batch size of the CUDA graph. If 'max_batch_size' is 0,
+        //@@         'batch_size' must be set to 0. Otherwise, 'batch_size' must
+        //@@         be set to value between 1 and 'max_batch_size'.
+        //@@
+        int32 batch_size = 1;
+
+        //@@      .. cpp:var:: map<string, Shape> input
+        //@@
+        //@@         The specification of the inputs. 'Shape' is the shape of
+        //@@         the input without batching dimension.
+        //@@
+        map<string, Shape> input = 2;
+      }
+
+      //@@      .. cpp:var:: int32 batch_size
+      //@@
+      //@@         The batch size of the CUDA graph. If 'max_batch_size' is 0,
+      //@@         'batch_size' must be set to 0. Otherwise, 'batch_size' must
+      //@@         be set to value between 1 and 'max_batch_size'.
+      //@@
+      int32 batch_size = 1;
+
+      //@@      .. cpp:var:: map<string, Shape> input
+      //@@
+      //@@         The specification of the inputs. 'Shape' is the shape of the
+      //@@         input without batching dimension.
+      //@@
+      map<string, Shape> input = 2;
+
+      //@@      .. cpp:var:: LowerBound graph_lower_bound
+      //@@
+      //@@         Specify the lower bound of the CUDA graph. Optional.
+      //@@         If specified, the graph can be used for input shapes and
+      //@@         batch sizes that are in closed interval between the lower
+      //@@         bound specification and graph specification. For dynamic
+      //@@         shape model, this allows CUDA graphs to be launched
+      //@@         frequently without capturing all possible shape combinations.
+      //@@         However, using graph for shape combinations different from
+      //@@         the one used for capturing introduces uninitialized data for
+      //@@         execution and it may distort the inference result if
+      //@@         the model is sensitive to uninitialized data.
+      //@@
+      LowerBound graph_lower_bound = 3;
+    }
+
+    //@@    .. cpp:var:: bool graphs
+    //@@
+    //@@       Use CUDA graphs API to capture model operations and execute
+    //@@       them more efficiently. Default value is false.
+    //@@       Currently only recognized by TensorRT backend.
+    //@@
+    bool graphs = 1;
+
+    //@@    .. cpp:var:: bool busy_wait_events
+    //@@
+    //@@       Use busy-waiting to synchronize CUDA events to achieve minimum
+    //@@       latency from event complete to host thread to be notified, with
+    //@@       the cost of high CPU load. Default value is false.
+    //@@       Currently only recognized by TensorRT backend.
+    //@@
+    bool busy_wait_events = 2;
+
+    //@@    .. cpp:var:: GraphSpec graph_spec (repeated)
+    //@@
+    //@@       Specification of the CUDA graph to be captured. If not specified
+    //@@       and 'graphs' is true, the default CUDA graphs will be captured
+    //@@       based on model settings.
+    //@@       Currently only recognized by TensorRT backend.
+    //@@
+    repeated GraphSpec graph_spec = 3;
+
+    //@@    .. cpp:var:: bool output_copy_stream
+    //@@
+    //@@       Uses a CUDA stream separate from the inference stream to copy the
+    //@@       output to host. However, be aware that setting this option to
+    //@@       true will lead to an increase in the memory consumption of the
+    //@@       model as Triton will allocate twice as much GPU memory for its
+    //@@       I/O tensor buffers. Default value is false.
+    //@@       Currently only recognized by TensorRT backend.
+    //@@
+    bool output_copy_stream = 4;
+  }
+
+  //@@
+  //@@  .. cpp:var:: message ExecutionAccelerators
+  //@@
+  //@@     Specify the preferred execution accelerators to be used to execute
+  //@@     the model. Currently only recognized by ONNX Runtime backend and
+  //@@     TensorFlow backend.
+  //@@
+  //@@     For ONNX Runtime backend, it will deploy the model with the execution
+  //@@     accelerators by priority, the priority is determined based on the
+  //@@     order that they are set, i.e. the provider at the front has highest
+  //@@     priority. Overall, the priority will be in the following order:
+  //@@         <gpu_execution_accelerator> (if instance is on GPU)
+  //@@         CUDA Execution Provider     (if instance is on GPU)
+  //@@         <cpu_execution_accelerator>
+  //@@         Default CPU Execution Provider
+  //@@
+  message ExecutionAccelerators
+  {
+    //@@
+    //@@  .. cpp:var:: message Accelerator
+    //@@
+    //@@     Specify the accelerator to be used to execute the model.
+    //@@     Accelerator with the same name may accept different parameters
+    //@@     depending on the backends.
+    //@@
+    message Accelerator
+    {
+      //@@    .. cpp:var:: string name
+      //@@
+      //@@       The name of the execution accelerator.
+      //@@
+      string name = 1;
+
+      //@@    .. cpp:var:: map<string, string> parameters
+      //@@
+      //@@       Additional paremeters used to configure the accelerator.
+      //@@
+      map<string, string> parameters = 2;
+    }
+
+    //@@    .. cpp:var:: Accelerator gpu_execution_accelerator (repeated)
+    //@@
+    //@@       The preferred execution provider to be used if the model instance
+    //@@       is deployed on GPU.
+    //@@
+    //@@       For ONNX Runtime backend, possible value is "tensorrt" as name,
+    //@@       and no parameters are required.
+    //@@
+    //@@       For TensorFlow backend, possible values are "tensorrt",
+    //@@       "auto_mixed_precision", "gpu_io".
+    //@@
+    //@@       For "tensorrt", the following parameters can be specified:
+    //@@         "precision_mode": The precision used for optimization.
+    //@@         Allowed values are "FP32" and "FP16". Default value is "FP32".
+    //@@
+    //@@         "max_cached_engines": The maximum number of cached TensorRT
+    //@@         engines in dynamic TensorRT ops. Default value is 100.
+    //@@
+    //@@         "minimum_segment_size": The smallest model subgraph that will
+    //@@         be considered for optimization by TensorRT. Default value is 3.
+    //@@
+    //@@         "max_workspace_size_bytes": The maximum GPU memory the model
+    //@@         can use temporarily during execution. Default value is 1GB.
+    //@@
+    //@@       For "auto_mixed_precision", no parameters are required. If set,
+    //@@       the model will try to use FP16 for better performance.
+    //@@       This optimization can not be set with "tensorrt".
+    //@@
+    //@@       For "gpu_io", no parameters are required. If set, the model will
+    //@@       be executed using TensorFlow Callable API to set input and output
+    //@@       tensors in GPU memory if possible, which can reduce data transfer
+    //@@       overhead if the model is used in ensemble. However, the Callable
+    //@@       object will be created on model creation and it will request all
+    //@@       outputs for every model execution, which may impact the
+    //@@       performance if a request does not require all outputs. This
+    //@@       optimization will only take affect if the model instance is
+    //@@       created with KIND_GPU.
+    //@@
+    repeated Accelerator gpu_execution_accelerator = 1;
+
+    //@@    .. cpp:var:: Accelerator cpu_execution_accelerator (repeated)
+    //@@
+    //@@       The preferred execution provider to be used if the model instance
+    //@@       is deployed on CPU.
+    //@@
+    //@@       For ONNX Runtime backend, possible value is "openvino" as name,
+    //@@       and no parameters are required.
+    //@@
+    repeated Accelerator cpu_execution_accelerator = 2;
+  }
+
+  //@@
+  //@@  .. cpp:var:: message PinnedMemoryBuffer
+  //@@
+  //@@     Specify whether to use a pinned memory buffer when transferring data
+  //@@     between non-pinned system memory and GPU memory. Using a pinned
+  //@@     memory buffer for system from/to GPU transfers will typically provide
+  //@@     increased performance. For example, in the common use case where the
+  //@@     request provides inputs and delivers outputs via non-pinned system
+  //@@     memory, if the model instance accepts GPU IOs, the inputs will be
+  //@@     processed by two copies: from non-pinned system memory to pinned
+  //@@     memory, and from pinned memory to GPU memory. Similarly, pinned
+  //@@     memory will be used for delivering the outputs.
+  //@@
+  message PinnedMemoryBuffer
+  {
+    //@@    .. cpp:var:: bool enable
+    //@@
+    //@@       Use pinned memory buffer. Default is true.
+    //@@
+    bool enable = 1;
+  }
+
+  //@@  .. cpp:var:: Graph graph
+  //@@
+  //@@     The graph optimization setting for the model. Optional.
+  //@@
+  Graph graph = 1;
+
+  //@@  .. cpp:var:: ModelPriority priority
+  //@@
+  //@@     The priority setting for the model. Optional.
+  //@@
+  ModelPriority priority = 2;
+
+  //@@  .. cpp:var:: Cuda cuda
+  //@@
+  //@@     CUDA-specific optimization settings. Optional.
+  //@@
+  Cuda cuda = 3;
+
+  //@@  .. cpp:var:: ExecutionAccelerators execution_accelerators
+  //@@
+  //@@     The accelerators used for the model. Optional.
+  //@@
+  ExecutionAccelerators execution_accelerators = 4;
+
+  //@@  .. cpp:var:: PinnedMemoryBuffer input_pinned_memory
+  //@@
+  //@@     Use pinned memory buffer when the data transfer for inputs
+  //@@     is between GPU memory and non-pinned system memory.
+  //@@     Default is true.
+  //@@
+  PinnedMemoryBuffer input_pinned_memory = 5;
+
+  //@@  .. cpp:var:: PinnedMemoryBuffer output_pinned_memory
+  //@@
+  //@@     Use pinned memory buffer when the data transfer for outputs
+  //@@     is between GPU memory and non-pinned system memory.
+  //@@     Default is true.
+  //@@
+  PinnedMemoryBuffer output_pinned_memory = 6;
+
+  //@@  .. cpp:var:: uint32 gather_kernel_buffer_threshold
+  //@@
+  //@@     The backend may use a gather kernel to gather input data if the
+  //@@     device has direct access to the source buffer and the destination
+  //@@     buffer. In such case, the gather kernel will be used only if the
+  //@@     number of buffers to be gathered is greater or equal to
+  //@@     the specifed value. If 0, the gather kernel will be disabled.
+  //@@     Default value is 0.
+  //@@     Currently only recognized by TensorRT backend.
+  //@@
+  uint32 gather_kernel_buffer_threshold = 7;
+
+  //@@  .. cpp:var:: bool eager_batching
+  //@@
+  //@@     Start preparing the next batch before the model instance is ready
+  //@@     for the next inference. This option can be used to overlap the
+  //@@     batch preparation with model execution, with the trade-off that
+  //@@     the next batch might be smaller than what it could have been.
+  //@@     Default value is false.
+  //@@     Currently only recognized by TensorRT backend.
+  //@@
+  bool eager_batching = 8;
+}
+
+//@@
+//@@.. cpp:var:: message ModelQueuePolicy
+//@@
+//@@   Queue policy for inference requests.
+//@@
+message ModelQueuePolicy
+{
+  //@@
+  //@@  .. cpp:enum:: TimeoutAction
+  //@@
+  //@@     The action applied to timed-out requests.
+  //@@
+  enum TimeoutAction {
+    //@@    .. cpp:enumerator:: Action::REJECT = 0
+    //@@
+    //@@       Reject the request and return error message accordingly.
+    //@@
+    REJECT = 0;
+
+    //@@    .. cpp:enumerator:: Action::DELAY = 1
+    //@@
+    //@@       Delay the request until all other requests at the same
+    //@@       (or higher) priority levels that have not reached their timeouts
+    //@@       are processed. A delayed request will eventually be processed,
+    //@@       but may be delayed indefinitely due to newly arriving requests.
+    //@@
+    DELAY = 1;
+  }
+
+  //@@
+  //@@  .. cpp:var:: TimeoutAction timeout_action
+  //@@
+  //@@     The action applied to timed-out request.
+  //@@     The default action is REJECT.
+  //@@
+  TimeoutAction timeout_action = 1;
+
+  //@@
+  //@@  .. cpp:var:: uint64 default_timeout_microseconds
+  //@@
+  //@@     The default timeout for every request, in microseconds.
+  //@@     The default value is 0 which indicates that no timeout is set.
+  //@@
+  uint64 default_timeout_microseconds = 2;
+
+  //@@
+  //@@  .. cpp:var:: bool allow_timeout_override
+  //@@
+  //@@     Whether individual request can override the default timeout value.
+  //@@     When true, individual requests can set a timeout that is less than
+  //@@     the default timeout value but may not increase the timeout.
+  //@@     The default value is false.
+  //@@
+  bool allow_timeout_override = 3;
+
+  //@@
+  //@@  .. cpp:var:: uint32 max_queue_size
+  //@@
+  //@@     The maximum queue size for holding requests. A request will be
+  //@@     rejected immediately if it can't be enqueued because the queue is
+  //@@     full. The default value is 0 which indicates that no maximum
+  //@@     queue size is enforced.
+  //@@
+  uint32 max_queue_size = 4;
+}
+
+//@@
+//@@.. cpp:var:: message ModelDynamicBatching
+//@@
+//@@   Dynamic batching configuration. These settings control how dynamic
+//@@   batching operates for the model.
+//@@
+message ModelDynamicBatching
+{
+  //@@  .. cpp:var:: int32 preferred_batch_size (repeated)
+  //@@
+  //@@     Preferred batch sizes for dynamic batching. If a batch of one of
+  //@@     these sizes can be formed it will be executed immediately.  If
+  //@@     not specified a preferred batch size will be chosen automatically
+  //@@     based on model and GPU characteristics.
+  //@@
+  repeated int32 preferred_batch_size = 1;
+
+  //@@  .. cpp:var:: uint64 max_queue_delay_microseconds
+  //@@
+  //@@     The maximum time, in microseconds, a request will be delayed in
+  //@@     the scheduling queue to wait for additional requests for
+  //@@     batching. Default is 0.
+  //@@
+  uint64 max_queue_delay_microseconds = 2;
+
+  //@@  .. cpp:var:: bool preserve_ordering
+  //@@
+  //@@     Should the dynamic batcher preserve the ordering of responses to
+  //@@     match the order of requests received by the scheduler. Default is
+  //@@     false. If true, the responses will be returned in the same order as
+  //@@     the order of requests sent to the scheduler. If false, the responses
+  //@@     may be returned in arbitrary order. This option is specifically
+  //@@     needed when a sequence of related inference requests (i.e. inference
+  //@@     requests with the same correlation ID) are sent to the dynamic
+  //@@     batcher to ensure that the sequence responses are in the correct
+  //@@     order.
+  //@@
+  bool preserve_ordering = 3;
+
+  //@@  .. cpp:var:: uint32 priority_levels
+  //@@
+  //@@     The number of priority levels to be enabled for the model,
+  //@@     the priority level starts from 1 and 1 is the highest priority.
+  //@@     Requests are handled in priority order with all priority 1 requests
+  //@@     processed before priority 2, all priority 2 requests processed before
+  //@@     priority 3, etc. Requests with the same priority level will be
+  //@@     handled in the order that they are received.
+  //@@
+  uint32 priority_levels = 4;
+
+  //@@  .. cpp:var:: uint32 default_priority_level
+  //@@
+  //@@     The priority level used for requests that don't specify their
+  //@@     priority. The value must be in the range [ 1, 'priority_levels' ].
+  //@@
+  uint32 default_priority_level = 5;
+
+  //@@  .. cpp:var:: ModelQueuePolicy default_queue_policy
+  //@@
+  //@@     The default queue policy used for requests that don't require
+  //@@     priority handling and requests that specify priority levels where
+  //@@     there is no specific policy given. If not specified, a policy with
+  //@@     default field values will be used.
+  //@@
+  ModelQueuePolicy default_queue_policy = 6;
+
+  //@@  .. cpp:var:: map<uint32, ModelQueuePolicy> priority_queue_policy
+  //@@
+  //@@     Specify the queue policy for the priority level. The default queue
+  //@@     policy will be used if a priority level doesn't specify a queue
+  //@@     policy.
+  //@@
+  map<uint32, ModelQueuePolicy> priority_queue_policy = 7;
+}
+
+//@@
+//@@.. cpp:var:: message ModelSequenceBatching
+//@@
+//@@   Sequence batching configuration. These settings control how sequence
+//@@   batching operates for the model.
+//@@
+message ModelSequenceBatching
+{
+  //@@  .. cpp:var:: message Control
+  //@@
+  //@@     A control is a signal that the sequence batcher uses to
+  //@@     communicate with a backend.
+  //@@
+  message Control
+  {
+    //@@
+    //@@    .. cpp:enum:: Kind
+    //@@
+    //@@       The kind of the control.
+    //@@
+    enum Kind {
+      //@@      .. cpp:enumerator:: Kind::CONTROL_SEQUENCE_START = 0
+      //@@
+      //@@         A new sequence is/is-not starting. If true a sequence is
+      //@@         starting, if false a sequence is continuing. Must
+      //@@         specify either int32_false_true, fp32_false_true or
+      //@@         bool_false_true for this control. This control is optional.
+      //@@
+      CONTROL_SEQUENCE_START = 0;
+
+      //@@      .. cpp:enumerator:: Kind::CONTROL_SEQUENCE_READY = 1
+      //@@
+      //@@         A sequence is/is-not ready for inference. If true the
+      //@@         input tensor data is valid and should be used. If false
+      //@@         the input tensor data is invalid and inferencing should
+      //@@         be "skipped". Must specify either int32_false_true,
+      //@@         fp32_false_true or bool_false_true for this control. This
+      //@@         control is optional.
+      //@@
+      CONTROL_SEQUENCE_READY = 1;
+
+      //@@      .. cpp:enumerator:: Kind::CONTROL_SEQUENCE_END = 2
+      //@@
+      //@@         A sequence is/is-not ending. If true a sequence is
+      //@@         ending, if false a sequence is continuing. Must specify
+      //@@         either int32_false_true, fp32_false_true or bool_false_true
+      //@@         for this control. This control is optional.
+      //@@
+      CONTROL_SEQUENCE_END = 2;
+
+      //@@      .. cpp:enumerator:: Kind::CONTROL_SEQUENCE_CORRID = 3
+      //@@
+      //@@         The correlation ID of the sequence. The correlation ID
+      //@@         is an uint64_t value that is communicated in whole or
+      //@@         in part by the tensor. The tensor's datatype must be
+      //@@         specified by data_type and must be TYPE_UINT64, TYPE_INT64,
+      //@@         TYPE_UINT32 or TYPE_INT32. If a 32-bit datatype is specified
+      //@@         the correlation ID will be truncated to the low-order 32
+      //@@         bits. This control is optional.
+      //@@
+      CONTROL_SEQUENCE_CORRID = 3;
+    }
+
+    //@@    .. cpp:var:: Kind kind
+    //@@
+    //@@       The kind of this control.
+    //@@
+    Kind kind = 1;
+
+    //@@    .. cpp:var:: int32 int32_false_true (repeated)
+    //@@
+    //@@       The control's true and false setting is indicated by setting
+    //@@       a value in an int32 tensor. The tensor must be a
+    //@@       1-dimensional tensor with size equal to the batch size of
+    //@@       the request. 'int32_false_true' must have two entries: the
+    //@@       first the false value and the second the true value.
+    //@@
+    repeated int32 int32_false_true = 2;
+
+    //@@    .. cpp:var:: float fp32_false_true (repeated)
+    //@@
+    //@@       The control's true and false setting is indicated by setting
+    //@@       a value in a fp32 tensor. The tensor must be a
+    //@@       1-dimensional tensor with size equal to the batch size of
+    //@@       the request. 'fp32_false_true' must have two entries: the
+    //@@       first the false value and the second the true value.
+    //@@
+    repeated float fp32_false_true = 3;
+
+    //@@    .. cpp:var:: bool bool_false_true (repeated)
+    //@@
+    //@@       The control's true and false setting is indicated by setting
+    //@@       a value in a bool tensor. The tensor must be a
+    //@@       1-dimensional tensor with size equal to the batch size of
+    //@@       the request. 'bool_false_true' must have two entries: the
+    //@@       first the false value and the second the true value.
+    //@@
+    repeated bool bool_false_true = 5;
+
+    //@@    .. cpp:var:: DataType data_type
+    //@@
+    //@@       The control's datatype.
+    //@@
+    DataType data_type = 4;
+  }
+
+  //@@  .. cpp:var:: message ControlInput
+  //@@
+  //@@     The sequence control values to communicate by a model input.
+  //@@
+  message ControlInput
+  {
+    //@@    .. cpp:var:: string name
+    //@@
+    //@@       The name of the model input.
+    //@@
+    string name = 1;
+
+    //@@    .. cpp:var:: Control control (repeated)
+    //@@
+    //@@       The control value(s) that should be communicated to the
+    //@@       model using this model input.
+    //@@
+    repeated Control control = 2;
+  }
+
+  //@@
+  //@@  .. cpp:var:: message InitialState
+  //@@
+  //@@     Settings used to initialize data for implicit state.
+  //@@
+  message InitialState
+  {
+    //@@      .. cpp:var:: DataType data_type
+    //@@
+    //@@         The data-type of the state.
+    //@@
+    DataType data_type = 1;
+
+    //@@      .. cpp:var:: int64 dims (repeated)
+    //@@
+    //@@         The shape of the state tensor, not including the batch dimension.
+    //@@
+    repeated int64 dims = 2;
+
+    //@@      .. cpp:var:: oneof state_data
+    //@@
+    //@@         Specify how the initial state data is generated.
+    //@@
+    oneof state_data
+    {
+      //@@
+      //@@      .. cpp:var:: bool zero_data
+      //@@
+      //@@         The identifier for using zeros as initial state data.
+      //@@         Note that the value of 'zero_data' will not be checked,
+      //@@         instead, zero data will be used as long as the field is set.
+      //@@
+      bool zero_data = 3;
+
+      //@@      .. cpp:var:: string data_file
+      //@@
+      //@@         The file whose content will be used as the initial data for
+      //@@         the state in row-major order. The file must be provided in
+      //@@         sub-directory 'initial_state' under the model directory.
+      //@@
+      string data_file = 4;
+    }
+
+    //@@  .. cpp:var:: string name
+    //@@
+    //@@     The name of the state initialization.
+    //@@
+    string name = 5;
+  }
+
+  //@@  .. cpp:var:: message State
+  //@@
+  //@@     An input / output pair of tensors that carry state for the sequence.
+  //@@
+  message State
+  {
+    //@@    .. cpp:var:: string input_name
+    //@@
+    //@@       The name of the model state input.
+    //@@
+    string input_name = 1;
+
+    //@@    .. cpp:var:: string output_name
+    //@@
+    //@@       The name of the model state output.
+    //@@
+    string output_name = 2;
+
+    //@@    .. cpp:var:: DataType data_type
+    //@@
+    //@@       The data-type of the state.
+    //@@
+    DataType data_type = 3;
+
+    //@@    .. cpp:var:: int64 dim (repeated)
+    //@@
+    //@@       The dimension.
+    //@@
+    repeated int64 dims = 4;
+
+    //@@  .. cpp:var:: InitialState initial_state (repeated)
+    //@@
+    //@@     The optional field to specify the initial state for the model.
+    //@@
+    repeated InitialState initial_state = 5;
+  }
+
+  //@@  .. cpp:var:: message StrategyDirect
+  //@@
+  //@@     The sequence batcher uses a specific, unique batch
+  //@@     slot for each sequence. All inference requests in a
+  //@@     sequence are directed to the same batch slot in the same
+  //@@     model instance over the lifetime of the sequence. This
+  //@@     is the default strategy.
+  //@@
+  message StrategyDirect
+  {
+    //@@    .. cpp:var:: uint64 max_queue_delay_microseconds
+    //@@
+    //@@       The maximum time, in microseconds, a candidate request
+    //@@       will be delayed in the sequence batch scheduling queue to
+    //@@       wait for additional requests for batching. Default is 0.
+    //@@
+    uint64 max_queue_delay_microseconds = 1;
+
+    //@@    .. cpp:var:: float minimum_slot_utilization
+    //@@
+    //@@       The minimum slot utilization that must be satisfied to
+    //@@       execute the batch before 'max_queue_delay_microseconds' expires.
+    //@@       For example, a value of 0.5 indicates that the batch should be
+    //@@       executed as soon as 50% or more of the slots are ready even if
+    //@@       the 'max_queue_delay_microseconds' timeout has not expired.
+    //@@       The default is 0.0, indicating that a batch will be executed
+    //@@       before 'max_queue_delay_microseconds' timeout expires if at least
+    //@@       one batch slot is ready. 'max_queue_delay_microseconds' will be
+    //@@       ignored unless minimum_slot_utilization is set to a non-zero
+    //@@       value.
+    //@@
+    float minimum_slot_utilization = 2;
+  }
+
+  //@@  .. cpp:var:: message StrategyOldest
+  //@@
+  //@@     The sequence batcher maintains up to 'max_candidate_sequences'
+  //@@     candidate sequences. 'max_candidate_sequences' can be greater
+  //@@     than the model's 'max_batch_size'. For inferencing the batcher
+  //@@     chooses from the candidate sequences up to 'max_batch_size'
+  //@@     inference requests. Requests are chosen in an oldest-first
+  //@@     manner across all candidate sequences. A given sequence is
+  //@@     not guaranteed to be assigned to the same batch slot for
+  //@@     all inference requests of that sequence.
+  //@@
+  message StrategyOldest
+  {
+    //@@    .. cpp:var:: int32 max_candidate_sequences
+    //@@
+    //@@       Maximum number of candidate sequences that the batcher
+    //@@       maintains. Excess seqences are kept in an ordered backlog
+    //@@       and become candidates when existing candidate sequences
+    //@@       complete.
+    //@@
+    int32 max_candidate_sequences = 1;
+
+    //@@    .. cpp:var:: int32 preferred_batch_size (repeated)
+    //@@
+    //@@       Preferred batch sizes for dynamic batching of candidate
+    //@@       sequences. If a batch of one of these sizes can be formed
+    //@@       it will be executed immediately. If not specified a
+    //@@       preferred batch size will be chosen automatically
+    //@@       based on model and GPU characteristics.
+    //@@
+    repeated int32 preferred_batch_size = 2;
+
+    //@@    .. cpp:var:: uint64 max_queue_delay_microseconds
+    //@@
+    //@@       The maximum time, in microseconds, a candidate request
+    //@@       will be delayed in the dynamic batch scheduling queue to
+    //@@       wait for additional requests for batching. Default is 0.
+    //@@
+    uint64 max_queue_delay_microseconds = 3;
+  }
+
+  //@@  .. cpp:var:: oneof strategy_choice
+  //@@
+  //@@     The strategy used by the sequence batcher. Default strategy
+  //@@     is 'direct'.
+  //@@
+  oneof strategy_choice
+  {
+    //@@    .. cpp:var:: StrategyDirect direct
+    //@@
+    //@@       StrategyDirect scheduling strategy.
+    //@@
+    StrategyDirect direct = 3;
+
+    //@@    .. cpp:var:: StrategyOldest oldest
+    //@@
+    //@@       StrategyOldest scheduling strategy.
+    //@@
+    StrategyOldest oldest = 4;
+  }
+
+  //@@  .. cpp:var:: uint64 max_sequence_idle_microseconds
+  //@@
+  //@@     The maximum time, in microseconds, that a sequence is allowed to
+  //@@     be idle before it is aborted. The inference server considers a
+  //@@     sequence idle when it does not have any inference request queued
+  //@@     for the sequence. If this limit is exceeded, the inference server
+  //@@     will free the sequence slot allocated by the sequence and make it
+  //@@     available for another sequence. If not specified (or specified as
+  //@@     zero) a default value of 1000000 (1 second) is used.
+  //@@
+  uint64 max_sequence_idle_microseconds = 1;
+
+  //@@  .. cpp:var:: ControlInput control_input (repeated)
+  //@@
+  //@@     The model input(s) that the server should use to communicate
+  //@@     sequence start, stop, ready and similar control values to the
+  //@@     model.
+  //@@
+  repeated ControlInput control_input = 2;
+
+  //@@  .. cpp:var:: State state (repeated)
+  //@@
+  //@@     The optional state that can be stored in Triton for performing
+  //@@     inference requests on a sequence. Each sequence holds an implicit
+  //@@     state local to itself. The output state tensor provided by the
+  //@@     model in 'output_name' field of the current inference request will
+  //@@     be transferred as an input tensor named 'input_name' in the next
+  //@@     request of the same sequence. The input state of the first request
+  //@@     in the sequence contains garbage data.
+  //@@
+  repeated State state = 5;
+}
+
+//@@
+//@@.. cpp:var:: message ModelEnsembling
+//@@
+//@@   Model ensembling configuration. These settings specify the models that
+//@@   compose the ensemble and how data flows between the models.
+//@@
+message ModelEnsembling
+{
+  //@@  .. cpp:var:: message Step
+  //@@
+  //@@     Each step specifies a model included in the ensemble,
+  //@@     maps ensemble tensor names to the model input tensors,
+  //@@     and maps model output tensors to ensemble tensor names
+  //@@
+  message Step
+  {
+    //@@  .. cpp:var:: string model_name
+    //@@
+    //@@     The name of the model to execute for this step of the ensemble.
+    //@@
+    string model_name = 1;
+
+    //@@  .. cpp:var:: int64 model_version
+    //@@
+    //@@     The version of the model to use for inference. If -1
+    //@@     the latest/most-recent version of the model is used.
+    //@@
+    int64 model_version = 2;
+
+    //@@  .. cpp:var:: map<string,string> input_map
+    //@@
+    //@@     Map from name of an input tensor on this step's model to ensemble
+    //@@     tensor name. The ensemble tensor must have the same data type and
+    //@@     shape as the model input. Each model input must be assigned to
+    //@@     one ensemble tensor, but the same ensemble tensor can be assigned
+    //@@     to multiple model inputs.
+    //@@
+    map<string, string> input_map = 3;
+
+    //@@  .. cpp:var:: map<string,string> output_map
+    //@@
+    //@@     Map from name of an output tensor on this step's model to ensemble
+    //@@     tensor name. The data type and shape of the ensemble tensor will
+    //@@     be inferred from the model output. It is optional to assign all
+    //@@     model outputs to ensemble tensors. One ensemble tensor name
+    //@@     can appear in an output map only once.
+    //@@
+    map<string, string> output_map = 4;
+  }
+
+  //@@  .. cpp:var:: Step step (repeated)
+  //@@
+  //@@     The models and the input / output mappings used within the ensemble.
+  //@@
+  repeated Step step = 1;
+}
+
+//@@
+//@@.. cpp:var:: message ModelParameter
+//@@
+//@@   A model parameter.
+//@@
+message ModelParameter
+{
+  //@@  .. cpp:var:: string string_value
+  //@@
+  //@@     The string value of the parameter.
+  //@@
+  string string_value = 1;
+}
+
+//@@
+//@@.. cpp:var:: message ModelWarmup
+//@@
+//@@   Settings used to construct the request sample for model warmup.
+//@@
+message ModelWarmup
+{
+  //@@
+  //@@  .. cpp:var:: message Input
+  //@@
+  //@@     Meta data associated with an input.
+  //@@
+  message Input
+  {
+    //@@    .. cpp:var:: DataType data_type
+    //@@
+    //@@       The data-type of the input.
+    //@@
+    DataType data_type = 1;
+
+    //@@    .. cpp:var:: int64 dims (repeated)
+    //@@
+    //@@       The shape of the input tensor, not including the batch dimension.
+    //@@
+    repeated int64 dims = 2;
+
+    //@@    .. cpp:var:: oneof input_data_type
+    //@@
+    //@@       Specify how the input data is generated. If the input has STRING
+    //@@       data type and 'random_data' is set, the data generation will fall
+    //@@       back to 'zero_data'.
+    //@@
+    oneof input_data_type
+    {
+      //@@
+      //@@    .. cpp:var:: bool zero_data
+      //@@
+      //@@       The identifier for using zeros as input data. Note that the
+      //@@       value of 'zero_data' will not be checked, instead, zero data
+      //@@       will be used as long as the field is set.
+      //@@
+      bool zero_data = 3;
+
+      //@@
+      //@@    .. cpp:var:: bool random_data
+      //@@
+      //@@       The identifier for using random data as input data. Note that
+      //@@       the value of 'random_data' will not be checked, instead,
+      //@@       random data will be used as long as the field is set.
+      //@@
+      bool random_data = 4;
+
+      //@@    .. cpp:var:: string input_data_file
+      //@@
+      //@@       The file whose content will be used as raw input data in
+      //@@       row-major order. The file must be provided in a sub-directory
+      //@@       'warmup' under the model directory. The file contents should be
+      //@@       in binary format. For TYPE_STRING data-type, an element is
+      //@@       represented by a 4-byte unsigned integer giving the length 
+      //@@       followed by the actual bytes.
+      //@@
+      string input_data_file = 5;
+    }
+  }
+
+  //@@  .. cpp:var:: string name
+  //@@
+  //@@     The name of the request sample.
+  //@@
+  string name = 1;
+
+  //@@  .. cpp:var:: uint32 batch_size
+  //@@
+  //@@     The batch size of the inference request. This must be >= 1. For
+  //@@     models that don't support batching, batch_size must be 1. If
+  //@@     batch_size > 1, the 'inputs' specified below will be duplicated to
+  //@@     match the batch size requested.
+  //@@
+  uint32 batch_size = 2;
+
+  //@@  .. cpp:var:: map<string, Input> inputs
+  //@@
+  //@@     The warmup meta data associated with every model input, including
+  //@@     control tensors.
+  //@@
+  map<string, Input> inputs = 3;
+
+  //@@  .. cpp:var:: uint32 count
+  //@@
+  //@@     The number of iterations that this warmup sample will be executed.
+  //@@     For example, if this field is set to 2, 2 model executions using this
+  //@@     sample will be scheduled for warmup. Default value is 0 which
+  //@@     indicates that this sample will be used only once.
+  //@@     Note that for sequence model, 'count' may not work well
+  //@@     because the model often expect a valid sequence of requests which
+  //@@     should be represented by a series of warmup samples. 'count > 1'
+  //@@     essentially "resends" one of the sample, which may invalidate the
+  //@@     sequence and result in unexpected warmup failure.
+  //@@
+  uint32 count = 4;
+}
+
+//@@
+//@@ .. cpp:var:: message ModelOperations
+//@@
+//@@    The metadata of libraries providing custom operations for this model.
+//@@
+message ModelOperations
+{
+  //@@  .. cpp:var:: string op_library_filename (repeated)
+  //@@
+  //@@     Optional paths of the libraries providing custom operations for
+  //@@     this model. Valid only for ONNX models.
+  //@@
+  repeated string op_library_filename = 1;
+}
+
+//@@
+//@@ .. cpp:var:: message ModelTransactionPolicy
+//@@
+//@@    The specification that describes the nature of transactions
+//@@    to be expected from the model.
+//@@
+message ModelTransactionPolicy
+{
+  //@@  .. cpp:var:: bool decoupled
+  //@@
+  //@@     Indicates whether responses generated by the model are decoupled with
+  //@@     the requests issued to it, which means the number of responses
+  //@@     generated by model may differ from number of requests issued, and
+  //@@     that the responses may be out of order relative to the order of
+  //@@     requests. The default is false, which means the model will generate
+  //@@     exactly one response for each request.
+  //@@
+  bool decoupled = 1;
+}
+
+//@@
+//@@.. cpp:var:: message ModelRepositoryAgents
+//@@
+//@@   The repository agents for the model.
+//@@
+message ModelRepositoryAgents
+{
+  //@@
+  //@@  .. cpp:var:: message Agent
+  //@@
+  //@@     A repository agent that should be invoked for the specified
+  //@@     repository actions for this model.
+  //@@
+  message Agent
+  {
+    //@@    .. cpp:var:: string name
+    //@@
+    //@@       The name of the agent.
+    //@@
+    string name = 1;
+
+    //@@    .. cpp:var:: map<string, string> parameters
+    //@@
+    //@@       The parameters for the agent.
+    //@@
+    map<string, string> parameters = 2;
+  }
+
+  //@@
+  //@@  .. cpp:var:: Agent agents (repeated)
+  //@@
+  //@@     The ordered list of agents for the model. These agents will be
+  //@@     invoked in order to respond to repository actions occuring for the
+  //@@     model.
+  //@@
+  repeated Agent agents = 1;
+}
+
+//@@
+//@@.. cpp:var:: message ModelResponseCache
+//@@
+//@@   The response cache setting for the model.
+//@@
+message ModelResponseCache
+{
+  //@@
+  //@@  .. cpp::var:: bool enable
+  //@@
+  //@@     Whether or not to use response cache for the model. If True, the
+  //@@     responses from the model are cached and when identical request
+  //@@     is encountered, instead of going through the model execution,
+  //@@     the response from the cache is utilized. By default, response
+  //@@     cache is disabled for the models.
+  //@@
+  bool enable = 1;
+}
+
+//@@
+//@@.. cpp:var:: message ModelConfig
+//@@
+//@@   A model configuration.
+//@@
+message ModelConfig
+{
+  //@@  .. cpp:var:: string name
+  //@@
+  //@@     The name of the model.
+  //@@
+  string name = 1;
+
+  //@@  .. cpp:var:: string platform
+  //@@
+  //@@     The framework for the model. Possible values are
+  //@@     "tensorrt_plan", "tensorflow_graphdef",
+  //@@     "tensorflow_savedmodel", "onnxruntime_onnx",
+  //@@     "pytorch_libtorch".
+  //@@
+  string platform = 2;
+
+  //@@  .. cpp:var:: string backend
+  //@@
+  //@@     The backend used by the model.
+  //@@
+  string backend = 17;
+
+  //@@  .. cpp:var:: ModelVersionPolicy version_policy
+  //@@
+  //@@     Policy indicating which version(s) of the model will be served.
+  //@@
+  ModelVersionPolicy version_policy = 3;
+
+  //@@  .. cpp:var:: int32 max_batch_size
+  //@@
+  //@@     Maximum batch size allowed for inference. This can only decrease
+  //@@     what is allowed by the model itself. A max_batch_size value of 0
+  //@@     indicates that batching is not allowed for the model and the
+  //@@     dimension/shape of the input and output tensors must exactly
+  //@@     match what is specified in the input and output configuration. A
+  //@@     max_batch_size value > 0 indicates that batching is allowed and
+  //@@     so the model expects the input tensors to have an additional
+  //@@     initial dimension for the batching that is not specified in the
+  //@@     input (for example, if the model supports batched inputs of
+  //@@     2-dimensional tensors then the model configuration will specify
+  //@@     the input shape as [ X, Y ] but the model will expect the actual
+  //@@     input tensors to have shape [ N, X, Y ]). For max_batch_size > 0
+  //@@     returned outputs will also have an additional initial dimension
+  //@@     for the batch.
+  //@@
+  int32 max_batch_size = 4;
+
+  //@@  .. cpp:var:: ModelInput input (repeated)
+  //@@
+  //@@     The inputs request by the model.
+  //@@
+  repeated ModelInput input = 5;
+
+  //@@  .. cpp:var:: ModelOutput output (repeated)
+  //@@
+  //@@     The outputs produced by the model.
+  //@@
+  repeated ModelOutput output = 6;
+
+  //@@  .. cpp:var:: BatchInput batch_input (repeated)
+  //@@
+  //@@     The model input(s) that the server should use to communicate
+  //@@     batch related values to the model.
+  //@@
+  repeated BatchInput batch_input = 20;
+
+  //@@  .. cpp:var:: BatchOutput batch_output (repeated)
+  //@@
+  //@@     The outputs produced by the model that requires special handling
+  //@@     by the model backend.
+  //@@
+  repeated BatchOutput batch_output = 21;
+
+  //@@  .. cpp:var:: ModelOptimizationPolicy optimization
+  //@@
+  //@@     Optimization configuration for the model. If not specified
+  //@@     then default optimization policy is used.
+  //@@
+  ModelOptimizationPolicy optimization = 12;
+
+  //@@  .. cpp:var:: oneof scheduling_choice
+  //@@
+  //@@     The scheduling policy for the model. If not specified the
+  //@@     default scheduling policy is used for the model. The default
+  //@@     policy is to execute each inference request independently.
+  //@@
+  oneof scheduling_choice
+  {
+    //@@    .. cpp:var:: ModelDynamicBatching dynamic_batching
+    //@@
+    //@@       If specified, enables the dynamic-batching scheduling
+    //@@       policy. With dynamic-batching the scheduler may group
+    //@@       together independent requests into a single batch to
+    //@@       improve inference throughput.
+    //@@
+    ModelDynamicBatching dynamic_batching = 11;
+
+    //@@    .. cpp:var:: ModelSequenceBatching sequence_batching
+    //@@
+    //@@       If specified, enables the sequence-batching scheduling
+    //@@       policy. With sequence-batching, inference requests
+    //@@       with the same correlation ID are routed to the same
+    //@@       model instance. Multiple sequences of inference requests
+    //@@       may be batched together into a single batch to
+    //@@       improve inference throughput.
+    //@@
+    ModelSequenceBatching sequence_batching = 13;
+
+    //@@    .. cpp:var:: ModelEnsembling ensemble_scheduling
+    //@@
+    //@@       If specified, enables the model-ensembling scheduling
+    //@@       policy. With model-ensembling, inference requests
+    //@@       will be processed according to the specification, such as an
+    //@@       execution sequence of models. The input specified in this model
+    //@@       config will be the input for the ensemble, and the output
+    //@@       specified will be the output of the ensemble.
+    //@@
+    ModelEnsembling ensemble_scheduling = 15;
+  }
+
+  //@@  .. cpp:var:: ModelInstanceGroup instance_group (repeated)
+  //@@
+  //@@     Instances of this model. If not specified, one instance
+  //@@     of the model will be instantiated on each available GPU.
+  //@@
+  repeated ModelInstanceGroup instance_group = 7;
+
+  //@@  .. cpp:var:: string default_model_filename
+  //@@
+  //@@     Optional filename of the model file to use if a
+  //@@     compute-capability specific model is not specified in
+  //@@     :cpp:var:`cc_model_filenames`. If not specified the default name
+  //@@     is 'model.graphdef', 'model.savedmodel', 'model.plan' or
+  //@@     'model.pt' depending on the model type.
+  //@@
+  string default_model_filename = 8;
+
+  //@@  .. cpp:var:: map<string,string> cc_model_filenames
+  //@@
+  //@@     Optional map from CUDA compute capability to the filename of
+  //@@     the model that supports that compute capability. The filename
+  //@@     refers to a file within the model version directory.
+  //@@
+  map<string, string> cc_model_filenames = 9;
+
+  //@@  .. cpp:var:: map<string,string> metric_tags
+  //@@
+  //@@     Optional metric tags. User-specific key-value pairs for metrics
+  //@@     reported for this model. These tags are applied to the metrics
+  //@@     reported on the HTTP metrics port.
+  //@@
+  map<string, string> metric_tags = 10;
+
+  //@@  .. cpp:var:: map<string,ModelParameter> parameters
+  //@@
+  //@@     Optional model parameters. User-specified parameter values.
+  //@@
+  map<string, ModelParameter> parameters = 14;
+
+  //@@  .. cpp:var:: ModelWarmup model_warmup (repeated)
+  //@@
+  //@@     Warmup setting of this model. If specified, all instances
+  //@@     will be run with the request samples in sequence before
+  //@@     serving the model.
+  //@@     This field can only be specified if the model is not an ensemble
+  //@@     model.
+  //@@
+  repeated ModelWarmup model_warmup = 16;
+
+  //@@  .. cpp:var:: ModelOperations model_operations
+  //@@
+  //@@     Optional metadata of the libraries providing custom operations for
+  //@@     this model.
+  //@@
+  ModelOperations model_operations = 18;
+
+  //@@  .. cpp:var:: ModelTransactionPolicy model_transaction_policy
+  //@@
+  //@@     Optional specification that describes the nature of transactions
+  //@@     to be expected from the model.
+  //@@
+  ModelTransactionPolicy model_transaction_policy = 19;
+
+  //@@  .. cpp:var:: ModelRepositoryAgents model_repository_agents
+  //@@
+  //@@     Optional specification of the agent(s) that should be invoked
+  //@@     with repository actions are performed for this model.
+  //@@
+  ModelRepositoryAgents model_repository_agents = 23;
+
+  //@@  .. cpp:var:: ModelResponseCache response_cache
+  //@@
+  //@@     Optional setting for utilizing the response cache for this
+  //@@     model.
+  //@@
+  ModelResponseCache response_cache = 24;
+}

3rdparty/common-r22.12/src/async_work_queue.cc

+// Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+//  * Redistributions of source code must retain the above copyright
+//    notice, this list of conditions and the following disclaimer.
+//  * Redistributions in binary form must reproduce the above copyright
+//    notice, this list of conditions and the following disclaimer in the
+//    documentation and/or other materials provided with the distribution.
+//  * Neither the name of NVIDIA CORPORATION nor the names of its
+//    contributors may be used to endorse or promote products derived
+//    from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "triton/common/async_work_queue.h"
+
+namespace triton { namespace common {
+
+AsyncWorkQueue::~AsyncWorkQueue()
+{
+  GetSingleton()->thread_pool_.reset();
+}
+
+AsyncWorkQueue*
+AsyncWorkQueue::GetSingleton()
+{
+  static AsyncWorkQueue singleton;
+  return &singleton;
+}
+
+Error
+AsyncWorkQueue::Initialize(size_t worker_count)
+{
+  if (worker_count < 1) {
+    return Error(
+        Error::Code::INVALID_ARG,
+        "Async work queue must be initialized with positive 'worker_count'");
+  }
+
+  static std::mutex init_mtx;
+  std::lock_guard<std::mutex> lk(init_mtx);
+
+  if (GetSingleton()->thread_pool_) {
+    return Error(
+        Error::Code::ALREADY_EXISTS,
+        "Async work queue has been initialized with " +
+            std::to_string(GetSingleton()->thread_pool_->Size()) +
+            " 'worker_count'");
+  }
+
+  GetSingleton()->thread_pool_.reset(new ThreadPool(worker_count));
+  return Error::Success;
+}
+
+size_t
+AsyncWorkQueue::WorkerCount()
+{
+  if (!GetSingleton()->thread_pool_) {
+    return 0;
+  }
+  return GetSingleton()->thread_pool_->Size();
+}
+
+Error
+AsyncWorkQueue::AddTask(std::function<void(void)>&& task)
+{
+  if (!GetSingleton()->thread_pool_) {
+    return Error(
+        Error::Code::UNAVAILABLE,
+        "Async work queue must be initialized before adding task");
+  }
+  GetSingleton()->thread_pool_->Enqueue(std::move(task));
+
+  return Error::Success;
+}
+
+void
+AsyncWorkQueue::Reset()
+{
+  // Reconstruct the singleton to reset it
+  GetSingleton()->~AsyncWorkQueue();
+  new (GetSingleton()) AsyncWorkQueue();
+}
+
+}}  // namespace triton::common

3rdparty/common-r22.12/src/error.cc

+// Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+//  * Redistributions of source code must retain the above copyright
+//    notice, this list of conditions and the following disclaimer.
+//  * Redistributions in binary form must reproduce the above copyright
+//    notice, this list of conditions and the following disclaimer in the
+//    documentation and/or other materials provided with the distribution.
+//  * Neither the name of NVIDIA CORPORATION nor the names of its
+//    contributors may be used to endorse or promote products derived
+//    from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "triton/common/error.h"
+
+namespace triton { namespace common {
+
+const Error Error::Success(Error::Code::SUCCESS);
+
+std::string
+Error::AsString() const
+{
+  std::string str(CodeString(code_));
+  str += ": " + msg_;
+  return str;
+}
+
+const char*
+Error::CodeString(const Code code)
+{
+  switch (code) {
+    case Error::Code::SUCCESS:
+      return "OK";
+    case Error::Code::UNKNOWN:
+      return "Unknown";
+    case Error::Code::INTERNAL:
+      return "Internal";
+    case Error::Code::NOT_FOUND:
+      return "Not found";
+    case Error::Code::INVALID_ARG:
+      return "Invalid argument";
+    case Error::Code::UNAVAILABLE:
+      return "Unavailable";
+    case Error::Code::UNSUPPORTED:
+      return "Unsupported";
+    case Error::Code::ALREADY_EXISTS:
+      return "Already exists";
+    default:
+      break;
+  }
+
+  return "<invalid code>";
+}
+
+}}  // namespace triton::common

3rdparty/common-r22.12/src/logging.cc

+// Copyright 2018-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+//  * Redistributions of source code must retain the above copyright
+//    notice, this list of conditions and the following disclaimer.
+//  * Redistributions in binary form must reproduce the above copyright
+//    notice, this list of conditions and the following disclaimer in the
+//    documentation and/or other materials provided with the distribution.
+//  * Neither the name of NVIDIA CORPORATION nor the names of its
+//    contributors may be used to endorse or promote products derived
+//    from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "triton/common/logging.h"
+
+#ifdef _WIN32
+// suppress the min and max definitions in Windef.h.
+#define NOMINMAX
+#include <Windows.h>
+#else
+#include <sys/time.h>
+#include <sys/types.h>
+#include <time.h>
+#include <unistd.h>
+#endif
+#include <algorithm>
+#include <iomanip>
+#include <iostream>
+
+namespace triton { namespace common {
+
+Logger gLogger_;
+
+Logger::Logger()
+    : enables_{true, true, true}, vlevel_(0), format_(Format::kDEFAULT)
+{
+}
+
+void
+Logger::Log(const std::string& msg)
+{
+  const std::lock_guard<std::mutex> lock(mutex_);
+  if (file_stream_.is_open()) {
+    file_stream_ << msg << std::endl;
+  } else {
+    std::cerr << msg << std::endl;
+  }
+}
+
+void
+Logger::Flush()
+{
+  std::cerr << std::flush;
+}
+
+
+const std::vector<char> LogMessage::level_name_{'E', 'W', 'I'};
+
+LogMessage::LogMessage(const char* file, int line, uint32_t level)
+{
+  std::string path(file);
+  size_t pos = path.rfind('/');
+  if (pos != std::string::npos) {
+    path = path.substr(pos + 1, std::string::npos);
+  }
+
+  // 'L' below is placeholder for showing log level
+  switch (gLogger_.LogFormat()) {
+    case Logger::Format::kDEFAULT: {
+      // LMMDD hh:mm:ss.ssssss
+#ifdef _WIN32
+      SYSTEMTIME system_time;
+      GetSystemTime(&system_time);
+      stream_ << level_name_[std::min(level, (uint32_t)Level::kINFO)]
+              << std::setfill('0') << std::setw(2) << system_time.wMonth
+              << std::setw(2) << system_time.wDay << ' ' << std::setw(2)
+              << system_time.wHour << ':' << std::setw(2) << system_time.wMinute
+              << ':' << std::setw(2) << system_time.wSecond << '.'
+              << std::setw(6) << system_time.wMilliseconds * 1000 << ' '
+              << static_cast<uint32_t>(GetCurrentProcessId()) << ' ' << path
+              << ':' << line << "] ";
+#else
+      struct timeval tv;
+      gettimeofday(&tv, NULL);
+      struct tm tm_time;
+      gmtime_r(((time_t*)&(tv.tv_sec)), &tm_time);
+      stream_ << level_name_[std::min(level, (uint32_t)Level::kINFO)]
+              << std::setfill('0') << std::setw(2) << (tm_time.tm_mon + 1)
+              << std::setw(2) << tm_time.tm_mday << ' ' << std::setw(2)
+              << tm_time.tm_hour << ':' << std::setw(2) << tm_time.tm_min << ':'
+              << std::setw(2) << tm_time.tm_sec << '.' << std::setw(6)
+              << tv.tv_usec << ' ' << static_cast<uint32_t>(getpid()) << ' '
+              << path << ':' << line << "] ";
+#endif
+      break;
+    }
+    case Logger::Format::kISO8601: {
+      // YYYY-MM-DDThh:mm:ssZ L
+#ifdef _WIN32
+      SYSTEMTIME system_time;
+      GetSystemTime(&system_time);
+      stream_ << system_time.wYear << '-' << std::setfill('0') << std::setw(2)
+              << system_time.wMonth << '-' << std::setw(2) << system_time.wDay
+              << 'T' << std::setw(2) << system_time.wHour << ':' << std::setw(2)
+              << system_time.wMinute << ':' << std::setw(2)
+              << system_time.wSecond << "Z "
+              << level_name_[std::min(level, (uint32_t)Level::kINFO)] << ' '
+              << static_cast<uint32_t>(GetCurrentProcessId()) << ' ' << path
+              << ':' << line << "] ";
+#else
+      struct timeval tv;
+      gettimeofday(&tv, NULL);
+      struct tm tm_time;
+      gmtime_r(((time_t*)&(tv.tv_sec)), &tm_time);
+      stream_ << (tm_time.tm_year + 1900) << '-' << std::setfill('0')
+              << std::setw(2) << (tm_time.tm_mon + 1) << '-' << std::setw(2)
+              << tm_time.tm_mday << 'T' << std::setw(2) << tm_time.tm_hour
+              << ':' << std::setw(2) << tm_time.tm_min << ':' << std::setw(2)
+              << tm_time.tm_sec << "Z "
+              << level_name_[std::min(level, (uint32_t)Level::kINFO)] << ' '
+              << static_cast<uint32_t>(getpid()) << ' ' << path << ':' << line
+              << "] ";
+#endif
+      break;
+    }
+  }
+}
+
+LogMessage::~LogMessage()
+{
+  gLogger_.Log(stream_.str());
+}
+
+}}  // namespace triton::common

3rdparty/common-r22.12/src/model_config.cc

+// Copyright 2018-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+//  * Redistributions of source code must retain the above copyright
+//    notice, this list of conditions and the following disclaimer.
+//  * Redistributions in binary form must reproduce the above copyright
+//    notice, this list of conditions and the following disclaimer in the
+//    documentation and/or other materials provided with the distribution.
+//  * Neither the name of NVIDIA CORPORATION nor the names of its
+//    contributors may be used to endorse or promote products derived
+//    from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "triton/common/model_config.h"
+
+namespace triton { namespace common {
+
+bool
+IsFixedSizeDataType(const inference::DataType dtype)
+{
+  return dtype != inference::DataType::TYPE_STRING;
+}
+
+size_t
+GetDataTypeByteSize(const inference::DataType dtype)
+{
+  switch (dtype) {
+    case inference::DataType::TYPE_BOOL:
+      return 1;
+    case inference::DataType::TYPE_UINT8:
+      return 1;
+    case inference::DataType::TYPE_UINT16:
+      return 2;
+    case inference::DataType::TYPE_UINT32:
+      return 4;
+    case inference::DataType::TYPE_UINT64:
+      return 8;
+    case inference::DataType::TYPE_INT8:
+      return 1;
+    case inference::DataType::TYPE_INT16:
+      return 2;
+    case inference::DataType::TYPE_INT32:
+      return 4;
+    case inference::DataType::TYPE_INT64:
+      return 8;
+    case inference::DataType::TYPE_FP16:
+      return 2;
+    case inference::DataType::TYPE_FP32:
+      return 4;
+    case inference::DataType::TYPE_FP64:
+      return 8;
+    case inference::DataType::TYPE_STRING:
+      return 0;
+    case inference::DataType::TYPE_BF16:
+      return 2;
+    default:
+      break;
+  }
+
+  return 0;
+}
+
+int64_t
+GetElementCount(const DimsList& dims)
+{
+  bool first = true;
+  int64_t cnt = 0;
+  for (auto dim : dims) {
+    if (dim == WILDCARD_DIM) {
+      return -1;
+    }
+
+    if (first) {
+      cnt = dim;
+      first = false;
+    } else {
+      cnt *= dim;
+    }
+  }
+
+  return cnt;
+}
+
+int64_t
+GetElementCount(const std::vector<int64_t>& dims)
+{
+  bool first = true;
+  int64_t cnt = 0;
+  for (auto dim : dims) {
+    if (dim == WILDCARD_DIM) {
+      return -1;
+    }
+
+    if (first) {
+      cnt = dim;
+      first = false;
+    } else {
+      cnt *= dim;
+    }
+  }
+
+  return cnt;
+}
+
+int64_t
+GetElementCount(const inference::ModelInput& mio)
+{
+  return GetElementCount(mio.dims());
+}
+
+int64_t
+GetElementCount(const inference::ModelOutput& mio)
+{
+  return GetElementCount(mio.dims());
+}
+
+int64_t
+GetByteSize(const inference::DataType& dtype, const DimsList& dims)
+{
+  size_t dt_size = GetDataTypeByteSize(dtype);
+  if (dt_size == 0) {
+    return -1;
+  }
+
+  int64_t cnt = GetElementCount(dims);
+  if (cnt == -1) {
+    return -1;
+  }
+
+  return cnt * dt_size;
+}
+
+int64_t
+GetByteSize(const inference::DataType& dtype, const std::vector<int64_t>& dims)
+{
+  size_t dt_size = GetDataTypeByteSize(dtype);
+  if (dt_size == 0) {
+    return -1;
+  }
+
+  int64_t cnt = GetElementCount(dims);
+  if (cnt == -1) {
+    return -1;
+  }
+
+  return cnt * dt_size;
+}
+
+int64_t
+GetByteSize(
+    const int batch_size, const inference::DataType& dtype,
+    const DimsList& dims)
+{
+  if (dims.size() == 0) {
+    return batch_size * GetDataTypeByteSize(dtype);
+  }
+
+  int64_t bs = GetByteSize(dtype, dims);
+  if (bs == -1) {
+    return -1;
+  }
+
+  return std::max(1, batch_size) * bs;
+}
+
+int64_t
+GetByteSize(
+    const int batch_size, const inference::DataType& dtype,
+    const std::vector<int64_t>& dims)
+{
+  if (dims.size() == 0) {
+    return batch_size * GetDataTypeByteSize(dtype);
+  }
+
+  int64_t bs = GetByteSize(dtype, dims);
+  if (bs == -1) {
+    return -1;
+  }
+
+  return std::max(1, batch_size) * bs;
+}
+
+int64_t
+GetByteSize(const inference::ModelInput& mio)
+{
+  return GetByteSize(mio.data_type(), mio.dims());
+}
+
+int64_t
+GetByteSize(const inference::ModelOutput& mio)
+{
+  return GetByteSize(mio.data_type(), mio.dims());
+}
+
+int
+GetCpuNiceLevel(const inference::ModelConfig& config)
+{
+  int nice = SCHEDULER_DEFAULT_NICE;
+  if (config.has_optimization()) {
+    switch (config.optimization().priority()) {
+      case inference::ModelOptimizationPolicy::PRIORITY_MAX:
+        nice = 0;
+        break;
+      case inference::ModelOptimizationPolicy::PRIORITY_MIN:
+        nice = 19;
+        break;
+      default:
+        nice = SCHEDULER_DEFAULT_NICE;
+        break;
+    }
+  }
+
+  return nice;
+}
+
+bool
+CompareDims(const DimsList& dims0, const DimsList& dims1)
+{
+  if (dims0.size() != dims1.size()) {
+    return false;
+  }
+
+  for (int i = 0; i < dims0.size(); ++i) {
+    if (dims0[i] != dims1[i]) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+bool
+CompareDims(
+    const std::vector<int64_t>& dims0, const std::vector<int64_t>& dims1)
+{
+  if (dims0.size() != dims1.size()) {
+    return false;
+  }
+
+  for (size_t i = 0; i < dims0.size(); ++i) {
+    if (dims0[i] != dims1[i]) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+bool
+CompareDimsWithWildcard(const DimsList& dims0, const DimsList& dims1)
+{
+  if (dims0.size() != dims1.size()) {
+    return false;
+  }
+
+  for (int i = 0; i < dims0.size(); ++i) {
+    if ((dims0[i] != WILDCARD_DIM) && (dims1[i] != WILDCARD_DIM) &&
+        (dims0[i] != dims1[i])) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+bool
+CompareDimsWithWildcard(
+    const DimsList& dims0, const std::vector<int64_t>& dims1)
+{
+  if (dims0.size() != (int64_t)dims1.size()) {
+    return false;
+  }
+
+  for (int i = 0; i < dims0.size(); ++i) {
+    if ((dims0[i] != WILDCARD_DIM) && (dims1[i] != WILDCARD_DIM) &&
+        (dims0[i] != dims1[i])) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+std::string
+DimsListToString(const DimsList& dims)
+{
+  bool first = true;
+
+  std::string str("[");
+  for (const auto& dim : dims) {
+    if (!first) {
+      str += ",";
+    }
+    str += std::to_string(dim);
+    first = false;
+  }
+
+  str += "]";
+  return str;
+}
+
+std::string
+DimsListToString(const std::vector<int64_t>& dims, const int start_idx)
+{
+  int idx = 0;
+
+  std::string str("[");
+  for (const auto& dim : dims) {
+    if (idx >= start_idx) {
+      if (idx > start_idx) {
+        str += ",";
+      }
+      str += std::to_string(dim);
+    }
+
+    idx++;
+  }
+
+  str += "]";
+  return str;
+}
+
+const char*
+DataTypeToProtocolString(const inference::DataType dtype)
+{
+  switch (dtype) {
+    case inference::DataType::TYPE_BOOL:
+      return "BOOL";
+    case inference::DataType::TYPE_UINT8:
+      return "UINT8";
+    case inference::DataType::TYPE_UINT16:
+      return "UINT16";
+    case inference::DataType::TYPE_UINT32:
+      return "UINT32";
+    case inference::DataType::TYPE_UINT64:
+      return "UINT64";
+    case inference::DataType::TYPE_INT8:
+      return "INT8";
+    case inference::DataType::TYPE_INT16:
+      return "INT16";
+    case inference::DataType::TYPE_INT32:
+      return "INT32";
+    case inference::DataType::TYPE_INT64:
+      return "INT64";
+    case inference::DataType::TYPE_FP16:
+      return "FP16";
+    case inference::DataType::TYPE_FP32:
+      return "FP32";
+    case inference::DataType::TYPE_FP64:
+      return "FP64";
+    case inference::DataType::TYPE_STRING:
+      return "BYTES";
+    case inference::DataType::TYPE_BF16:
+      return "BF16";
+    default:
+      break;
+  }
+
+  return "<invalid>";
+}
+
+inference::DataType
+ProtocolStringToDataType(const std::string& dtype)
+{
+  return ProtocolStringToDataType(dtype.c_str(), dtype.size());
+}
+
+inference::DataType
+ProtocolStringToDataType(const char* dtype, size_t len)
+{
+  if (len < 4 || len > 6) {
+    return inference::DataType::TYPE_INVALID;
+  }
+
+  if ((*dtype == 'I') && (len != 6)) {
+    if ((dtype[1] == 'N') && (dtype[2] == 'T')) {
+      if ((dtype[3] == '8') && (len == 4)) {
+        return inference::DataType::TYPE_INT8;
+      } else if ((dtype[3] == '1') && (dtype[4] == '6')) {
+        return inference::DataType::TYPE_INT16;
+      } else if ((dtype[3] == '3') && (dtype[4] == '2')) {
+        return inference::DataType::TYPE_INT32;
+      } else if ((dtype[3] == '6') && (dtype[4] == '4')) {
+        return inference::DataType::TYPE_INT64;
+      }
+    }
+  } else if ((*dtype == 'U') && (len != 4)) {
+    if ((dtype[1] == 'I') && (dtype[2] == 'N') && (dtype[3] == 'T')) {
+      if ((dtype[4] == '8') && (len == 5)) {
+        return inference::DataType::TYPE_UINT8;
+      } else if ((dtype[4] == '1') && (dtype[5] == '6')) {
+        return inference::DataType::TYPE_UINT16;
+      } else if ((dtype[4] == '3') && (dtype[5] == '2')) {
+        return inference::DataType::TYPE_UINT32;
+      } else if ((dtype[4] == '6') && (dtype[5] == '4')) {
+        return inference::DataType::TYPE_UINT64;
+      }
+    }
+  } else if ((*dtype == 'F') && (dtype[1] == 'P') && (len == 4)) {
+    if ((dtype[2] == '1') && (dtype[3] == '6')) {
+      return inference::DataType::TYPE_FP16;
+    } else if ((dtype[2] == '3') && (dtype[3] == '2')) {
+      return inference::DataType::TYPE_FP32;
+    } else if ((dtype[2] == '6') && (dtype[3] == '4')) {
+      return inference::DataType::TYPE_FP64;
+    }
+  } else if (*dtype == 'B') {
+    switch (dtype[1]) {
+      case 'Y':
+        if (!strcmp(dtype + 2, "TES")) {
+          return inference::DataType::TYPE_STRING;
+        }
+        break;
+      case 'O':
+        if (!strcmp(dtype + 2, "OL")) {
+          return inference::DataType::TYPE_BOOL;
+        }
+        break;
+      case 'F':
+        if (!strcmp(dtype + 2, "16")) {
+          return inference::DataType::TYPE_BF16;
+        }
+        break;
+    }
+  }
+
+  return inference::DataType::TYPE_INVALID;
+}
+
+}}  // namespace triton::common

3rdparty/common-r22.12/src/table_printer.cc

+// Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+//  * Redistributions of source code must retain the above copyright
+//    notice, this list of conditions and the following disclaimer.
+//  * Redistributions in binary form must reproduce the above copyright
+//    notice, this list of conditions and the following disclaimer in the
+//    documentation and/or other materials provided with the distribution.
+//  * Neither the name of NVIDIA CORPORATION nor the names of its
+//    contributors may be used to endorse or promote products derived
+//    from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "triton/common/table_printer.h"
+
+#ifdef _WIN32
+// suppress the min and max definitions in Windef.h.
+#define NOMINMAX
+#include <Windows.h>
+#else
+#include <sys/ioctl.h>
+#include <unistd.h>
+#endif
+#include <algorithm>
+#include <iomanip>
+#include <iostream>
+#include <memory>
+#include <numeric>
+#include <sstream>
+#include <string>
+#include <vector>
+
+namespace triton { namespace common {
+
+//
+// ASCII table printer.
+//
+void
+TablePrinter::InsertRow(const std::vector<std::string>& row)
+{
+  std::vector<std::vector<std::string>> table_row;
+
+  // Number of lines in each field in the record
+  size_t max_height = 0;
+
+  // Update max length of data items in each row
+  for (size_t i = 0; i < row.size(); ++i) {
+    table_row.push_back(std::vector<std::string>{});
+    std::stringstream ss(row[i]);
+    std::string line;
+
+    size_t max_width = 0;
+    while (std::getline(ss, line, '\n')) {
+      table_row[i].push_back(line);
+      if (line.size() > max_width)
+        max_width = line.size();
+    }
+
+    if (max_width > max_widths_[i])
+      max_widths_[i] = max_width;
+
+    size_t number_of_lines = table_row[i].size();
+    if (max_height < number_of_lines)
+      max_height = number_of_lines;
+  }
+
+  max_heights_.push_back(max_height);
+  data_.emplace_back(table_row);
+}
+
+void
+TablePrinter::FairShare()
+{
+  // initialize original index locations
+  size_t array_size = max_widths_.size();
+  std::vector<size_t> idx(array_size);
+  iota(idx.begin(), idx.end(), 0);
+
+  stable_sort(idx.begin(), idx.end(), [this](size_t i1, size_t i2) {
+    return this->max_widths_[i1] < this->max_widths_[i2];
+  });
+
+  size_t loop_index = 1;
+  for (auto itr = idx.begin(); itr != idx.end(); ++itr) {
+    // If a column is not using all the space allocated to it
+    if (max_widths_[*itr] < shares_[*itr]) {
+      float excess = shares_[*itr] - max_widths_[*itr];
+      shares_[*itr] -= excess;
+
+      if (itr == idx.end() - 1)
+        break;
+      auto update_itr = idx.begin() + (itr - idx.begin() + 1);
+
+      // excess amount of unused space that must be distributed evenly to the
+      // next columns
+      float excess_per_column = excess / (array_size - loop_index);
+
+      for (; update_itr != idx.end(); ++update_itr) {
+        shares_[*update_itr] += excess_per_column;
+        excess -= excess_per_column;
+      }
+    }
+    ++loop_index;
+  }
+
+  // Remove any decimal shares
+  for (auto itr = idx.begin(); itr != idx.end(); ++itr) {
+    shares_[*itr] = (size_t)shares_[*itr];
+  }
+
+  // For each record
+  for (size_t i = 0; i < data_.size(); i++) {
+    auto current_row = data_[i];
+
+    // For each field in the record
+    for (size_t j = 0; j < current_row.size(); j++) {
+      // For each line in the record
+      for (size_t line_index = 0; line_index < current_row[j].size();
+           line_index++) {
+        std::string line = current_row[j][line_index];
+        size_t num_rows = (line.size() + shares_[j] - 1) / shares_[j];
+
+        // If the number of rows required for this record is larger than 1, we
+        // will break that line and put it in multiple lines
+        if (num_rows > 1) {
+          // Remove the multi-line field, it will be replaced by the line
+          // that can fits the column size
+          data_[i][j].erase(data_[i][j].begin() + line_index);
+          for (size_t k = 0; k < num_rows; k++) {
+            size_t start_index =
+                std::min((size_t)(k * shares_[j]), line.size());
+            size_t end_index =
+                std::min((size_t)((k + 1) * shares_[j]), line.size());
+            data_[i][j].insert(
+                data_[i][j].begin() + line_index + k,
+                line.substr(start_index, end_index - start_index));
+          }
+
+          // We need to advance the index for the splitted lines.
+          line_index += num_rows - 1;
+        }
+
+        if (max_heights_[i] < (num_rows - 1 + current_row[j].size()))
+          max_heights_[i] += num_rows - 1;
+      }
+    }
+  }
+}
+
+void
+TablePrinter::AddRow(std::stringstream& table, size_t row_index)
+{
+  auto row = data_[row_index];
+  size_t max_height = max_heights_[row_index];
+
+  for (size_t j = 0; j < max_height; j++) {
+    table << "|" << std::left;
+
+    for (size_t i = 0; i < row.size(); i++) {
+      if (j < row[i].size())
+        table << " " << std::setw(shares_[i]) << row[i][j] << " |";
+      else
+        table << " " << std::setw(shares_[i]) << " "
+              << " |";
+    }
+
+    // Do not add new line if this is the last row of this record
+    if (j != max_height - 1)
+      table << "\n";
+  }
+  table << "\n";
+}
+
+void
+TablePrinter::AddRowDivider(std::stringstream& table)
+{
+  table << "+";
+  for (const auto& share : shares_) {
+    for (size_t i = 0; i < share + 2; i++) table << "-";
+    table << "+";
+  }
+  table << "\n";
+}
+
+std::string
+TablePrinter::PrintTable()
+{
+  std::stringstream table;
+  table << "\n";
+
+  FairShare();
+
+  AddRowDivider(table);
+  // Add table headers
+  AddRow(table, 0);
+  AddRowDivider(table);
+
+  for (size_t j = 1; j < data_.size(); j++) {
+    AddRow(table, j);
+  }
+
+  AddRowDivider(table);
+
+  return table.str();
+}
+
+// TablePrinter will take the ownership of `headers`.
+TablePrinter::TablePrinter(const std::vector<std::string>& headers)
+{
+  // terminal size
+  size_t column_size = 500;
+#ifdef _WIN32
+  CONSOLE_SCREEN_BUFFER_INFO csbi;
+  int ret = GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &csbi);
+  if (ret && (csbi.dwSize.X != 0)) {
+    column_size = csbi.dwSize.X;
+  }
+#else
+  struct winsize terminal_size;
+  int status = ioctl(STDOUT_FILENO, TIOCGWINSZ, &terminal_size);
+  if ((status == 0) && (terminal_size.ws_col != 0)) {
+    column_size = terminal_size.ws_col;
+  }
+#endif
+
+  for (size_t i = 0; i < headers.size(); ++i) {
+    max_widths_.emplace_back(0);
+  }
+
+  // Calculate fair share of every column
+  size_t number_of_columns = headers.size();
+
+  // Terminal width is the actual terminal width minus two times spaces
+  // required before and after each column and number of columns plus 1 for
+  // the pipes between the columns
+  size_t terminal_width =
+      column_size - (2 * number_of_columns) - (number_of_columns + 1);
+  int equal_share = terminal_width / headers.size();
+
+  for (size_t i = 0; i < headers.size(); ++i) {
+    shares_.emplace_back(equal_share);
+    terminal_width -= equal_share;
+  }
+
+  InsertRow(headers);
+}
+
+}}  // namespace triton::common

3rdparty/common-r22.12/src/thread_pool.cc

+// Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+//  * Redistributions of source code must retain the above copyright
+//    notice, this list of conditions and the following disclaimer.
+//  * Redistributions in binary form must reproduce the above copyright
+//    notice, this list of conditions and the following disclaimer in the
+//    documentation and/or other materials provided with the distribution.
+//  * Neither the name of NVIDIA CORPORATION nor the names of its
+//    contributors may be used to endorse or promote products derived
+//    from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "triton/common/thread_pool.h"
+#include <stdexcept>
+
+namespace triton { namespace common {
+
+ThreadPool::ThreadPool(size_t thread_count)
+{
+  if (!thread_count) {
+    throw std::invalid_argument("Thread count must be greater than zero.");
+  }
+
+  // Define infinite loop for each thread to wait for a task to complete
+  const auto worker_loop = [this]() {
+    while (true) {
+      Task task;
+      {
+        std::unique_lock<std::mutex> lk(queue_mtx_);
+        // Wake if there's a task to do, or the pool has been stopped.
+        cv_.wait(lk, [&]() { return !task_queue_.empty() || stop_; });
+        // Exit condition
+        if (stop_ && task_queue_.empty()) {
+          break;
+        }
+        task = std::move(task_queue_.front());
+        task_queue_.pop();
+      }
+
+      // Execute task - ensure function has a valid target
+      if (task) {
+        task();
+      }
+    }
+  };
+
+  workers_.reserve(thread_count);
+  for (size_t i = 0; i < thread_count; ++i) {
+    workers_.emplace_back(worker_loop);
+  }
+}
+
+ThreadPool::~ThreadPool()
+{
+  {
+    std::lock_guard<std::mutex> lk(queue_mtx_);
+    // Signal to each worker that it should exit loop when tasks are finished
+    stop_ = true;
+  }
+  // Wake all threads to clean up
+  cv_.notify_all();
+  for (auto& t : workers_) {
+    t.join();
+  }
+}
+
+void
+ThreadPool::Enqueue(Task&& task)
+{
+  {
+    std::lock_guard<std::mutex> lk(queue_mtx_);
+    // Don't accept more work if pool is shutting down
+    if (stop_) {
+      return;
+    }
+    task_queue_.push(std::move(task));
+  }
+  // Only wake one thread per task
+  // Todo: DLIS-3859 if ThreadPool gets used more.
+  cv_.notify_one();
+}
+
+}}  // namespace triton::common

3rdparty/common-r22.12/tools/format.py

+#!/usr/bin/python
+
+# Copyright (c) 2018-2020, NVIDIA CORPORATION. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import argparse
+import os
+import subprocess
+import yapf
+
+FLAGS = None
+FORMAT_EXTS = ('proto', 'cc', 'cu', 'h')
+SKIP_PATHS = ('tools',)
+
+
+def visit(path):
+    if FLAGS.verbose:
+        print("visiting " + path)
+
+    valid_ext = False
+    python_file = False
+    for ext in FORMAT_EXTS:
+        if path.endswith('.' + ext):
+            valid_ext = True
+            break
+    if path.endswith('.py'):
+        valid_ext = True
+        python_file = True
+    if not valid_ext:
+        if FLAGS.verbose:
+            print("skipping due to extension: " + path)
+        return True
+
+    for skip in SKIP_PATHS:
+        if path.startswith(skip):
+            if FLAGS.verbose:
+                print("skipping due to path prefix: " + path)
+            return True
+    if python_file:
+        yapf.yapflib.yapf_api.FormatFile(path,
+                                         in_place=True,
+                                         style_config='google')
+        return True
+    else:
+        args = ['clang-format-6.0', '--style=file', '-i']
+        if FLAGS.verbose:
+            args.append('-verbose')
+        args.append(path)
+
+        ret = subprocess.call(args)
+        if ret != 0:
+            print("format failed for " + path)
+            return False
+
+    return True
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-v',
+                        '--verbose',
+                        action="store_true",
+                        required=False,
+                        default=False,
+                        help='Enable verbose output')
+    parser.add_argument('paths',
+                        type=str,
+                        nargs='*',
+                        default=None,
+                        help='Directories or files to format')
+    FLAGS = parser.parse_args()
+
+    # Check the version of yapf. Needs a consistent version
+    # of yapf to prevent unneccessary changes in the code.
+    if (yapf.__version__ != '0.30.0'):
+        print("Needs yapf 0.30.0, but got yapf {}".format(yapf.__version__))
+
+    if (FLAGS.paths is None) or (len(FLAGS.paths) == 0):
+        parser.print_help()
+        exit(1)
+
+    ret = True
+    for path in FLAGS.paths:
+        if not os.path.isdir(path):
+            if not visit(path):
+                ret = False
+        else:
+            for root, dirs, files in os.walk(path):
+                for name in files:
+                    if not visit(os.path.join(root, name)):
+                        ret = False
+
+    exit(0 if ret else 1)

3rdparty/common-r22.12/tools/pre-commit

+#!/bin/bash
+# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+###############################################################################
+#
+# Git pre-commit hook for Triton related projects
+#
+# To install this hook for a project, copy "pre-commit" and "format.py" into
+# ".git/hooks/" directory of the project
+#
+###############################################################################
+
+###############################################################################
+#
+# Run formatter script
+#
+###############################################################################
+
+# Repo root
+GIT_REPO_ROOT=$(git rev-parse --show-toplevel)
+
+PYTHON_CMD=python3
+FORMATTER_PY=${GIT_REPO_ROOT}/.git/hooks/format.py
+
+CHANGED_FILES="$(git --no-pager diff --name-status --no-color --cached | awk '{ if (match($1, /R[0-9]+/)) { print $3 } else if ($1 != "D") { print $2 } }')"
+
+echo "Running Python auto-format..."
+for CHANGED_FILE in $CHANGED_FILES;
+do
+    ${PYTHON_CMD} ${FORMATTER_PY} ${GIT_REPO_ROOT}/${CHANGED_FILE}
+    git add ${GIT_REPO_ROOT}/${CHANGED_FILE}
+done

3rdparty/core-r22.12/.clang-format

+---
+BasedOnStyle: Google
+
+IndentWidth: 2
+ContinuationIndentWidth: 4
+UseTab: Never
+MaxEmptyLinesToKeep: 2
+
+SortIncludes: true
+CompactNamespaces: true
+ReflowComments: true
+
+DerivePointerAlignment: false
+PointerAlignment: Left
+
+AllowShortIfStatementsOnASingleLine: false
+AllowShortBlocksOnASingleLine: false
+AllowShortFunctionsOnASingleLine: Inline
+
+AlwaysBreakAfterReturnType: TopLevelDefinitions
+AlignAfterOpenBracket: AlwaysBreak
+BreakBeforeBraces: Custom
+BraceWrapping:
+  AfterClass: false
+  AfterControlStatement: false
+  AfterEnum: false
+  AfterFunction: true
+  AfterNamespace: false
+  AfterStruct: false
+  AfterUnion: false
+  BeforeCatch: true
+
+BinPackArguments: true
+BinPackParameters: true
+ConstructorInitializerAllOnOneLineOrOnePerLine: false
+
+IndentCaseLabels: true
\ No newline at end of file

3rdparty/core-r22.12/.gitignore

+/build
+/.vscode
+*.so

3rdparty/core-r22.12/LICENSE

+Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+ * Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+ * Neither the name of NVIDIA CORPORATION nor the names of its
+   contributors may be used to endorse or promote products derived
+   from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

3rdparty/core-r22.12/README.md

+<!--
+# Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+-->
+
+[![License](https://img.shields.io/badge/License-BSD3-lightgrey.svg)](https://opensource.org/licenses/BSD-3-Clause)
+
+# Triton Inference Server Core
+
+This repository holds the source code and headers for the library that
+implements the core functionality of Triton. The *core* library can be
+built as described below and used directly via its [C
+API](https://github.com/triton-inference-server/server/blob/main/docs/customization_guide/inference_protocols.md#in-process-triton-server-api). To
+be useful the core library must be paired with one or more backends.
+You can learn more about backends in the [backend
+repo](https://github.com/triton-inference-server/backend).
+
+Typically you do not build or use the core library on its own, but as
+part of the *tritonserver* executable. The *tritonserver* executable
+is built in the [server
+repo](https://github.com/triton-inference-server/server) as described
+in the [server build
+documentation](https://github.com/triton-inference-server/server/blob/main/docs/customization_guide/build.md).
+
+Ask questions or report problems in the main Triton [issues
+page](https://github.com/triton-inference-server/server/issues).
+
+## Build the Triton Core Library
+
+Before building the Triton core library, your build system must
+install the required dependencies described in the [build
+documentation](https://github.com/triton-inference-server/server/blob/main/docs/customization_guide/build.md). For
+example, if you are building the core library with GPU support
+(-DTRITON_ENABLE_GPU=ON), then you must install the CUDA, cuDNN, and
+TensorRT dependencies required for the version of Triton you are
+building.
+
+To build, first clone the release branch matching the Triton release
+you are interest in (*rxx.yy*), or the *main* branch to build the
+top-of-tree. The Triton core library is built with CMake.
+
+```
+$ mkdir build
+$ cd build
+$ cmake -DCMAKE_INSTALL_PREFIX:PATH=`pwd`/install -DTRITON_CORE_HEADERS_ONLY=OFF ..
+$ make install
+```
+
+When the build completes, the install directory will contain the
+Triton core shared library (install/lib/libtritonserver.so on Linux,
+install/bin/tritonserver.dll on Windows), and the core library headers
+files in install/include/triton/core.
+
+### Build a Release Branch
+
+The following required Triton repositories will be pulled and used in
+the build. By default the "main" branch/tag will be used for each repo
+but the listed CMake argument can be used to override.
+
+* triton-inference-server/third_party: -DTRITON_THIRD_PARTY_REPO_TAG=[tag]
+* triton-inference-server/common: -DTRITON_COMMON_REPO_TAG=[tag]
+
+You will need to override if you are building from a release
+branch. For example, if you are building the r22.03 version of Triton,
+you would clone the r22.03 branch of the core repo and use the
+following cmake command.
+
+```
+$ cmake -DTRITON_THIRD_PARTY_REPO_TAG=r22.03 -DTRITON_COMMON_REPO_TAG=r22.03 -DTRITON_CORE_HEADERS_ONLY=OFF ..
+```
+
+### Build Options
+
+The [CMakeLists.txt](CMakeLists.txt) file contains the options
+available when build the core library. For example, to build the core
+library with the default settings plus S3 cloud storage and ensembling
+support use the following command.
+
+```
+$ cmake -DTRITON_CORE_HEADERS_ONLY=OFF -DTRITON_ENABLE_S3=ON -DTRITON_ENABLE_ENSEMBLE=ON ..
+```

3rdparty/core-r22.12/cmake/TritonCoreConfig.cmake.in

+# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+include(CMakeFindDependencyMacro)
+
+get_filename_component(
+  TRITONCORE_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH
+)
+
+list(APPEND CMAKE_MODULE_PATH ${TRITONCORE_CMAKE_DIR})
+
+if(NOT TARGET TritonCore::triton-core-serverapi)
+  include("${TRITONCORE_CMAKE_DIR}/TritonCoreTargets.cmake")
+endif()

3rdparty/core-r22.12/include/triton/core/tritonbackend.h

+// Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+//  * Redistributions of source code must retain the above copyright
+//    notice, this list of conditions and the following disclaimer.
+//  * Redistributions in binary form must reproduce the above copyright
+//    notice, this list of conditions and the following disclaimer in the
+//    documentation and/or other materials provided with the distribution.
+//  * Neither the name of NVIDIA CORPORATION nor the names of its
+//    contributors may be used to endorse or promote products derived
+//    from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#pragma once
+
+#include <stddef.h>
+#include <stdint.h>
+#include "triton/core/tritonserver.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef _COMPILING_TRITONBACKEND
+#if defined(_MSC_VER)
+#define TRITONBACKEND_DECLSPEC __declspec(dllexport)
+#define TRITONBACKEND_ISPEC __declspec(dllimport)
+#elif defined(__GNUC__)
+#define TRITONBACKEND_DECLSPEC __attribute__((__visibility__("default")))
+#define TRITONBACKEND_ISPEC
+#else
+#define TRITONBACKEND_DECLSPEC
+#define TRITONBACKEND_ISPEC
+#endif
+#else
+#if defined(_MSC_VER)
+#define TRITONBACKEND_DECLSPEC __declspec(dllimport)
+#define TRITONBACKEND_ISPEC __declspec(dllexport)
+#else
+#define TRITONBACKEND_DECLSPEC
+#define TRITONBACKEND_ISPEC
+#endif
+#endif
+
+struct TRITONBACKEND_MemoryManager;
+struct TRITONBACKEND_Input;
+struct TRITONBACKEND_Output;
+struct TRITONBACKEND_State;
+struct TRITONBACKEND_Request;
+struct TRITONBACKEND_ResponseFactory;
+struct TRITONBACKEND_Response;
+struct TRITONBACKEND_Backend;
+struct TRITONBACKEND_Model;
+struct TRITONBACKEND_ModelInstance;
+struct TRITONBACKEND_BackendAttribute;
+
+///
+/// TRITONBACKEND API Version
+///
+/// The TRITONBACKEND API is versioned with major and minor version
+/// numbers. Any change to the API that does not impact backwards
+/// compatibility (for example, adding a non-required function)
+/// increases the minor version number. Any change that breaks
+/// backwards compatibility (for example, deleting or changing the
+/// behavior of a function) increases the major version number. A
+/// backend should check that the API version used to compile the
+/// backend is compatible with the API version of the Triton server
+/// that it is running in. This is typically done by code similar to
+/// the following which makes sure that the major versions are equal
+/// and that the minor version of Triton is >= the minor version used
+/// to build the backend.
+///
+///   uint32_t api_version_major, api_version_minor;
+///   TRITONBACKEND_ApiVersion(&api_version_major, &api_version_minor);
+///   if ((api_version_major != TRITONBACKEND_API_VERSION_MAJOR) ||
+///       (api_version_minor < TRITONBACKEND_API_VERSION_MINOR)) {
+///     return TRITONSERVER_ErrorNew(
+///       TRITONSERVER_ERROR_UNSUPPORTED,
+///       "triton backend API version does not support this backend");
+///   }
+///
+#define TRITONBACKEND_API_VERSION_MAJOR 1
+#define TRITONBACKEND_API_VERSION_MINOR 10
+
+/// Get the TRITONBACKEND API version supported by Triton. This value
+/// can be compared against the TRITONBACKEND_API_VERSION_MAJOR and
+/// TRITONBACKEND_API_VERSION_MINOR used to build the backend to
+/// ensure that Triton is compatible with the backend.
+///
+/// \param major Returns the TRITONBACKEND API major version supported
+/// by Triton.
+/// \param minor Returns the TRITONBACKEND API minor version supported
+/// by Triton.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ApiVersion(
+    uint32_t* major, uint32_t* minor);
+
+/// TRITONBACKEND_ArtifactType
+///
+/// The ways that the files that make up a backend or model are
+/// communicated to the backend.
+///
+///   TRITONBACKEND_ARTIFACT_FILESYSTEM: The model or backend
+///     artifacts are made available to Triton via a locally
+///     accessible filesystem. The backend can access these files
+///     using an appropriate system API.
+///
+typedef enum TRITONBACKEND_artifacttype_enum {
+  TRITONBACKEND_ARTIFACT_FILESYSTEM
+} TRITONBACKEND_ArtifactType;
+
+
+///
+/// TRITONBACKEND_MemoryManager
+///
+/// Object representing an memory manager that is capable of
+/// allocating and otherwise managing different memory types. For
+/// improved performance Triton maintains pools for GPU and CPU-pinned
+/// memory and the memory manager allows backends to access those
+/// pools.
+///
+
+/// Allocate a contiguous block of memory of a specific type using a
+/// memory manager. Two error codes have specific interpretations for
+/// this function:
+///
+///   TRITONSERVER_ERROR_UNSUPPORTED: Indicates that Triton is
+///     incapable of allocating the requested memory type and memory
+///     type ID. Requests for the memory type and ID will always fail
+///     no matter 'byte_size' of the request.
+///
+///   TRITONSERVER_ERROR_UNAVAILABLE: Indicates that Triton can
+///      allocate the memory type and ID but that currently it cannot
+///      allocate a contiguous block of memory of the requested
+///      'byte_size'.
+///
+/// \param manager The memory manager.
+/// \param buffer Returns the allocated memory.
+/// \param memory_type The type of memory to allocate.
+/// \param memory_type_id The ID associated with the memory type to
+/// allocate. For GPU memory this indicates the device ID of the GPU
+/// to allocate from.
+/// \param byte_size The size of memory to allocate, in bytes.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_MemoryManagerAllocate(
+    TRITONBACKEND_MemoryManager* manager, void** buffer,
+    const TRITONSERVER_MemoryType memory_type, const int64_t memory_type_id,
+    const uint64_t byte_size);
+
+/// Free a buffer that was previously allocated with
+/// TRITONBACKEND_MemoryManagerAllocate. The call must provide the
+/// same values for 'memory_type' and 'memory_type_id' as were used
+/// when the buffer was allocate or else the behavior is undefined.
+///
+/// \param manager The memory manager.
+/// \param buffer The allocated memory buffer to free.
+/// \param memory_type The type of memory of the buffer.
+/// \param memory_type_id The ID associated with the memory type of
+/// the buffer.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_MemoryManagerFree(
+    TRITONBACKEND_MemoryManager* manager, void* buffer,
+    const TRITONSERVER_MemoryType memory_type, const int64_t memory_type_id);
+
+///
+/// TRITONBACKEND_Input
+///
+/// Object representing an input tensor.
+///
+
+/// Get the name and properties of an input tensor. The returned
+/// strings and other properties are owned by the input, not the
+/// caller, and so should not be modified or freed.
+///
+/// \param input The input tensor.
+/// \param name If non-nullptr, returns the tensor name.
+/// \param datatype If non-nullptr, returns the tensor datatype.
+/// \param shape If non-nullptr, returns the tensor shape.
+/// \param dim_count If non-nullptr, returns the number of dimensions
+/// in the tensor shape.
+/// \param byte_size If non-nullptr, returns the size of the available
+/// data for the tensor, in bytes. This size reflects the actual data
+/// available, and does not necessarily match what is
+/// expected/required for the tensor given its shape and datatype. It
+/// is the responsibility of the backend to handle mismatches in these
+/// sizes appropriately.
+/// \param buffer_count If non-nullptr, returns the number of buffers
+/// holding the contents of the tensor. These buffers are accessed
+/// using TRITONBACKEND_InputBuffer.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_InputProperties(
+    TRITONBACKEND_Input* input, const char** name,
+    TRITONSERVER_DataType* datatype, const int64_t** shape,
+    uint32_t* dims_count, uint64_t* byte_size, uint32_t* buffer_count);
+
+/// Get the name and properties of an input tensor associated with a given
+/// host policy. If there are no input buffers for the specified  host policy,
+/// the properties of the fallback input buffers are returned. The returned
+/// strings and other properties are owned by the input, not the caller, and so
+/// should not be modified or freed.
+///
+/// \param input The input tensor.
+/// \param host_policy_name The host policy name. Fallback input properties
+/// will be return if nullptr is provided.
+/// \param name If non-nullptr, returns the tensor name.
+/// \param datatype If non-nullptr, returns the tensor datatype.
+/// \param shape If non-nullptr, returns the tensor shape.
+/// \param dim_count If non-nullptr, returns the number of dimensions
+/// in the tensor shape.
+/// \param byte_size If non-nullptr, returns the size of the available
+/// data for the tensor, in bytes. This size reflects the actual data
+/// available, and does not necessarily match what is
+/// expected/required for the tensor given its shape and datatype. It
+/// is the responsibility of the backend to handle mismatches in these
+/// sizes appropriately.
+/// \param buffer_count If non-nullptr, returns the number of buffers
+/// holding the contents of the tensor. These buffers are accessed
+/// using TRITONBACKEND_InputBufferForHostPolicy.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error*
+TRITONBACKEND_InputPropertiesForHostPolicy(
+    TRITONBACKEND_Input* input, const char* host_policy_name, const char** name,
+    TRITONSERVER_DataType* datatype, const int64_t** shape,
+    uint32_t* dims_count, uint64_t* byte_size, uint32_t* buffer_count);
+
+/// Get a buffer holding (part of) the tensor data for an input. For a
+/// given input the number of buffers composing the input are found
+/// from 'buffer_count' returned by TRITONBACKEND_InputProperties. The
+/// returned buffer is owned by the input and so should not be
+/// modified or freed by the caller. The lifetime of the buffer
+/// matches that of the input and so the buffer should not be accessed
+/// after the input tensor object is released.
+///
+/// \param input The input tensor.
+/// \param index The index of the buffer. Must be 0 <= index <
+/// buffer_count, where buffer_count is the value returned by
+/// TRITONBACKEND_InputProperties.
+/// \param buffer Returns a pointer to a contiguous block of data for
+/// the named input.
+/// \param buffer_byte_size Returns the size, in bytes, of 'buffer'.
+/// \param memory_type Acts as both input and output. On input gives
+/// the buffer memory type preferred by the function caller.  Returns
+/// the actual memory type of 'buffer'.
+/// \param memory_type_id Acts as both input and output. On input
+/// gives the buffer memory type id preferred by the function caller.
+/// Returns the actual memory type id of 'buffer'.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_InputBuffer(
+    TRITONBACKEND_Input* input, const uint32_t index, const void** buffer,
+    uint64_t* buffer_byte_size, TRITONSERVER_MemoryType* memory_type,
+    int64_t* memory_type_id);
+
+/// Get a buffer holding (part of) the tensor data for an input for a specific
+/// host policy. If there are no input buffers specified for this host policy,
+/// the fallback input buffer is returned.
+/// For a given input the number of buffers composing the input are found
+/// from 'buffer_count' returned by TRITONBACKEND_InputPropertiesForHostPolicy.
+/// The returned buffer is owned by the input and so should not be modified or
+/// freed by the caller. The lifetime of the buffer matches that of the input
+/// and so the buffer should not be accessed after the input tensor object is
+/// released.
+///
+/// \param input The input tensor.
+/// \param host_policy_name The host policy name. Fallback input buffer
+/// will be return if nullptr is provided.
+/// \param index The index of the buffer. Must be 0 <= index <
+/// buffer_count, where buffer_count is the value returned by
+/// TRITONBACKEND_InputPropertiesForHostPolicy.
+/// \param buffer Returns a pointer to a contiguous block of data for
+/// the named input.
+/// \param buffer_byte_size Returns the size, in bytes, of 'buffer'.
+/// \param memory_type Acts as both input and output. On input gives
+/// the buffer memory type preferred by the function caller.  Returns
+/// the actual memory type of 'buffer'.
+/// \param memory_type_id Acts as both input and output. On input
+/// gives the buffer memory type id preferred by the function caller.
+/// Returns the actual memory type id of 'buffer'.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error*
+TRITONBACKEND_InputBufferForHostPolicy(
+    TRITONBACKEND_Input* input, const char* host_policy_name,
+    const uint32_t index, const void** buffer, uint64_t* buffer_byte_size,
+    TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id);
+
+/// Get the buffer attributes associated with the given input buffer. For a
+/// given input the number of buffers composing the input are found from
+/// 'buffer_count' returned by TRITONBACKEND_InputProperties. The returned
+/// 'buffer_attributes' is owned by the input and so should not be modified or
+/// freed by the caller. The lifetime of the 'buffer_attributes' matches that of
+/// the input and so the 'buffer_attributes' should not be accessed after the
+/// input tensor object is released.
+///
+/// \param input The input tensor.
+/// \param index The index of the buffer. Must be 0 <= index < buffer_count,
+/// where buffer_count is the value returned by TRITONBACKEND_InputProperties.
+/// \param buffer Returns a pointer to a contiguous block of data for
+/// the named input.
+/// \param buffer_attributes Returns the attributes for the given buffer.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_InputBufferAttributes(
+    TRITONBACKEND_Input* input, const uint32_t index, const void** buffer,
+    TRITONSERVER_BufferAttributes** buffer_attributes);
+
+///
+/// TRITONBACKEND_Output
+///
+/// Object representing a response output tensor.
+///
+
+/// Get a buffer to use to hold the tensor data for the output. The
+/// returned buffer is owned by the output and so should not be freed
+/// by the caller. The caller can and should fill the buffer with the
+/// output data for the tensor. The lifetime of the buffer matches
+/// that of the output and so the buffer should not be accessed after
+/// the output tensor object is released.
+///
+/// \param buffer Returns a pointer to a buffer where the contents of
+/// the output tensor should be placed.
+/// \param buffer_byte_size The size, in bytes, of the buffer required
+/// by the caller.
+/// \param memory_type Acts as both input and output. On input gives
+/// the buffer memory type preferred by the caller.  Returns the
+/// actual memory type of 'buffer'.
+/// \param memory_type_id Acts as both input and output. On input
+/// gives the buffer memory type id preferred by the caller. Returns
+/// the actual memory type id of 'buffer'.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_OutputBuffer(
+    TRITONBACKEND_Output* output, void** buffer,
+    const uint64_t buffer_byte_size, TRITONSERVER_MemoryType* memory_type,
+    int64_t* memory_type_id);
+
+/// Get the buffer attributes associated with the given output buffer. The
+/// returned 'buffer_attributes' is owned by the output and so should not be
+/// modified or freed by the caller. The lifetime of the 'buffer_attributes'
+/// matches that of the output and so the 'buffer_attributes' should not be
+/// accessed after the output tensor object is released. This function must be
+/// called after the TRITONBACKEND_OutputBuffer otherwise it might contain
+/// incorrect data.
+///
+/// \param output The output tensor.
+/// \param buffer_attributes Returns the attributes for the output buffer.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_OutputBufferAttributes(
+    TRITONBACKEND_Output* output,
+    TRITONSERVER_BufferAttributes** buffer_attributes);
+
+///
+/// TRITONBACKEND_Request
+///
+/// Object representing an inference request.
+///
+
+/// Get the ID of the request. Can be nullptr if request doesn't have
+/// an ID. The returned string is owned by the request, not the
+/// caller, and so should not be modified or freed.
+///
+/// \param request The inference request.
+/// \param id Returns the ID.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestId(
+    TRITONBACKEND_Request* request, const char** id);
+
+/// Get the correlation ID of the request if it is an unsigned integer.
+/// Zero indicates that the request does not have a correlation ID.
+/// Returns failure if correlation ID for given request is not an unsigned
+/// integer.
+///
+/// \param request The inference request.
+/// \param id Returns the correlation ID.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestCorrelationId(
+    TRITONBACKEND_Request* request, uint64_t* id);
+
+/// Get the correlation ID of the request if it is a string.
+/// Empty string indicates that the request does not have a correlation ID.
+/// Returns error if correlation ID for given request is not a string.
+///
+/// \param request The inference request.
+/// \param id Returns the correlation ID.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error*
+TRITONBACKEND_RequestCorrelationIdString(
+    TRITONBACKEND_Request* request, const char** id);
+
+/// Get the flag(s) associated with a request. On return 'flags' holds
+/// a bitwise-or of all flag values, see TRITONSERVER_RequestFlag for
+/// available flags.
+///
+/// \param request The inference request.
+/// \param flags Returns the flags.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestFlags(
+    TRITONBACKEND_Request* request, uint32_t* flags);
+
+/// Get the number of input tensors specified in the request.
+///
+/// \param request The inference request.
+/// \param count Returns the number of input tensors.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestInputCount(
+    TRITONBACKEND_Request* request, uint32_t* count);
+
+/// Get the name of an input tensor. The caller does not own
+/// the returned string and must not modify or delete it. The lifetime
+/// of the returned string extends only as long as 'request'.
+///
+/// \param request The inference request.
+/// \param index The index of the input tensor. Must be 0 <= index <
+/// count, where count is the value returned by
+/// TRITONBACKEND_RequestInputCount.
+/// \param input_name Returns the name of the input tensor
+/// corresponding to the index.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestInputName(
+    TRITONBACKEND_Request* request, const uint32_t index,
+    const char** input_name);
+
+/// Get a named request input. The lifetime of the returned input
+/// object matches that of the request and so the input object should
+/// not be accessed after the request object is released.
+///
+/// \param request The inference request.
+/// \param name The name of the input.
+/// \param input Returns the input corresponding to the name.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestInput(
+    TRITONBACKEND_Request* request, const char* name,
+    TRITONBACKEND_Input** input);
+
+/// Get a request input by index. The order of inputs in a given
+/// request is not necessarily consistent with other requests, even if
+/// the requests are in the same batch. As a result, you can not
+/// assume that an index obtained from one request will point to the
+/// same input in a different request.
+///
+/// The lifetime of the returned input object matches that of the
+/// request and so the input object should not be accessed after the
+/// request object is released.
+///
+/// \param request The inference request.
+/// \param index The index of the input tensor. Must be 0 <= index <
+/// count, where count is the value returned by
+/// TRITONBACKEND_RequestInputCount.
+/// \param input Returns the input corresponding to the index.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestInputByIndex(
+    TRITONBACKEND_Request* request, const uint32_t index,
+    TRITONBACKEND_Input** input);
+
+/// Get the number of output tensors requested to be returned in the
+/// request.
+///
+/// \param request The inference request.
+/// \param count Returns the number of output tensors.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestOutputCount(
+    TRITONBACKEND_Request* request, uint32_t* count);
+
+/// Get the name of a requested output tensor. The caller does not own
+/// the returned string and must not modify or delete it. The lifetime
+/// of the returned string extends only as long as 'request'.
+///
+/// \param request The inference request.
+/// \param index The index of the requested output tensor. Must be 0
+/// <= index < count, where count is the value returned by
+/// TRITONBACKEND_RequestOutputCount.
+/// \param output_name Returns the name of the requested output tensor
+/// corresponding to the index.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestOutputName(
+    TRITONBACKEND_Request* request, const uint32_t index,
+    const char** output_name);
+
+/// Returns the preferred memory type and memory type ID of the output buffer
+/// for the request. As much as possible, Triton will attempt to return
+/// the same memory_type and memory_type_id values that will be returned by
+/// the subsequent call to TRITONBACKEND_OutputBuffer, however, the backend must
+/// be capable of handling cases where the values differ.
+///
+/// \param request The request.
+/// \param name The name of the output tensor. This is optional
+/// and it should be set to nullptr to indicate that the tensor name has
+/// not determined.
+/// \param byte_size The expected size of the buffer. This is optional
+/// and it should be set to nullptr to indicate that the byte size has
+/// not determined.
+/// \param memory_type Acts as both input and output. On input gives
+/// the memory type preferred by the caller. Returns memory type preferred
+/// by Triton, taken account of the caller preferred type.
+/// \param memory_type_id Acts as both input and output. On input gives
+/// the memory type ID preferred by the caller. Returns memory type ID preferred
+/// by Triton, taken account of the caller preferred type ID.
+/// \return a TRITONSERVER_Error object if a failure occurs.
+/// A TRITONSERVER_ERROR_UNAVAILABLE error indicates that the properties are not
+/// available, other error codes indicate an error.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error*
+TRITONBACKEND_RequestOutputBufferProperties(
+    TRITONBACKEND_Request* request, const char* name, size_t* byte_size,
+    TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id);
+
+/// Release the request. The request should be released when it is no
+/// longer needed by the backend. If this call returns with an error
+/// (i.e. non-nullptr) then the request was not released and ownership
+/// remains with the backend. If this call returns with success, the
+/// 'request' object is no longer owned by the backend and must not be
+/// used. Any tensor names, data types, shapes, input tensors,
+/// etc. returned by TRITONBACKEND_Request* functions for this request
+/// are no longer valid. If a persistent copy of that data is required
+/// it must be created before calling this function.
+///
+/// \param request The inference request.
+/// \param release_flags Flags indicating what type of request release
+/// should be performed. \see TRITONSERVER_RequestReleaseFlag. \see
+/// TRITONSERVER_InferenceRequestReleaseFn_t.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_RequestRelease(
+    TRITONBACKEND_Request* request, uint32_t release_flags);
+
+///
+/// TRITONBACKEND_ResponseFactory
+///
+/// Object representing an inference response factory. Using a
+/// response factory is not required; instead a response can be
+/// generated directly from a TRITONBACKEND_Request object using
+/// TRITONBACKEND_ResponseNew(). A response factory allows a request
+/// to be released before all responses have been sent. Releasing a
+/// request as early as possible releases all input tensor data and
+/// therefore may be desirable in some cases.
+
+/// Create the response factory associated with a request.
+///
+/// \param factory Returns the new response factory.
+/// \param request The inference request.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ResponseFactoryNew(
+    TRITONBACKEND_ResponseFactory** factory, TRITONBACKEND_Request* request);
+
+/// Destroy a response factory.
+///
+/// \param factory The response factory.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ResponseFactoryDelete(
+    TRITONBACKEND_ResponseFactory* factory);
+
+/// Send response flags without a corresponding response.
+///
+/// \param factory The response factory.
+/// \param send_flags Flags to send. \see
+/// TRITONSERVER_ResponseCompleteFlag. \see
+/// TRITONSERVER_InferenceResponseCompleteFn_t.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error*
+TRITONBACKEND_ResponseFactorySendFlags(
+    TRITONBACKEND_ResponseFactory* factory, const uint32_t send_flags);
+
+///
+/// TRITONBACKEND_Response
+///
+/// Object representing an inference response. For a given request,
+/// the backend must carefully manage the lifecycle of responses
+/// generated for that request to ensure that the output tensor
+/// buffers are allocated correctly. When a response is created with
+/// TRITONBACKEND_ResponseNew or TRITONBACKEND_ResponseNewFromFactory,
+/// all the outputs and corresponding buffers must be created for that
+/// response using TRITONBACKEND_ResponseOutput and
+/// TRITONBACKEND_OutputBuffer *before* another response is created
+/// for the request. For a given response, outputs can be created in
+/// any order but they must be created sequentially/sychronously (for
+/// example, the backend cannot use multiple threads to simultaneously
+/// add multiple outputs to a response).
+///
+/// The above requirement applies only to responses being generated
+/// for a given request. The backend may generate responses in
+/// parallel on multiple threads as long as those responses are for
+/// different requests.
+///
+/// This order of response creation must be strictly followed. But,
+/// once response(s) are created they do not need to be sent
+/// immediately, nor do they need to be sent in the order they were
+/// created. The backend may even delete a created response instead of
+/// sending it by using TRITONBACKEND_ResponseDelete.
+
+/// Create a response for a request.
+///
+/// \param response Returns the new response.
+/// \param request The request.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ResponseNew(
+    TRITONBACKEND_Response** response, TRITONBACKEND_Request* request);
+
+/// Create a response using a factory.
+///
+/// \param response Returns the new response.
+/// \param factory The response factory.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ResponseNewFromFactory(
+    TRITONBACKEND_Response** response, TRITONBACKEND_ResponseFactory* factory);
+
+/// Destroy a response. It is not necessary to delete a response if
+/// TRITONBACKEND_ResponseSend is called as that function transfers
+/// ownership of the response object to Triton.
+///
+/// \param response The response.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ResponseDelete(
+    TRITONBACKEND_Response* response);
+
+/// Set a string parameter in the response.
+///
+/// \param response The response.
+/// \param name The name of the parameter.
+/// \param value The value of the parameter.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error*
+TRITONBACKEND_ResponseSetStringParameter(
+    TRITONBACKEND_Response* response, const char* name, const char* value);
+
+/// Set an integer parameter in the response.
+///
+/// \param response The response.
+/// \param name The name of the parameter.
+/// \param value The value of the parameter.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error*
+TRITONBACKEND_ResponseSetIntParameter(
+    TRITONBACKEND_Response* response, const char* name, const int64_t value);
+
+/// Set an boolean parameter in the response.
+///
+/// \param response The response.
+/// \param name The name of the parameter.
+/// \param value The value of the parameter.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error*
+TRITONBACKEND_ResponseSetBoolParameter(
+    TRITONBACKEND_Response* response, const char* name, const bool value);
+
+/// Create an output tensor in the response. The lifetime of the
+/// returned output tensor object matches that of the response and so
+/// the output tensor object should not be accessed after the response
+/// object is deleted.
+///
+/// \param response The response.
+/// \param output Returns the new response output.
+/// \param name The name of the output tensor.
+/// \param datatype The datatype of the output tensor.
+/// \param shape The shape of the output tensor.
+/// \param dims_count The number of dimensions in the output tensor
+/// shape.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ResponseOutput(
+    TRITONBACKEND_Response* response, TRITONBACKEND_Output** output,
+    const char* name, const TRITONSERVER_DataType datatype,
+    const int64_t* shape, const uint32_t dims_count);
+
+/// Send a response. Calling this function transfers ownership of the
+/// response object to Triton. The caller must not access or delete
+/// the response object after calling this function.
+///
+/// \param response The response.
+/// \param send_flags Flags associated with the response. \see
+/// TRITONSERVER_ResponseCompleteFlag. \see
+/// TRITONSERVER_InferenceResponseCompleteFn_t.
+/// \param error The TRITONSERVER_Error to send if the response is an
+/// error, or nullptr if the response is successful.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ResponseSend(
+    TRITONBACKEND_Response* response, const uint32_t send_flags,
+    TRITONSERVER_Error* error);
+
+///
+/// TRITONBACKEND_State
+///
+/// Object representing a state.
+///
+
+/// Create a state in the request. The returned state object is only valid
+/// before the TRITONBACKEND_StateUpdate is called. The state should not be
+/// freed by the caller. If TRITONBACKEND_StateUpdate is not called, the
+/// lifetime of the state matches the lifetime of the request. If the state name
+/// does not exist in the "state" section of the model configuration, the state
+/// will not be created and an error will be returned. If this function is
+/// called when sequence batching is not enabled or there is no 'states' section
+/// in the sequence batching section of the model configuration, this call will
+/// return an error.
+///
+/// \param state Returns the new state.
+/// \param request The request.
+/// \param name The name of the state.
+/// \param datatype The datatype of the state.
+/// \param shape The shape of the state.
+/// \param dims_count The number of dimensions in the state shape.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_StateNew(
+    TRITONBACKEND_State** state, TRITONBACKEND_Request* request,
+    const char* name, const TRITONSERVER_DataType datatype,
+    const int64_t* shape, const uint32_t dims_count);
+
+/// Update the state for the sequence. Calling this function will replace the
+/// state stored for this seqeunce in Triton with 'state' provided in the
+/// function argument. If this function is called when sequence batching is not
+/// enabled or there is no 'states' section in the sequence batching section of
+/// the model configuration, this call will return an error. The backend is not
+/// required to call this function. If the backend doesn't call
+/// TRITONBACKEND_StateUpdate function, this particular state for the sequence
+/// will not be updated and the next inference request in the sequence will use
+/// the same state as the current inference request.
+///
+/// \param state The state.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_StateUpdate(
+    TRITONBACKEND_State* state);
+
+/// Get a buffer to use to hold the tensor data for the state. The returned
+/// buffer is owned by the state and so should not be freed by the caller. The
+/// caller can and should fill the buffer with the state data. The buffer must
+/// not be accessed by the backend after TRITONBACKEND_StateUpdate is called.
+/// The caller should fill the buffer before calling TRITONBACKEND_StateUpdate.
+///
+/// \param state The state.
+/// \param buffer Returns a pointer to a buffer where the contents of the state
+/// should be placed.
+/// \param buffer_byte_size The size, in bytes, of the buffer required
+/// by the caller.
+/// \param memory_type Acts as both input and output. On input gives
+/// the buffer memory type preferred by the caller.  Returns the
+/// actual memory type of 'buffer'.
+/// \param memory_type_id Acts as both input and output. On input
+/// gives the buffer memory type id preferred by the caller. Returns
+/// the actual memory type id of 'buffer'.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_StateBuffer(
+    TRITONBACKEND_State* state, void** buffer, const uint64_t buffer_byte_size,
+    TRITONSERVER_MemoryType* memory_type, int64_t* memory_type_id);
+
+/// Get the buffer attributes associated with the given state buffer.
+/// The returned 'buffer_attributes' is owned by the state and so should not be
+/// modified or freed by the caller. The lifetime of the 'buffer_attributes'
+/// matches that of the state.
+///
+/// \param state The state.
+/// \param buffer_attributes Returns the buffer attributes for the given state.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_StateBufferAttributes(
+    TRITONBACKEND_State* state,
+    TRITONSERVER_BufferAttributes** buffer_attributes);
+
+///
+/// TRITONBACKEND_Backend
+///
+/// Object representing a backend.
+///
+
+/// TRITONBACKEND_ExecutionPolicy
+///
+/// Types of execution policy that can be implemented by a backend.
+///
+///   TRITONBACKEND_EXECUTION_BLOCKING: An instance of the model
+///     blocks in TRITONBACKEND_ModelInstanceExecute until it is ready
+///     to handle another inference. Upon returning from
+///     TRITONBACKEND_ModelInstanceExecute, Triton may immediately
+///     call TRITONBACKEND_ModelInstanceExecute for the same instance
+///     to execute a new batch of requests. Thus, most backends using
+///     this policy will not return from
+///     TRITONBACKEND_ModelInstanceExecute until all responses have
+///     been sent and all requests have been released. This is the
+///     default execution policy.
+///
+///   TRITONBACKEND_EXECUTION_DEVICE_BLOCKING: An instance, A, of the
+///     model blocks in TRITONBACKEND_ModelInstanceExecute if the
+///     device associated with the instance is unable to handle
+///     another inference. Even if another instance, B, associated
+///     with the device, is available and ready to perform an
+///     inference, Triton will not invoke
+///     TRITONBACKEND_ModeInstanceExecute for B until A returns from
+///     TRITONBACKEND_ModelInstanceExecute. Triton will not be blocked
+///     from calling TRITONBACKEND_ModelInstanceExecute for instance
+///     C, which is associated with a different device than A and B,
+///     even if A or B has not returned from
+///     TRITONBACKEND_ModelInstanceExecute. This execution policy is
+///     typically used by a backend that can cooperatively execute
+///     multiple model instances on the same device.
+///
+typedef enum TRITONBACKEND_execpolicy_enum {
+  TRITONBACKEND_EXECUTION_BLOCKING,
+  TRITONBACKEND_EXECUTION_DEVICE_BLOCKING
+} TRITONBACKEND_ExecutionPolicy;
+
+/// Get the name of the backend. The caller does not own the returned
+/// string and must not modify or delete it. The lifetime of the
+/// returned string extends only as long as 'backend'.
+///
+/// \param backend The backend.
+/// \param name Returns the name of the backend.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_BackendName(
+    TRITONBACKEND_Backend* backend, const char** name);
+
+/// Get the backend configuration.  The 'backend_config' message is
+/// owned by Triton and should not be modified or freed by the caller.
+///
+/// The backend configuration, as JSON, is:
+///
+///   {
+///     "cmdline" : {
+///       "<setting>" : "<value>",
+///       ...
+///     }
+///   }
+///
+/// \param backend The backend.
+/// \param backend_config Returns the backend configuration as a message.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_BackendConfig(
+    TRITONBACKEND_Backend* backend, TRITONSERVER_Message** backend_config);
+
+/// Get the execution policy for this backend. By default the
+/// execution policy is TRITONBACKEND_EXECUTION_BLOCKING.
+///
+/// \param backend The backend.
+/// \param policy Returns the execution policy.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_BackendExecutionPolicy(
+    TRITONBACKEND_Backend* backend, TRITONBACKEND_ExecutionPolicy* policy);
+
+/// Set the execution policy for this backend. By default the
+/// execution policy is TRITONBACKEND_EXECUTION_BLOCKING. Triton reads
+/// the backend's execution policy after calling
+/// TRITONBACKEND_Initialize, so to be recognized changes to the
+/// execution policy must be made in TRITONBACKEND_Initialize.
+/// Also, note that if using sequence batcher for the model, Triton will
+/// use TRITONBACKEND_EXECUTION_BLOCKING policy irrespective of the
+/// policy specified by this setter function.
+///
+/// \param backend The backend.
+/// \param policy The execution policy.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error*
+TRITONBACKEND_BackendSetExecutionPolicy(
+    TRITONBACKEND_Backend* backend, TRITONBACKEND_ExecutionPolicy policy);
+
+/// Get the location of the files that make up the backend
+/// implementation. This location contains the backend shared library
+/// and any other files located with the shared library. The
+/// 'location' communicated depends on how the backend is being
+/// communicated to Triton as indicated by 'artifact_type'.
+///
+///   TRITONBACKEND_ARTIFACT_FILESYSTEM: The backend artifacts are
+///     made available to Triton via the local filesytem. 'location'
+///     returns the full path to the directory containing this
+///     backend's artifacts. The returned string is owned by Triton,
+///     not the caller, and so should not be modified or freed.
+///
+/// \param backend The backend.
+/// \param artifact_type Returns the artifact type for the backend.
+/// \param path Returns the location.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_BackendArtifacts(
+    TRITONBACKEND_Backend* backend, TRITONBACKEND_ArtifactType* artifact_type,
+    const char** location);
+
+/// Get the memory manager associated with a backend.
+///
+/// \param backend The backend.
+/// \param manager Returns the memory manager.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_BackendMemoryManager(
+    TRITONBACKEND_Backend* backend, TRITONBACKEND_MemoryManager** manager);
+
+/// Get the user-specified state associated with the backend. The
+/// state is completely owned and managed by the backend.
+///
+/// \param backend The backend.
+/// \param state Returns the user state, or nullptr if no user state.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_BackendState(
+    TRITONBACKEND_Backend* backend, void** state);
+
+/// Set the user-specified state associated with the backend. The
+/// state is completely owned and managed by the backend.
+///
+/// \param backend The backend.
+/// \param state The user state, or nullptr if no user state.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_BackendSetState(
+    TRITONBACKEND_Backend* backend, void* state);
+
+///
+/// TRITONBACKEND_Model
+///
+/// Object representing a model implemented using the backend.
+///
+
+/// Get the name of the model. The returned string is owned by the
+/// model object, not the caller, and so should not be modified or
+/// freed.
+///
+/// \param model The model.
+/// \param name Returns the model name.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelName(
+    TRITONBACKEND_Model* model, const char** name);
+
+/// Get the version of the model.
+///
+/// \param model The model.
+/// \param version Returns the model version.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelVersion(
+    TRITONBACKEND_Model* model, uint64_t* version);
+
+/// Get the location of the files that make up the model. The
+/// 'location' communicated depends on how the model is being
+/// communicated to Triton as indicated by 'artifact_type'.
+///
+///   TRITONBACKEND_ARTIFACT_FILESYSTEM: The model artifacts are made
+///     available to Triton via the local filesytem. 'location'
+///     returns the full path to the directory in the model repository
+///     that contains this model's artifacts. The returned string is
+///     owned by Triton, not the caller, and so should not be modified
+///     or freed.
+///
+/// \param model The model.
+/// \param artifact_type Returns the artifact type for the model.
+/// \param path Returns the location.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelRepository(
+    TRITONBACKEND_Model* model, TRITONBACKEND_ArtifactType* artifact_type,
+    const char** location);
+
+/// Get the model configuration. The caller takes ownership of the
+/// message object and must call TRITONSERVER_MessageDelete to release
+/// the object. The configuration is available via this call even
+/// before the model is loaded and so can be used in
+/// TRITONBACKEND_ModelInitialize. TRITONSERVER_ServerModelConfig
+/// returns equivalent information but is not useable until after the
+/// model loads.
+///
+/// \param model The model.
+/// \param config_version The model configuration will be returned in
+/// a format matching this version. If the configuration cannot be
+/// represented in the requested version's format then an error will
+/// be returned. Currently only version 1 is supported.
+/// \param model_config Returns the model configuration as a message.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelConfig(
+    TRITONBACKEND_Model* model, const uint32_t config_version,
+    TRITONSERVER_Message** model_config);
+
+/// Whether the backend should attempt to auto-complete the model configuration.
+/// If true, the model should fill the inputs, outputs, and max batch size in
+/// the model configuration if incomplete. If the model configuration is
+/// changed,  the new configuration must be reported to Triton using
+/// TRITONBACKEND_ModelSetConfig.
+///
+/// \param model The model.
+/// \param auto_complete_config Returns whether the backend should auto-complete
+/// the model configuration.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error*
+TRITONBACKEND_ModelAutoCompleteConfig(
+    TRITONBACKEND_Model* model, bool* auto_complete_config);
+
+/// Set the model configuration in Triton server. This API should only be called
+/// when the backend implements the auto-completion of model configuration
+/// and TRITONBACKEND_ModelAutoCompleteConfig returns true in
+/// auto_complete_config. Only the inputs, outputs, max batch size, and
+/// scheduling choice can be changed. A caveat being scheduling choice can only
+/// be changed if none is previously set. Any other changes to the model
+/// configuration will be ignored by Triton. This function can only be called
+/// from TRITONBACKEND_ModelInitialize, calling in any other context will result
+/// in an error being returned. Additionally, Triton server can add some of the
+/// missing fields in the provided config with this call. The backend must get
+/// the complete configuration again by using TRITONBACKEND_ModelConfig.
+/// TRITONBACKEND_ModelSetConfig does not take ownership of the message object
+/// and so the caller should call TRITONSERVER_MessageDelete to release the
+/// object once the function returns.
+///
+/// \param model The model.
+/// \param config_version The format version of the model configuration.
+/// If the configuration is not represented in the version's format
+/// then an error will be returned. Currently only version 1 is supported.
+/// \param model_config The updated model configuration as a message.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelSetConfig(
+    TRITONBACKEND_Model* model, const uint32_t config_version,
+    TRITONSERVER_Message* model_config);
+
+/// Get the TRITONSERVER_Server object that this model is being served
+/// by.
+///
+/// \param model The model.
+/// \param server Returns the server.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelServer(
+    TRITONBACKEND_Model* model, TRITONSERVER_Server** server);
+
+/// Get the backend used by the model.
+///
+/// \param model The model.
+/// \param model Returns the backend object.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelBackend(
+    TRITONBACKEND_Model* model, TRITONBACKEND_Backend** backend);
+
+/// Get the user-specified state associated with the model. The
+/// state is completely owned and managed by the backend.
+///
+/// \param model The model.
+/// \param state Returns the user state, or nullptr if no user state.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelState(
+    TRITONBACKEND_Model* model, void** state);
+
+/// Set the user-specified state associated with the model. The
+/// state is completely owned and managed by the backend.
+///
+/// \param model The model.
+/// \param state The user state, or nullptr if no user state.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelSetState(
+    TRITONBACKEND_Model* model, void* state);
+
+///
+/// TRITONBACKEND_ModelInstance
+///
+/// Object representing a model instance implemented using the
+/// backend.
+///
+
+/// Get the name of the model instance. The returned string is owned by the
+/// model object, not the caller, and so should not be modified or
+/// freed.
+///
+/// \param instance The model instance.
+/// \param name Returns the instance name.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceName(
+    TRITONBACKEND_ModelInstance* instance, const char** name);
+
+/// Get the kind of the model instance.
+///
+/// \param instance The model instance.
+/// \param kind Returns the instance kind.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceKind(
+    TRITONBACKEND_ModelInstance* instance,
+    TRITONSERVER_InstanceGroupKind* kind);
+
+/// Get the device ID of the model instance.
+///
+/// \param instance The model instance.
+/// \param device_id Returns the instance device ID.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceDeviceId(
+    TRITONBACKEND_ModelInstance* instance, int32_t* device_id);
+
+/// Get the host policy setting.  The 'host_policy' message is
+/// owned by Triton and should not be modified or freed by the caller.
+///
+/// The host policy setting, as JSON, is:
+///
+///   {
+///     "<host_policy>" : {
+///       "<setting>" : "<value>",
+///       ...
+///     }
+///   }
+///
+/// \param instance The model instance.
+/// \param host_policy Returns the host policy setting as a message.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error*
+TRITONBACKEND_ModelInstanceHostPolicy(
+    TRITONBACKEND_ModelInstance* instance, TRITONSERVER_Message** host_policy);
+
+/// Whether the model instance is passive.
+///
+/// \param instance The model instance.
+/// \param is_passive Returns true if the instance is passive, false otherwise
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceIsPassive(
+    TRITONBACKEND_ModelInstance* instance, bool* is_passive);
+
+/// Get the number of optimization profiles to be loaded for the instance.
+///
+/// \param instance The model instance.
+/// \param count Returns the number of optimization profiles.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error*
+TRITONBACKEND_ModelInstanceProfileCount(
+    TRITONBACKEND_ModelInstance* instance, uint32_t* count);
+
+/// Get the name of optimization profile. The caller does not own
+/// the returned string and must not modify or delete it. The lifetime
+/// of the returned string extends only as long as 'instance'.
+///
+/// \param instance The model instance.
+/// \param index The index of the optimization profile. Must be 0
+/// <= index < count, where count is the value returned by
+/// TRITONBACKEND_ModelInstanceProfileCount.
+/// \param profile_name Returns the name of the optimization profile
+/// corresponding to the index.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error*
+TRITONBACKEND_ModelInstanceProfileName(
+    TRITONBACKEND_ModelInstance* instance, const uint32_t index,
+    const char** profile_name);
+
+/// Get the number of secondary devices configured for the instance.
+///
+/// \param instance The model instance.
+/// \param count Returns the number of secondary devices.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error*
+TRITONBACKEND_ModelInstanceSecondaryDeviceCount(
+    TRITONBACKEND_ModelInstance* instance, uint32_t* count);
+
+/// Get the properties of indexed secondary device. The returned
+/// strings and other properties are owned by the instance, not the
+/// caller, and so should not be modified or freed.
+///
+/// \param instance The model instance.
+/// \param index The index of the secondary device. Must be 0
+/// <= index < count, where count is the value returned by
+/// TRITONBACKEND_ModelInstanceSecondaryDeviceCount.
+/// \param kind Returns the kind of secondary device corresponding
+/// to the index.
+/// \param id Returns the id of secondary device corresponding to the index.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error*
+TRITONBACKEND_ModelInstanceSecondaryDeviceProperties(
+    TRITONBACKEND_ModelInstance* instance, uint32_t index, const char** kind,
+    int64_t* id);
+
+/// Get the model associated with a model instance.
+///
+/// \param instance The model instance.
+/// \param backend Returns the model object.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceModel(
+    TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Model** model);
+
+/// Get the user-specified state associated with the model
+/// instance. The state is completely owned and managed by the
+/// backend.
+///
+/// \param instance The model instance.
+/// \param state Returns the user state, or nullptr if no user state.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceState(
+    TRITONBACKEND_ModelInstance* instance, void** state);
+
+/// Set the user-specified state associated with the model
+/// instance. The state is completely owned and managed by the
+/// backend.
+///
+/// \param instance The model instance.
+/// \param state The user state, or nullptr if no user state.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceSetState(
+    TRITONBACKEND_ModelInstance* instance, void* state);
+
+/// Record statistics for an inference request.
+///
+/// Set 'success' true to indicate that the inference request
+/// completed successfully. In this case all timestamps should be
+/// non-zero values reported in nanoseconds and should be collected
+/// using std::chrono::steady_clock::now().time_since_epoch() or the equivalent.
+/// Set 'success' to false to indicate that the inference request failed
+/// to complete successfully. In this case all timestamps values are
+/// ignored.
+///
+/// For consistency of measurement across different backends, the
+/// timestamps should be collected at the following points during
+/// TRITONBACKEND_ModelInstanceExecute.
+///
+///   TRITONBACKEND_ModelInstanceExecute()
+///     CAPTURE TIMESPACE (exec_start_ns)
+///     < process input tensors to prepare them for inference
+///       execution, including copying the tensors to/from GPU if
+///       necessary>
+///     CAPTURE TIMESPACE (compute_start_ns)
+///     < perform inference computations to produce outputs >
+///     CAPTURE TIMESPACE (compute_end_ns)
+///     < allocate output buffers and extract output tensors, including
+///       copying the tensors to/from GPU if necessary>
+///     CAPTURE TIMESPACE (exec_end_ns)
+///     return
+///
+/// Note that these statistics are associated with a valid
+/// TRITONBACKEND_Request object and so must be reported before the
+/// request is released. For backends that release the request before
+/// all response(s) are sent, these statistics cannot capture
+/// information about the time required to produce the response.
+///
+/// \param instance The model instance.
+/// \param request The inference request that statistics are being
+/// reported for.
+/// \param success True if the inference request completed
+/// successfully, false if it failed to complete.
+/// \param exec_start_ns Timestamp for the start of execution.
+/// \param compute_start_ns Timestamp for the start of execution
+/// computations.
+/// \param compute_end_ns Timestamp for the end of execution
+/// computations.
+/// \param exec_end_ns Timestamp for the end of execution.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error*
+TRITONBACKEND_ModelInstanceReportStatistics(
+    TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Request* request,
+    const bool success, const uint64_t exec_start_ns,
+    const uint64_t compute_start_ns, const uint64_t compute_end_ns,
+    const uint64_t exec_end_ns);
+
+/// Record statistics for the execution of an entire batch of
+/// inference requests.
+///
+/// All timestamps should be non-zero values reported in nanoseconds
+/// and should be collected using
+/// std::chrono::steady_clock::now().time_since_epoch() or the equivalent.
+/// See TRITONBACKEND_ModelInstanceReportStatistics for more information about
+/// the timestamps.
+///
+/// 'batch_size' is the sum of the batch sizes for the individual
+/// requests that were delivered together in the call to
+/// TRITONBACKEND_ModelInstanceExecute. For example, if three requests
+/// are passed to TRITONBACKEND_ModelInstanceExecute and those
+/// requests have batch size 1, 2, and 3; then 'batch_size' should be
+/// set to 6.
+///
+/// \param instance The model instance.
+/// \param batch_size Combined batch size of all the individual
+/// requests executed in the batch.
+/// \param exec_start_ns Timestamp for the start of execution.
+/// \param compute_start_ns Timestamp for the start of execution
+/// computations.
+/// \param compute_end_ns Timestamp for the end of execution
+/// computations.
+/// \param exec_end_ns Timestamp for the end of execution.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_DECLSPEC TRITONSERVER_Error*
+TRITONBACKEND_ModelInstanceReportBatchStatistics(
+    TRITONBACKEND_ModelInstance* instance, const uint64_t batch_size,
+    const uint64_t exec_start_ns, const uint64_t compute_start_ns,
+    const uint64_t compute_end_ns, const uint64_t exec_end_ns);
+
+///
+/// The following functions can be implemented by a backend. Functions
+/// indicated as required must be implemented or the backend will fail
+/// to load.
+///
+
+/// Initialize a backend. This function is optional, a backend is not
+/// required to implement it. This function is called once when a
+/// backend is loaded to allow the backend to initialize any state
+/// associated with the backend. A backend has a single state that is
+/// shared across all models that use the backend.
+///
+/// \param backend The backend.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_Initialize(
+    TRITONBACKEND_Backend* backend);
+
+/// Finalize for a backend. This function is optional, a backend is
+/// not required to implement it. This function is called once, just
+/// before the backend is unloaded. All state associated with the
+/// backend should be freed and any threads created for the backend
+/// should be exited/joined before returning from this function.
+///
+/// \param backend The backend.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_Finalize(
+    TRITONBACKEND_Backend* backend);
+
+/// Initialize for a model. This function is optional, a backend is
+/// not required to implement it. This function is called once when a
+/// model that uses the backend is loaded to allow the backend to
+/// initialize any state associated with the model. The backend should
+/// also examine the model configuration to determine if the
+/// configuration is suitable for the backend. Any errors reported by
+/// this function will prevent the model from loading.
+///
+/// \param model The model.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_ModelInitialize(
+    TRITONBACKEND_Model* model);
+
+/// Finalize for a model. This function is optional, a backend is not
+/// required to implement it. This function is called once for a
+/// model, just before the model is unloaded from Triton. All state
+/// associated with the model should be freed and any threads created
+/// for the model should be exited/joined before returning from this
+/// function.
+///
+/// \param model The model.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_ModelFinalize(
+    TRITONBACKEND_Model* model);
+
+/// Initialize for a model instance. This function is optional, a
+/// backend is not required to implement it. This function is called
+/// once when a model instance is created to allow the backend to
+/// initialize any state associated with the instance.
+///
+/// \param instance The model instance.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceInitialize(
+    TRITONBACKEND_ModelInstance* instance);
+
+/// Finalize for a model instance. This function is optional, a
+/// backend is not required to implement it. This function is called
+/// once for an instance, just before the corresponding model is
+/// unloaded from Triton. All state associated with the instance
+/// should be freed and any threads created for the instance should be
+/// exited/joined before returning from this function.
+///
+/// \param instance The model instance.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceFinalize(
+    TRITONBACKEND_ModelInstance* instance);
+
+/// Execute a batch of one or more requests on a model instance. This
+/// function is required. Triton will not perform multiple
+/// simultaneous calls to this function for a given model 'instance';
+/// however, there may be simultaneous calls for different model
+/// instances (for the same or different models).
+///
+/// If an error is returned the ownership of the request objects
+/// remains with Triton and the backend must not retain references to
+/// the request objects or access them in any way.
+///
+/// If success is returned, ownership of the request objects is
+/// transferred to the backend and it is then responsible for creating
+/// responses and releasing the request objects. Note that even though
+/// ownership of the request objects is transferred to the backend, the
+/// ownership of the buffer holding request pointers is returned back
+/// to Triton upon return from TRITONBACKEND_ModelInstanceExecute. If
+/// any request objects need to be maintained beyond
+/// TRITONBACKEND_ModelInstanceExecute, then the pointers must be copied
+/// out of the array within TRITONBACKEND_ModelInstanceExecute.
+///
+/// \param instance The model instance.
+/// \param requests The requests.
+/// \param request_count The number of requests in the batch.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_ModelInstanceExecute(
+    TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Request** requests,
+    const uint32_t request_count);
+
+/// Query the backend for different model attributes. This function is optional,
+/// a backend is not required to implement it. The backend is also not required
+/// to set all backend attribute listed. This function is called when
+/// Triton requires further backend / model information to perform operations.
+/// This function may be called multiple times within the lifetime of the
+/// backend (between TRITONBACKEND_Initialize and TRITONBACKEND_Finalize).
+/// The backend may return error to indicate failure to set the backend
+/// attributes, and the attributes specified in the same function call will be
+/// ignored. Triton will update the specified attributes if 'nullptr' is
+/// returned.
+///
+/// \param backend The backend.
+/// \param backend_attributes Return the backend attribute.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONBACKEND_ISPEC TRITONSERVER_Error* TRITONBACKEND_GetBackendAttribute(
+    TRITONBACKEND_Backend* backend,
+    TRITONBACKEND_BackendAttribute* backend_attributes);
+
+/// TRITONBACKEND_BackendAttribute
+///
+/// API to modify attributes associated with a backend.
+///
+
+/// Add the preferred instance group of the backend. This function
+/// can be called multiple times to cover different instance group kinds that
+/// the backend supports, given the priority order that the first call describes
+/// the most preferred group. In the case where instance group are not
+/// explicitly provided, Triton will use this attribute to create model
+/// deployment that aligns more with the backend preference.
+///
+/// \param backend_attributes The backend attributes object.
+/// \param kind The kind of the instance group.
+/// \param count The number of instances per device. Triton default will be used
+/// if 0 is provided.
+/// \param device_ids The devices where instances should be available. Triton
+/// default will be used if 'nullptr' is provided.
+/// \param id_count The number of devices in 'device_ids'.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONSERVER_DECLSPEC TRITONSERVER_Error*
+TRITONBACKEND_BackendAttributeAddPreferredInstanceGroup(
+    TRITONBACKEND_BackendAttribute* backend_attributes,
+    const TRITONSERVER_InstanceGroupKind kind, const uint64_t count,
+    const uint64_t* device_ids, const uint64_t id_count);
+
+#ifdef __cplusplus
+}
+#endif

3rdparty/core-r22.12/include/triton/core/tritonrepoagent.h

+// Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+//  * Redistributions of source code must retain the above copyright
+//    notice, this list of conditions and the following disclaimer.
+//  * Redistributions in binary form must reproduce the above copyright
+//    notice, this list of conditions and the following disclaimer in the
+//    documentation and/or other materials provided with the distribution.
+//  * Neither the name of NVIDIA CORPORATION nor the names of its
+//    contributors may be used to endorse or promote products derived
+//    from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#pragma once
+
+#include <stddef.h>
+#include <stdint.h>
+#include "triton/core/tritonserver.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef _COMPILING_TRITONREPOAGENT
+#if defined(_MSC_VER)
+#define TRITONREPOAGENT_DECLSPEC __declspec(dllexport)
+#define TRITONREPOAGENT_ISPEC __declspec(dllimport)
+#elif defined(__GNUC__)
+#define TRITONREPOAGENT_DECLSPEC __attribute__((__visibility__("default")))
+#define TRITONREPOAGENT_ISPEC
+#else
+#define TRITONREPOAGENT_DECLSPEC
+#define TRITONREPOAGENT_ISPEC
+#endif
+#else
+#if defined(_MSC_VER)
+#define TRITONREPOAGENT_DECLSPEC __declspec(dllimport)
+#define TRITONREPOAGENT_ISPEC __declspec(dllexport)
+#else
+#define TRITONREPOAGENT_DECLSPEC
+#define TRITONREPOAGENT_ISPEC
+#endif
+#endif
+
+struct TRITONREPOAGENT_Agent;
+struct TRITONREPOAGENT_AgentModel;
+
+///
+/// TRITONREPOAGENT API Version
+///
+/// The TRITONREPOAGENT API is versioned with major and minor version
+/// numbers. Any change to the API that does not impact backwards
+/// compatibility (for example, adding a non-required function)
+/// increases the minor version number. Any change that breaks
+/// backwards compatibility (for example, deleting or changing the
+/// behavior of a function) increases the major version number. A
+/// repository agent should check that the API version used to compile
+/// the agent is compatible with the API version of the Triton server
+/// that it is running in. This is typically done by code similar to
+/// the following which makes sure that the major versions are equal
+/// and that the minor version of Triton is >= the minor version used
+/// to build the agent.
+///
+///   uint32_t api_version_major, api_version_minor;
+///   TRITONREPOAGENT_ApiVersion(&api_version_major, &api_version_minor);
+///   if ((api_version_major != TRITONREPOAGENT_API_VERSION_MAJOR) ||
+///       (api_version_minor < TRITONREPOAGENT_API_VERSION_MINOR)) {
+///     return TRITONSERVER_ErrorNew(
+///       TRITONSERVER_ERROR_UNSUPPORTED,
+///       "triton repository agent API version does not support this agent");
+///   }
+///
+#define TRITONREPOAGENT_API_VERSION_MAJOR 0
+#define TRITONREPOAGENT_API_VERSION_MINOR 1
+
+/// Get the TRITONREPOAGENT API version supported by Triton. This
+/// value can be compared against the
+/// TRITONREPOAGENT_API_VERSION_MAJOR and
+/// TRITONREPOAGENT_API_VERSION_MINOR used to build the agent to
+/// ensure that Triton is compatible with the agent.
+///
+/// \param major Returns the TRITONREPOAGENT API major version supported
+/// by Triton.
+/// \param minor Returns the TRITONREPOAGENT API minor version supported
+/// by Triton.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONREPOAGENT_DECLSPEC TRITONSERVER_Error* TRITONREPOAGENT_ApiVersion(
+    uint32_t* major, uint32_t* minor);
+
+/// TRITONREPOAGENT_ArtifactType
+///
+/// The ways that the files that make up a model's repository content
+/// are communicated between Triton and the agent.
+///
+///   TRITONREPOAGENT_ARTIFACT_FILESYSTEM: The model artifacts are
+///     communicated to and from the repository agent via a locally
+///     accessible filesystem. The agent can access these files using
+///     an appropriate filesystem API.
+///
+///   TRITONREPOAGENT_ARTIFACT_REMOTE_FILESYSTEM: The model artifacts are
+///     communicated to and from the repository agent via a remote filesystem.
+///     The remote filesystem path follows the same convention as is used for
+///     repository paths, for example, "s3://" prefix indicates an S3 path.
+///
+typedef enum TRITONREPOAGENT_artifacttype_enum {
+  TRITONREPOAGENT_ARTIFACT_FILESYSTEM,
+  TRITONREPOAGENT_ARTIFACT_REMOTE_FILESYSTEM
+} TRITONREPOAGENT_ArtifactType;
+
+/// TRITONREPOAGENT_ActionType
+///
+/// Types of repository actions that can be handled by an agent.
+/// The lifecycle of a TRITONREPOAGENT_AgentModel begins with a call to
+/// TRITONREPOAGENT_ModelInitialize and ends with a call to
+/// TRITONREPOAGENT_ModelFinalize. Between those calls the current lifecycle
+/// state of the model is communicated by calls to TRITONREPOAGENT_ModelAction.
+/// Possible lifecycles are:
+///
+/// LOAD -> LOAD_COMPLETE -> UNLOAD -> UNLOAD_COMPLETE
+/// LOAD -> LOAD_FAIL
+///
+///   TRITONREPOAGENT_ACTION_LOAD: A model is being loaded.
+///
+///   TRITONREPOAGENT_ACTION_LOAD_COMPLETE: The model load completed
+///     successfully and the model is now loaded.
+///
+///   TRITONREPOAGENT_ACTION_LOAD_FAIL: The model load did not complete
+///     successfully. The model is not loaded.
+///
+///   TRITONREPOAGENT_ACTION_UNLOAD: The model is being unloaded.
+///
+///   TRITONREPOAGENT_ACTION_UNLOAD_COMPLETE: The model unload is complete.
+///
+typedef enum TRITONREPOAGENT_actiontype_enum {
+  TRITONREPOAGENT_ACTION_LOAD,
+  TRITONREPOAGENT_ACTION_LOAD_COMPLETE,
+  TRITONREPOAGENT_ACTION_LOAD_FAIL,
+  TRITONREPOAGENT_ACTION_UNLOAD,
+  TRITONREPOAGENT_ACTION_UNLOAD_COMPLETE
+} TRITONREPOAGENT_ActionType;
+
+/// Get the location of the files that make up the model. The
+/// 'location' communicated depends on how the model is being
+/// communicated to the agent as indicated by 'artifact_type'.
+///
+///   TRITONREPOAGENT_ARTIFACT_FILESYSTEM: The model artifacts are
+///     made available to the agent via the local
+///     filesytem. 'location' returns the full path to the directory
+///     in the model repository that contains the model's
+///     artifacts. The returned location string is owned by Triton,
+///     not the caller, and so should not be modified or freed. The
+///     contents of the directory are owned by Triton, not the agent,
+///     and so the agent should not delete or modify the contents. Use
+///     TRITONREPOAGENT_RepositoryAcquire to get a location that can be
+///     used to modify the model repository contents.
+///
+///   TRITONREPOAGENT_ARTIFACT_REMOTE_FILESYSTEM: The model artifacts are
+///     made available to the agent via a remote filesystem.
+///     'location' returns the full path to the remote directory that contains
+///     the model's artifacts. The returned location string is owned by Triton,
+///     not the caller, and so should not be modified or freed. The contents of
+///     the remote directory are owned by Triton, not the agent,
+///     and so the agent should not delete or modify the contents.
+///     Use TRITONREPOAGENT_ModelRepositoryLocationAcquire to get a location
+///     that can be used to write updated model repository contents.
+///
+/// \param agent The agent.
+/// \param model The model.
+/// \param artifact_type Returns the artifact type for the location.
+/// \param path Returns the location.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONREPOAGENT_DECLSPEC TRITONSERVER_Error*
+TRITONREPOAGENT_ModelRepositoryLocation(
+    TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model,
+    TRITONREPOAGENT_ArtifactType* artifact_type, const char** location);
+
+/// Acquire a location where the agent can produce a new version of
+/// the model repository files. This is a convenience method to create
+/// a temporary directory for the agent. The agent is responsible for
+/// calling TRITONREPOAGENT_ModelRepositoryLocationDelete in
+/// TRITONREPOAGENT_ModelFinalize to delete the location. Initially the
+/// acquired location is empty. The 'location' communicated depends on
+/// the requested 'artifact_type'.
+///
+///   TRITONREPOAGENT_ARTIFACT_FILESYSTEM: The location is a directory
+///     on the local filesystem. 'location' returns the full path to
+///     an empty directory that the agent should populate with the
+///     model's artifacts. The returned location string is owned by
+///     Triton, not the agent, and so should not be modified or freed.
+///
+/// \param agent The agent.
+/// \param model The model.
+/// \param artifact_type The artifact type for the location.
+/// \param path Returns the location.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONREPOAGENT_DECLSPEC TRITONSERVER_Error*
+TRITONREPOAGENT_ModelRepositoryLocationAcquire(
+    TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model,
+    const TRITONREPOAGENT_ArtifactType artifact_type, const char** location);
+
+/// Discard and release ownership of a previously acquired location
+/// and its contents. The agent must not access or modify the location
+/// or its contents after this call.
+///
+/// \param agent The agent.
+/// \param model The model.
+/// \param path The location to release.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONREPOAGENT_DECLSPEC TRITONSERVER_Error*
+TRITONREPOAGENT_ModelRepositoryLocationRelease(
+    TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model,
+    const char* location);
+
+/// Inform Triton that the specified repository location should be used for
+/// the model in place of the original model repository. This method can only be
+/// called when TRITONREPOAGENT_ModelAction is invoked with
+/// TRITONREPOAGENT_ACTION_LOAD. The 'location' The 'location'
+/// communicated depends on how the repository is being
+/// communicated to Triton as indicated by 'artifact_type'.
+///
+///   TRITONREPOAGENT_ARTIFACT_FILESYSTEM: The model artifacts are
+///     made available to Triton via the local filesytem. 'location' returns
+///     the full path to the directory. Ownership of the contents of the
+///     returned directory are transferred to Triton and the agent should not
+///     modified or freed the contents until TRITONREPOAGENT_ModelFinalize.
+///     The local filesystem directory can be created using
+///     TRITONREPOAGENT_ModelReopsitroyLocationAcquire or the agent can use
+///     its own local filesystem API.
+///
+///   TRITONREPOAGENT_ARTIFACT_REMOTE_FILESYSTEM: The model artifacts are
+///     made available to Triton via a remote filesystem. 'location' returns
+///     the full path to the remote filesystem directory. Ownership of the
+///     contents of the returned directory are transferred to Triton and
+///     the agent should not modified or freed the contents until
+///     TRITONREPOAGENT_ModelFinalize.
+///
+/// \param agent The agent.
+/// \param model The model.
+/// \param artifact_type The artifact type for the location.
+/// \param path Returns the location.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONREPOAGENT_DECLSPEC TRITONSERVER_Error*
+TRITONREPOAGENT_ModelRepositoryUpdate(
+    TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model,
+    const TRITONREPOAGENT_ArtifactType artifact_type, const char* location);
+
+/// Get the number of agent parameters defined for a model.
+///
+/// \param agent The agent.
+/// \param model The model.
+/// \param count Returns the number of input tensors.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONREPOAGENT_DECLSPEC TRITONSERVER_Error*
+TRITONREPOAGENT_ModelParameterCount(
+    TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model,
+    uint32_t* count);
+
+/// Get a parameter name and value. The caller does not own the
+/// returned strings and must not modify or delete them.
+///
+/// \param agent The agent.
+/// \param model The model.
+/// \param index The index of the parameter. Must be 0 <= index <
+/// count, where count is the value returned by
+/// TRITONREPOAGENT_ModelParameterCount.
+/// \param parameter_name Returns the name of the parameter.
+/// \param parameter_value Returns the value of the parameter.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONREPOAGENT_DECLSPEC TRITONSERVER_Error* TRITONREPOAGENT_ModelParameter(
+    TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model,
+    const uint32_t index, const char** parameter_name,
+    const char** parameter_value);
+
+/// Get the model configuration. The caller takes ownership of the
+/// message object and must call TRITONSERVER_MessageDelete to release
+/// the object. If the model repository does not contain a
+/// config.pbtxt file then 'model_config' is returned as nullptr.
+///
+/// \param agent The agent.
+/// \param model The model.
+/// \param config_version The model configuration will be returned in
+/// a format matching this version. If the configuration cannot be
+/// represented in the requested version's format then an error will
+/// be returned. Currently only version 1 is supported.
+/// \param model_config Returns the model configuration as a message.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONREPOAGENT_DECLSPEC TRITONSERVER_Error* TRITONREPOAGENT_ModelConfig(
+    TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model,
+    const uint32_t config_version, TRITONSERVER_Message** model_config);
+
+/// Get the user-specified state associated with the model.
+///
+/// \param model The agent model.
+/// \param state Returns the user state, or nullptr if no user state.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONREPOAGENT_DECLSPEC TRITONSERVER_Error* TRITONREPOAGENT_ModelState(
+    TRITONREPOAGENT_AgentModel* model, void** state);
+
+/// Set the user-specified state associated with the model.
+///
+/// \param model The agent model.
+/// \param state The user state, or nullptr if no user state.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONREPOAGENT_DECLSPEC TRITONSERVER_Error* TRITONREPOAGENT_ModelSetState(
+    TRITONREPOAGENT_AgentModel* model, void* state);
+
+/// Get the user-specified state associated with the agent.
+///
+/// \param agent The agent.
+/// \param state Returns the user state, or nullptr if no user state.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONREPOAGENT_DECLSPEC TRITONSERVER_Error* TRITONREPOAGENT_State(
+    TRITONREPOAGENT_Agent* agent, void** state);
+
+/// Set the user-specified state associated with the agent.
+///
+/// \param agent The agent.
+/// \param state The user state, or nullptr if no user state.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONREPOAGENT_DECLSPEC TRITONSERVER_Error* TRITONREPOAGENT_SetState(
+    TRITONREPOAGENT_Agent* agent, void* state);
+
+///
+/// The following functions can be implemented by an agent. Functions
+/// indicated as required must be implemented or the agent will fail
+/// to load.
+///
+
+/// Initialize an agent. This function is optional. This function is
+/// called once when an agent is loaded to allow the agent to
+/// initialize any state associated with the agent. An agent has a
+/// single state that is shared across all invocations of the agent.
+///
+/// \param agent The agent.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONREPOAGENT_ISPEC TRITONSERVER_Error* TRITONREPOAGENT_Initialize(
+    TRITONREPOAGENT_Agent* agent);
+
+/// Finalize for an agent. This function is optional. This function is
+/// called once, just before the agent is unloaded. All state
+/// associated with the agent should be freed and any threads created
+/// for the agent should be exited/joined before returning from this
+/// function.
+///
+/// \param agent The agent.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONREPOAGENT_ISPEC TRITONSERVER_Error* TRITONREPOAGENT_Finalize(
+    TRITONREPOAGENT_Agent* agent);
+
+/// Initialize a model associated with an agent. This function is optional.
+/// This function is called once when an agent model's lifecycle begins to allow
+/// the agent model to initialize any state associated with it. An agent model
+/// has a single state that is shared across all the lifecycle of the agent
+/// model.
+///
+/// \param agent The agent to be associated with the model.
+/// \param model The model.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONREPOAGENT_ISPEC TRITONSERVER_Error* TRITONREPOAGENT_ModelInitialize(
+    TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model);
+
+/// Finalize for a model. This function is optional. This function is
+/// called once, just before the end of the agent model's lifecycle. All state
+/// associated with the agent model should be freed and any threads created
+/// for the agent model should be exited/joined before returning from this
+/// function. If the model acquired a model location using
+/// TRITONREPOAGENT_ModelRepositoryLocationAcquire, it must call
+/// TRITONREPOAGENT_ModelRepositoryLocationRelease to release that location.
+///
+/// \param agent The agent associated with the model.
+/// \param model The model.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONREPOAGENT_ISPEC TRITONSERVER_Error* TRITONREPOAGENT_ModelFinalize(
+    TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model);
+
+/// Handle an action for a specified model. This function is
+/// required. Triton will not perform multiple simultaneous calls to
+/// this function for a given agent and model; however, there may be
+/// simultaneous calls for the agent for different models.
+///
+/// If the agent does not handle the action the agent should
+/// immediately return success (nullptr).
+///
+/// Any modification to the model's repository must be made when 'action_type'
+/// is TRITONREPOAGENT_ACTION_LOAD.
+/// To modify the model's repository the agent must either acquire a mutable
+/// location via TRITONREPOAGENT_ModelRepositoryLocationAcquire
+/// or its own managed location, report the location to Triton via
+/// TRITONREPOAGENT_ModelRepositoryUpdate, and then return
+/// success (nullptr). If the agent does not need to make any changes
+/// to the model repository it should not call
+/// TRITONREPOAGENT_ModelRepositoryUpdate and then return success.
+/// To indicate that a model load should fail return a non-success status.
+///
+/// \param agent The agent.
+/// \param model The model that is the target of the action.
+/// \action_type The type of action the agent should handle for the model.
+/// \return a TRITONSERVER_Error indicating success or failure.
+TRITONREPOAGENT_ISPEC TRITONSERVER_Error* TRITONREPOAGENT_ModelAction(
+    TRITONREPOAGENT_Agent* agent, TRITONREPOAGENT_AgentModel* model,
+    const TRITONREPOAGENT_ActionType action_type);
+
+#ifdef __cplusplus
+}
+#endif