{
  "openapi": "3.0.3",
  "info": {
    "title": "Text Generation Inference",
    "description": "Text Generation Webserver",
    "contact": {
      "name": "Olivier Dehaene"
    },
    "license": {
      "name": "Apache 2.0",
      "url": "https://www.apache.org/licenses/LICENSE-2.0"
    },
    "version": "2.0.1"
  },
  "paths": {
    "/": {
      "post": {
        "tags": [
          "Text Generation Inference"
        ],
        "summary": "Generate tokens if `stream == false` or a stream of token if `stream == true`",
        "description": "Generate tokens if `stream == false` or a stream of token if `stream == true`",
        "operationId": "compat_generate",
        "requestBody": {
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/CompatGenerateRequest"
              }
            }
          },
          "required": true
        },
        "responses": {
          "200": {
            "description": "Generated Text",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/GenerateResponse"
                }
              },
              "text/event-stream": {
                "schema": {
                  "$ref": "#/components/schemas/StreamResponse"
                }
              }
            }
          },
          "422": {
            "description": "Input validation error",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/ErrorResponse"
                },
                "example": {
                  "error": "Input validation error"
                }
              }
            }
          },
          "424": {
            "description": "Generation Error",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/ErrorResponse"
                },
                "example": {
                  "error": "Request failed during generation"
                }
              }
            }
          },
          "429": {
            "description": "Model is overloaded",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/ErrorResponse"
                },
                "example": {
                  "error": "Model is overloaded"
                }
              }
            }
          },
          "500": {
            "description": "Incomplete generation",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/ErrorResponse"
                },
                "example": {
                  "error": "Incomplete generation"
                }
              }
            }
          }
        }
      }
    },
    "/generate": {
      "post": {
        "tags": [
          "Text Generation Inference"
        ],
        "summary": "Generate tokens",
        "description": "Generate tokens",
        "operationId": "generate",
        "requestBody": {
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/GenerateRequest"
              }
            }
          },
          "required": true
        },
        "responses": {
          "200": {
            "description": "Generated Text",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/GenerateResponse"
                }
              }
            }
          },
          "422": {
            "description": "Input validation error",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/ErrorResponse"
                },
                "example": {
                  "error": "Input validation error"
                }
              }
            }
          },
          "424": {
            "description": "Generation Error",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/ErrorResponse"
                },
                "example": {
                  "error": "Request failed during generation"
                }
              }
            }
          },
          "429": {
            "description": "Model is overloaded",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/ErrorResponse"
                },
                "example": {
                  "error": "Model is overloaded"
                }
              }
            }
          },
          "500": {
            "description": "Incomplete generation",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/ErrorResponse"
                },
                "example": {
                  "error": "Incomplete generation"
                }
              }
            }
          }
        }
      }
    },
    "/generate_stream": {
      "post": {
        "tags": [
          "Text Generation Inference"
        ],
        "summary": "Generate a stream of token using Server-Sent Events",
        "description": "Generate a stream of token using Server-Sent Events",
        "operationId": "generate_stream",
        "requestBody": {
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/GenerateRequest"
              }
            }
          },
          "required": true
        },
        "responses": {
          "200": {
            "description": "Generated Text",
            "content": {
              "text/event-stream": {
                "schema": {
                  "$ref": "#/components/schemas/StreamResponse"
                }
              }
            }
          },
          "422": {
            "description": "Input validation error",
            "content": {
              "text/event-stream": {
                "schema": {
                  "$ref": "#/components/schemas/ErrorResponse"
                },
                "example": {
                  "error": "Input validation error"
                }
              }
            }
          },
          "424": {
            "description": "Generation Error",
            "content": {
              "text/event-stream": {
                "schema": {
                  "$ref": "#/components/schemas/ErrorResponse"
                },
                "example": {
                  "error": "Request failed during generation"
                }
              }
            }
          },
          "429": {
            "description": "Model is overloaded",
            "content": {
              "text/event-stream": {
                "schema": {
                  "$ref": "#/components/schemas/ErrorResponse"
                },
                "example": {
                  "error": "Model is overloaded"
                }
              }
            }
          },
          "500": {
            "description": "Incomplete generation",
            "content": {
              "text/event-stream": {
                "schema": {
                  "$ref": "#/components/schemas/ErrorResponse"
                },
                "example": {
                  "error": "Incomplete generation"
                }
              }
            }
          }
        }
      }
    },
    "/health": {
      "get": {
        "tags": [
          "Text Generation Inference"
        ],
        "summary": "Health check method",
        "description": "Health check method",
        "operationId": "health",
        "responses": {
          "200": {
            "description": "Everything is working fine"
          },
          "503": {
            "description": "Text generation inference is down",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/ErrorResponse"
                },
                "example": {
                  "error": "unhealthy",
                  "error_type": "healthcheck"
                }
              }
            }
          }
        }
      }
    },
    "/info": {
      "get": {
        "tags": [
          "Text Generation Inference"
        ],
        "summary": "Text Generation Inference endpoint info",
        "description": "Text Generation Inference endpoint info",
        "operationId": "get_model_info",
        "responses": {
          "200": {
            "description": "Served model info",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/Info"
                }
              }
            }
          }
        }
      }
    },
    "/metrics": {
      "get": {
        "tags": [
          "Text Generation Inference"
        ],
        "summary": "Prometheus metrics scrape endpoint",
        "description": "Prometheus metrics scrape endpoint",
        "operationId": "metrics",
        "responses": {
          "200": {
            "description": "Prometheus Metrics",
            "content": {
              "text/plain": {
                "schema": {
                  "type": "string"
                }
              }
            }
          }
        }
      }
    },
    "/tokenize": {
      "post": {
        "tags": [
          "Text Generation Inference"
        ],
        "summary": "Tokenize inputs",
        "description": "Tokenize inputs",
        "operationId": "tokenize",
        "requestBody": {
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/GenerateRequest"
              }
            }
          },
          "required": true
        },
        "responses": {
          "200": {
            "description": "Tokenized ids",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/TokenizeResponse"
                }
              }
            }
          },
          "404": {
            "description": "No tokenizer found",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/ErrorResponse"
                },
                "example": {
                  "error": "No fast tokenizer available"
                }
              }
            }
          }
        }
      }
    },
    "/v1/chat/completions": {
      "post": {
        "tags": [
          "Text Generation Inference"
        ],
        "summary": "Generate tokens",
        "description": "Generate tokens",
        "operationId": "chat_completions",
        "requestBody": {
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/ChatRequest"
              }
            }
          },
          "required": true
        },
        "responses": {
          "200": {
            "description": "Generated Chat Completion",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/ChatCompletion"
                }
              },
              "text/event-stream": {
                "schema": {
                  "$ref": "#/components/schemas/ChatCompletionChunk"
                }
              }
            }
          },
          "422": {
            "description": "Input validation error",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/ErrorResponse"
                },
                "example": {
                  "error": "Input validation error"
                }
              }
            }
          },
          "424": {
            "description": "Generation Error",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/ErrorResponse"
                },
                "example": {
                  "error": "Request failed during generation"
                }
              }
            }
          },
          "429": {
            "description": "Model is overloaded",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/ErrorResponse"
                },
                "example": {
                  "error": "Model is overloaded"
                }
              }
            }
          },
          "500": {
            "description": "Incomplete generation",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/ErrorResponse"
                },
                "example": {
                  "error": "Incomplete generation"
                }
              }
            }
          }
        }
      }
    },
    "/v1/completions": {
      "post": {
        "tags": [
          "Text Generation Inference"
        ],
        "summary": "Generate tokens",
        "description": "Generate tokens",
        "operationId": "completions",
        "requestBody": {
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/CompletionRequest"
              }
            }
          },
          "required": true
        },
        "responses": {
          "200": {
            "description": "Generated Chat Completion",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/Completion"
                }
              },
              "text/event-stream": {
                "schema": {
                  "$ref": "#/components/schemas/CompletionCompleteChunk"
                }
              }
            }
          },
          "422": {
            "description": "Input validation error",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/ErrorResponse"
                },
                "example": {
                  "error": "Input validation error"
                }
              }
            }
          },
          "424": {
            "description": "Generation Error",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/ErrorResponse"
                },
                "example": {
                  "error": "Request failed during generation"
                }
              }
            }
          },
          "429": {
            "description": "Model is overloaded",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/ErrorResponse"
                },
                "example": {
                  "error": "Model is overloaded"
                }
              }
            }
          },
          "500": {
            "description": "Incomplete generation",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/ErrorResponse"
                },
                "example": {
                  "error": "Incomplete generation"
                }
              }
            }
          }
        }
      }
    }
  },
  "components": {
    "schemas": {
      "BestOfSequence": {
        "type": "object",
        "required": [
          "generated_text",
          "finish_reason",
          "generated_tokens",
          "prefill",
          "tokens"
        ],
        "properties": {
          "finish_reason": {
            "$ref": "#/components/schemas/FinishReason"
          },
          "generated_text": {
            "type": "string",
            "example": "test"
          },
          "generated_tokens": {
            "type": "integer",
            "format": "int32",
            "example": 1,
            "minimum": 0
          },
          "prefill": {
            "type": "array",
            "items": {
              "$ref": "#/components/schemas/PrefillToken"
            }
          },
          "seed": {
            "type": "integer",
            "format": "int64",
            "example": 42,
            "nullable": true,
            "minimum": 0
          },
          "tokens": {
            "type": "array",
            "items": {
              "$ref": "#/components/schemas/Token"
            }
          },
          "top_tokens": {
            "type": "array",
            "items": {
              "type": "array",
              "items": {
                "$ref": "#/components/schemas/Token"
              }
            }
          }
        }
      },
      "ChatCompletion": {
        "type": "object",
        "required": [
          "id",
          "object",
          "created",
          "model",
          "system_fingerprint",
          "choices",
          "usage"
        ],
        "properties": {
          "choices": {
            "type": "array",
            "items": {
              "$ref": "#/components/schemas/ChatCompletionComplete"
            }
          },
          "created": {
            "type": "integer",
            "format": "int64",
            "example": "1706270835",
            "minimum": 0
          },
          "id": {
            "type": "string"
          },
          "model": {
            "type": "string",
            "example": "mistralai/Mistral-7B-Instruct-v0.2"
          },
          "object": {
            "type": "string"
          },
          "system_fingerprint": {
            "type": "string"
          },
          "usage": {
            "$ref": "#/components/schemas/Usage"
          }
        }
      },
      "ChatCompletionChoice": {
        "type": "object",
        "required": [
          "index",
          "delta"
        ],
        "properties": {
          "delta": {
            "$ref": "#/components/schemas/ChatCompletionDelta"
          },
          "finish_reason": {
            "type": "string",
            "nullable": true
          },
          "index": {
            "type": "integer",
            "format": "int32",
            "minimum": 0
          },
          "logprobs": {
            "allOf": [
              {
                "$ref": "#/components/schemas/ChatCompletionLogprobs"
              }
            ],
            "nullable": true
          }
        }
      },
      "ChatCompletionChunk": {
        "type": "object",
        "required": [
          "id",
          "object",
          "created",
          "model",
          "system_fingerprint",
          "choices"
        ],
        "properties": {
          "choices": {
            "type": "array",
            "items": {
              "$ref": "#/components/schemas/ChatCompletionChoice"
            }
          },
          "created": {
            "type": "integer",
            "format": "int64",
            "example": "1706270978",
            "minimum": 0
          },
          "id": {
            "type": "string"
          },
          "model": {
            "type": "string",
            "example": "mistralai/Mistral-7B-Instruct-v0.2"
          },
          "object": {
            "type": "string"
          },
          "system_fingerprint": {
            "type": "string"
          }
        }
      },
      "ChatCompletionComplete": {
        "type": "object",
        "required": [
          "index",
          "message",
          "finish_reason"
        ],
        "properties": {
          "finish_reason": {
            "type": "string"
          },
          "index": {
            "type": "integer",
            "format": "int32",
            "minimum": 0
          },
          "logprobs": {
            "allOf": [
              {
                "$ref": "#/components/schemas/ChatCompletionLogprobs"
              }
            ],
            "nullable": true
          },
          "message": {
            "$ref": "#/components/schemas/Message"
          }
        }
      },
      "ChatCompletionDelta": {
        "type": "object",
        "required": [
          "role"
        ],
        "properties": {
          "content": {
            "type": "string",
            "example": "What is Deep Learning?",
            "nullable": true
          },
          "role": {
            "type": "string",
            "example": "user"
          },
          "tool_calls": {
            "allOf": [
              {
                "$ref": "#/components/schemas/DeltaToolCall"
              }
            ],
            "nullable": true
          }
        }
      },
      "ChatCompletionLogprob": {
        "type": "object",
        "required": [
          "token",
          "logprob",
          "top_logprobs"
        ],
        "properties": {
          "logprob": {
            "type": "number",
            "format": "float"
          },
          "token": {
            "type": "string"
          },
          "top_logprobs": {
            "type": "array",
            "items": {
              "$ref": "#/components/schemas/ChatCompletionTopLogprob"
            }
          }
        }
      },
      "ChatCompletionLogprobs": {
        "type": "object",
        "required": [
          "content"
        ],
        "properties": {
          "content": {
            "type": "array",
            "items": {
              "$ref": "#/components/schemas/ChatCompletionLogprob"
            }
          }
        }
      },
      "ChatCompletionTopLogprob": {
        "type": "object",
        "required": [
          "token",
          "logprob"
        ],
        "properties": {
          "logprob": {
            "type": "number",
            "format": "float"
          },
          "token": {
            "type": "string"
          }
        }
      },
      "ChatRequest": {
        "type": "object",
        "required": [
          "model",
          "messages"
        ],
        "properties": {
          "frequency_penalty": {
            "type": "number",
            "format": "float",
            "description": "Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far,\ndecreasing the model's likelihood to repeat the same line verbatim.",
            "example": "1.0",
            "nullable": true
          },
          "logit_bias": {
            "type": "array",
            "items": {
              "type": "number",
              "format": "float"
            },
            "description": "UNUSED\nModify the likelihood of specified tokens appearing in the completion. Accepts a JSON object that maps tokens\n(specified by their token ID in the tokenizer) to an associated bias value from -100 to 100. Mathematically,\nthe bias is added to the logits generated by the model prior to sampling. The exact effect will vary per model,\nbut values between -1 and 1 should decrease or increase likelihood of selection; values like -100 or 100 should\nresult in a ban or exclusive selection of the relevant token.",
            "nullable": true
          },
          "logprobs": {
            "type": "boolean",
            "description": "Whether to return log probabilities of the output tokens or not. If true, returns the log probabilities of each\noutput token returned in the content of message.",
            "example": "false",
            "nullable": true
          },
          "max_tokens": {
            "type": "integer",
            "format": "int32",
            "description": "The maximum number of tokens that can be generated in the chat completion.",
            "example": "32",
            "nullable": true,
            "minimum": 0
          },
          "messages": {
            "type": "array",
            "items": {
              "$ref": "#/components/schemas/Message"
            },
            "description": "A list of messages comprising the conversation so far.",
            "example": "[{\"role\": \"user\", \"content\": \"What is Deep Learning?\"}]"
          },
          "model": {
            "type": "string",
            "description": "[UNUSED] ID of the model to use. See the model endpoint compatibility table for details on which models work with the Chat API.",
            "example": "mistralai/Mistral-7B-Instruct-v0.2"
          },
          "n": {
            "type": "integer",
            "format": "int32",
            "description": "UNUSED\nHow many chat completion choices to generate for each input message. Note that you will be charged based on the\nnumber of generated tokens across all of the choices. Keep n as 1 to minimize costs.",
            "example": "2",
            "nullable": true,
            "minimum": 0
          },
          "presence_penalty": {
            "type": "number",
            "format": "float",
            "description": "Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far,\nincreasing the model's likelihood to talk about new topics",
            "example": 0.1,
            "nullable": true
          },
          "seed": {
            "type": "integer",
            "format": "int64",
            "example": 42,
            "nullable": true,
            "minimum": 0
          },
          "stop": {
            "type": "array",
            "items": {
              "type": "string"
            },
            "description": "Up to 4 sequences where the API will stop generating further tokens.",
            "example": "null",
            "nullable": true
          },
          "stream": {
            "type": "boolean"
          },
          "temperature": {
            "type": "number",
            "format": "float",
            "description": "What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while\nlower values like 0.2 will make it more focused and deterministic.\n\nWe generally recommend altering this or `top_p` but not both.",
            "example": 1.0,
            "nullable": true
          },
          "tool_choice": {
            "allOf": [
              {
                "$ref": "#/components/schemas/ToolType"
              }
            ],
            "nullable": true
          },
          "tool_prompt": {
            "type": "string",
            "description": "A prompt to be appended before the tools",
            "example": "\"You will be presented with a JSON schema representing a set of tools.\nIf the user request lacks of sufficient information to make a precise tool selection: Do not invent any tool's properties, instead notify with an error message.\n\nJSON Schema:\n\"",
            "nullable": true
          },
          "tools": {
            "type": "array",
            "items": {
              "$ref": "#/components/schemas/Tool"
            },
            "description": "A list of tools the model may call. Currently, only functions are supported as a tool. Use this to provide a list of\nfunctions the model may generate JSON inputs for.",
            "example": "null",
            "nullable": true
          },
          "top_logprobs": {
            "type": "integer",
            "format": "int32",
            "description": "An integer between 0 and 5 specifying the number of most likely tokens to return at each token position, each with\nan associated log probability. logprobs must be set to true if this parameter is used.",
            "example": "5",
            "nullable": true,
            "minimum": 0
          },
          "top_p": {
            "type": "number",
            "format": "float",
            "description": "An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the\ntokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered.",
            "example": 0.95,
            "nullable": true
          }
        }
      },
      "CompatGenerateRequest": {
        "type": "object",
        "required": [
          "inputs"
        ],
        "properties": {
          "inputs": {
            "type": "string",
            "example": "My name is Olivier and I"
          },
          "parameters": {
            "$ref": "#/components/schemas/GenerateParameters"
          },
          "stream": {
            "type": "boolean",
            "default": "false"
          }
        }
      },
      "CompletionComplete": {
        "type": "object",
        "required": [
          "index",
          "text",
          "finish_reason"
        ],
        "properties": {
          "finish_reason": {
            "type": "string"
          },
          "index": {
            "type": "integer",
            "format": "int32",
            "minimum": 0
          },
          "logprobs": {
            "type": "array",
            "items": {
              "type": "number",
              "format": "float"
            },
            "nullable": true
          },
          "text": {
            "type": "string"
          }
        }
      },
      "CompletionCompleteChunk": {
        "type": "object",
        "required": [
          "id",
          "object",
          "created",
          "choices",
          "model",
          "system_fingerprint"
        ],
        "properties": {
          "choices": {
            "type": "array",
            "items": {
              "$ref": "#/components/schemas/CompletionComplete"
            }
          },
          "created": {
            "type": "integer",
            "format": "int64",
            "minimum": 0
          },
          "id": {
            "type": "string"
          },
          "model": {
            "type": "string"
          },
          "object": {
            "type": "string"
          },
          "system_fingerprint": {
            "type": "string"
          }
        }
      },
      "CompletionRequest": {
        "type": "object",
        "required": [
          "model",
          "prompt"
        ],
        "properties": {
          "frequency_penalty": {
            "type": "number",
            "format": "float",
            "description": "Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far,\ndecreasing the model's likelihood to repeat the same line verbatim.",
            "example": "1.0",
            "nullable": true
          },
          "max_tokens": {
            "type": "integer",
            "format": "int32",
            "description": "The maximum number of tokens that can be generated in the chat completion.",
            "default": "32",
            "nullable": true,
            "minimum": 0
          },
          "model": {
            "type": "string",
            "description": "UNUSED\nID of the model to use. See the model endpoint compatibility table for details on which models work with the Chat API.",
            "example": "mistralai/Mistral-7B-Instruct-v0.2"
          },
          "prompt": {
            "type": "array",
            "items": {
              "type": "string"
            },
            "description": "The prompt to generate completions for.",
            "example": "What is Deep Learning?"
          },
          "repetition_penalty": {
            "type": "number",
            "format": "float",
            "nullable": true
          },
          "seed": {
            "type": "integer",
            "format": "int64",
            "example": 42,
            "nullable": true,
            "minimum": 0
          },
          "stream": {
            "type": "boolean"
          },
          "suffix": {
            "type": "string",
            "description": "The text to append to the prompt. This is useful for completing sentences or generating a paragraph of text.\nplease see the completion_template field in the model's tokenizer_config.json file for completion template.",
            "nullable": true
          },
          "temperature": {
            "type": "number",
            "format": "float",
            "description": "What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while\nlower values like 0.2 will make it more focused and deterministic. We generally recommend altering this or `top_p` but not both.",
            "example": 1.0,
            "nullable": true
          },
          "top_p": {
            "type": "number",
            "format": "float",
            "description": "An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the\ntokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered.",
            "example": 0.95,
            "nullable": true
          }
        }
      },
      "DeltaToolCall": {
        "type": "object",
        "required": [
          "index",
          "id",
          "type",
          "function"
        ],
        "properties": {
          "function": {
            "$ref": "#/components/schemas/Function"
          },
          "id": {
            "type": "string"
          },
          "index": {
            "type": "integer",
            "format": "int32",
            "minimum": 0
          },
          "type": {
            "type": "string"
          }
        }
      },
      "Details": {
        "type": "object",
        "required": [
          "finish_reason",
          "generated_tokens",
          "prefill",
          "tokens"
        ],
        "properties": {
          "best_of_sequences": {
            "type": "array",
            "items": {
              "$ref": "#/components/schemas/BestOfSequence"
            },
            "nullable": true
          },
          "finish_reason": {
            "$ref": "#/components/schemas/FinishReason"
          },
          "generated_tokens": {
            "type": "integer",
            "format": "int32",
            "example": 1,
            "minimum": 0
          },
          "prefill": {
            "type": "array",
            "items": {
              "$ref": "#/components/schemas/PrefillToken"
            }
          },
          "seed": {
            "type": "integer",
            "format": "int64",
            "example": 42,
            "nullable": true,
            "minimum": 0
          },
          "tokens": {
            "type": "array",
            "items": {
              "$ref": "#/components/schemas/Token"
            }
          },
          "top_tokens": {
            "type": "array",
            "items": {
              "type": "array",
              "items": {
                "$ref": "#/components/schemas/Token"
              }
            }
          }
        }
      },
      "ErrorResponse": {
        "type": "object",
        "required": [
          "error",
          "error_type"
        ],
        "properties": {
          "error": {
            "type": "string"
          },
          "error_type": {
            "type": "string"
          }
        }
      },
      "FinishReason": {
        "type": "string",
        "enum": [
          "length",
          "eos_token",
          "stop_sequence"
        ],
        "example": "Length"
      },
      "Function": {
        "type": "object",
        "required": [
          "arguments"
        ],
        "properties": {
          "arguments": {
            "type": "string"
          },
          "name": {
            "type": "string",
            "nullable": true
          }
        }
      },
      "FunctionDefinition": {
        "type": "object",
        "required": [
          "name",
          "arguments"
        ],
        "properties": {
          "arguments": {},
          "description": {
            "type": "string",
            "nullable": true
          },
          "name": {
            "type": "string"
          }
        }
      },
      "GenerateParameters": {
        "type": "object",
        "properties": {
          "best_of": {
            "type": "integer",
            "default": "null",
            "example": 1,
            "nullable": true,
            "minimum": 0,
            "exclusiveMinimum": 0
          },
          "decoder_input_details": {
            "type": "boolean",
            "default": "false"
          },
          "details": {
            "type": "boolean",
            "default": "true"
          },
          "do_sample": {
            "type": "boolean",
            "default": "false",
            "example": true
          },
          "frequency_penalty": {
            "type": "number",
            "format": "float",
            "default": "null",
            "example": 0.1,
            "nullable": true,
            "exclusiveMinimum": -2
          },
          "grammar": {
            "allOf": [
              {
                "$ref": "#/components/schemas/GrammarType"
              }
            ],
            "default": "null",
            "nullable": true
          },
          "max_new_tokens": {
            "type": "integer",
            "format": "int32",
            "default": "100",
            "example": "20",
            "nullable": true,
            "minimum": 0
          },
          "repetition_penalty": {
            "type": "number",
            "format": "float",
            "default": "null",
            "example": 1.03,
            "nullable": true,
            "exclusiveMinimum": 0
          },
          "return_full_text": {
            "type": "boolean",
            "default": "null",
            "example": false,
            "nullable": true
          },
          "seed": {
            "type": "integer",
            "format": "int64",
            "default": "null",
            "example": "null",
            "nullable": true,
            "minimum": 0,
            "exclusiveMinimum": 0
          },
          "stop": {
            "type": "array",
            "items": {
              "type": "string"
            },
            "example": [
              "photographer"
            ],
            "maxItems": 4
          },
          "temperature": {
            "type": "number",
            "format": "float",
            "default": "null",
            "example": 0.5,
            "nullable": true,
            "exclusiveMinimum": 0
          },
          "top_k": {
            "type": "integer",
            "format": "int32",
            "default": "null",
            "example": 10,
            "nullable": true,
            "exclusiveMinimum": 0
          },
          "top_n_tokens": {
            "type": "integer",
            "format": "int32",
            "default": "null",
            "example": 5,
            "nullable": true,
            "minimum": 0,
            "exclusiveMinimum": 0
          },
          "top_p": {
            "type": "number",
            "format": "float",
            "default": "null",
            "example": 0.95,
            "nullable": true,
            "maximum": 1,
            "exclusiveMinimum": 0
          },
          "truncate": {
            "type": "integer",
            "default": "null",
            "example": "null",
            "nullable": true,
            "minimum": 0
          },
          "typical_p": {
            "type": "number",
            "format": "float",
            "default": "null",
            "example": 0.95,
            "nullable": true,
            "maximum": 1,
            "exclusiveMinimum": 0
          },
          "watermark": {
            "type": "boolean",
            "default": "false",
            "example": true
          }
        }
      },
      "GenerateRequest": {
        "type": "object",
        "required": [
          "inputs"
        ],
        "properties": {
          "inputs": {
            "type": "string",
            "example": "My name is Olivier and I"
          },
          "parameters": {
            "$ref": "#/components/schemas/GenerateParameters"
          }
        }
      },
      "GenerateResponse": {
        "type": "object",
        "required": [
          "generated_text"
        ],
        "properties": {
          "details": {
            "allOf": [
              {
                "$ref": "#/components/schemas/Details"
              }
            ],
            "nullable": true
          },
          "generated_text": {
            "type": "string",
            "example": "test"
          }
        }
      },
      "GrammarType": {
        "oneOf": [
          {
            "type": "object",
            "required": [
              "type",
              "value"
            ],
            "properties": {
              "type": {
                "type": "string",
                "enum": [
                  "json"
                ]
              },
              "value": {
                "description": "A string that represents a [JSON Schema](https://json-schema.org/).\n\nJSON Schema is a declarative language that allows to annotate JSON documents\nwith types and descriptions."
              }
            }
          },
          {
            "type": "object",
            "required": [
              "type",
              "value"
            ],
            "properties": {
              "type": {
                "type": "string",
                "enum": [
                  "regex"
                ]
              },
              "value": {
                "type": "string"
              }
            }
          }
        ],
        "discriminator": {
          "propertyName": "type"
        }
      },
      "Info": {
        "type": "object",
        "required": [
          "model_id",
          "model_dtype",
          "model_device_type",
          "max_concurrent_requests",
          "max_best_of",
          "max_stop_sequences",
          "max_input_length",
          "max_total_tokens",
          "waiting_served_ratio",
          "max_batch_total_tokens",
          "max_waiting_tokens",
          "validation_workers",
          "max_client_batch_size",
          "version"
        ],
        "properties": {
          "docker_label": {
            "type": "string",
            "example": "null",
            "nullable": true
          },
          "max_batch_size": {
            "type": "integer",
            "example": "null",
            "nullable": true,
            "minimum": 0
          },
          "max_batch_total_tokens": {
            "type": "integer",
            "format": "int32",
            "example": "32000",
            "minimum": 0
          },
          "max_best_of": {
            "type": "integer",
            "example": "2",
            "minimum": 0
          },
          "max_client_batch_size": {
            "type": "integer",
            "example": "32",
            "minimum": 0
          },
          "max_concurrent_requests": {
            "type": "integer",
            "description": "Router Parameters",
            "example": "128",
            "minimum": 0
          },
          "max_input_length": {
            "type": "integer",
            "example": "1024",
            "minimum": 0
          },
          "max_stop_sequences": {
            "type": "integer",
            "example": "4",
            "minimum": 0
          },
          "max_total_tokens": {
            "type": "integer",
            "example": "2048",
            "minimum": 0
          },
          "max_waiting_tokens": {
            "type": "integer",
            "example": "20",
            "minimum": 0
          },
          "model_device_type": {
            "type": "string",
            "example": "cuda"
          },
          "model_dtype": {
            "type": "string",
            "example": "torch.float16"
          },
          "model_id": {
            "type": "string",
            "description": "Model info",
            "example": "bigscience/blomm-560m"
          },
          "model_pipeline_tag": {
            "type": "string",
            "example": "text-generation",
            "nullable": true
          },
          "model_sha": {
            "type": "string",
            "example": "e985a63cdc139290c5f700ff1929f0b5942cced2",
            "nullable": true
          },
          "sha": {
            "type": "string",
            "example": "null",
            "nullable": true
          },
          "validation_workers": {
            "type": "integer",
            "example": "2",
            "minimum": 0
          },
          "version": {
            "type": "string",
            "description": "Router Info",
            "example": "0.5.0"
          },
          "waiting_served_ratio": {
            "type": "number",
            "format": "float",
            "example": "1.2"
          }
        }
      },
      "Message": {
        "type": "object",
        "required": [
          "role"
        ],
        "properties": {
          "content": {
            "type": "string",
            "example": "My name is David and I",
            "nullable": true
          },
          "name": {
            "type": "string",
            "example": "\"David\"",
            "nullable": true
          },
          "role": {
            "type": "string",
            "example": "user"
          },
          "tool_calls": {
            "type": "array",
            "items": {
              "$ref": "#/components/schemas/ToolCall"
            },
            "nullable": true
          }
        }
      },
      "PrefillToken": {
        "type": "object",
        "required": [
          "id",
          "text",
          "logprob"
        ],
        "properties": {
          "id": {
            "type": "integer",
            "format": "int32",
            "example": 0,
            "minimum": 0
          },
          "logprob": {
            "type": "number",
            "format": "float",
            "example": -0.34,
            "nullable": true
          },
          "text": {
            "type": "string",
            "example": "test"
          }
        }
      },
      "SimpleToken": {
        "type": "object",
        "required": [
          "id",
          "text",
          "start",
          "stop"
        ],
        "properties": {
          "id": {
            "type": "integer",
            "format": "int32",
            "example": 0,
            "minimum": 0
          },
          "start": {
            "type": "integer",
            "example": 0,
            "minimum": 0
          },
          "stop": {
            "type": "integer",
            "example": 2,
            "minimum": 0
          },
          "text": {
            "type": "string",
            "example": "test"
          }
        }
      },
      "StreamDetails": {
        "type": "object",
        "required": [
          "finish_reason",
          "generated_tokens"
        ],
        "properties": {
          "finish_reason": {
            "$ref": "#/components/schemas/FinishReason"
          },
          "generated_tokens": {
            "type": "integer",
            "format": "int32",
            "example": 1,
            "minimum": 0
          },
          "seed": {
            "type": "integer",
            "format": "int64",
            "example": 42,
            "nullable": true,
            "minimum": 0
          }
        }
      },
      "StreamResponse": {
        "type": "object",
        "required": [
          "index",
          "token"
        ],
        "properties": {
          "details": {
            "allOf": [
              {
                "$ref": "#/components/schemas/StreamDetails"
              }
            ],
            "default": "null",
            "nullable": true
          },
          "generated_text": {
            "type": "string",
            "default": "null",
            "example": "test",
            "nullable": true
          },
          "index": {
            "type": "integer",
            "format": "int32",
            "minimum": 0
          },
          "token": {
            "$ref": "#/components/schemas/Token"
          },
          "top_tokens": {
            "type": "array",
            "items": {
              "$ref": "#/components/schemas/Token"
            }
          }
        }
      },
      "Token": {
        "type": "object",
        "required": [
          "id",
          "text",
          "logprob",
          "special"
        ],
        "properties": {
          "id": {
            "type": "integer",
            "format": "int32",
            "example": 0,
            "minimum": 0
          },
          "logprob": {
            "type": "number",
            "format": "float",
            "example": -0.34,
            "nullable": true
          },
          "special": {
            "type": "boolean",
            "example": "false"
          },
          "text": {
            "type": "string",
            "example": "test"
          }
        }
      },
      "TokenizeResponse": {
        "type": "array",
        "items": {
          "$ref": "#/components/schemas/SimpleToken"
        }
      },
      "Tool": {
        "type": "object",
        "required": [
          "type",
          "function"
        ],
        "properties": {
          "function": {
            "$ref": "#/components/schemas/FunctionDefinition"
          },
          "type": {
            "type": "string",
            "example": "function"
          }
        }
      },
      "ToolCall": {
        "type": "object",
        "required": [
          "id",
          "type",
          "function"
        ],
        "properties": {
          "function": {
            "$ref": "#/components/schemas/FunctionDefinition"
          },
          "id": {
            "type": "integer",
            "format": "int32",
            "minimum": 0
          },
          "type": {
            "type": "string"
          }
        }
      },
      "ToolType": {
        "oneOf": [
          {
            "type": "object",
            "required": [
              "FunctionName"
            ],
            "properties": {
              "FunctionName": {
                "type": "string"
              }
            }
          },
          {
            "type": "string",
            "enum": [
              "OneOf"
            ]
          }
        ]
      },
      "Usage": {
        "type": "object",
        "required": [
          "prompt_tokens",
          "completion_tokens",
          "total_tokens"
        ],
        "properties": {
          "completion_tokens": {
            "type": "integer",
            "format": "int32",
            "minimum": 0
          },
          "prompt_tokens": {
            "type": "integer",
            "format": "int32",
            "minimum": 0
          },
          "total_tokens": {
            "type": "integer",
            "format": "int32",
            "minimum": 0
          }
        }
      }
    }
  },
  "tags": [
    {
      "name": "Text Generation Inference",
      "description": "Hugging Face Text Generation Inference API"
    }
  ]
}
