# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from typing import List, Optional from pydantic import BaseModel, Field TokenIdType = int # TODO: move these to common for all LLMs once we adopt dynamo-run # derived from lib/llm/src/protocols/common/preprocessor.rs class StopConditions(BaseModel): max_tokens: Optional[int] = None stop: Optional[List[str]] = None stop_token_ids_hidden: Optional[List[TokenIdType]] = None min_tokens: Optional[int] = None ignore_eos: Optional[bool] = None class SamplingOptions(BaseModel): n: Optional[int] = None best_of: Optional[int] = None presence_penalty: Optional[float] = None frequency_penalty: Optional[float] = None repetition_penalty: Optional[float] = None temperature: Optional[float] = None top_p: Optional[float] = None top_k: Optional[int] = None min_p: Optional[float] = None use_beam_search: Optional[bool] = None length_penalty: Optional[float] = None seed: Optional[int] = None class PreprocessedRequest(BaseModel): token_ids: List[TokenIdType] stop_conditions: StopConditions sampling_options: SamplingOptions eos_token_ids: List[TokenIdType] = Field(default_factory=list) mdc_sum: Optional[str] = None annotations: List[str] = Field(default_factory=list)