skip responding eos token

Signed-off-by: Ceng23333 <441651826@qq.com>

python/infinilm/server/inference_server.py

@@ -293,6 +293,14 @@ class InferenceServer:
                     req.mark_canceled()
                     break
 
+                # Skip EOS token text for OpenAI API compatibility
+                # Check if this token is an EOS token by comparing token_id with eos_token_ids
+                eos_token_ids = self.engine.engine.eos_token_ids
+                is_eos_token = (
+                    eos_token_ids and token_output.token_id in eos_token_ids
+                )
+
+                if not is_eos_token and token_output.token_text:
                     # Send token
                     chunk = json.dumps(
                         chunk_json(
@@ -374,6 +382,14 @@ class InferenceServer:
                     req.mark_canceled()
                     break
 
+                # Skip EOS token text for OpenAI API compatibility
+                # Check if this token is an EOS token by comparing token_id with eos_token_ids
+                eos_token_ids = self.engine.engine.eos_token_ids
+                is_eos_token = (
+                    eos_token_ids and token_output.token_id in eos_token_ids
+                )
+
+                if not is_eos_token:
                     output_text += token_output.token_text
 
                 if token_output.finished:

test/bench/test_benchmark.py

@@ -4,7 +4,6 @@ import argparse
 import time
 import re
 import csv
-from datasets import load_dataset, Dataset
 import numpy as np
 import infinicore
 from infinilm.modeling_utils import load_model_state_dict_by_file
@@ -12,6 +11,7 @@ from infinilm.distributed import DistConfig
 from infinilm.cache import StaticKVCacheConfig
 from infinilm.infer_engine import GenerationConfig, InferEngine
 from infinilm.cache import StaticKVCacheConfig
+from datasets import load_dataset, Dataset
 from abc import ABC, abstractmethod