skip responding eos token

Signed-off-by: Ceng23333 <441651826@qq.com>

python/infinilm/server/inference_server.py

@@ -293,14 +293,22 @@ class InferenceServer:
                     req.mark_canceled()
                     break
 
-                # Send token
-                chunk = json.dumps(
-                    chunk_json(
-                        request_id, content=token_output.token_text, model=self.model_id
-                    ),
-                    ensure_ascii=False,
+                # Skip EOS token text for OpenAI API compatibility
+                # Check if this token is an EOS token by comparing token_id with eos_token_ids
+                eos_token_ids = self.engine.engine.eos_token_ids
+                is_eos_token = (
+                    eos_token_ids and token_output.token_id in eos_token_ids
                 )
-                yield f"data: {chunk}\n\n"
+
+                if not is_eos_token and token_output.token_text:
+                    # Send token
+                    chunk = json.dumps(
+                        chunk_json(
+                            request_id, content=token_output.token_text, model=self.model_id
+                        ),
+                        ensure_ascii=False,
+                    )
+                    yield f"data: {chunk}\n\n"
 
                 if token_output.finished:
                     finish_reason = self._convert_finish_reason(
@@ -374,7 +382,15 @@ class InferenceServer:
                     req.mark_canceled()
                     break
 
-                output_text += token_output.token_text
+                # Skip EOS token text for OpenAI API compatibility
+                # Check if this token is an EOS token by comparing token_id with eos_token_ids
+                eos_token_ids = self.engine.engine.eos_token_ids
+                is_eos_token = (
+                    eos_token_ids and token_output.token_id in eos_token_ids
+                )
+
+                if not is_eos_token:
+                    output_text += token_output.token_text
 
                 if token_output.finished:
                     break

test/bench/test_benchmark.py

@@ -4,7 +4,6 @@ import argparse
 import time
 import re
 import csv
-from datasets import load_dataset, Dataset
 import numpy as np
 import infinicore
 from infinilm.modeling_utils import load_model_state_dict_by_file
@@ -12,6 +11,7 @@ from infinilm.distributed import DistConfig
 from infinilm.cache import StaticKVCacheConfig
 from infinilm.infer_engine import GenerationConfig, InferEngine
 from infinilm.cache import StaticKVCacheConfig
+from datasets import load_dataset, Dataset
 from abc import ABC, abstractmethod