Skip to content

GitLab

You need to sign in or sign up before continuing.
Unverified Commit 6463070b authored by wang jiahao's avatar wang jiahao Committed by GitHub
Browse files

Merge pull request #1064 from kvcache-ai/fix-temperature 

fix temperature=0, flashinfer sample error
parents aac0c91d ec03bcbd
Hide whitespace changes
Inline Side-by-side
Showing with 2 additions and 0 deletions
ktransformers/server/backend/interfaces/balance_serve.py
View file @ 6463070b
...@@ -367,6 +367,8 @@ class BalanceServeInterface(BackendInterfaceBase): ...@@ -367,6 +367,8 @@ class BalanceServeInterface(BackendInterfaceBase):
#@TODO add server #@TODO add server
stop_criteria = [self.tokenizer.encode(self.tokenizer.eos_token, add_special_tokens=False),self.tokenizer.encode("<|im_end|>")] stop_criteria = [self.tokenizer.encode(self.tokenizer.eos_token, add_special_tokens=False),self.tokenizer.encode("<|im_end|>")]
query_add.stop_criteria = stop_criteria query_add.stop_criteria = stop_criteria
if temperature == 0:
temperature = 0.0001
query_add.sample_options.temperature = temperature query_add.sample_options.temperature = temperature
if top_p == 0: if top_p == 0:
top_p = 0.0001 top_p = 0.0001
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment