Skip to content

GitLab

You need to sign in or sign up before continuing.
Unverified Commit cb8ac1b0 authored by lvhan028's avatar lvhan028 Committed by GitHub
Browse files

fix crash when conversation history out of limit (#28)

parent c16b857b
Show whitespace changes
Inline Side-by-side
Showing with 5 additions and 0 deletions
llmdeploy/serve/fastertransformer/chatbot.py
View file @ cb8ac1b0
...@@ -168,6 +168,8 @@ class Chatbot: ...@@ -168,6 +168,8 @@ class Chatbot:
sequence_start, sequence_start,
sequence_end): sequence_end):
yield status, res, tokens yield status, res, tokens
if status.value < 0:
return
self._session.histories = \ self._session.histories = \
self._session.histories + self._session.prompt + \ self._session.histories + self._session.prompt + \
self._session.response self._session.response
...@@ -332,7 +334,10 @@ class Chatbot: ...@@ -332,7 +334,10 @@ class Chatbot:
f'history tokens {session.sequence_length}, ' \ f'history tokens {session.sequence_length}, ' \
f'request length {request_output_len}' f'request length {request_output_len}'
yield StatusCode.TRITON_SESSION_OUT_OF_LIMIT, errmsg, 0 yield StatusCode.TRITON_SESSION_OUT_OF_LIMIT, errmsg, 0
return
logger.info(f'session {session.session_id}, ' logger.info(f'session {session.session_id}, '
f'max length: {self.cfg.session_len}, '
f'input tokens: {input_tokens}, ' f'input tokens: {input_tokens}, '
f'request tokens: {request_output_len}, ' f'request tokens: {request_output_len}, '
f'history tokens: {session.sequence_length}') f'history tokens: {session.sequence_length}')
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment