Skip to content

GitLab

"vscode:/vscode.git/clone" did not exist on "cfa6337960683d8c6919bfb3be6e04389dfdb3c6"
Unverified Commit dc5eba86 authored by Sergio Perez's avatar Sergio Perez Committed by GitHub
Browse files

fix error when appending eot_token_id for generate_until tasks (#1699)

parent 8b326be7
Hide whitespace changes
Inline Side-by-side
Showing with 2 additions and 2 deletions
lm_eval/models/nemo_lm.py
View file @ dc5eba86
...@@ -485,8 +485,8 @@ class NeMoLM(LM): ...@@ -485,8 +485,8 @@ class NeMoLM(LM):
def get_until(req_args): def get_until(req_args):
until = req_args.get("until", []) until = req_args.get("until", [])
until = deepcopy(until) # prevent from modifying req_args for cache_key until = deepcopy(until) # prevent from modifying req_args for cache_key
if self.eot_token_id not in until: if self.tokenizer.ids_to_tokens([self.eot_token_id])[0] not in until:
until.append(self.eot_token_id) until.append(self.tokenizer.ids_to_tokens([self.eot_token_id])[0])
return until return until
def _collate(x): def _collate(x):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment