Unverified Commit dc5eba86 authored by Sergio Perez's avatar Sergio Perez Committed by GitHub
Browse files

fix error when appending eot_token_id for generate_until tasks (#1699)

parent 8b326be7
......@@ -485,8 +485,8 @@ class NeMoLM(LM):
def get_until(req_args):
until = req_args.get("until", [])
until = deepcopy(until) # prevent from modifying req_args for cache_key
if self.eot_token_id not in until:
until.append(self.eot_token_id)
if self.tokenizer.ids_to_tokens([self.eot_token_id])[0] not in until:
until.append(self.tokenizer.ids_to_tokens([self.eot_token_id])[0])
return until
def _collate(x):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment