"docs/source/vscode:/vscode.git/clone" did not exist on "e2dc558e9c8eae3409481ca74efc15e07f8e90c8"
Unverified Commit e30ceae0 authored by Nicolas Patry's avatar Nicolas Patry Committed by GitHub
Browse files

small fix to remove unused eos in processor when it's not used. (#23408)

parent 2f424d79
...@@ -1175,7 +1175,11 @@ class LlamaConverter(SpmConverter): ...@@ -1175,7 +1175,11 @@ class LlamaConverter(SpmConverter):
single = f"{(bos+':0 ') * add_bos}$A:0{(' '+eos+':0') * add_eos}" single = f"{(bos+':0 ') * add_bos}$A:0{(' '+eos+':0') * add_eos}"
pair = f"{single}{(' '+bos+':1') * add_bos} $B:1{(' '+eos+':1') * add_eos}" pair = f"{single}{(' '+bos+':1') * add_bos} $B:1{(' '+eos+':1') * add_eos}"
special_tokens = [(bos, bos_token_id), (eos, eos_token_id)] special_tokens = []
if add_bos:
special_tokens.append((bos, bos_token_id))
if add_eos:
special_tokens.append((eos, eos_token_id))
return processors.TemplateProcessing(single=single, pair=pair, special_tokens=special_tokens) return processors.TemplateProcessing(single=single, pair=pair, special_tokens=special_tokens)
else: else:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment