Skip to content

GitLab

Unverified Commit efd319b3 authored by Jonathan Tow's avatar Jonathan Tow Committed by GitHub
Browse files

Merge pull request #14 from bigscience-workshop/fix-gpt-j 

Add `eot_token` property to gpt-j
parents f65e196e 32d0658c
Hide whitespace changes
Inline Side-by-side
Showing with 4 additions and 0 deletions
lm_eval/models/gptj.py
View file @ efd319b3
...@@ -39,6 +39,10 @@ class GPTJLM(BaseLM): ...@@ -39,6 +39,10 @@ class GPTJLM(BaseLM):
# if gpus > 1: # if gpus > 1:
# self.gptj = nn.DataParallel(self.gptj) # self.gptj = nn.DataParallel(self.gptj)
@property
def eot_token(self):
return self.tokenizer.eos_token
@property @property
def eot_token_id(self): def eot_token_id(self):
# we use EOT because end of *text* is more accurate for what we're doing than end of *sentence* # we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment