Remove num_tokens

lm_eval/base.py

@@ -20,16 +20,6 @@ class LM(abc.ABC):
         """
         pass
 
-    @classmethod
-    def num_tokens(cls, string):
-        """Return the number of tokens in a string, based on tokenization
-
-        :param string: str
-            Input string
-        :return: int
-        """
-        pass
-
     @classmethod
     def create_from_arg_string(cls, arg_string):
         """Constructor method, in case models need additional arguments

lm_eval/models/dummy.py

@@ -5,8 +5,5 @@ from . import MODEL_REGISTRY
 @MODEL_REGISTRY.register("dummy")
 class DummyLM(LM):
 
-    def generate(self, context, max_gen_length):
-        return "lol"
-
     def loglikelihood(self, context, continuation):
         return 0.0

lm_eval/models/gpt2.py

@@ -28,6 +28,3 @@ class GPT2LM(LM):
         logits = F.log_softmax(self.gpt2(inp)[0], dim=-1)[:, ctxlen - 1:-1]  # [batch, seq, vocab]
 
         return torch.gather(logits, 2, cont_toks.unsqueeze(-1)).squeeze(-1)
-
-    def num_tokens(self, string):
-        return len(self.tokenizer.tokenize(string))

lm_eval/models/gpt3.py

@@ -47,4 +47,3 @@ class GPT3LM(LM):
         logprobs = response.choices[0]["logprobs"]["token_logprobs"]
         continuation_logprobs = logprobs[ctxlen:]
         return sum(continuation_logprobs)
-