misc fixes

lm_eval/api/model.py

@@ -104,3 +104,17 @@ class LM(abc.ABC):
         args = utils.simple_parse_args_string(arg_string)
         args2 = {k: v for k, v in additional_config.items() if v is not None}
         return cls(**args, **args2)
+
+    @property
+    def rank(self):
+        # used in the case of parallelism. Hardcoded to
+        # ensure no errors arise using API models which do
+        # not support multi-device parallelism nor expect it.
+        return 0
+
+    @property
+    def world_size(self):
+        # used in the case of parallelism. Hardcoded to
+        # ensure no errors arise using API models which do
+        # not support multi-device parallelism nor expect it.
+        return 1

lm_eval/models/anthropic_llms.py

 import os
-from lm_eval.base import BaseLM
+from lm_eval.api.model import LM
+from lm_eval.api.registry import register_model
 from tqdm import tqdm
 import time
 
@@ -37,6 +38,7 @@ def anthropic_completion(
             backoff_time *= 1.5
 
 
+@register_model("anthropic")
 class AnthropicLM(BaseLM):
     REQ_CHUNK_SIZE = 20
 

lm_eval/models/openai_completions.py

@@ -58,7 +58,7 @@ def oa_completion(**kwargs):
 
 
 @register_model("openai", "openai-completions", "gooseai")
-class GPT3LM(LM):
+class OpenaiCompletionsLM(LM):
     REQ_CHUNK_SIZE = 20
 
     def __init__(self, engine, truncate=False):

lm_eval/tasks/lambada/lambada_openai.yaml

@@ -16,6 +16,6 @@ metric_list:
   - metric: perplexity
     aggregation: perplexity
     higher_is_better: false
-  - metric: accuracy
+  - metric: acc
     aggregation: mean
     higher_is_better: true

lm_eval/tasks/lambada/lambada_standard.yaml

@@ -17,6 +17,6 @@ metric_list:
   - metric: perplexity
     aggregation: perplexity
     higher_is_better: false
-  - metric: accuracy
+  - metric: acc
     aggregation: mean
     higher_is_better: true

lm_eval/tasks/lambada_cloze/lambada_openai_cloze.yaml

@@ -15,6 +15,6 @@ metric_list:
   - metric: perplexity
     aggregation: perplexity
     higher_is_better: false
-  - metric: accuracy
+  - metric: acc
     aggregation: mean
     higher_is_better: true