Implement fewshot description experiment

scripts/fewshot_description_experiment.py

+import argparse
+import json
+import numpy as np
+import random
+import itertools
+import collections
+import logging
+
+from lm_eval import models, tasks, evaluator, base
+
+logging.getLogger("openai").setLevel(logging.WARNING)
+
+
+fewshot_descriptions = [
+    "foo",
+    "bar"
+]
+
+task = "lambada"
+num_fewshot = 0
+model = "gpt2"
+model_args = ""
+limit = None
+no_cache = False
+
+
+class CustomDescTask:
+    def __init__(self, task, desc):
+        self.task = task
+        self.desc = desc
+
+        def fewshot_description():
+            return self.desc
+        
+        self.task.fewshot_description = fewshot_description
+
+    def __getattr__(self, attr):
+        return getattr(self.task, attr)
+
+
+def main():
+    random.seed(42)
+    np.random.seed(42)
+
+    lm = models.get_model(model).create_from_arg_string(model_args)
+    
+    if limit:
+        print("WARNING: --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.")
+
+    if not no_cache:
+        lm = base.CachingLM(lm, 'lm_cache/' + model + '_' + model_args.replace('=', '-').replace(',', '_') + '.db')
+
+    task_dict = tasks.get_task_dict([task])
+
+    for desc in fewshot_descriptions:
+        custom_task_dict = {k: CustomDescTask(v, desc) for k, v in task_dict.items()}
+
+        results = evaluator.evaluate(lm, custom_task_dict, True, num_fewshot, limit)
+
+        dumped = json.dumps(results, indent=2)
+
+        print('Description:', desc)
+        print(dumped)
+
+        # MAKE TABLE
+        from pytablewriter import MarkdownTableWriter
+
+        writer = MarkdownTableWriter()
+        writer.headers = ["Task", "Metric", "Value"]
+
+        values = []
+
+        for k, dic in results.items():
+            for m, v in dic.items():
+                values.append([k, m, '%.4f' % v])
+                k = ""
+        writer.value_matrix = values
+
+        print(writer.dumps())
+
+
+if __name__ == "__main__":
+    main()