Merge remote-tracking branch 'upstream/big-refactor' into seq2seq-refactor

lm_eval/tasks/wikitext/preprocess_wikitext.py

 import re
 
+
 def wikitext_detokenizer(doc):
     string = doc["page"]
     # contractions

lm_eval/tasks/wikitext/wikitext.yaml

 group:
   - wikitext_group
-task: wikitext_yaml
+task: default
 dataset_path: EleutherAI/wikitext_document_level
 dataset_name: wikitext-2-raw-v1
 output_type: loglikelihood_rolling
@@ -14,11 +14,5 @@ should_decontaminate: true
 doc_to_decontamination_query: "{{page}}"
 metric_list:
   - metric: word_perplexity
-    aggregation: weighted_perplexity
-    higher_is_better: false
   - metric: byte_perplexity
-    aggregation: weighted_perplexity
-    higher_is_better: false
   - metric: bits_per_byte
-    aggregation: bits_per_byte
-    higher_is_better: false
\ No newline at end of file

lm_eval/utils.py

@@ -157,22 +157,32 @@ def make_table(result_dict):
 
     md_writer = MarkdownTableWriter()
     latex_writer = LatexTableWriter()
-    md_writer.headers = ["Task", "Version", "Metric", "Value", "", "Stderr"]
-    latex_writer.headers = ["Task", "Version", "Metric", "Value", "", "Stderr"]
+    md_writer.headers = ["Task", "Version", "Filter", "Metric", "Value", "", "Stderr"]
+    latex_writer.headers = [
+        "Task",
+        "Version",
+        "Filter",
+        "Metric",
+        "Value",
+        "",
+        "Stderr",
+    ]
 
     values = []
 
     for k, dic in result_dict["results"].items():
         version = result_dict["versions"][k]
-        for m, v in dic.items():
+        for (mf), v in dic.items():
+            m, _, f = mf.partition(",")
+            print(m, f)
             if m.endswith("_stderr"):
                 continue
 
-            if m + "_stderr" in dic:
-                se = dic[m + "_stderr"]
-                values.append([k, version, m, "%.4f" % v, "±", "%.4f" % se])
+            if m + "_stderr" + "," + f in dic:
+                se = dic[m + "_stderr" + "," + f]
+                values.append([k, version, f, m, "%.4f" % v, "±", "%.4f" % se])
             else:
-                values.append([k, version, m, "%.4f" % v, "", ""])
+                values.append([k, version, f, m, "%.4f" % v, "", ""])
             k = ""
             version = ""
     md_writer.value_matrix = values

main.py

@@ -19,7 +19,9 @@ class MultiChoice:
         for value in values.split(","):
             if len(fnmatch.filter(self.choices, value)) == 0:
                 eval_logger.warning("{} is not in task list.".format(value))
-                # eval_logger.info(f"{choices} is this")
+                eval_logger.info(f"Available tasks to choose:")
+                for choice in self.choices:
+                    eval_logger.info(f"    {choice}")
 
         return True