Merge remote-tracking branch 'origin/big-refactor' into big-refactor_dp

lm_eval/tasks/triviaqa/default.yaml

@@ -27,3 +27,5 @@ metric_list:
     higher_is_better: true
     ignore_case: true
     ignore_punctuation: true
+metadata:
+  - version: 2.0

lm_eval/tasks/truthfulqa/truthfulqa_gen.yaml

@@ -75,3 +75,5 @@ metric_list:
   - metric: rougeL_diff
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 2.0

lm_eval/tasks/truthfulqa/truthfulqa_mc1.yaml

@@ -32,3 +32,5 @@ metric_list:
   - metric: acc
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 2.0

lm_eval/tasks/truthfulqa/truthfulqa_mc2.yaml

@@ -9,3 +9,5 @@ metric_list:
   - metric: acc
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 2.0

lm_eval/tasks/unscramble/anagrams1.yaml

@@ -16,3 +16,5 @@ metric_list:
     higher_is_better: true
     ignore_case: false
     ignore_punctuation: false
+metadata:
+  - version: 1.0

lm_eval/tasks/unscramble/anagrams2.yaml

@@ -16,3 +16,5 @@ metric_list:
     higher_is_better: true
     ignore_case: false
     ignore_punctuation: false
+metadata:
+  - version: 1.0

lm_eval/tasks/unscramble/cycle_letters.yaml

@@ -16,3 +16,5 @@ metric_list:
     higher_is_better: true
     ignore_case: false
     ignore_punctuation: false
+metadata:
+  - version: 1.0

lm_eval/tasks/unscramble/random_insertion.yaml

@@ -16,3 +16,5 @@ metric_list:
     higher_is_better: true
     ignore_case: false
     ignore_punctuation: false
+metadata:
+  - version: 1.0

lm_eval/tasks/unscramble/reversed_words.yaml

@@ -16,3 +16,5 @@ metric_list:
     higher_is_better: true
     ignore_case: false
     ignore_punctuation: false
+metadata:
+  - version: 1.0

lm_eval/tasks/webqs/webqs.yaml

@@ -16,3 +16,5 @@ metric_list:
   - metric: exact_match
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 1.0

lm_eval/tasks/wikitext/wikitext.yaml

@@ -14,3 +14,5 @@ metric_list:
   - metric: word_perplexity
   - metric: byte_perplexity
   - metric: bits_per_byte
+metadata:
+  - version: 2.0

lm_eval/tasks/winogrande/default.yaml

@@ -13,3 +13,5 @@ metric_list:
   - metric: acc
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 1.0

lm_eval/tasks/wmt2016/ro_en-t5_prompt.yaml

@@ -15,3 +15,5 @@ metric_list:
   - metric: !function metrics.bleu
     aggregation: !function metrics.agg_bleu
     higher_is_better: true
+metadata:
+  - version: 0.0

lm_eval/tasks/wsc273/default.yaml

@@ -13,3 +13,5 @@ metric_list:
   - metric: acc
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 1.0

lm_eval/tasks/xcopa/default_et.yaml

@@ -10,3 +10,5 @@ doc_to_target: label
 doc_to_choice: !function utils.doc_to_choice
 metric_list:
   - metric: acc
+metadata:
+  - version: 1.0

lm_eval/tasks/xnli/xnli_common_yaml

@@ -15,3 +15,5 @@ metric_list:
   - metric: acc
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 1.0

lm_eval/tasks/xstorycloze/default_ar.yaml

@@ -14,3 +14,5 @@ metric_list:
   - metric: acc
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 1.0

lm_eval/tasks/xwinograd/xwinograd_common_yaml

@@ -16,3 +16,5 @@ metric_list:
   - metric: acc
     aggregation: mean
     higher_is_better: true
+metadata:
+  - version: 1.0

lm_eval/utils.py

@@ -339,31 +339,27 @@ def make_table(result_dict, column: str = "results"):
     elif column == "groups":
         column_name = "Groups"
 
-    md_writer = MarkdownTableWriter()
-    latex_writer = LatexTableWriter()
-    md_writer.headers = [
-        column_name,
-        "Version",
-        "Filter",
-        "Metric",
-        "Value",
-        "",
-        "Stderr",
-    ]
-    latex_writer.headers = [
+    all_headers = [
         column_name,
         "Version",
         "Filter",
+        "n-shot",
         "Metric",
         "Value",
         "",
         "Stderr",
     ]
 
+    md_writer = MarkdownTableWriter()
+    latex_writer = LatexTableWriter()
+    md_writer.headers = all_headers
+    latex_writer.headers = all_headers
+
     values = []
 
     for k, dic in result_dict[column].items():
         version = result_dict["versions"][k]
+        n = str(result_dict["n-shot"][k])
 
         if "alias" in dic:
             k = dic.pop("alias")
@@ -375,9 +371,9 @@ def make_table(result_dict, column: str = "results"):
 
             if m + "_stderr" + "," + f in dic:
                 se = dic[m + "_stderr" + "," + f]
-                values.append([k, version, f, m, "%.4f" % v, "±", "%.4f" % se])
+                values.append([k, version, f, n, m, "%.4f" % v, "±", "%.4f" % se])
             else:
-                values.append([k, version, f, m, "%.4f" % v, "", ""])
+                values.append([k, version, f, n, m, "%.4f" % v, "", ""])
             k = ""
             version = ""
     md_writer.value_matrix = values