fix alllll the merge conflicts

lm_eval/tasks/xcopa/_xcopa.yaml

+group: xcopa
+task:
+  - xcopa_et
+  - xcopa_ht
+  - xcopa_id
+  - xcopa_it
+  - xcopa_qu
+  - xcopa_sw
+  - xcopa_ta
+  - xcopa_th
+  - xcopa_tr
+  - xcopa_vi
+  - xcopa_zh
+aggregate_metric_list:
+  - metric: acc
+    aggregation: mean
+    weight_by_size: True
+metadata:
+  version: 1.0

lm_eval/tasks/xcopa/default_et.yaml

-group: xcopa
 task: xcopa_et
 dataset_path: xcopa
 dataset_name: et

lm_eval/tasks/xnli/_xnli.yaml

+group: xnli
+task:
+  - xnli_ar
+  - xnli_bg
+  - xnli_de
+  - xnli_el
+  - xnli_en
+  - xnli_es
+  - xnli_fr
+  - xnli_hi
+  - xnli_ru
+  - xnli_sw
+  - xnli_th
+  - xnli_tr
+  - xnli_ur
+  - xnli_vi
+  - xnli_zh
+aggregate_metric_list:
+  - metric: acc
+    aggregation: mean
+    weight_by_size: true
+metadata:
+  version: 1.0

lm_eval/tasks/xnli/xnli_common_yaml

 # This file will be included in the generated language-specific task configs.
 # It doesn't have a yaml file extension as it is not meant to be imported directly
 # by the harness.
-group: xnli
 task: null
 dataset_path: xnli
 dataset_name: null

lm_eval/tasks/xnli_eu/README.md

@@ -24,9 +24,9 @@ Homepage: https://github.com/hitz-zentroa/xnli-eu
 }
 ```
 
-### Groups and Tasks
+### Groups, Tags, and Tasks
 
-#### Groups
+#### Tags
 
 * `xnli_eu_mt_native`: Includes MT and Native variants of the XNLIeu dataset.
 

lm_eval/tasks/xnli_eu/xnli_common_yaml

-group: xnli
 task: null
 dataset_path: xnli
 dataset_name: null

lm_eval/tasks/xnli_eu/xnli_eu_mt.yaml

 include: xnli_eu.yaml
-group: xnli_eu_mt_native
+tag: xnli_eu_mt_native
 task: xnli_eu_mt
 dataset_name: eu_mt

lm_eval/tasks/xnli_eu/xnli_eu_native.yaml

 include: xnli_eu.yaml
-group: xnli_eu_mt_native
+tag: xnli_eu_mt_native
 task: xnli_eu_native
 training_split: null
 validation_split: null

lm_eval/tasks/xstorycloze/_xstorycloze.yaml

+group: xstorycloze
+task:
+  - xstorycloze_ar
+  - xstorycloze_en
+  - xstorycloze_es
+  - xstorycloze_eu
+  - xstorycloze_hi
+  - xstorycloze_id
+  - xstorycloze_my
+  - xstorycloze_ru
+  - xstorycloze_sw
+  - xstorycloze_te
+  - xstorycloze_zh
+aggregate_metric_list:
+  - metric: acc
+    aggregation: mean
+    weight_by_size: true
+metadata:
+  version: 1.0

lm_eval/tasks/xstorycloze/default_ar.yaml

-group: xstorycloze
 task: xstorycloze_ar
 dataset_path: juletxara/xstory_cloze
 dataset_name: ar

lm_eval/tasks/xwinograd/_xwinograd.yaml

+group: xwinograd
+task:
+  - xwinograd_en
+  - xwinograd_fr
+  - xwinograd_jp
+  - xwinograd_pt
+  - xwinograd_ru
+  - xwinograd_zh
+aggregate_metric_list:
+  - metric: acc
+    aggregation: mean
+    weight_by_size: true
+metadata:
+  version: 1.0

lm_eval/tasks/xwinograd/xwinograd_common_yaml

 # This file will be included in the generated language-specific task configs.
 # It doesn't have a yaml file extension as it is not meant to be imported directly
 # by the harness.
-group:
-  - xwinograd
 dataset_path: Muennighoff/xwinograd
 dataset_name: null  # Overridden by language-specific config.
 output_type: multiple_choice

lm_eval/utils.py

@@ -338,7 +338,9 @@ def make_table(result_dict, column: str = "results", sort_results: bool = False)
 
     keys = result_dict[column].keys()
     if sort_results:
-        # sort entries alphabetically
+        # sort entries alphabetically by task or group name.
+        # NOTE: we default here to false, because order matters for multi-level table printing a la mmlu.
+        # sorting here would mess that up
         keys = sorted(keys)
     for k in keys:
         dic = result_dict[column][k]

pyproject.toml

@@ -39,7 +39,6 @@ dependencies = [
     "dill",
     "word2number",
     "more_itertools",
-    "shortuuid",
 ]
 
 [tool.setuptools.packages.find]

templates/new_yaml_task/README.md

@@ -17,12 +17,16 @@ Homepage: `homepage to the benchmark's website goes here, if applicable`
 BibTeX-formatted citation goes here
 ```
 
-### Groups and Tasks
+### Groups, Tags, and Tasks
 
 #### Groups
 
 * `group_name`: `Short description`
 
+#### Tags
+
+* `tag_name`: `Short description`
+
 #### Tasks
 
 * `task_name`: `1-sentence description of what this particular task does`