rename

lm_eval/tasks/ruler/cwe.yaml

-include: niah_1.yaml
+include: niah_single_1.yaml
 task: ruler_cwe
 download_dataset: !function cwe_utils.get_cw_dataset
 generation_kwargs:

lm_eval/tasks/ruler/fwe.yaml

-include: niah_1.yaml
+include: niah_single_1.yaml
 task: ruler_fwe
 download_dataset: !function fwe_utils.fwe_download
 generation_kwargs:

lm_eval/tasks/ruler/niah_4.yaml → lm_eval/tasks/ruler/niah_multikey_1.yaml

-task: niah_4
-include: niah_1.yaml
+task: niah_multikey_1
+include: niah_single_1.yaml
 download_dataset: !function utils.niah_multikey_1

lm_eval/tasks/ruler/niah_5.yaml → lm_eval/tasks/ruler/niah_multikey_2.yaml

-task: niah_5
-include: niah_1.yaml
+task: niah_multikey_2
+include: niah_single_1.yaml
 download_dataset: !function utils.niah_multikey_2

lm_eval/tasks/ruler/niah_6.yaml → lm_eval/tasks/ruler/niah_multikey_3.yaml

-task: niah_6
-include: niah_1.yaml
+task: niah_multikey_3
+include: niah_single_1.yaml
 download_dataset: !function utils.niah_multikey_3

lm_eval/tasks/ruler/niah_8.yaml → lm_eval/tasks/ruler/niah_multiquery.yaml

-task: niah_8
-include: niah_1.yaml
+task: niah_multiquery
+include: niah_single_1.yaml
 download_dataset: !function utils.niah_multiquery

lm_eval/tasks/ruler/niah_7.yaml → lm_eval/tasks/ruler/niah_multivalue.yaml

-task: niah_7
-include: niah_1.yaml
+task: niah_multivalue
+include: niah_single_1.yaml
 download_dataset: !function utils.niah_multivalue

lm_eval/tasks/ruler/niah_1.yaml → lm_eval/tasks/ruler/niah_single_1.yaml

 tag:
-  - ruler
-task: niah_1
+  - longcxt
+task: niah_single_1
 dataset_path: ""
 dataset_name: ""
 output_type: generate_until
@@ -10,7 +10,6 @@ doc_to_text: "{{input}}"
 doc_to_target: "{{outputs[0]}}"
 gen_prefix: "{{gen_prefix}}"
 process_results: !function utils.process_results
-
 metric_list:
   - metric: "4096"
     aggregation: !function utils.aggregate_metrics

lm_eval/tasks/ruler/niah_2.yaml → lm_eval/tasks/ruler/niah_single_2.yaml

-task: niah_2
-include: niah_1.yaml
+task: niah_single_2
+include: niah_single_1.yaml
 download_dataset: !function utils.niah_single_2

lm_eval/tasks/ruler/niah_3.yaml → lm_eval/tasks/ruler/niah_single_3.yaml

-task: niah_3
-include: niah_1.yaml
+task: niah_single_3
+include: niah_single_1.yaml
 download_dataset: !function utils.niah_single_3

lm_eval/tasks/ruler/qa_squad.yaml

-include: niah_1.yaml
+include: niah_single_1.yaml
 task: ruler_qa_squad
 download_dataset: !function qa_utils.get_squad
 test_split: test

lm_eval/tasks/ruler/qa_utils.py

@@ -15,7 +15,7 @@
 
 import itertools  # noqa: I001
 import random
-from functools import cache, partial
+from functools import cache
 
 import datasets
 import requests
@@ -237,5 +237,13 @@ def get_qa_dataset(ds, **kwargs):
     }
 
 
-get_squad = partial(get_qa_dataset, "squad")
-get_hotpotqa = partial(get_qa_dataset, "hotpotqa")
+def get_squad(**kwargs):
+    return get_qa_dataset("squad", **kwargs)
+
+
+def get_hotpotqa(**kwargs):
+    return get_qa_dataset("hotpotqa", **kwargs)
+
+
+# get_squad = lambda **kwargs: partial(get_qa_dataset, "squad")(**kwargs)
+# get_hotpotqa = lambda **kwargs: partial(get_qa_dataset, "hotpotqa")(**kwargs)

lm_eval/tasks/ruler/ruler.yaml

+group: ruler
+task:
+  - niah_single_1
+  - niah_single_2
+  - niah_single_3
+  - niah_multikey_1
+  - niah_multikey_2
+  - niah_multikey_3
+  - niah_multiquery
+  - niah_multivalue
+  - ruler_vt
+  - ruler_cwe
+  - ruler_fwe
+  - ruler_qa_squad
+  - ruler_qa_hotpot
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: False
+metadata:
+  version: 1

lm_eval/tasks/ruler/vt.yaml

-include: niah_1.yaml
+include: niah_single_1.yaml
 task: ruler_vt
 download_dataset: !function vt_utils.get_vt_dataset
 generation_kwargs: