update

lm_eval/tasks/scrolls/narrativeqa/default.yaml

@@ -3,13 +3,14 @@ task: scrolls_narrativeqa
 dataset_name: narrative_qa
 output_type: greedy_until
 process_docs: !function ../preprocessors.process_docs_prepended_question
-doc_to_text: "{{text}}\n\nHypothesis: {{question}}\nConclusion:"
-doc_to_target: "{{outputs[0]}}"
-doc_to_choice: ["Not mentioned", "Entailment", "Contradiction"]
+doc_to_text: "{{text}}\n\nQuestion: {{question}}\nAnswer:"
+doc_to_target: "{{outputs| join(', ')}}"
 should_decontaminate: true
 doc_to_decontamination_query: input
 metric_list:
   - metric: f1
+    aggregation: !function ../metrics.agg_f1
+    higher_is_better: true
 generation_kwargs:
   until:
     - "\n"

lm_eval/tasks/scrolls/qasper/default.yaml

-group: scrolls
-task: scrolls_qasper
-dataset_path: tau/scrolls
-dataset_name: qasper
-output_type: multiple_choice
-training_split: train
-validation_split: validation
-process_docs: !function ../preprocessors.process_docs_prepended_question
-doc_to_text: "{{text}}\n\nHypothesis: {{question}}\nConclusion:"
-doc_to_target: "{{outputs[0]}}"
-doc_to_choice: ["yes", "no"]
-should_decontaminate: true
-doc_to_decontamination_query: input
-metric_list:
-  - metric: f1

lm_eval/tasks/scrolls/quality/utils.py

 import re
 from functools import partial
 
-from ..preprocessors import process_docs_prepended_question
+import sys
+sys.path.append('..')
+from preprocessors import process_docs_prepended_question
 
 def process_docs(dataset):
 

lm_eval/tasks/scrolls/scroll_multiplechoice_task_yaml

@@ -3,7 +3,7 @@ dataset_path: tau/scrolls
 output_type: multiple_choice
 training_split: train
 validation_split: validation
-process_docs: !function ../preprocessors.process_docs
+process_docs: !function preprocessors.process_docs
 should_decontaminate: true
 doc_to_decontamination_query: input
 metric_list:

lm_eval/tasks/scrolls/scroll_summary_task_yaml

@@ -3,13 +3,21 @@ dataset_path: tau/scrolls
 output_type: greedy_until
 training_split: train
 validation_split: validation
-process_docs: !function ../preprocessors.process_docs
+process_docs: !function preprocessors.process_docs
 doc_to_text: "{{input}}\n\nQuestion: What is a summary of the preceding text?\nAnswer:"
-doc_to_target: "{{outputs.join(", ")}}"
+doc_to_target: "{{outputs|join(', ')}}"
 should_decontaminate: true
 doc_to_decontamination_query: input
 metric_list:
-  - metric: f1
+  - metric: !function metrics.rouge1
+    aggregation: mean
+    higher_is_better: true
+  - metric: !function metrics.rouge2
+    aggregation: mean
+    higher_is_better: true
+  - metric: !function metrics.rougeL
+    aggregation: mean
+    higher_is_better: true
 generation_kwargs:
   until:
     - "\n"

lm_eval/tasks/scrolls/summscreenfd/default.yaml

 include: ../scroll_summary_task_yaml
-task: scrolls_govreport
+task: scrolls_summscreenfd
 dataset_path: tau/scrolls
 dataset_name: summ_screen_fd

lm_eval/utils.py

@@ -432,8 +432,7 @@ def load_yaml_config(yaml_path):
                 # If not found, assume the included yaml
                 # is in the same dir as the original yaml
                 if not os.path.isfile(path):
-                    path = os.path.join(yaml_dir, path)
-
+                    path = os.path.normpath(os.path.join(yaml_dir, path))
                 try:
                     included_yaml_config = load_yaml_config(path)
                     final_yaml_config.update(included_yaml_config)