removed unused files for now

lm_eval/tasks/truthfulqa/alternative_worlds/prompt_variation/style_03/truthfulqa_mc1.yaml

-task: truthfulqa_mc1_alt_pv_03
-dataset_path: truthful_qa
-dataset_name: multiple_choice
-output_type: multiple_choice
-training_split: null
-validation_split: validation
-test_split: null
-num_fewshot: 0
-doc_to_choice: "Question: {{mc2_targets.choices}}\nAnswer:"
-doc_to_target: 0
-doc_to_choice: "{{mc1_targets.choices}}"
-should_decontaminate: True
-doc_to_decontamination_query: question
-metric_list:
-  - metric: acc
-    aggregation: mean
-    higher_is_better: true

lm_eval/tasks/truthfulqa/alternative_worlds/prompt_variation/style_03/truthfulqa_mc2.yaml

-include: truthfulqa_mc1.yaml
-task: truthfulqa_mc2_alt_pv_03
-doc_to_target: 0
-doc_to_choice: "{{mc2_targets.choices}}"
-process_results: !function ../../../utils.process_results_mc2
-should_decontaminate: True
-doc_to_decontamination_query: question
-metric_list:
-  - metric: acc
-    aggregation: mean
-    higher_is_better: true

lm_eval/tasks/truthfulqa/alternative_worlds/truthfulqa_mc1.yaml

-group:
-  - truthfulqa
-task: truthfulqa_mc1
-dataset_path: truthful_qa
-dataset_name: multiple_choice
-output_type: multiple_choice
-training_split: null
-validation_split: validation
-test_split: null
-num_fewshot: 0
-doc_to_text: "{{question}}"
-doc_to_target: 0
-doc_to_choice: "{{mc1_targets.choices}}"
-should_decontaminate: True
-doc_to_decontamination_query: question
-metric_list:
-  - metric: acc
-    aggregation: mean
-    higher_is_better: true

lm_eval/tasks/truthfulqa/alternative_worlds/truthfulqa_mc2.yaml

-include: truthfulqa_mc1.yaml
-task: truthfulqa_mc2
-doc_to_target: 0
-doc_to_choice: "{{mc2_targets.choices}}"
-process_results: !function utils.process_results_mc2
-should_decontaminate: True
-doc_to_decontamination_query: question
-metric_list:
-  - metric: acc
-    aggregation: mean
-    higher_is_better: true

lm_eval/utils.py

@@ -497,11 +497,20 @@ def import_function(loader, node):
     return function
 
 
-# Add the import_function constructor to the YAML loader
-yaml.add_constructor("!function", import_function)
+def ignore_constructor(loader, node):
+    return node
+
+
+def simple_load_yaml_config(yaml_path=None, yaml_config=None, yaml_dir=None):
+    yaml.add_constructor("!function", ignore_constructor)
+    with open(yaml_path, "rb") as file:
+        yaml_config = yaml.full_load(file)
+    return yaml_config
 
 
 def load_yaml_config(yaml_path=None, yaml_config=None, yaml_dir=None):
+    # Add the import_function constructor to the YAML loader
+    yaml.add_constructor("!function", import_function)
     if yaml_config is None:
         with open(yaml_path, "rb") as file:
             yaml_config = yaml.full_load(file)

pyproject.toml

@@ -21,7 +21,7 @@ license = { "text" = "MIT" }
 dependencies = [
     "accelerate>=0.21.0",
     "evaluate",
-    "datasets>=2.0.0",
+    "datasets>=2.14.0",
     "evaluate>=0.4.0",
     "jsonlines",
     "numexpr",