Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evaluation-harness into toxicity

results/xglm/xglm-7.5B/xglm-7.5B_mathematical_reasoning_few_shot_5-shot.json

-{
-  "results": {
-    "math_num_theory": {
-      "acc": 0.0,
-      "acc_stderr": 0.0
-    },
-    "gsm8k": {
-      "acc": 0.002274450341167551,
-      "acc_stderr": 0.0013121578148674316
-    },
-    "math_geometry": {
-      "acc": 0.0,
-      "acc_stderr": 0.0
-    },
-    "drop": {
-      "em": 0.05421560402684564,
-      "em_stderr": 0.002318984649948223,
-      "f1": 0.08962458053691245,
-      "f1_stderr": 0.0026401926224488034
-    },
-    "math_prealgebra": {
-      "acc": 0.0,
-      "acc_stderr": 0.0
-    },
-    "math_counting_and_prob": {
-      "acc": 0.0,
-      "acc_stderr": 0.0
-    },
-    "math_precalc": {
-      "acc": 0.0,
-      "acc_stderr": 0.0
-    },
-    "math_intermediate_algebra": {
-      "acc": 0.0,
-      "acc_stderr": 0.0
-    },
-    "math_algebra": {
-      "acc": 0.0,
-      "acc_stderr": 0.0
-    },
-    "mathqa": {
-      "acc": 0.23986599664991626,
-      "acc_stderr": 0.007816818250028128,
-      "acc_norm": 0.23517587939698492,
-      "acc_norm_stderr": 0.0077638612776946255
-    }
-  },
-  "versions": {
-    "math_num_theory": 1,
-    "gsm8k": 0,
-    "math_geometry": 1,
-    "drop": 1,
-    "math_prealgebra": 1,
-    "math_counting_and_prob": 1,
-    "math_precalc": 1,
-    "math_intermediate_algebra": 1,
-    "math_algebra": 1,
-    "mathqa": 0
-  },
-  "config": {
-    "model": "hf-causal-experimental",
-    "model_args": "pretrained=facebook/xglm-7.5B,use_accelerate=True",
-    "num_fewshot": 5,
-    "batch_size": "auto",
-    "device": "cuda:0",
-    "no_cache": true,
-    "limit": null,
-    "bootstrap_iters": 100000,
-    "description_dict": {}
-  }
-}

results/xglm/xglm-7.5B/xglm-7.5B_pawsx_0-shot.json

-{
-  "results": {
-    "pawsx_en": {
-      "acc": 0.5885,
-      "acc_stderr": 0.011006563824537298
-    },
-    "pawsx_es": {
-      "acc": 0.528,
-      "acc_stderr": 0.011165587094621537
-    },
-    "pawsx_fr": {
-      "acc": 0.518,
-      "acc_stderr": 0.011175886999478619
-    },
-    "pawsx_zh": {
-      "acc": 0.513,
-      "acc_stderr": 0.01117935548207038
-    },
-    "pawsx_ja": {
-      "acc": 0.52,
-      "acc_stderr": 0.011174185930778312
-    },
-    "pawsx_de": {
-      "acc": 0.559,
-      "acc_stderr": 0.011105006104468736
-    },
-    "pawsx_ko": {
-      "acc": 0.4595,
-      "acc_stderr": 0.011146389370464362
-    }
-  },
-  "versions": {
-    "pawsx_en": 0,
-    "pawsx_es": 0,
-    "pawsx_fr": 0,
-    "pawsx_zh": 0,
-    "pawsx_ja": 0,
-    "pawsx_de": 0,
-    "pawsx_ko": 0
-  },
-  "config": {
-    "model": "hf-causal-experimental",
-    "model_args": "pretrained=facebook/xglm-7.5B,use_accelerate=True",
-    "num_fewshot": 0,
-    "batch_size": "auto",
-    "device": "cuda",
-    "no_cache": true,
-    "limit": null,
-    "bootstrap_iters": 100000,
-    "description_dict": {}
-  }
-}

results/xglm/xglm-7.5B/xglm-7.5B_xcopa_0-shot.json

-{
-  "results": {
-    "xcopa_et": {
-      "acc": 0.612,
-      "acc_stderr": 0.021814300984787635
-    },
-    "xcopa_th": {
-      "acc": 0.594,
-      "acc_stderr": 0.02198396209008634
-    },
-    "xcopa_qu": {
-      "acc": 0.488,
-      "acc_stderr": 0.02237662679792717
-    },
-    "xcopa_ta": {
-      "acc": 0.544,
-      "acc_stderr": 0.02229623834840705
-    },
-    "xcopa_zh": {
-      "acc": 0.638,
-      "acc_stderr": 0.0215136625275824
-    },
-    "xcopa_vi": {
-      "acc": 0.702,
-      "acc_stderr": 0.02047511809298897
-    },
-    "xcopa_sw": {
-      "acc": 0.6,
-      "acc_stderr": 0.021930844120728505
-    },
-    "xcopa_it": {
-      "acc": 0.636,
-      "acc_stderr": 0.021539170637317685
-    },
-    "xcopa_tr": {
-      "acc": 0.584,
-      "acc_stderr": 0.022064943313928848
-    },
-    "xcopa_id": {
-      "acc": 0.694,
-      "acc_stderr": 0.0206295699983454
-    },
-    "xcopa_ht": {
-      "acc": 0.574,
-      "acc_stderr": 0.022136577335085637
-    }
-  },
-  "versions": {
-    "xcopa_et": 0,
-    "xcopa_th": 0,
-    "xcopa_qu": 0,
-    "xcopa_ta": 0,
-    "xcopa_zh": 0,
-    "xcopa_vi": 0,
-    "xcopa_sw": 0,
-    "xcopa_it": 0,
-    "xcopa_tr": 0,
-    "xcopa_id": 0,
-    "xcopa_ht": 0
-  },
-  "config": {
-    "model": "hf-causal-experimental",
-    "model_args": "pretrained=facebook/xglm-7.5B,use_accelerate=True",
-    "num_fewshot": 0,
-    "batch_size": "auto",
-    "device": "cuda",
-    "no_cache": true,
-    "limit": null,
-    "bootstrap_iters": 100000,
-    "description_dict": {}
-  }
-}

results/xglm/xglm-7.5B/xglm-7.5B_xnli_0-shot.json

-{
-  "results": {
-    "xnli_ar": {
-      "acc": 0.3337325349301397,
-      "acc_stderr": 0.00666266628252267
-    },
-    "xnli_bg": {
-      "acc": 0.4489021956087824,
-      "acc_stderr": 0.007027723874210379
-    },
-    "xnli_de": {
-      "acc": 0.48982035928143713,
-      "acc_stderr": 0.0070632481147059134
-    },
-    "xnli_el": {
-      "acc": 0.40658682634730536,
-      "acc_stderr": 0.006940323712177368
-    },
-    "xnli_en": {
-      "acc": 0.5385229540918164,
-      "acc_stderr": 0.0070437128985425335
-    },
-    "xnli_es": {
-      "acc": 0.47704590818363274,
-      "acc_stderr": 0.007057263845316342
-    },
-    "xnli_fr": {
-      "acc": 0.4694610778443114,
-      "acc_stderr": 0.007051522651006734
-    },
-    "xnli_hi": {
-      "acc": 0.4720558882235529,
-      "acc_stderr": 0.007053670508441103
-    },
-    "xnli_ru": {
-      "acc": 0.46327345309381235,
-      "acc_stderr": 0.007045628330322907
-    },
-    "xnli_sw": {
-      "acc": 0.45828343313373254,
-      "acc_stderr": 0.007040080446339805
-    },
-    "xnli_th": {
-      "acc": 0.437125748502994,
-      "acc_stderr": 0.007008633817895695
-    },
-    "xnli_tr": {
-      "acc": 0.4626746506986028,
-      "acc_stderr": 0.007045000071900887
-    },
-    "xnli_ur": {
-      "acc": 0.42095808383233535,
-      "acc_stderr": 0.006975878576227385
-    },
-    "xnli_vi": {
-      "acc": 0.46327345309381235,
-      "acc_stderr": 0.007045628330322896
-    },
-    "xnli_zh": {
-      "acc": 0.3536926147704591,
-      "acc_stderr": 0.006755492859492898
-    }
-  },
-  "versions": {
-    "xnli_ar": 0,
-    "xnli_bg": 0,
-    "xnli_de": 0,
-    "xnli_el": 0,
-    "xnli_en": 0,
-    "xnli_es": 0,
-    "xnli_fr": 0,
-    "xnli_hi": 0,
-    "xnli_ru": 0,
-    "xnli_sw": 0,
-    "xnli_th": 0,
-    "xnli_tr": 0,
-    "xnli_ur": 0,
-    "xnli_vi": 0,
-    "xnli_zh": 0
-  },
-  "config": {
-    "model": "hf-causal-experimental",
-    "model_args": "pretrained=facebook/xglm-7.5B,use_accelerate=True",
-    "num_fewshot": 0,
-    "batch_size": "auto",
-    "device": "cuda",
-    "no_cache": true,
-    "limit": null,
-    "bootstrap_iters": 100000,
-    "description_dict": {}
-  }
-}

results/xglm/xglm-7.5B/xglm-7.5B_xstory_cloze_0-shot.json

-{
-  "results": {
-    "xstory_cloze_es": {
-      "acc": 0.6406353408338848,
-      "acc_stderr": 0.012347659802101675
-    },
-    "xstory_cloze_zh": {
-      "acc": 0.5890138980807412,
-      "acc_stderr": 0.012661578894368948
-    },
-    "xstory_cloze_sw": {
-      "acc": 0.5929847782925215,
-      "acc_stderr": 0.012642664836816926
-    },
-    "xstory_cloze_en": {
-      "acc": 0.6982131039046989,
-      "acc_stderr": 0.011812877848905303
-    },
-    "xstory_cloze_hi": {
-      "acc": 0.5876902713434812,
-      "acc_stderr": 0.012667694122397068
-    },
-    "xstory_cloze_ar": {
-      "acc": 0.5618795499669094,
-      "acc_stderr": 0.012768206616277757
-    },
-    "xstory_cloze_eu": {
-      "acc": 0.5771012574454004,
-      "acc_stderr": 0.0127132250091262
-    },
-    "xstory_cloze_id": {
-      "acc": 0.6293845135671741,
-      "acc_stderr": 0.012428861084065903
-    },
-    "xstory_cloze_ru": {
-      "acc": 0.6353408338848445,
-      "acc_stderr": 0.012386781532906161
-    },
-    "xstory_cloze_te": {
-      "acc": 0.6022501654533422,
-      "acc_stderr": 0.012595197856703525
-    },
-    "xstory_cloze_my": {
-      "acc": 0.57114493712773,
-      "acc_stderr": 0.01273620271314778
-    }
-  },
-  "versions": {
-    "xstory_cloze_es": 0,
-    "xstory_cloze_zh": 0,
-    "xstory_cloze_sw": 0,
-    "xstory_cloze_en": 0,
-    "xstory_cloze_hi": 0,
-    "xstory_cloze_ar": 0,
-    "xstory_cloze_eu": 0,
-    "xstory_cloze_id": 0,
-    "xstory_cloze_ru": 0,
-    "xstory_cloze_te": 0,
-    "xstory_cloze_my": 0
-  },
-  "config": {
-    "model": "hf-causal-experimental",
-    "model_args": "pretrained=facebook/xglm-7.5B,use_accelerate=True",
-    "num_fewshot": 0,
-    "batch_size": "auto",
-    "device": "cuda",
-    "no_cache": true,
-    "limit": null,
-    "bootstrap_iters": 100000,
-    "description_dict": {}
-  }
-}

results/xglm/xglm-7.5B/xglm-7.5B_xwinograd_0-shot.json

-{
-  "results": {
-    "xwinograd_zh": {
-      "acc": 0.7281746031746031,
-      "acc_stderr": 0.01983712759311063
-    },
-    "xwinograd_ru": {
-      "acc": 0.6317460317460317,
-      "acc_stderr": 0.027219500732466696
-    },
-    "xwinograd_pt": {
-      "acc": 0.6730038022813688,
-      "acc_stderr": 0.028982074243683254
-    },
-    "xwinograd_en": {
-      "acc": 0.7948387096774193,
-      "acc_stderr": 0.008376626547826555
-    },
-    "xwinograd_jp": {
-      "acc": 0.6496350364963503,
-      "acc_stderr": 0.01541389159576608
-    },
-    "xwinograd_fr": {
-      "acc": 0.6506024096385542,
-      "acc_stderr": 0.05265151356440471
-    }
-  },
-  "versions": {
-    "xwinograd_zh": 0,
-    "xwinograd_ru": 0,
-    "xwinograd_pt": 0,
-    "xwinograd_en": 0,
-    "xwinograd_jp": 0,
-    "xwinograd_fr": 0
-  },
-  "config": {
-    "model": "hf-causal-experimental",
-    "model_args": "pretrained=facebook/xglm-7.5B,use_accelerate=True",
-    "num_fewshot": 0,
-    "batch_size": "auto",
-    "device": "cuda",
-    "no_cache": true,
-    "limit": null,
-    "bootstrap_iters": 100000,
-    "description_dict": {}
-  }
-}

scripts/build_benchmark.py

+import os
+import yaml
+import argparse
+
+from tqdm import tqdm
+from promptsource.templates import DatasetTemplates
+
+from lm_eval import utils
+
+# from lm_eval.api.registry import ALL_TASKS
+from lm_eval.logger import eval_logger
+
+# from lm_eval.tasks import include_task_folder
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--benchmark_name", required=True)
+    parser.add_argument("--benchmark_path", required=True)
+    parser.add_argument("--task_save_path", default="lm_eval/tasks/")
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+
+    args = parse_args()
+
+    with open(args.benchmark_path) as file:
+        TASK_LIST = yaml.full_load(file)
+        for task in tqdm(TASK_LIST):
+            eval_logger.info(f"Processing {task}")
+
+            dataset_name = task["dataset_path"]
+            if "dataset_name" in task:
+                subset_name = task["dataset_name"]
+                file_subdir = f"{dataset_name}/{subset_name}"
+            else:
+                subset_name = None
+                file_subdir = f"{dataset_name}"
+
+            file_path = os.path.join(args.task_save_path, file_subdir, "promptsource/")
+
+            os.makedirs(file_path, exist_ok=True)
+
+            if subset_name is None:
+                prompts = DatasetTemplates(dataset_name=dataset_name)
+            else:
+                prompts = DatasetTemplates(
+                    dataset_name=dataset_name, subset_name=subset_name
+                )
+
+            for idx, prompt_name in enumerate(prompts.all_template_names):
+                full_file_name = f"promptsource_{idx}.yaml"
+                config_dict = {
+                    "group": args.benchmark_name,
+                    "include": "promptsource_template.yaml",
+                    "use_prompts": f"promptsource:{prompt_name}",
+                }
+
+                file_save_path = os.path.join(file_path, full_file_name)
+                eval_logger.info(f"Save to {file_save_path}")
+                with open(file_save_path, "w") as yaml_file:
+                    yaml.dump(config_dict, yaml_file)

tests/extra/test_utils.py

+import json
+from typing import List
+from lm_eval.utils import load_yaml_config
+from pathlib import Path
+
+
+FILE_PATH = file_path = ".github/outputs/tasks_all_changed_and_modified_files.txt"
+
+
+def load_changed_files(file_path: str = FILE_PATH) -> List[str]:
+    with open(file_path, "r") as f:
+        return [l for line in f.readlines() for l in line.strip().split(" ")]
+
+
+def parser(full_path: List[str]) -> List[str]:
+    _output = set()
+    for x in full_path:
+        if x.endswith(".yaml"):
+            _output.add(load_yaml_config(x)["task"])
+        elif x.endswith(".py"):
+            path = [str(x) for x in (list(Path(x).parent.glob("*.yaml")))]
+            _output |= {load_yaml_config(x)["task"] for x in path}
+    return list(_output)