Merge branch 'big-refactor' of https://github.com/EleutherAI/lm-evaluation-harness into add-readme

lm_eval/tasks/xnli/xnli_th.yaml

+# Generated by utils.py
+dataset_name: th
+doc_to_choice: '{{[premise+", ถูกต้อง? ใช่, "+hypothesis,premise+", ถูกต้อง? ดังนั้น,
+  "+hypothesis,premise+", ถูกต้อง? ไม่, "+hypothesis]}}'
+doc_to_text: ''
+include: xnli_common_yaml
+task: xnli_th

lm_eval/tasks/xnli/xnli_tr.yaml

+# Generated by utils.py
+dataset_name: tr
+doc_to_choice: '{{[premise+", doğru? Evet, "+hypothesis,premise+", doğru? Böylece,
+  "+hypothesis,premise+", doğru? Hayır, "+hypothesis]}}'
+doc_to_text: ''
+include: xnli_common_yaml
+task: xnli_tr

lm_eval/tasks/xnli/xnli_ur.yaml

+# Generated by utils.py
+dataset_name: ur
+doc_to_choice: '{{[premise+", صحیح? جی ہاں, "+hypothesis,premise+", صحیح? اس لئے,
+  "+hypothesis,premise+", صحیح? نہیں, "+hypothesis]}}'
+doc_to_text: ''
+include: xnli_common_yaml
+task: xnli_ur

lm_eval/tasks/xnli/xnli_vi.yaml

+# Generated by utils.py
+dataset_name: vi
+doc_to_choice: '{{[premise+", đúng? Vâng, "+hypothesis,premise+", đúng? Vì vậy, "+hypothesis,premise+",
+  đúng? Không, "+hypothesis]}}'
+doc_to_text: ''
+include: xnli_common_yaml
+task: xnli_vi

lm_eval/tasks/xnli/xnli_zh.yaml

+# Generated by utils.py
+dataset_name: zh
+doc_to_choice: '{{[premise+", 正确? 是的, "+hypothesis,premise+", 正确? 所以, "+hypothesis,premise+",
+  正确? 不是的, "+hypothesis]}}'
+doc_to_text: ''
+include: xnli_common_yaml
+task: xnli_zh

lm_eval/tasks/xstorycloze/default_ar.yaml

+group: xstorycloze
+task: xstorycloze_ar
+dataset_path: juletxara/xstory_cloze
+dataset_name: ar
+output_type: multiple_choice
+training_split: train
+validation_split: eval
+doc_to_text: "{{[input_sentence_1, input_sentence_2, input_sentence_3, input_sentence_4]|join(' ')}}"
+doc_to_target: "{{answer_right_ending-1}}"
+doc_to_choice: "{{[sentence_quiz1, sentence_quiz2]}}"
+should_decontaminate: true
+doc_to_decontamination_query: "{{[input_sentence_1, input_sentence_2, input_sentence_3, input_sentence_4]|join(' ')}}"
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true

lm_eval/tasks/xstorycloze/default_en.yaml

+include: default_ar.yaml
+task: xstorycloze_en
+dataset_name: en

lm_eval/tasks/xstorycloze/default_es.yaml

+include: default_ar.yaml
+task: xstorycloze_es
+dataset_name: es

lm_eval/tasks/xstorycloze/default_eu.yaml

+include: default_ar.yaml
+task: xstorycloze_eu
+dataset_name: eu

lm_eval/tasks/xstorycloze/default_hi.yaml

+include: default_ar.yaml
+task: xstorycloze_hi
+dataset_name: hi

lm_eval/tasks/xstorycloze/default_id.yaml

+include: default_ar.yaml
+task: xstorycloze_id
+dataset_name: id

lm_eval/tasks/xstorycloze/default_my.yaml

+include: default_ar.yaml
+task: xstorycloze_my
+dataset_name: my

lm_eval/tasks/xstorycloze/default_ru.yaml

+include: default_ar.yaml
+task: xstorycloze_ru
+dataset_name: ru

lm_eval/tasks/xstorycloze/default_sw.yaml

+include: default_ar.yaml
+task: xstorycloze_sw
+dataset_name: sw

lm_eval/tasks/xstorycloze/default_te.yaml

+include: default_ar.yaml
+task: xstorycloze_te
+dataset_name: te

lm_eval/tasks/xstorycloze/default_zh.yaml

+include: default_ar.yaml
+task: xstorycloze_zh
+dataset_name: zh

setup.py

 import setuptools
+import itertools
 
 with open("README.md", "r", encoding="utf-8") as fh:
     long_description = fh.read()
 
+
+extras_require = {
+    "dev": ["black", "flake8", "pre-commit", "pytest", "pytest-cov"],
+    "linting": [
+        "flake8",
+        "pylint",
+        "mypy",
+        "pre-commit",
+    ],
+    "testing": ["pytest", "pytest-cov", "pytest-xdist"],
+    "multilingual": ["nagisa>=0.2.7", "jieba>=0.42.1"],
+    "sentencepiece": ["sentencepiece>=0.1.98", "protobuf>=4.22.1"],
+    "promptsource": [
+        "promptsource @ git+https://github.com/bigscience-workshop/promptsource.git#egg=promptsource"
+    ],
+    "gptq": ["auto-gptq[triton] @ git+https://github.com/PanQiWei/AutoGPTQ"],
+    "anthropic": ["anthropic"],
+    "openai": ["openai", "tiktoken"],
+}
+extras_require["all"] = list(itertools.chain.from_iterable(extras_require.values()))
+
+
 setuptools.setup(
     name="lm_eval",
     version="1.0.0",
@@ -50,22 +73,5 @@ setuptools.setup(
         "transformers>=4.1",
         "zstandard",
     ],
-    extras_require={
-        "dev": ["black", "flake8", "pre-commit", "pytest", "pytest-cov"],
-        "linting": [
-            "flake8",
-            "pylint",
-            "mypy",
-            "pre-commit",
-        ],
-        "testing": ["pytest", "pytest-cov", "pytest-xdist"],
-        "multilingual": ["nagisa>=0.2.7", "jieba>=0.42.1"],
-        "sentencepiece": ["sentencepiece>=0.1.98", "protobuf>=4.22.1"],
-        "promptsource": [
-            "promptsource @ git+https://github.com/bigscience-workshop/promptsource.git#egg=promptsource"
-        ],
-        "gptq": ["auto-gptq[triton] @ git+https://github.com/PanQiWei/AutoGPTQ"],
-        "anthropic": ["anthropic"],
-        "openai": ["openai", "tiktoken"],
-    },
+    extras_require=extras_require,
 )