From fc5019ead53c45119c522c62e8eea2daa837c56e Mon Sep 17 00:00:00 2001 From: Baber Abbasi <92168766+baberabb@users.noreply.github.com> Date: Tue, 29 Apr 2025 11:54:28 +0500 Subject: [PATCH 01/46] use np.NaN (#2937) --- lm_eval/tasks/bbq/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lm_eval/tasks/bbq/utils.py b/lm_eval/tasks/bbq/utils.py index 8ec6b01b..f00c7c6c 100644 --- a/lm_eval/tasks/bbq/utils.py +++ b/lm_eval/tasks/bbq/utils.py @@ -4,6 +4,9 @@ import datasets import numpy as np +if np.__version__ >= "2.0": + np.NaN = np.nan + # Possible unknown responses, copied from the HELM implementation UNKNOWN_RESPONSES = [ "Unknown", -- GitLab From ab618f01a1d770b1c0f48f1835db360dbf14b948 Mon Sep 17 00:00:00 2001 From: Alexandre Marques Date: Tue, 6 May 2025 10:22:12 -0400 Subject: [PATCH 02/46] Add support for enable_thinking argument in vllm model, set default to False (#2947) --- lm_eval/models/vllm_causallms.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lm_eval/models/vllm_causallms.py b/lm_eval/models/vllm_causallms.py index 866039fd..13a0b359 100644 --- a/lm_eval/models/vllm_causallms.py +++ b/lm_eval/models/vllm_causallms.py @@ -68,6 +68,7 @@ class VLLM(TemplateLM): device: str = "cuda", data_parallel_size: int = 1, lora_local_path: str = None, + enable_thinking: bool = False, **kwargs, ): super().__init__() @@ -129,6 +130,7 @@ class VLLM(TemplateLM): add_bos_token=add_bos_token, ) self.tokenizer = configure_pad_token(self.tokenizer, model_config=self._config) + self.enable_thinking = enable_thinking self.add_bos_token = add_bos_token if "gemma" in pretrained.lower(): self.add_bos_token = True @@ -209,6 +211,7 @@ class VLLM(TemplateLM): add_generation_prompt=add_generation_prompt, continue_final_message=not add_generation_prompt, chat_template=self.hf_chat_template, + enable_thinking=self.enable_thinking, ) return chat_templated -- GitLab From 71f2954bfcb3dbaf7b33ca67066dedea2de7aa17 Mon Sep 17 00:00:00 2001 From: Vladislav Mikhailov <43072268+vmkhlv@users.noreply.github.com> Date: Tue, 6 May 2025 17:00:20 +0200 Subject: [PATCH 03/46] Added NorEval, a novel Norwegian benchmark (#2919) * added noreval * added a checklist for noreval * run pre-commit * changed imports and added short noreval description * fixed norsumm path * refactored multi-folder tasks * refactored multi-folder tasks --- .gitignore | 1 + lm_eval/tasks/README.md | 315 +++++++++--------- lm_eval/tasks/noreval/README.md | 84 +++++ lm_eval/tasks/noreval/ask_gec/README.md | 28 ++ lm_eval/tasks/noreval/ask_gec/_ask_gec_yaml | 15 + lm_eval/tasks/noreval/ask_gec/ask_gec_p0.yaml | 3 + lm_eval/tasks/noreval/ask_gec/ask_gec_p1.yaml | 3 + lm_eval/tasks/noreval/ask_gec/ask_gec_p2.yaml | 3 + lm_eval/tasks/noreval/ask_gec/ask_gec_p3.yaml | 3 + lm_eval/tasks/noreval/ask_gec/ask_gec_p4.yaml | 3 + lm_eval/tasks/noreval/ask_gec/errant.py | 106 ++++++ lm_eval/tasks/noreval/ncb/ncb.yaml | 13 + .../noreval/norbelebele/_norbelebele_yaml | 18 + .../noreval/norbelebele/norbelebele_p0.yaml | 4 + .../noreval/norbelebele/norbelebele_p1.yaml | 4 + .../noreval/norbelebele/norbelebele_p2.yaml | 4 + .../noreval/norbelebele/norbelebele_p3.yaml | 5 + .../noreval/norbelebele/norbelebele_p4.yaml | 4 + .../norcommonsenseqa/_norcommonsenseqa_yaml | 15 + .../nno/norcommonsenseqa_nno_p0.yaml | 6 + .../nno/norcommonsenseqa_nno_p1.yaml | 6 + .../nno/norcommonsenseqa_nno_p2.yaml | 6 + .../nno/norcommonsenseqa_nno_p3.yaml | 6 + .../nno/norcommonsenseqa_nno_p4.yaml | 6 + .../nob/norcommonsenseqa_nob_p0.yaml | 6 + .../nob/norcommonsenseqa_nob_p1.yaml | 6 + .../nob/norcommonsenseqa_nob_p2.yaml | 6 + .../nob/norcommonsenseqa_nob_p3.yaml | 6 + .../nob/norcommonsenseqa_nob_p4.yaml | 6 + lm_eval/tasks/noreval/norec/_norec_yaml | 14 + .../norec_document/norec_document_p0.yaml | 6 + .../norec_document/norec_document_p1.yaml | 6 + .../norec_document/norec_document_p2.yaml | 6 + .../norec_document/norec_document_p3.yaml | 6 + .../norec_document/norec_document_p4.yaml | 6 + .../norec_sentence/norec_sentence_p0.yaml | 6 + .../norec_sentence/norec_sentence_p1.yaml | 6 + .../norec_sentence/norec_sentence_p2.yaml | 6 + .../norec_sentence/norec_sentence_p3.yaml | 6 + .../norec_sentence/norec_sentence_p4.yaml | 6 + lm_eval/tasks/noreval/norec/utils.py | 13 + lm_eval/tasks/noreval/noreval.jpg | Bin 0 -> 699330 bytes lm_eval/tasks/noreval/noridiom/_noridiom_yaml | 23 ++ .../noreval/noridiom/nno/noridiom_nno_p0.yaml | 5 + .../noreval/noridiom/nno/noridiom_nno_p1.yaml | 5 + .../noreval/noridiom/nno/noridiom_nno_p2.yaml | 5 + .../noreval/noridiom/nno/noridiom_nno_p3.yaml | 5 + .../noreval/noridiom/nno/noridiom_nno_p4.yaml | 5 + .../noreval/noridiom/nob/noridiom_nob_p0.yaml | 5 + .../noreval/noridiom/nob/noridiom_nob_p1.yaml | 5 + .../noreval/noridiom/nob/noridiom_nob_p2.yaml | 5 + .../noreval/noridiom/nob/noridiom_nob_p3.yaml | 5 + .../noreval/noridiom/nob/noridiom_nob_p4.yaml | 5 + lm_eval/tasks/noreval/noridiom/utils.py | 44 +++ .../noreval/noropenbookqa/_noropenbookqa_yaml | 16 + .../nno/noropenbookqa_nno_p0.yaml | 6 + .../nno/noropenbookqa_nno_p1.yaml | 6 + .../nno/noropenbookqa_nno_p2.yaml | 6 + .../nno/noropenbookqa_nno_p3.yaml | 6 + .../nno/noropenbookqa_nno_p4.yaml | 6 + .../nob/noropenbookqa_nob_p0.yaml | 6 + .../nob/noropenbookqa_nob_p1.yaml | 6 + .../nob/noropenbookqa_nob_p2.yaml | 6 + .../nob/noropenbookqa_nob_p3.yaml | 6 + .../nob/noropenbookqa_nob_p4.yaml | 6 + lm_eval/tasks/noreval/noropenbookqa/utils.py | 5 + lm_eval/tasks/noreval/norquad/_norquad_yaml | 25 ++ lm_eval/tasks/noreval/norquad/norquad_p0.yaml | 3 + lm_eval/tasks/noreval/norquad/norquad_p1.yaml | 3 + lm_eval/tasks/noreval/norquad/norquad_p2.yaml | 3 + lm_eval/tasks/noreval/norquad/norquad_p3.yaml | 3 + lm_eval/tasks/noreval/norquad/norquad_p4.yaml | 3 + lm_eval/tasks/noreval/norquad/utils.py | 62 ++++ .../norrewrite_instruct.yaml | 20 ++ lm_eval/tasks/noreval/norsumm/_norsumm_yaml | 35 ++ .../noreval/norsumm/nno/norsumm_nno_p0.yaml | 5 + .../noreval/norsumm/nno/norsumm_nno_p1.yaml | 5 + .../noreval/norsumm/nno/norsumm_nno_p2.yaml | 6 + .../noreval/norsumm/nno/norsumm_nno_p3.yaml | 6 + .../noreval/norsumm/nno/norsumm_nno_p4.yaml | 5 + .../noreval/norsumm/nno/norsumm_nno_p5.yaml | 5 + .../noreval/norsumm/nob/norsumm_nob_p0.yaml | 5 + .../noreval/norsumm/nob/norsumm_nob_p1.yaml | 5 + .../noreval/norsumm/nob/norsumm_nob_p2.yaml | 6 + .../noreval/norsumm/nob/norsumm_nob_p3.yaml | 6 + .../noreval/norsumm/nob/norsumm_nob_p4.yaml | 5 + .../noreval/norsumm/nob/norsumm_nob_p5.yaml | 5 + lm_eval/tasks/noreval/norsumm/utils.py | 126 +++++++ .../norsummarize_instruct.yaml | 20 ++ .../generation/_nortruthfulqa_gen_yaml | 54 +++ .../nno/nortruthfulqa_gen_nno_p0.yaml | 5 + .../nno/nortruthfulqa_gen_nno_p1.yaml | 5 + .../nno/nortruthfulqa_gen_nno_p2.yaml | 5 + .../nno/nortruthfulqa_gen_nno_p3.yaml | 5 + .../nno/nortruthfulqa_gen_nno_p4.yaml | 5 + .../nob/nortruthfulqa_gen_nob_p0.yaml | 5 + .../nob/nortruthfulqa_gen_nob_p1.yaml | 5 + .../nob/nortruthfulqa_gen_nob_p2.yaml | 5 + .../nob/nortruthfulqa_gen_nob_p3.yaml | 5 + .../nob/nortruthfulqa_gen_nob_p4.yaml | 5 + .../noreval/nortruthfulqa/generation/utils.py | 152 +++++++++ .../multiple_choice/_nortruthfulqa_mc_yaml | 14 + .../nno/nortruthfulqa_mc_nno_p0.yaml | 5 + .../nno/nortruthfulqa_mc_nno_p1.yaml | 5 + .../nno/nortruthfulqa_mc_nno_p2.yaml | 6 + .../nno/nortruthfulqa_mc_nno_p3.yaml | 6 + .../nno/nortruthfulqa_mc_nno_p4.yaml | 5 + .../multiple_choice/nno/utils.py | 35 ++ .../nob/nortruthfulqa_mc_nob_p0.yaml | 5 + .../nob/nortruthfulqa_mc_nob_p1.yaml | 5 + .../nob/nortruthfulqa_mc_nob_p2.yaml | 6 + .../nob/nortruthfulqa_mc_nob_p3.yaml | 6 + .../nob/nortruthfulqa_mc_nob_p4.yaml | 5 + .../multiple_choice/nob/utils.py | 35 ++ .../noreval/nrk_quiz_qa/_nrk_quiz_qa_yaml | 16 + .../nrk_quiz_qa/nno/nrk_quiz_qa_nno_p0.yaml | 6 + .../nrk_quiz_qa/nno/nrk_quiz_qa_nno_p1.yaml | 6 + .../nrk_quiz_qa/nno/nrk_quiz_qa_nno_p2.yaml | 6 + .../nrk_quiz_qa/nno/nrk_quiz_qa_nno_p3.yaml | 6 + .../nrk_quiz_qa/nno/nrk_quiz_qa_nno_p4.yaml | 6 + .../tasks/noreval/nrk_quiz_qa/nno/utils.py | 44 +++ .../nrk_quiz_qa/nob/nrk_quiz_qa_nob_p0.yaml | 6 + .../nrk_quiz_qa/nob/nrk_quiz_qa_nob_p1.yaml | 6 + .../nrk_quiz_qa/nob/nrk_quiz_qa_nob_p2.yaml | 6 + .../nrk_quiz_qa/nob/nrk_quiz_qa_nob_p3.yaml | 6 + .../nrk_quiz_qa/nob/nrk_quiz_qa_nob_p4.yaml | 6 + .../tasks/noreval/nrk_quiz_qa/nob/utils.py | 46 +++ lm_eval/tasks/noreval/tatoeba/_tatoeba_yaml | 19 ++ .../tatoeba_eng_nno/tatoeba_eng_nno_p0.yaml | 6 + .../tatoeba_eng_nno/tatoeba_eng_nno_p1.yaml | 6 + .../tatoeba_eng_nno/tatoeba_eng_nno_p2.yaml | 6 + .../tatoeba_eng_nno/tatoeba_eng_nno_p3.yaml | 6 + .../tatoeba_eng_nob/tatoeba_eng_nob_p0.yaml | 6 + .../tatoeba_eng_nob/tatoeba_eng_nob_p1.yaml | 6 + .../tatoeba_eng_nob/tatoeba_eng_nob_p2.yaml | 6 + .../tatoeba_eng_nob/tatoeba_eng_nob_p3.yaml | 6 + .../tatoeba_nno_eng/tatoeba_nno_eng_p0.yaml | 6 + .../tatoeba_nno_eng/tatoeba_nno_eng_p1.yaml | 6 + .../tatoeba_nno_eng/tatoeba_nno_eng_p2.yaml | 6 + .../tatoeba_nno_eng/tatoeba_nno_eng_p3.yaml | 6 + .../tatoeba_nob_eng/tatoeba_nob_eng_p0.yaml | 6 + .../tatoeba_nob_eng/tatoeba_nob_eng_p1.yaml | 6 + .../tatoeba_nob_eng/tatoeba_nob_eng_p2.yaml | 6 + .../tatoeba_nob_eng/tatoeba_nob_eng_p3.yaml | 6 + 144 files changed, 1860 insertions(+), 158 deletions(-) create mode 100644 lm_eval/tasks/noreval/README.md create mode 100644 lm_eval/tasks/noreval/ask_gec/README.md create mode 100644 lm_eval/tasks/noreval/ask_gec/_ask_gec_yaml create mode 100644 lm_eval/tasks/noreval/ask_gec/ask_gec_p0.yaml create mode 100644 lm_eval/tasks/noreval/ask_gec/ask_gec_p1.yaml create mode 100644 lm_eval/tasks/noreval/ask_gec/ask_gec_p2.yaml create mode 100644 lm_eval/tasks/noreval/ask_gec/ask_gec_p3.yaml create mode 100644 lm_eval/tasks/noreval/ask_gec/ask_gec_p4.yaml create mode 100644 lm_eval/tasks/noreval/ask_gec/errant.py create mode 100644 lm_eval/tasks/noreval/ncb/ncb.yaml create mode 100644 lm_eval/tasks/noreval/norbelebele/_norbelebele_yaml create mode 100644 lm_eval/tasks/noreval/norbelebele/norbelebele_p0.yaml create mode 100644 lm_eval/tasks/noreval/norbelebele/norbelebele_p1.yaml create mode 100644 lm_eval/tasks/noreval/norbelebele/norbelebele_p2.yaml create mode 100644 lm_eval/tasks/noreval/norbelebele/norbelebele_p3.yaml create mode 100644 lm_eval/tasks/noreval/norbelebele/norbelebele_p4.yaml create mode 100644 lm_eval/tasks/noreval/norcommonsenseqa/_norcommonsenseqa_yaml create mode 100644 lm_eval/tasks/noreval/norcommonsenseqa/nno/norcommonsenseqa_nno_p0.yaml create mode 100644 lm_eval/tasks/noreval/norcommonsenseqa/nno/norcommonsenseqa_nno_p1.yaml create mode 100644 lm_eval/tasks/noreval/norcommonsenseqa/nno/norcommonsenseqa_nno_p2.yaml create mode 100644 lm_eval/tasks/noreval/norcommonsenseqa/nno/norcommonsenseqa_nno_p3.yaml create mode 100644 lm_eval/tasks/noreval/norcommonsenseqa/nno/norcommonsenseqa_nno_p4.yaml create mode 100644 lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p0.yaml create mode 100644 lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p1.yaml create mode 100644 lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p2.yaml create mode 100644 lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p3.yaml create mode 100644 lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p4.yaml create mode 100644 lm_eval/tasks/noreval/norec/_norec_yaml create mode 100644 lm_eval/tasks/noreval/norec/norec_document/norec_document_p0.yaml create mode 100644 lm_eval/tasks/noreval/norec/norec_document/norec_document_p1.yaml create mode 100644 lm_eval/tasks/noreval/norec/norec_document/norec_document_p2.yaml create mode 100644 lm_eval/tasks/noreval/norec/norec_document/norec_document_p3.yaml create mode 100644 lm_eval/tasks/noreval/norec/norec_document/norec_document_p4.yaml create mode 100644 lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p0.yaml create mode 100644 lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p1.yaml create mode 100644 lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p2.yaml create mode 100644 lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p3.yaml create mode 100644 lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p4.yaml create mode 100644 lm_eval/tasks/noreval/norec/utils.py create mode 100644 lm_eval/tasks/noreval/noreval.jpg create mode 100644 lm_eval/tasks/noreval/noridiom/_noridiom_yaml create mode 100644 lm_eval/tasks/noreval/noridiom/nno/noridiom_nno_p0.yaml create mode 100644 lm_eval/tasks/noreval/noridiom/nno/noridiom_nno_p1.yaml create mode 100644 lm_eval/tasks/noreval/noridiom/nno/noridiom_nno_p2.yaml create mode 100644 lm_eval/tasks/noreval/noridiom/nno/noridiom_nno_p3.yaml create mode 100644 lm_eval/tasks/noreval/noridiom/nno/noridiom_nno_p4.yaml create mode 100644 lm_eval/tasks/noreval/noridiom/nob/noridiom_nob_p0.yaml create mode 100644 lm_eval/tasks/noreval/noridiom/nob/noridiom_nob_p1.yaml create mode 100644 lm_eval/tasks/noreval/noridiom/nob/noridiom_nob_p2.yaml create mode 100644 lm_eval/tasks/noreval/noridiom/nob/noridiom_nob_p3.yaml create mode 100644 lm_eval/tasks/noreval/noridiom/nob/noridiom_nob_p4.yaml create mode 100644 lm_eval/tasks/noreval/noridiom/utils.py create mode 100644 lm_eval/tasks/noreval/noropenbookqa/_noropenbookqa_yaml create mode 100644 lm_eval/tasks/noreval/noropenbookqa/nno/noropenbookqa_nno_p0.yaml create mode 100644 lm_eval/tasks/noreval/noropenbookqa/nno/noropenbookqa_nno_p1.yaml create mode 100644 lm_eval/tasks/noreval/noropenbookqa/nno/noropenbookqa_nno_p2.yaml create mode 100644 lm_eval/tasks/noreval/noropenbookqa/nno/noropenbookqa_nno_p3.yaml create mode 100644 lm_eval/tasks/noreval/noropenbookqa/nno/noropenbookqa_nno_p4.yaml create mode 100644 lm_eval/tasks/noreval/noropenbookqa/nob/noropenbookqa_nob_p0.yaml create mode 100644 lm_eval/tasks/noreval/noropenbookqa/nob/noropenbookqa_nob_p1.yaml create mode 100644 lm_eval/tasks/noreval/noropenbookqa/nob/noropenbookqa_nob_p2.yaml create mode 100644 lm_eval/tasks/noreval/noropenbookqa/nob/noropenbookqa_nob_p3.yaml create mode 100644 lm_eval/tasks/noreval/noropenbookqa/nob/noropenbookqa_nob_p4.yaml create mode 100644 lm_eval/tasks/noreval/noropenbookqa/utils.py create mode 100644 lm_eval/tasks/noreval/norquad/_norquad_yaml create mode 100644 lm_eval/tasks/noreval/norquad/norquad_p0.yaml create mode 100644 lm_eval/tasks/noreval/norquad/norquad_p1.yaml create mode 100644 lm_eval/tasks/noreval/norquad/norquad_p2.yaml create mode 100644 lm_eval/tasks/noreval/norquad/norquad_p3.yaml create mode 100644 lm_eval/tasks/noreval/norquad/norquad_p4.yaml create mode 100644 lm_eval/tasks/noreval/norquad/utils.py create mode 100644 lm_eval/tasks/noreval/norrewrite-instruct/norrewrite_instruct.yaml create mode 100644 lm_eval/tasks/noreval/norsumm/_norsumm_yaml create mode 100644 lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p0.yaml create mode 100644 lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p1.yaml create mode 100644 lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p2.yaml create mode 100644 lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p3.yaml create mode 100644 lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p4.yaml create mode 100644 lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p5.yaml create mode 100644 lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p0.yaml create mode 100644 lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p1.yaml create mode 100644 lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p2.yaml create mode 100644 lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p3.yaml create mode 100644 lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p4.yaml create mode 100644 lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p5.yaml create mode 100644 lm_eval/tasks/noreval/norsumm/utils.py create mode 100644 lm_eval/tasks/noreval/norsummarize-instruct/norsummarize_instruct.yaml create mode 100644 lm_eval/tasks/noreval/nortruthfulqa/generation/_nortruthfulqa_gen_yaml create mode 100644 lm_eval/tasks/noreval/nortruthfulqa/generation/nno/nortruthfulqa_gen_nno_p0.yaml create mode 100644 lm_eval/tasks/noreval/nortruthfulqa/generation/nno/nortruthfulqa_gen_nno_p1.yaml create mode 100644 lm_eval/tasks/noreval/nortruthfulqa/generation/nno/nortruthfulqa_gen_nno_p2.yaml create mode 100644 lm_eval/tasks/noreval/nortruthfulqa/generation/nno/nortruthfulqa_gen_nno_p3.yaml create mode 100644 lm_eval/tasks/noreval/nortruthfulqa/generation/nno/nortruthfulqa_gen_nno_p4.yaml create mode 100644 lm_eval/tasks/noreval/nortruthfulqa/generation/nob/nortruthfulqa_gen_nob_p0.yaml create mode 100644 lm_eval/tasks/noreval/nortruthfulqa/generation/nob/nortruthfulqa_gen_nob_p1.yaml create mode 100644 lm_eval/tasks/noreval/nortruthfulqa/generation/nob/nortruthfulqa_gen_nob_p2.yaml create mode 100644 lm_eval/tasks/noreval/nortruthfulqa/generation/nob/nortruthfulqa_gen_nob_p3.yaml create mode 100644 lm_eval/tasks/noreval/nortruthfulqa/generation/nob/nortruthfulqa_gen_nob_p4.yaml create mode 100644 lm_eval/tasks/noreval/nortruthfulqa/generation/utils.py create mode 100644 lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/_nortruthfulqa_mc_yaml create mode 100644 lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/nortruthfulqa_mc_nno_p0.yaml create mode 100644 lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/nortruthfulqa_mc_nno_p1.yaml create mode 100644 lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/nortruthfulqa_mc_nno_p2.yaml create mode 100644 lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/nortruthfulqa_mc_nno_p3.yaml create mode 100644 lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/nortruthfulqa_mc_nno_p4.yaml create mode 100644 lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/utils.py create mode 100644 lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/nortruthfulqa_mc_nob_p0.yaml create mode 100644 lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/nortruthfulqa_mc_nob_p1.yaml create mode 100644 lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/nortruthfulqa_mc_nob_p2.yaml create mode 100644 lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/nortruthfulqa_mc_nob_p3.yaml create mode 100644 lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/nortruthfulqa_mc_nob_p4.yaml create mode 100644 lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/utils.py create mode 100644 lm_eval/tasks/noreval/nrk_quiz_qa/_nrk_quiz_qa_yaml create mode 100644 lm_eval/tasks/noreval/nrk_quiz_qa/nno/nrk_quiz_qa_nno_p0.yaml create mode 100644 lm_eval/tasks/noreval/nrk_quiz_qa/nno/nrk_quiz_qa_nno_p1.yaml create mode 100644 lm_eval/tasks/noreval/nrk_quiz_qa/nno/nrk_quiz_qa_nno_p2.yaml create mode 100644 lm_eval/tasks/noreval/nrk_quiz_qa/nno/nrk_quiz_qa_nno_p3.yaml create mode 100644 lm_eval/tasks/noreval/nrk_quiz_qa/nno/nrk_quiz_qa_nno_p4.yaml create mode 100644 lm_eval/tasks/noreval/nrk_quiz_qa/nno/utils.py create mode 100644 lm_eval/tasks/noreval/nrk_quiz_qa/nob/nrk_quiz_qa_nob_p0.yaml create mode 100644 lm_eval/tasks/noreval/nrk_quiz_qa/nob/nrk_quiz_qa_nob_p1.yaml create mode 100644 lm_eval/tasks/noreval/nrk_quiz_qa/nob/nrk_quiz_qa_nob_p2.yaml create mode 100644 lm_eval/tasks/noreval/nrk_quiz_qa/nob/nrk_quiz_qa_nob_p3.yaml create mode 100644 lm_eval/tasks/noreval/nrk_quiz_qa/nob/nrk_quiz_qa_nob_p4.yaml create mode 100644 lm_eval/tasks/noreval/nrk_quiz_qa/nob/utils.py create mode 100644 lm_eval/tasks/noreval/tatoeba/_tatoeba_yaml create mode 100644 lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nno/tatoeba_eng_nno_p0.yaml create mode 100644 lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nno/tatoeba_eng_nno_p1.yaml create mode 100644 lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nno/tatoeba_eng_nno_p2.yaml create mode 100644 lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nno/tatoeba_eng_nno_p3.yaml create mode 100644 lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nob/tatoeba_eng_nob_p0.yaml create mode 100644 lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nob/tatoeba_eng_nob_p1.yaml create mode 100644 lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nob/tatoeba_eng_nob_p2.yaml create mode 100644 lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nob/tatoeba_eng_nob_p3.yaml create mode 100644 lm_eval/tasks/noreval/tatoeba/tatoeba_nno_eng/tatoeba_nno_eng_p0.yaml create mode 100644 lm_eval/tasks/noreval/tatoeba/tatoeba_nno_eng/tatoeba_nno_eng_p1.yaml create mode 100644 lm_eval/tasks/noreval/tatoeba/tatoeba_nno_eng/tatoeba_nno_eng_p2.yaml create mode 100644 lm_eval/tasks/noreval/tatoeba/tatoeba_nno_eng/tatoeba_nno_eng_p3.yaml create mode 100644 lm_eval/tasks/noreval/tatoeba/tatoeba_nob_eng/tatoeba_nob_eng_p0.yaml create mode 100644 lm_eval/tasks/noreval/tatoeba/tatoeba_nob_eng/tatoeba_nob_eng_p1.yaml create mode 100644 lm_eval/tasks/noreval/tatoeba/tatoeba_nob_eng/tatoeba_nob_eng_p2.yaml create mode 100644 lm_eval/tasks/noreval/tatoeba/tatoeba_nob_eng/tatoeba_nob_eng_p3.yaml diff --git a/.gitignore b/.gitignore index 56bb8038..d04100d0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.DS_Store env *.pyc output/ diff --git a/lm_eval/tasks/README.md b/lm_eval/tasks/README.md index 9b35aea8..c4fc5a55 100644 --- a/lm_eval/tasks/README.md +++ b/lm_eval/tasks/README.md @@ -5,165 +5,164 @@ For more information, including a full list of task names and their precise meanings or sources, follow the links provided to the individual README.md files for each subfolder. -| Task Family | Description | Language(s) | -|--------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------| -| [aclue](aclue/README.md) | Tasks focusing on ancient Chinese language understanding and cultural aspects. | Ancient Chinese | -| [acp_bench](acpbench/README.md) | Tasks evaluating the reasoning ability about Action, Change, and Planning | English | -| [aexams](aexams/README.md) | Tasks in Arabic related to various academic exams covering a range of subjects. | Arabic | -| [agieval](agieval/README.md) | Tasks involving historical data or questions related to history and historical texts. | English, Chinese | -| [anli](anli/README.md) | Adversarial natural language inference tasks designed to test model robustness. | English | -| [arabic_leaderboard_complete](arabic_leaderboard_complete/README.md) | A full version of the tasks in the Open Arabic LLM Leaderboard, focusing on the evaluation of models that reflect the characteristics of Arabic language understanding and comprehension, culture, and heritage. Note that some of these tasks are machine-translated. | Arabic (Some MT) | -| [arabic_leaderboard_light](arabic_leaderboard_light/README.md) | A light version of the tasks in the Open Arabic LLM Leaderboard (i.e., 10% samples of the test set in the original benchmarks), focusing on the evaluation of models that reflect the characteristics of Arabic language understanding and comprehension, culture, and heritage. Note that some of these tasks are machine-translated. | Arabic (Some MT) | -| [arabicmmlu](arabicmmlu/README.md) | Localized Arabic version of MMLU with multiple-choice questions from 40 subjects. | Arabic | -| [AraDICE](aradice/README.md) | A collection of multiple tasks carefully designed to evaluate dialectal and cultural capabilities in large language models (LLMs). | Arabic | -| [arc](arc/README.md) | Tasks involving complex reasoning over a diverse set of questions. | English | -| [arithmetic](arithmetic/README.md) | Tasks involving numerical computations and arithmetic reasoning. | English | -| [asdiv](asdiv/README.md) | Tasks involving arithmetic and mathematical reasoning challenges. | English | -| [babi](babi/README.md) | Tasks designed as question and answering challenges based on simulated stories. | English | -| [basque_bench](basque_bench/README.md) | Collection of tasks in Basque encompassing various evaluation areas. | Basque | -| [basqueglue](basqueglue/README.md) | Tasks designed to evaluate language understanding in Basque language. | Basque | -| [bbh](bbh/README.md) | Tasks focused on deep semantic understanding through hypothesization and reasoning. | English, German | -| [bbq](bbq/README.md) | A question-answering benchmark designed to measure social biases in language models across various demographic categories and contexts. | English | -| [belebele](belebele/README.md) | Language understanding tasks in a variety of languages and scripts. | Multiple (122 languages) | -| benchmarks | General benchmarking tasks that test a wide range of language understanding capabilities. | | -| [bertaqa](bertaqa/README.md) | Local Basque cultural trivia QA tests in English and Basque languages. | English, Basque, Basque (MT) | -| [bigbench](bigbench/README.md) | Broad tasks from the BIG-bench benchmark designed to push the boundaries of large models. | Multiple | -| [blimp](blimp/README.md) | Tasks testing grammatical phenomena to evaluate language model's linguistic capabilities. | English | -| [careqa](careqa/README.md) | Multiple choice and open-ended medical question answering based on the Spanish Specialised Healthcare Training (MIR) exams. | English, Spanish | -| [catalan_bench](catalan_bench/README.md) | Collection of tasks in Catalan encompassing various evaluation areas. | Catalan | -| [ceval](ceval/README.md) | Tasks that evaluate language understanding and reasoning in an educational context. | Chinese | -| [cmmlu](cmmlu/README.md) | Multi-subject multiple choice question tasks for comprehensive academic assessment. | Chinese | -| code_x_glue | Tasks that involve understanding and generating code across multiple programming languages. | Go, Java, JS, PHP, Python, Ruby | -| [commonsense_qa](commonsense_qa/README.md) | CommonsenseQA, a multiple-choice QA dataset for measuring commonsense knowledge. | English | -| [copal_id](copal_id/README.md) United States | Indonesian causal commonsense reasoning dataset that captures local nuances. | Indonesian | -| [coqa](coqa/README.md) | Conversational question answering tasks to test dialog understanding. | English | -| [crows_pairs](crows_pairs/README.md) | Tasks designed to test model biases in various sociodemographic groups. | English, French | -| csatqa | Tasks related to SAT and other standardized testing questions for academic assessment. | Korean | -| [darija_bench](darija_bench/README.md) | Traditional NLP tasks (Translation, Summariation, etc..) for Moroccan Darija | Moroccan Darija (some MT) | -| [darijahellaswag](darijahellaswag/README.md) | Moroccan Darija version of HellaSwag. | Moroccan Darija (MT) | -| [darijammlu](darijammlu/README.md) | Multiple-choice QA in Moroccan Darija (an Arabic dialect). | Moroccan Darija (MT) | -| [drop](drop/README.md) | Tasks requiring numerical reasoning, reading comprehension, and question answering. | English | -| [eq_bench](eq_bench/README.md) | Tasks focused on equality and ethics in question answering and decision-making. | English | -| [eus_exams](eus_exams/README.md) | Tasks based on various professional and academic exams in the Basque language. | Basque | -| [eus_proficiency](eus_proficiency/README.md) | Tasks designed to test proficiency in the Basque language across various topics. | Basque | -| [eus_reading](eus_reading/README.md) | Reading comprehension tasks specifically designed for the Basque language. | Basque | -| [eus_trivia](eus_trivia/README.md) | Trivia and knowledge testing tasks in the Basque language. | Basque | -| [evalita_LLM](evalita_llm/README.md) | A native Italian benchmark with diverse tasks formats and multiple prompts. | Italian | -| [fda](fda/README.md) | Tasks for extracting key-value pairs from FDA documents to test information extraction. | English | -| [fld](fld/README.md) | Tasks involving free-form and directed dialogue understanding. | English | -| [french_bench](french_bench/README.md) | Set of tasks designed to assess language model performance in French. | French | -| [galician_bench](galician_bench/README.md) | Collection of tasks in Galician encompassing various evaluation areas. | Galician | -| [global_mmlu](global_mmlu/README.md) | Collection of culturally sensitive and culturally agnostic MMLU tasks in 15 languages with human translations or post-edits. | Multiple (15 languages) | -| [glue](glue/README.md) | General Language Understanding Evaluation benchmark to test broad language abilities. | English | -| [gpqa](gpqa/README.md) | Tasks designed for general public question answering and knowledge verification. | English | -| [gsm8k](gsm8k/README.md) | A benchmark of grade school math problems aimed at evaluating reasoning capabilities. | English | -| [groundcocoa](groundcocoa/README.md) | A benchmark evaluating the conditional and compositional reasoning of language models using a grounding task. | English | -| [haerae](haerae/README.md) | Tasks focused on assessing detailed factual and historical knowledge. | Korean | -| [headqa](headqa/README.md) | A high-level education-based question answering dataset to test specialized knowledge. | Spanish, English | -| [hellaswag](hellaswag/README.md) | Tasks to predict the ending of stories or scenarios, testing comprehension and creativity. | English | -| [hendrycks_ethics](hendrycks_ethics/README.md) | Tasks designed to evaluate the ethical reasoning capabilities of models. | English | -| [hendrycks_math](hendrycks_math/README.md) | Mathematical problem-solving tasks to test numerical reasoning and problem-solving. | English | -| [histoires_morales](histoires_morales/README.md) | A dataset of structured narratives that describe normative and norm-divergent actions taken by individuals to accomplish certain intentions in concrete situations. | French (Some MT) | -| [hrm8k](hrm8k/README.md) | A challenging bilingual math reasoning benchmark for Korean and English. | Korean (Some MT), English (Some MT) | -| [humaneval](humaneval/README.md) | Code generation task that measure functional correctness for synthesizing programs from docstrings. | Python | -| [ifeval](ifeval/README.md) | Interactive fiction evaluation tasks for narrative understanding and reasoning. | English | -| [inverse_scaling](inverse_scaling/README.md) | Multiple-choice tasks from the Inverse Scaling Prize, designed to find settings where larger language models perform worse. | English | -| [japanese_leaderboard](japanese_leaderboard/README.md) | Japanese language understanding tasks to benchmark model performance on various linguistic aspects. | Japanese | -| [jsonschema_bench](jsonschema_bench/README.md) | Evaluate the ability of LLMs to generate JSON objects that conform to a given JSON schema, including API, configuration files, and other structured data formats. | JSON | -| [kbl](kbl/README.md) | Korean Benchmark for Legal Language Understanding. | Korean | -| [kmmlu](kmmlu/README.md) | Knowledge-based multi-subject multiple choice questions for academic evaluation. | Korean | -| [kobest](kobest/README.md) | A collection of tasks designed to evaluate understanding in Korean language. | Korean | -| [kormedmcqa](kormedmcqa/README.md) | Medical question answering tasks in Korean to test specialized domain knowledge. | Korean | -| [lambada](lambada/README.md) | Tasks designed to predict the endings of text passages, testing language prediction skills. | English | -| [lambada_cloze](lambada_cloze/README.md) | Cloze-style LAMBADA dataset. | English | -| [lambada_multilingual](lambada_multilingual/README.md) | Multilingual LAMBADA dataset. This is a legacy version of the multilingual dataset, and users should instead use `lambada_multilingual_stablelm`. | German, English, Spanish, French, Italian | -| [lambada_multilingual_stablelm](lambada_multilingual_stablelm/README.md) | Multilingual LAMBADA dataset. Users should prefer evaluating on this version of the multilingual dataset instead of on `lambada_multilingual`. | German, English, Spanish, French, Italian, Dutch, Portuguese | -| [leaderboard](leaderboard/README.md) | Task group used by Hugging Face's [Open LLM Leaderboard v2](https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard). Those tasks are static and will not change through time | English | -| [lingoly](lingoly/README.md) | Challenging logical reasoning benchmark in low-resource languages with controls for memorization | English, Multilingual | -| [llama3](llama3/README.md) | Evals reproducing those provided by the LLAMA team in the Hugging Face repo (instruct) | English, Multilingual | -| [logiqa](logiqa/README.md) | Logical reasoning tasks requiring advanced inference and deduction. | English, Chinese | -| [logiqa2](logiqa2/README.md) | Large-scale logical reasoning dataset adapted from the Chinese Civil Service Examination. | English, Chinese | -| [longbench](longbench/README.md) | LongBench evaluates language models' ability to understand lengthy texts across multiple tasks and languages. | English, Chinese | -| [mastermind](mastermind/README.md) | Reasoning benchmark based on the board game of Mastermind. | English | -| [mathqa](mathqa/README.md) | Question answering tasks involving mathematical reasoning and problem-solving. | English | -| [mbpp](mbpp/README.md) | A benchmark designed to measure the ability to synthesize short Python programs from natural language descriptions. | Python | -| [meddialog](meddialog/README.md) | Medical open-ended QA and Question Entailment stemming from the MedDialog dataset. | English | -| [medtext](medtext/README.md) | Medical open-ended QA from the MedText Clinical Notes dataset. | English | -| [mimic_repsum](mimic_repsum/README.md) | Medical report summarization from the MIMIC-III dataset. | English | -| [mc_taco](mc_taco/README.md) | Question-answer pairs that require temporal commonsense comprehension. | English | -| [med_concepts_qa](med_concepts_qa/README.md) | Benchmark for evaluating LLMs on their abilities to interpret medical codes and distinguish between medical concept. | English | -| [metabench](metabench/README.md) | Distilled versions of six popular benchmarks which are highly predictive of overall benchmark performance and of a single general ability latent trait. | English | -| [mediqa_qa2019](mediqa_qa2019/README.md) | Open-ended healthcare question answering benchmark from the MEDIQA 2019 challenge. | English | -| medmcqa | Medical multiple choice questions assessing detailed medical knowledge. | English | -| medqa | Multiple choice question answering based on the United States Medical License Exams. | | -| [meqsum](meqsum/README.md) | Healtcare Question Entailment benchmark from the MeqSum dataset. | | -| [mgsm](mgsm/README.md) | Benchmark of multilingual grade-school math problems. | Spanish, French, German, Russian, Chinese, Japanese, Thai, Swahili, Bengali, Telugu | -| [minerva_math](minerva_math/README.md) | Mathematics-focused tasks requiring numerical reasoning and problem-solving skills. | English | -| [mlqa](mlqa/README.md) | MultiLingual Question Answering benchmark dataset for evaluating cross-lingual question answering performance. | English, Arabic, German, Spanish, Hindi, Vietnamese, Simplified Chinese | -| [mmlu](mmlu/README.md) | Massive Multitask Language Understanding benchmark for broad domain language evaluation. Several variants are supported. | English | -| [mmlu_pro](mmlu_pro/README.md) | A refined set of MMLU, integrating more challenging, reasoning-focused questions and expanding the choice set from four to ten options. | English | -| [mmlu-pro-plus](mmlu-pro-plus/README.md) | A new test set for evaluating shortcut learning and higher-order reasoning of LLMs. | English | -| [mmlu_prox](mmlu_prox/README.md) | A multilingual benchmark that extends MMLU-Pro to multiple typologically diverse languages with human validation. | English, Japanese, Chinese, Korean, French, German, Spanish, Portuguese, Swahili, Thai, Arabic, Hindi, Bengali | -| [mmlusr](mmlusr/README.md) | Variation of MMLU designed to be more rigorous. | English | -| model_written_evals | Evaluation tasks auto-generated for evaluating a collection of AI Safety concerns. | | -| [moral_stories](moral_stories/README.md) | A crowd-sourced dataset of structured narratives that describe normative and norm-divergent actions taken by individuals to accomplish certain intentions in concrete situations. | English | -| [mts_dialog](mts_dialog/README.md) | Open-ended healthcare QA from the MTS-Dialog dataset. | English | -| [mutual](mutual/README.md) | A retrieval-based dataset for multi-turn dialogue reasoning. | English | -| [nq_open](nq_open/README.md) | Open domain question answering tasks based on the Natural Questions dataset. | English | -| [okapi/arc_multilingual](okapi/arc_multilingual/README.md) | Tasks that involve reading comprehension and information retrieval challenges. | Multiple (31 languages) **Machine Translated.** | -| [okapi/hellaswag_multilingual](okapi/hellaswag_multilingual/README.md) | Tasks that involve reading comprehension and information retrieval challenges. | Multiple (30 languages) **Machine Translated.** | -| okapi/mmlu_multilingual | Tasks that involve reading comprehension and information retrieval challenges. | Multiple (34 languages) **Machine Translated.** | -| [okapi/truthfulqa_multilingual](okapi/truthfulqa_multilingual/README.md) | Tasks that involve reading comprehension and information retrieval challenges. | Multiple (31 languages) **Machine Translated.** | -| [olaph](olaph/README.md) | Open-ended medical factuality Question Answering from the OLAPH dataset. | English | -| [openbookqa](openbookqa/README.md) | Open-book question answering tasks that require external knowledge and reasoning. | English | -| [paloma](paloma/README.md) | Paloma is a comprehensive benchmark designed to evaluate open language models across a wide range of domains, ranging from niche artist communities to mental health forums on Reddit. | English | -| [paws-x](paws-x/README.md) | Paraphrase Adversaries from Word Scrambling, focusing on cross-lingual capabilities. | English, French, Spanish, German, Chinese, Japanese, Korean | -| [pile](pile/README.md) | Open source language modelling data set that consists of 22 smaller, high-quality datasets. | English | -| [pile_10k](pile_10k/README.md) | The first 10K elements of The Pile, useful for debugging models trained on it. | English | -| [piqa](piqa/README.md) | Physical Interaction Question Answering tasks to test physical commonsense reasoning. | English | -| [polemo2](polemo2/README.md) | Sentiment analysis and emotion detection tasks based on Polish language data. | Polish | -| [portuguese_bench](portuguese_bench/README.md) | Collection of tasks in European Portuguese encompassing various evaluation areas. | Portuguese | -| [prost](prost/README.md) | Tasks requiring understanding of professional standards and ethics in various domains. | English | -| [pubmedqa](pubmedqa/README.md) | Question answering tasks based on PubMed research articles for biomedical understanding. | English | -| [qa4mre](qa4mre/README.md) | Question Answering for Machine Reading Evaluation, assessing comprehension and reasoning. | English | -| [qasper](qasper/README.md) | Question Answering dataset based on academic papers, testing in-depth scientific knowledge. | English | -| [race](race/README.md) | Reading comprehension assessment tasks based on English exams in China. | English | -| realtoxicityprompts | Tasks to evaluate language models for generating text with potential toxicity. | | -| [ruler](ruler/README.md) | RULER is a benchmark for testing how well language models handle long pieces of text. Requires custom arg (see readme) | English | -| [sciq](sciq/README.md) | Science Question Answering tasks to assess understanding of scientific concepts. | English | -| [score](score/README.md) | Systematic consistency and robustness evaluation for LLMs on 3 datasets(MMLU-Pro, Agi Eval and MATH) | English | -| [scrolls](scrolls/README.md) | Tasks that involve long-form reading comprehension across various domains. | English | -| [simple_cooccurrence_bias](simple_cooccurrence_bias/README.md) | A metric that evaluates language models for biases based on stereotypical word associations and co-occurrences in text. | English | -| [siqa](siqa/README.md) | Social Interaction Question Answering to evaluate common sense and social reasoning. | English | -| [spanish_bench](spanish_bench/README.md) | Collection of tasks in Spanish encompassing various evaluation areas. | Spanish | -| [squad_completion](squad_completion/README.md) | A variant of the SQuAD question answering task designed for zero-shot evaluation of small LMs. | English | -| [squadv2](squadv2/README.md) | Stanford Question Answering Dataset version 2, a reading comprehension benchmark. | English | -| [storycloze](storycloze/README.md) | Tasks to predict story endings, focusing on narrative logic and coherence. | English | -| [super_glue](super_glue/README.md) | A suite of challenging tasks designed to test a range of language understanding skills. | English | -| [swag](swag/README.md) | Situations With Adversarial Generations, predicting the next event in videos. | English | -| [swde](swde/README.md) | Information extraction tasks from semi-structured web pages. | English | -| [tinyBenchmarks](tinyBenchmarks/README.md) | Evaluation of large language models with fewer examples using tiny versions of popular benchmarks. | English | -| [tmmluplus](tmmluplus/README.md) | An extended set of tasks under the TMMLU framework for broader academic assessments. | Traditional Chinese | -| [toxigen](toxigen/README.md) | Tasks designed to evaluate language models on their propensity to generate toxic content. | English | -| [translation](translation/README.md) | Tasks focused on evaluating the language translation capabilities of models. | Arabic, English, Spanish, Basque, Hindi, Indonesian, Burmese, Russian, Swahili, Telugu, Chinese | -| [triviaqa](triviaqa/README.md) | A large-scale dataset for trivia question answering to test general knowledge. | English | -| [truthfulqa](truthfulqa/README.md) | A QA task aimed at evaluating the truthfulness and factual accuracy of model responses. | English | -| [turkishmmlu](turkishmmlu/README.md) | A multiple-choice QA test modeled after MMLU, written in Turkish based on Turkish high-school level exams. | Turkish | -| [unitxt](unitxt/README.md) | A number of tasks implemented using the unitxt library for flexible, shareable, and reusable data preparation and evaluation for generative AI. | English | -| [unscramble](unscramble/README.md) | Tasks involving the rearrangement of scrambled sentences to test syntactic understanding. | English | -| [webqs](webqs/README.md) | Web-based question answering tasks designed to evaluate internet search and retrieval. | English | -| [wikitext](wikitext/README.md) | Tasks based on text from Wikipedia articles to assess language modeling and generation. | English | -| [winogender](winogender/README.md) | A diagnostic dataset that tests for gender bias in coreference resolution by measuring how models associate pronouns with different occupations. | English | -| [winogrande](winogrande/README.md) | A large-scale dataset for coreference resolution, inspired by the Winograd Schema Challenge. | English | -| [wmdp](wmdp/README.md) | A benchmark with the objective of minimizing performance, based on potentially-sensitive multiple-choice knowledge questions. | English | -| [wmt2016](wmt2016/README.md) | Tasks from the WMT 2016 shared task, focusing on translation between multiple languages. | English, Czech, German, Finnish, Russian, Romanian, Turkish | -| [wsc273](wsc273/README.md) | The Winograd Schema Challenge, a test of commonsense reasoning and coreference resolution. | English | -| [xcopa](xcopa/README.md) | Cross-lingual Choice of Plausible Alternatives, testing reasoning in multiple languages. | Estonian, Haitian, Indonesian, Italian, Quechua, Swahili, Tamil, Thai, Turkish, Vietnamese, Chinese | +| Task Family | Description | Language(s) | +|--------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------| +| [aclue](aclue/README.md) | Tasks focusing on ancient Chinese language understanding and cultural aspects. | Ancient Chinese | +| [acp_bench](acpbench/README.md) | Tasks evaluating the reasoning ability about Action, Change, and Planning | English | +| [aexams](aexams/README.md) | Tasks in Arabic related to various academic exams covering a range of subjects. | Arabic | +| [agieval](agieval/README.md) | Tasks involving historical data or questions related to history and historical texts. | English, Chinese | +| [anli](anli/README.md) | Adversarial natural language inference tasks designed to test model robustness. | English | +| [arabic_leaderboard_complete](arabic_leaderboard_complete/README.md) | A full version of the tasks in the Open Arabic LLM Leaderboard, focusing on the evaluation of models that reflect the characteristics of Arabic language understanding and comprehension, culture, and heritage. Note that some of these tasks are machine-translated. | Arabic (Some MT) | +| [arabic_leaderboard_light](arabic_leaderboard_light/README.md) | A light version of the tasks in the Open Arabic LLM Leaderboard (i.e., 10% samples of the test set in the original benchmarks), focusing on the evaluation of models that reflect the characteristics of Arabic language understanding and comprehension, culture, and heritage. Note that some of these tasks are machine-translated. | Arabic (Some MT) | +| [arabicmmlu](arabicmmlu/README.md) | Localized Arabic version of MMLU with multiple-choice questions from 40 subjects. | Arabic | +| [AraDICE](aradice/README.md) | A collection of multiple tasks carefully designed to evaluate dialectal and cultural capabilities in large language models (LLMs). | Arabic | +| [arc](arc/README.md) | Tasks involving complex reasoning over a diverse set of questions. | English | +| [arithmetic](arithmetic/README.md) | Tasks involving numerical computations and arithmetic reasoning. | English | +| [asdiv](asdiv/README.md) | Tasks involving arithmetic and mathematical reasoning challenges. | English | +| [babi](babi/README.md) | Tasks designed as question and answering challenges based on simulated stories. | English | +| [basque_bench](basque_bench/README.md) | Collection of tasks in Basque encompassing various evaluation areas. | Basque | +| [basqueglue](basqueglue/README.md) | Tasks designed to evaluate language understanding in Basque language. | Basque | +| [bbh](bbh/README.md) | Tasks focused on deep semantic understanding through hypothesization and reasoning. | English, German | +| [bbq](bbq/README.md) | A question-answering benchmark designed to measure social biases in language models across various demographic categories and contexts. | English | +| [belebele](belebele/README.md) | Language understanding tasks in a variety of languages and scripts. | Multiple (122 languages) | +| benchmarks | General benchmarking tasks that test a wide range of language understanding capabilities. | | +| [bertaqa](bertaqa/README.md) | Local Basque cultural trivia QA tests in English and Basque languages. | English, Basque, Basque (MT) | +| [bigbench](bigbench/README.md) | Broad tasks from the BIG-bench benchmark designed to push the boundaries of large models. | Multiple | +| [blimp](blimp/README.md) | Tasks testing grammatical phenomena to evaluate language model's linguistic capabilities. | English | +| [careqa](careqa/README.md) | Multiple choice and open-ended medical question answering based on the Spanish Specialised Healthcare Training (MIR) exams. | English, Spanish | +| [catalan_bench](catalan_bench/README.md) | Collection of tasks in Catalan encompassing various evaluation areas. | Catalan | +| [ceval](ceval/README.md) | Tasks that evaluate language understanding and reasoning in an educational context. | Chinese | +| [cmmlu](cmmlu/README.md) | Multi-subject multiple choice question tasks for comprehensive academic assessment. | Chinese | +| code_x_glue | Tasks that involve understanding and generating code across multiple programming languages. | Go, Java, JS, PHP, Python, Ruby | +| [commonsense_qa](commonsense_qa/README.md) | CommonsenseQA, a multiple-choice QA dataset for measuring commonsense knowledge. | English | +| [copal_id](copal_id/README.md) United States | Indonesian causal commonsense reasoning dataset that captures local nuances. | Indonesian | +| [coqa](coqa/README.md) | Conversational question answering tasks to test dialog understanding. | English | +| [crows_pairs](crows_pairs/README.md) | Tasks designed to test model biases in various sociodemographic groups. | English, French | +| csatqa | Tasks related to SAT and other standardized testing questions for academic assessment. | Korean | +| [darija_bench](darija_bench/README.md) | Traditional NLP tasks (Translation, Summariation, etc..) for Moroccan Darija | Moroccan Darija (some MT) | +| [darijahellaswag](darijahellaswag/README.md) | Moroccan Darija version of HellaSwag. | Moroccan Darija (MT) | +| [darijammlu](darijammlu/README.md)| Multiple-choice QA in Moroccan Darija (an Arabic dialect). | Moroccan Darija (MT) | +| [drop](drop/README.md) | Tasks requiring numerical reasoning, reading comprehension, and question answering. | English | +| [eq_bench](eq_bench/README.md) | Tasks focused on equality and ethics in question answering and decision-making. | English | +| [eus_exams](eus_exams/README.md) | Tasks based on various professional and academic exams in the Basque language. | Basque | +| [eus_proficiency](eus_proficiency/README.md) | Tasks designed to test proficiency in the Basque language across various topics. | Basque | +| [eus_reading](eus_reading/README.md) | Reading comprehension tasks specifically designed for the Basque language. | Basque | +| [eus_trivia](eus_trivia/README.md) | Trivia and knowledge testing tasks in the Basque language. | Basque | +| [evalita_LLM](evalita_llm/README.md) | A native Italian benchmark with diverse tasks formats and multiple prompts. | Italian | +| [fda](fda/README.md) | Tasks for extracting key-value pairs from FDA documents to test information extraction. | English | +| [fld](fld/README.md) | Tasks involving free-form and directed dialogue understanding. | English | +| [french_bench](french_bench/README.md) | Set of tasks designed to assess language model performance in French. | French | +| [galician_bench](galician_bench/README.md) | Collection of tasks in Galician encompassing various evaluation areas. | Galician | +| [global_mmlu](global_mmlu/README.md) | Collection of culturally sensitive and culturally agnostic MMLU tasks in 15 languages with human translations or post-edits. | Multiple (15 languages) | +| [glue](glue/README.md) | General Language Understanding Evaluation benchmark to test broad language abilities. | English | +| [gpqa](gpqa/README.md) | Tasks designed for general public question answering and knowledge verification. | English | +| [gsm8k](gsm8k/README.md) | A benchmark of grade school math problems aimed at evaluating reasoning capabilities. | English | +| [groundcocoa](groundcocoa/README.md) | A benchmark evaluating the conditional and compositional reasoning of language models using a grounding task. | English | +| [haerae](haerae/README.md) | Tasks focused on assessing detailed factual and historical knowledge. | Korean | +| [headqa](headqa/README.md) | A high-level education-based question answering dataset to test specialized knowledge. | Spanish, English | +| [hellaswag](hellaswag/README.md) | Tasks to predict the ending of stories or scenarios, testing comprehension and creativity. | English | +| [hendrycks_ethics](hendrycks_ethics/README.md) | Tasks designed to evaluate the ethical reasoning capabilities of models. | English | +| [hendrycks_math](hendrycks_math/README.md) | Mathematical problem-solving tasks to test numerical reasoning and problem-solving. | English | +| [histoires_morales](histoires_morales/README.md) | A dataset of structured narratives that describe normative and norm-divergent actions taken by individuals to accomplish certain intentions in concrete situations. | French (Some MT) | +| [hrm8k](hrm8k/README.md) | A challenging bilingual math reasoning benchmark for Korean and English. | Korean (Some MT), English (Some MT) | +| [humaneval](humaneval/README.md) | Code generation task that measure functional correctness for synthesizing programs from docstrings. | Python | +| [ifeval](ifeval/README.md) | Interactive fiction evaluation tasks for narrative understanding and reasoning. | English | +| [inverse_scaling](inverse_scaling/README.md) | Multiple-choice tasks from the Inverse Scaling Prize, designed to find settings where larger language models perform worse. | English | +| [japanese_leaderboard](japanese_leaderboard/README.md) | Japanese language understanding tasks to benchmark model performance on various linguistic aspects. | Japanese | +| [jsonschema_bench](jsonschema_bench/README.md) | Evaluate the ability of LLMs to generate JSON objects that conform to a given JSON schema, including API, configuration files, and other structured data formats. | JSON | +| [kbl](kbl/README.md) | Korean Benchmark for Legal Language Understanding. | Korean | +| [kmmlu](kmmlu/README.md) | Knowledge-based multi-subject multiple choice questions for academic evaluation. | Korean | +| [kobest](kobest/README.md) | A collection of tasks designed to evaluate understanding in Korean language. | Korean | +| [kormedmcqa](kormedmcqa/README.md) | Medical question answering tasks in Korean to test specialized domain knowledge. | Korean | +| [lambada](lambada/README.md) | Tasks designed to predict the endings of text passages, testing language prediction skills. | English | +| [lambada_cloze](lambada_cloze/README.md) | Cloze-style LAMBADA dataset. | English | +| [lambada_multilingual](lambada_multilingual/README.md) | Multilingual LAMBADA dataset. This is a legacy version of the multilingual dataset, and users should instead use `lambada_multilingual_stablelm`. | German, English, Spanish, French, Italian | +| [lambada_multilingual_stablelm](lambada_multilingual_stablelm/README.md) | Multilingual LAMBADA dataset. Users should prefer evaluating on this version of the multilingual dataset instead of on `lambada_multilingual`. | German, English, Spanish, French, Italian, Dutch, Portuguese | +| [leaderboard](leaderboard/README.md) | Task group used by Hugging Face's [Open LLM Leaderboard v2](https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard). Those tasks are static and will not change through time | English | +| [lingoly](lingoly/README.md) | Challenging logical reasoning benchmark in low-resource languages with controls for memorization | English, Multilingual | +| [logiqa](logiqa/README.md) | Logical reasoning tasks requiring advanced inference and deduction. | English, Chinese | +| [logiqa2](logiqa2/README.md) | Large-scale logical reasoning dataset adapted from the Chinese Civil Service Examination. | English, Chinese | +| [mastermind](mastermind/README.md) | Reasoning benchmark based on the board game of Mastermind. | English | +| [mathqa](mathqa/README.md) | Question answering tasks involving mathematical reasoning and problem-solving. | English | +| [mbpp](mbpp/README.md) | A benchmark designed to measure the ability to synthesize short Python programs from natural language descriptions. | Python | +| [meddialog](meddialog/README.md) | Medical open-ended QA and Question Entailment stemming from the MedDialog dataset. | English | +| [medtext](medtext/README.md) | Medical open-ended QA from the MedText Clinical Notes dataset. | English | +| [mimic_repsum](mimic_repsum/README.md) | Medical report summarization from the MIMIC-III dataset. | English | +| [mc_taco](mc_taco/README.md) | Question-answer pairs that require temporal commonsense comprehension. | English | +| [med_concepts_qa](med_concepts_qa/README.md) | Benchmark for evaluating LLMs on their abilities to interpret medical codes and distinguish between medical concept. | English | +| [metabench](metabench/README.md) | Distilled versions of six popular benchmarks which are highly predictive of overall benchmark performance and of a single general ability latent trait. | English | +| [mediqa_qa2019](mediqa_qa2019/README.md) | Open-ended healthcare question answering benchmark from the MEDIQA 2019 challenge. | English | +| medmcqa | Medical multiple choice questions assessing detailed medical knowledge. | English | +| medqa | Multiple choice question answering based on the United States Medical License Exams. | | +| [meqsum](meqsum/README.md) | Healtcare Question Entailment benchmark from the MeqSum dataset. | | +| [mgsm](mgsm/README.md) | Benchmark of multilingual grade-school math problems. | Spanish, French, German, Russian, Chinese, Japanese, Thai, Swahili, Bengali, Telugu | +| [minerva_math](minerva_math/README.md) | Mathematics-focused tasks requiring numerical reasoning and problem-solving skills. | English | +| [mlqa](mlqa/README.md) | MultiLingual Question Answering benchmark dataset for evaluating cross-lingual question answering performance. | English, Arabic, German, Spanish, Hindi, Vietnamese, Simplified Chinese | +| [mmlu](mmlu/README.md) | Massive Multitask Language Understanding benchmark for broad domain language evaluation. Several variants are supported. | English | +| [mmlu_pro](mmlu_pro/README.md) | A refined set of MMLU, integrating more challenging, reasoning-focused questions and expanding the choice set from four to ten options. | English | +| [mmlu-pro-plus](mmlu-pro-plus/README.md) | A new test set for evaluating shortcut learning and higher-order reasoning of LLMs. | English | +| [mmlu_prox](mmlu_prox/README.md) | A multilingual benchmark that extends MMLU-Pro to multiple typologically diverse languages with human validation. | English, Japanese, Chinese, Korean, French, German, Spanish, Portuguese, Swahili, Thai, Arabic, Hindi, Bengali | +| [mmlusr](mmlusr/README.md) | Variation of MMLU designed to be more rigorous. | English | +| model_written_evals | Evaluation tasks auto-generated for evaluating a collection of AI Safety concerns. | | +| [moral_stories](moral_stories/README.md) | A crowd-sourced dataset of structured narratives that describe normative and norm-divergent actions taken by individuals to accomplish certain intentions in concrete situations. | English | +| [mts_dialog](mts_dialog/README.md) | Open-ended healthcare QA from the MTS-Dialog dataset. | English | +| [mutual](mutual/README.md) | A retrieval-based dataset for multi-turn dialogue reasoning. | English | +| [noreval](noreval/README.md) | A human-created Norwegian language understanding and generation benchmark. | Norwegian (Bokmål and Nynorsk) | +| [nq_open](nq_open/README.md) | Open domain question answering tasks based on the Natural Questions dataset. | English | +| [okapi/arc_multilingual](okapi/arc_multilingual/README.md) | Tasks that involve reading comprehension and information retrieval challenges. | Multiple (31 languages) **Machine Translated.** | +| [okapi/hellaswag_multilingual](okapi/hellaswag_multilingual/README.md) | Tasks that involve reading comprehension and information retrieval challenges. | Multiple (30 languages) **Machine Translated.** | +| okapi/mmlu_multilingual | Tasks that involve reading comprehension and information retrieval challenges. | Multiple (34 languages) **Machine Translated.** | +| [okapi/truthfulqa_multilingual](okapi/truthfulqa_multilingual/README.md) | Tasks that involve reading comprehension and information retrieval challenges. | Multiple (31 languages) **Machine Translated.** | +| [olaph](olaph/README.md) | Open-ended medical factuality Question Answering from the OLAPH dataset. | English | +| [openbookqa](openbookqa/README.md) | Open-book question answering tasks that require external knowledge and reasoning. | English | +| [paloma](paloma/README.md) | Paloma is a comprehensive benchmark designed to evaluate open language models across a wide range of domains, ranging from niche artist communities to mental health forums on Reddit. | English | +| [paws-x](paws-x/README.md) | Paraphrase Adversaries from Word Scrambling, focusing on cross-lingual capabilities. | English, French, Spanish, German, Chinese, Japanese, Korean | +| [pile](pile/README.md) | Open source language modelling data set that consists of 22 smaller, high-quality datasets. | English | +| [pile_10k](pile_10k/README.md) | The first 10K elements of The Pile, useful for debugging models trained on it. | English | +| [piqa](piqa/README.md) | Physical Interaction Question Answering tasks to test physical commonsense reasoning. | English | +| [polemo2](polemo2/README.md) | Sentiment analysis and emotion detection tasks based on Polish language data. | Polish | +| [portuguese_bench](portuguese_bench/README.md) | Collection of tasks in European Portuguese encompassing various evaluation areas. | Portuguese | +| [prost](prost/README.md) | Tasks requiring understanding of professional standards and ethics in various domains. | English | +| [pubmedqa](pubmedqa/README.md) | Question answering tasks based on PubMed research articles for biomedical understanding. | English | +| [qa4mre](qa4mre/README.md) | Question Answering for Machine Reading Evaluation, assessing comprehension and reasoning. | English | +| [qasper](qasper/README.md) | Question Answering dataset based on academic papers, testing in-depth scientific knowledge. | English | +| [race](race/README.md) | Reading comprehension assessment tasks based on English exams in China. | English | +| realtoxicityprompts | Tasks to evaluate language models for generating text with potential toxicity. | | +| [ruler](ruler/README.md) | RULER is a benchmark for testing how well language models handle long pieces of text. Requires custom arg (see readme) | English | +| [sciq](sciq/README.md) | Science Question Answering tasks to assess understanding of scientific concepts. | English | +| [score](score/README.md) | Systematic consistency and robustness evaluation for LLMs on 3 datasets(MMLU-Pro, Agi Eval and MATH) | English | +| [scrolls](scrolls/README.md) | Tasks that involve long-form reading comprehension across various domains. | English | +| [simple_cooccurrence_bias](simple_cooccurrence_bias/README.md) | A metric that evaluates language models for biases based on stereotypical word associations and co-occurrences in text. | English | +| [siqa](siqa/README.md) | Social Interaction Question Answering to evaluate common sense and social reasoning. | English | +| [spanish_bench](spanish_bench/README.md) | Collection of tasks in Spanish encompassing various evaluation areas. | Spanish | +| [squad_completion](squad_completion/README.md) | A variant of the SQuAD question answering task designed for zero-shot evaluation of small LMs. | English | +| [squadv2](squadv2/README.md) | Stanford Question Answering Dataset version 2, a reading comprehension benchmark. | English | +| [storycloze](storycloze/README.md) | Tasks to predict story endings, focusing on narrative logic and coherence. | English | +| [super_glue](super_glue/README.md) | A suite of challenging tasks designed to test a range of language understanding skills. | English | +| [swag](swag/README.md) | Situations With Adversarial Generations, predicting the next event in videos. | English | +| [swde](swde/README.md) | Information extraction tasks from semi-structured web pages. | English | +| [tinyBenchmarks](tinyBenchmarks/README.md) | Evaluation of large language models with fewer examples using tiny versions of popular benchmarks. | English | +| [tmmluplus](tmmluplus/README.md) | An extended set of tasks under the TMMLU framework for broader academic assessments. | Traditional Chinese | +| [toxigen](toxigen/README.md) | Tasks designed to evaluate language models on their propensity to generate toxic content. | English | +| [translation](translation/README.md) | Tasks focused on evaluating the language translation capabilities of models. | Arabic, English, Spanish, Basque, Hindi, Indonesian, Burmese, Russian, Swahili, Telugu, Chinese | +| [triviaqa](triviaqa/README.md) | A large-scale dataset for trivia question answering to test general knowledge. | English | +| [truthfulqa](truthfulqa/README.md) | A QA task aimed at evaluating the truthfulness and factual accuracy of model responses. | English | +| [turkishmmlu](turkishmmlu/README.md) | A multiple-choice QA test modeled after MMLU, written in Turkish based on Turkish high-school level exams. | Turkish | +| [unitxt](unitxt/README.md) | A number of tasks implemented using the unitxt library for flexible, shareable, and reusable data preparation and evaluation for generative AI. | English | +| [unscramble](unscramble/README.md) | Tasks involving the rearrangement of scrambled sentences to test syntactic understanding. | English | +| [webqs](webqs/README.md) | Web-based question answering tasks designed to evaluate internet search and retrieval. | English | +| [wikitext](wikitext/README.md) | Tasks based on text from Wikipedia articles to assess language modeling and generation. | English | +| [winogender](winogender/README.md) | A diagnostic dataset that tests for gender bias in coreference resolution by measuring how models associate pronouns with different occupations. | English | +| [winogrande](winogrande/README.md) | A large-scale dataset for coreference resolution, inspired by the Winograd Schema Challenge. | English | +| [wmdp](wmdp/README.md) | A benchmark with the objective of minimizing performance, based on potentially-sensitive multiple-choice knowledge questions. | English | +| [wmt2016](wmt2016/README.md) | Tasks from the WMT 2016 shared task, focusing on translation between multiple languages. | English, Czech, German, Finnish, Russian, Romanian, Turkish | +| [wsc273](wsc273/README.md) | The Winograd Schema Challenge, a test of commonsense reasoning and coreference resolution. | English | +| [xcopa](xcopa/README.md) | Cross-lingual Choice of Plausible Alternatives, testing reasoning in multiple languages. | Estonian, Haitian, Indonesian, Italian, Quechua, Swahili, Tamil, Thai, Turkish, Vietnamese, Chinese | | [xnli](xnli/README.md) | Cross-Lingual Natural Language Inference to test understanding across different languages. | Arabic, Bulgarian, German, Greek, English, Spanish, French, Hindi, Russian, Swahili, Thai, Turkish, Urdu, Vietnamese, Chinese | -| [xnli_eu](xnli_eu/README.md) | Cross-lingual Natural Language Inference tasks in Basque. | Basque | -| [xquad](xquad/README.md) | Cross-lingual Question Answering Dataset in multiple languages. | Arabic, German, Greek, English, Spanish, Hindi, Romanian, Russian, Thai, Turkish, Vietnamese, Chinese | -| [xstorycloze](xstorycloze/README.md) | Cross-lingual narrative understanding tasks to predict story endings in multiple languages. | Russian, Simplified Chinese, Spanish, Arabic, Hindi, Indonesian, Telugu, Swahili, Basque, Burmese | -| [xwinograd](xwinograd/README.md) | Cross-lingual Winograd schema tasks for coreference resolution in multiple languages. | English, French, Japanese, Portuguese, Russian, Chinese | +| [xnli_eu](xnli_eu/README.md) | Cross-lingual Natural Language Inference tasks in Basque. | Basque | +| [xquad](xquad/README.md) | Cross-lingual Question Answering Dataset in multiple languages. | Arabic, German, Greek, English, Spanish, Hindi, Romanian, Russian, Thai, Turkish, Vietnamese, Chinese | +| [xstorycloze](xstorycloze/README.md) | Cross-lingual narrative understanding tasks to predict story endings in multiple languages. | Russian, Simplified Chinese, Spanish, Arabic, Hindi, Indonesian, Telugu, Swahili, Basque, Burmese | +| [xwinograd](xwinograd/README.md) | Cross-lingual Winograd schema tasks for coreference resolution in multiple languages. | English, French, Japanese, Portuguese, Russian, Chinese | ## Multimodal Tasks | Task Family | Description | Modality | diff --git a/lm_eval/tasks/noreval/README.md b/lm_eval/tasks/noreval/README.md new file mode 100644 index 00000000..b7338577 --- /dev/null +++ b/lm_eval/tasks/noreval/README.md @@ -0,0 +1,84 @@ +# 🇳🇴 NorEval + +### Paper + +* Title: `NorEval: A Norwegian Language Understanding and Generation Evaluation Benchmark` +* Abstract: [arxiv.org/abs/2504.07749](https://arxiv.org/abs/2504.07749) +* Homepage: [github.com/ltgoslo/noreval](https://github.com/ltgoslo/noreval/tree/main) + +![noreval](noreval.jpg) + +**Overview of the NorEval design.** 😼 denotes datasets used in [NorBench](https://aclanthology.org/2023.nodalida-1.61/), [NLEBench](https://aclanthology.org/2024.emnlp-main.317/), [ScandEval](https://aclanthology.org/2023.nodalida-1.20/), and [SEB](https://proceedings.neurips.cc/paper_files/paper/2024/file/4746bb91bd073ec7eef930d5775122ba-Paper-Datasets_and_Benchmarks_Track.pdf); 🚀 represents datasets that have not been used in the existing Norwegian benchmarks; and 😎 denotes our novel datasets introduced as part of NorEval. EN=English; BM=Norwegian Bokmål; NN=Norwegian Nynorsk. + +🇳🇴 NorEval is a multi-task Norwegian language understanding and generation evaluation benchmark that combines 19 existing peer-reviewed datasets with five datasets created from scratch. NorEval covers nine diverse task categories: sentiment analysis, Norwegian language knowledge, Norwegian-specific \& world knowledge, machine reading comprehension, commonsense reasoning, machine translation, text summarization, instruction following, and truthfulness. Our main evaluation principles are: + +- 🌐 **Linguistic diversity**: support for both of the official written standards of Norwegian: Bokmål and Nynorsk (the minority variant). +- 📊 **Task diversity**: coverage of various least addressed tasks for Norwegian. In particular, only three out of 24 NorEval datasets are included in existing Norwegian benchmarks to date: [NorBench](https://aclanthology.org/2023.nodalida-1.61/), [NLEBench](https://aclanthology.org/2024.emnlp-main.317/), [ScandEval](https://aclanthology.org/2023.nodalida-1.20/), and [SEB](https://proceedings.neurips.cc/paper_files/paper/2024/file/4746bb91bd073ec7eef930d5775122ba-Paper-Datasets_and_Benchmarks_Track.pdf). +- 🧠 **Data quality**: focus on only peer-reviewed human-created datasets to ensure reliable evaluation in the context of the Norwegian language, culture, and values. +- 📏 **Prompt sensitivity**: evaluation across 100+ human-written prompts to account for the prompt sensitivity. +- 👩🏻‍🔬 **Standardized evaluation**: integration of NorEval into LM Evaluation Harness for flexible and reproducible evaluation. + + +### Tasks + +|Name |Bokmål | Nynorsk |*k*-shot | Task type | Task category | +|:---|:---|:---|:---|:---|:---| +|[NoReC Sentence](https://huggingface.co/datasets/ltg/norec_sentence) |```norec_sentence``` | ❌ |✅ |Text classification| Sentiment analysis | +|[NoReC Document](https://huggingface.co/datasets/ltg/norec_document) |```norec_document``` | ❌ |✅ |Text classification| Sentiment analysis | +|[NCB](https://huggingface.co/datasets/hcfa/ncb) |```ncb```| ❌ | ❌ |Sentence ranking| Norwegian language knowledge | +|[NorIdiom](https://huggingface.co/datasets/Sprakbanken/Norwegian_idioms) |```noridiom_nob``` | ```noridiom_nno``` | ❌ |Sentence completion| Norwegian language knowledge | +|[Belebele](https://huggingface.co/datasets/facebook/belebele) |```norbelebele```| ❌|❌ |Multiple-choice question answering| Machine reading comprehension | +|[NRK-Quiz-QA](https://huggingface.co/datasets/ltg/nrk_quiz_qa) |```nrk_quiz_qa_nob```| ```nrk_quiz_qa_nno```| ❌ |Multiple-choice question answering| Norwegian-specific & world knowledge | +|[NorOpenBookQA](https://huggingface.co/datasets/ltg/noropenbookqa) |```noropenbookqa_nob```| ```noropenbookqa_nno``` |✅ |Multiple-choice question answering| Norwegian-specific & world knowledge | +|[NorCommonsenseQA](https://huggingface.co/datasets/ltg/norcommonsenseqa) |```norcommonsenseqa_nob```| ```norcommonsenseqa_nno``` |❌ |Multiple-choice question answering|Commonsense reasoning | +|[NorTruthfulQA Multiple choice](https://huggingface.co/datasets/ltg/nortruthfulqa_mc) |```nortruthfulqa_mc_nob```| ```nortruthfulqa_mc_nno``` |❌ |Multiple-choice question answering |Truthfulness | +|[NorQuAD](https://huggingface.co/datasets/ltg/norquad) |```norquad```| ❌ | ✅ |Generative question answering |Machine reading comprehension | +|[NorTruthfulQA Generation](https://huggingface.co/datasets/ltg/nortruthfulqa_gen) |```nortruthfulqa_gen_nob```| ```nortruthfulqa_gen_nno``` | ❌ | Generative question answering|Truthfulness | +|[ASK-GEC](https://huggingface.co/datasets/ltg/ask-gec) |```ask_gec```| ❌ |✅ |Sequence-to-sequence generation|Norwegian language knowledge | +|[NorSumm](https://huggingface.co/datasets/SamiaT/NorSumm) |```norsumm_nob``` | ```norsumm_nno``` |✅ |Sequence-to-sequence generation|Text summarization | +|[Tatoeba (English → Bokmål/Nynorsk)](https://huggingface.co/datasets/Helsinki-NLP/tatoeba_mt) | ```tatoeba_eng_nob```| ```tatoeba_eng_nno``` |✅ |Sequence-to-sequence generation|Machine translation | +|[Tatoeba (Bokmål/Nynorsk → English)](https://huggingface.co/datasets/Helsinki-NLP/tatoeba_mt) | ```tatoeba_nob_eng```| ```tatoeba_nno_eng``` |✅ |Sequence-to-sequence generation|Machine translation | +|[NorRewrite-Instruct](https://huggingface.co/datasets/ltg/norrewrite-instruct) |```norrewrite_instruct``` |❌ |❌ |Sequence-to-sequence generation|Instruction following| +|[NorSummarize-Instruct](https://huggingface.co/datasets/ltg/norsummarize-instruct) |```norsummarize_instruct``` |❌ |❌ |Sequence-to-sequence generation|Instruction following| + +
+Table description + +* **Name**: a dataset name with a HuggingFace link. +* **Bokmål**: the LM Evaluation Harness task name for the Norwegian Bokmål dataset. +* **Nynorsk**: the LM Evaluation Harness task name for the Norwegian Nynorsk dataset, if available. +* **k-shot**: the support for *k*-shot evaluation regimes with *k* > 0. We follow the original datasets' design and focus mainly on the zero-shot evaluation by default. + * ✅ means that the user can run the evaluation in both zero-shot and *k*-shot regimes. + * ❌ denotes that only the zero-shot evaluation regime is available due to the lack of the training or validation set to sample the demonstration examples from. Technically, *k*-shot evaluation on the test set is possible using sampling without replacement, given that the model is not proprietary and not accessed via an API. +* **Task type**: the task type. +* **Task category**: the task category. + +
+ +##### Comments on Belebele +Belebele for Norwegian Bokmål is already available in LM Evaluation Harness as `belebele_nob_Latn`. However, our version (`norbelebele`) supports five prompt templates written by Norwegian native speakers, which are different from the default prompt template used in Belebele. + + + +### Citation + +``` +@article{mikhailov2025noreval, + title={NorEval: A Norwegian Language Understanding and Generation Evaluation Benchmark}, + author={Mikhailov, Vladislav and Enstad, Tita and Samuel, David and Farseth{\aa}s, Hans Christian and Kutuzov, Andrey and Velldal, Erik and {\O}vrelid, Lilja}, + journal={arXiv preprint arXiv:2504.07749}, + year={2025} +} +``` + +### Checklist + +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [ ] If yes, does the original paper provide a reference implementation? + * [ ] Yes, original implementation contributed by author of the benchmark + +If other tasks on this dataset are already supported: +* [x] Is the "Main" variant of this task clearly denoted? +* [x] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/lm_eval/tasks/noreval/ask_gec/README.md b/lm_eval/tasks/noreval/ask_gec/README.md new file mode 100644 index 00000000..35de80b3 --- /dev/null +++ b/lm_eval/tasks/noreval/ask_gec/README.md @@ -0,0 +1,28 @@ +### Evaluation example + +Here, we use the `--predict_only` argument and compute the performance metrics as described below. + +**Step 1: Generate the predictions** + +```bash +lm_eval \ + --model hf \ + --model_args pretrained=AI-Sweden-Models/Llama-3-8B \ + --tasks ask_gec \ + --output results/ask_gec/0-shot/ \ + --log_samples \ + --show_config \ + --write_out \ + --predict_only \ + --batch_size auto \ + --num_fewshot 0 +``` + +**Step 2: Evaluate the predictions with ERRANT** + +* Please refer to the installation instructions [here](https://github.com/chrisjbryant/errant/tree/main). +* Run the following: + ```bash + python3 ask_gec/errant.py --fpath results/ask_gec/0-shot/AI-Sweden-Models__Llama-3-8B/samples_ask_gec_p0_2025-01-28T01-08-13.454441.jsonl --out_fdir results/ask_gec/0-shot/AI-Sweden-Models__Llama-3-8B/ + ``` +* The results will be saved as `results/ask_gec/0-shot/AI-Sweden-Models__Llama-3-8B/samples_ask_gec_p0_2025-01-28T01-08-13.454441_errant.json` diff --git a/lm_eval/tasks/noreval/ask_gec/_ask_gec_yaml b/lm_eval/tasks/noreval/ask_gec/_ask_gec_yaml new file mode 100644 index 00000000..deffb7c9 --- /dev/null +++ b/lm_eval/tasks/noreval/ask_gec/_ask_gec_yaml @@ -0,0 +1,15 @@ +tag: ask_gec +dataset_path: ltg/ask-gec +output_type: generate_until +training_split: train +validation_split: validation +test_split: test +doc_to_target: correction +generation_kwargs: + until: + - "\n" + do_sample: false + num_beams: 1 + max_new_tokens: 256 +metadata: + version: 1.0 diff --git a/lm_eval/tasks/noreval/ask_gec/ask_gec_p0.yaml b/lm_eval/tasks/noreval/ask_gec/ask_gec_p0.yaml new file mode 100644 index 00000000..383160bc --- /dev/null +++ b/lm_eval/tasks/noreval/ask_gec/ask_gec_p0.yaml @@ -0,0 +1,3 @@ +task: ask_gec_p0 +doc_to_text: "Tekst: {{source}}\nKorreksjon:" +include: _ask_gec_yaml diff --git a/lm_eval/tasks/noreval/ask_gec/ask_gec_p1.yaml b/lm_eval/tasks/noreval/ask_gec/ask_gec_p1.yaml new file mode 100644 index 00000000..24f176c3 --- /dev/null +++ b/lm_eval/tasks/noreval/ask_gec/ask_gec_p1.yaml @@ -0,0 +1,3 @@ +task: ask_gec_p1 +doc_to_text: "Tekst: {{source}}\nRettet versjon:" +include: _ask_gec_yaml diff --git a/lm_eval/tasks/noreval/ask_gec/ask_gec_p2.yaml b/lm_eval/tasks/noreval/ask_gec/ask_gec_p2.yaml new file mode 100644 index 00000000..e8da2018 --- /dev/null +++ b/lm_eval/tasks/noreval/ask_gec/ask_gec_p2.yaml @@ -0,0 +1,3 @@ +task: ask_gec_p2 +doc_to_text: "Skriv om følgende tekst slik at den blir grammatisk korrekt: {{source}}\nKorreksjon:" +include: _ask_gec_yaml diff --git a/lm_eval/tasks/noreval/ask_gec/ask_gec_p3.yaml b/lm_eval/tasks/noreval/ask_gec/ask_gec_p3.yaml new file mode 100644 index 00000000..def64d44 --- /dev/null +++ b/lm_eval/tasks/noreval/ask_gec/ask_gec_p3.yaml @@ -0,0 +1,3 @@ +task: ask_gec_p3 +doc_to_text: "Original versjon: {{source}}\nKorrekturlest og rettet versjon:" +include: _ask_gec_yaml diff --git a/lm_eval/tasks/noreval/ask_gec/ask_gec_p4.yaml b/lm_eval/tasks/noreval/ask_gec/ask_gec_p4.yaml new file mode 100644 index 00000000..81c04c6c --- /dev/null +++ b/lm_eval/tasks/noreval/ask_gec/ask_gec_p4.yaml @@ -0,0 +1,3 @@ +task: ask_gec_p4 +doc_to_text: "Rett opp grammatiske feil i denne teksten: {{source}}\nKorreksjon:" +include: _ask_gec_yaml diff --git a/lm_eval/tasks/noreval/ask_gec/errant.py b/lm_eval/tasks/noreval/ask_gec/errant.py new file mode 100644 index 00000000..89721659 --- /dev/null +++ b/lm_eval/tasks/noreval/ask_gec/errant.py @@ -0,0 +1,106 @@ +import argparse +import json +import os +import subprocess + +import pandas as pd + + +def parse_args(): + """ + Parses arguments. + Returns: + Arguments containing the names of the prediction file and the file directory to for saving the evaluation results. + """ + parser = argparse.ArgumentParser() + parser.add_argument( + "--fpath", + type=str, + help="path to a model output file in the lm-evaluation-harness format.", + ) + parser.add_argument( + "--out_fdir", + type=str, + help="path to an output directory for saving the results.", + ) + args = parser.parse_args() + return args + + +def read_examples(fpath: str): + """ + Reads examples from the prediction file. + Args: + fpath: A path to the prediction file. + Returns: + Lists of the sources, targets, and predictions. + """ + examples = pd.read_json(fpath, lines=True) + sources, targets, predictions = [], [], [] + for i, example in examples.iterrows(): + sources.append(example["doc"]["source"]) + targets.append(example["doc"]["correction"]) + predictions.append(example["resps"][0][0].replace("\n\n", "\n")) + return sources, targets, predictions + + +def save_results(fpath: str, obj: dict): + """ + Saves the evaluation results. + Args: + fpath: A path for the output file for saving the results. + obj: The evaluation results. + """ + with open(fpath, "w+", encoding="utf-8") as out: + json.dump(obj, out, indent=3) + + +def evaluate(fpath: str, out_fpath: str): + """ + Runs the evaluation based on the ERRANT performance metric. + Args: + fpath: A path to the prediction file. + out_Fpath: A path for the output file for saving the results. + """ + tmp_name = fpath.replace(".jsonl", "").replace("/", "-") + os.makedirs("tmp", exist_ok=True) + sources, targets, predictions = read_examples(fpath=fpath) + with open(f"tmp/{tmp_name}_sources.txt", "w+") as f: + f.write("\n".join(sources)) + with open(f"tmp/{tmp_name}_targets.txt", "w+") as f: + f.write("\n".join(targets)) + with open(f"tmp/{tmp_name}_predictions.txt", "w+") as f: + f.write("\n".join(predictions)) + subprocess.run( + f"errant_parallel -orig tmp/{tmp_name}_sources.txt -cor tmp/{tmp_name}_targets.txt -out tmp/{tmp_name}_targets.m2 -lev -tok", + shell=True, + ) + subprocess.run( + f"errant_parallel -orig tmp/{tmp_name}_sources.txt -cor tmp/{tmp_name}_predictions.txt -out tmp/{tmp_name}_predictions.m2 -lev -tok", + shell=True, + ) + output = subprocess.check_output( + f"errant_compare -ref tmp/{tmp_name}_targets.m2 -hyp tmp/{tmp_name}_predictions.m2", + shell=True, + ) + f_05 = float(output.decode().strip().split("\n")[-2].split()[-1].strip()) + print(f"Prediction fpath: {fpath}\n\nERRANT: {f_05}", flush=True) + print(f"Saving to: {out_fpath}", flush=True) + save_results(obj={"errant": f_05}, fpath=out_fpath) + subprocess.run(f"rm tmp/{tmp_name}_*", shell=True) + + +def main(): + args = parse_args() + fpath = args.fpath + print(f"Out: {args.out_fdir}", flush=True) + out_fpath = fpath.replace(".jsonl", "_errant.json") + evaluate(fpath=fpath, out_fpath=out_fpath) + + +if __name__ == "__main__": + print( + "\nWARNING: make sure you have ERRANT installed to run the evaluation! Available here: https://github.com/chrisjbryant/errant\n\n", + flush=True, + ) + main() diff --git a/lm_eval/tasks/noreval/ncb/ncb.yaml b/lm_eval/tasks/noreval/ncb/ncb.yaml new file mode 100644 index 00000000..0150c25a --- /dev/null +++ b/lm_eval/tasks/noreval/ncb/ncb.yaml @@ -0,0 +1,13 @@ +task: ncb +dataset_path: hcfa/ncb +output_type: multiple_choice +test_split: train +doc_to_text: "" +doc_to_target: 0 +doc_to_choice: "{{[correct, wrong]}}" +num_fewshot: 0 +metric_list: + - metric: acc + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/noreval/norbelebele/_norbelebele_yaml b/lm_eval/tasks/noreval/norbelebele/_norbelebele_yaml new file mode 100644 index 00000000..41f6b113 --- /dev/null +++ b/lm_eval/tasks/noreval/norbelebele/_norbelebele_yaml @@ -0,0 +1,18 @@ +tag: norbelebele +dataset_path: facebook/belebele +dataset_name: nob_Latn +test_split: test +fewshot_split: test +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_target: "{{['1', '2', '3', '4'].index(correct_answer_num)}}" +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/noreval/norbelebele/norbelebele_p0.yaml b/lm_eval/tasks/noreval/norbelebele/norbelebele_p0.yaml new file mode 100644 index 00000000..a1037ef7 --- /dev/null +++ b/lm_eval/tasks/noreval/norbelebele/norbelebele_p0.yaml @@ -0,0 +1,4 @@ +task: norbelebele_p0 +include: _norbelebele_yaml +doc_to_text: "Tekst: {{flores_passage}}\nSpørsmål: {{question}}\nA: {{mc_answer1}}\nB: {{mc_answer2}}\nC: {{mc_answer3}}\nD: {{mc_answer4}}\nSvar:" +doc_to_choice: ["A", "B", "C", "D"] diff --git a/lm_eval/tasks/noreval/norbelebele/norbelebele_p1.yaml b/lm_eval/tasks/noreval/norbelebele/norbelebele_p1.yaml new file mode 100644 index 00000000..43cc76c0 --- /dev/null +++ b/lm_eval/tasks/noreval/norbelebele/norbelebele_p1.yaml @@ -0,0 +1,4 @@ +task: norbelebele_p1 +include: _norbelebele_yaml +doc_to_text: "Bakgrunn: {{flores_passage}}\nSpørsmål:{{question}}\nSvaralternativer:\n- {{mc_answer1}}\n- {{mc_answer2}}\n- {{mc_answer3}}\n- {{mc_answer4}}\nRiktig svar:" +doc_to_choice: "{{[mc_answer1, mc_answer2, mc_answer3, mc_answer4]}}" diff --git a/lm_eval/tasks/noreval/norbelebele/norbelebele_p2.yaml b/lm_eval/tasks/noreval/norbelebele/norbelebele_p2.yaml new file mode 100644 index 00000000..8ff96f0e --- /dev/null +++ b/lm_eval/tasks/noreval/norbelebele/norbelebele_p2.yaml @@ -0,0 +1,4 @@ +task: norbelebele_p2 +include: _norbelebele_yaml +doc_to_text: "{{question}}\nHvilket av følgende mulige svar er det riktige?\nA: {{mc_answer1}}\nB: {{mc_answer2}}\nC: {{mc_answer3}}\nD: {{mc_answer4}}\nSvar:" +doc_to_choice: ["A", "B", "C", "D"] diff --git a/lm_eval/tasks/noreval/norbelebele/norbelebele_p3.yaml b/lm_eval/tasks/noreval/norbelebele/norbelebele_p3.yaml new file mode 100644 index 00000000..6fedf28a --- /dev/null +++ b/lm_eval/tasks/noreval/norbelebele/norbelebele_p3.yaml @@ -0,0 +1,5 @@ +task: norbelebele_p3 +include: _norbelebele_yaml +doc_to_text: "Svar på følgende spørsmål: {{question}}\nSvaret skal baseres på følgende tekst:\n{{flores_passage}}\nVelg et svar fra denne listen:\n– {{mc_answer1}}\n– {{mc_answer2}},\n– {{mc_answer3}}\n– {{mc_answer4}}" +doc_to_choice: "{{[mc_answer1, mc_answer2, mc_answer3, mc_answer4]}}" +target_delimiter: "\n" diff --git a/lm_eval/tasks/noreval/norbelebele/norbelebele_p4.yaml b/lm_eval/tasks/noreval/norbelebele/norbelebele_p4.yaml new file mode 100644 index 00000000..0285168e --- /dev/null +++ b/lm_eval/tasks/noreval/norbelebele/norbelebele_p4.yaml @@ -0,0 +1,4 @@ +task: norbelebele_p4 +include: _norbelebele_yaml +doc_to_text: "{{flores_passage}}\n\n{{question}}\n\nA: {{mc_answer1}}\nB: {{mc_answer2}}\nC: {{mc_answer3}}\nD: {{mc_answer4}}\n\nEr det riktige svaret A, B, C, eller D?" +doc_to_choice: ["A", "B", "C", "D"] diff --git a/lm_eval/tasks/noreval/norcommonsenseqa/_norcommonsenseqa_yaml b/lm_eval/tasks/noreval/norcommonsenseqa/_norcommonsenseqa_yaml new file mode 100644 index 00000000..a17c01fe --- /dev/null +++ b/lm_eval/tasks/noreval/norcommonsenseqa/_norcommonsenseqa_yaml @@ -0,0 +1,15 @@ +dataset_path: ltg/norcommonsenseqa +output_type: multiple_choice +training_split: null +validation_split: null +test_split: train +doc_to_target: "{{choices.label.index(answer)}}" +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/noreval/norcommonsenseqa/nno/norcommonsenseqa_nno_p0.yaml b/lm_eval/tasks/noreval/norcommonsenseqa/nno/norcommonsenseqa_nno_p0.yaml new file mode 100644 index 00000000..06e59c2c --- /dev/null +++ b/lm_eval/tasks/noreval/norcommonsenseqa/nno/norcommonsenseqa_nno_p0.yaml @@ -0,0 +1,6 @@ +tag: norcommonsenseqa_nno +dataset_name: nn +task: norcommonsenseqa_nno_p0 +include: ../_norcommonsenseqa_yaml +doc_to_text: "Spørsmål: {{question}}\n\nSvar:" +doc_to_choice: "{{choices.text}}" diff --git a/lm_eval/tasks/noreval/norcommonsenseqa/nno/norcommonsenseqa_nno_p1.yaml b/lm_eval/tasks/noreval/norcommonsenseqa/nno/norcommonsenseqa_nno_p1.yaml new file mode 100644 index 00000000..db7ac9d9 --- /dev/null +++ b/lm_eval/tasks/noreval/norcommonsenseqa/nno/norcommonsenseqa_nno_p1.yaml @@ -0,0 +1,6 @@ +tag: norcommonsenseqa_nno +dataset_name: nn +task: norcommonsenseqa_nno_p1 +include: ../_norcommonsenseqa_yaml +doc_to_text: "{{question}}\nKva av følgande moglege svar er det rette?\nA: {{choices.text[0]}}\nB: {{choices.text[1]}}\nC: {{choices.text[2]}}\nD: {{choices.text[3]}}\nE: {{choices.text[4]}}\n\nSvar:" +doc_to_choice: "{{choices.label}}" diff --git a/lm_eval/tasks/noreval/norcommonsenseqa/nno/norcommonsenseqa_nno_p2.yaml b/lm_eval/tasks/noreval/norcommonsenseqa/nno/norcommonsenseqa_nno_p2.yaml new file mode 100644 index 00000000..2477eca8 --- /dev/null +++ b/lm_eval/tasks/noreval/norcommonsenseqa/nno/norcommonsenseqa_nno_p2.yaml @@ -0,0 +1,6 @@ +tag: norcommonsenseqa_nno +dataset_name: nn +task: norcommonsenseqa_nno_p2 +include: ../_norcommonsenseqa_yaml +doc_to_text: "Gitt alternativa under, kva er svaret på følgande spørsmål: {{question}}\n\nAlternativ:\n- {{choices.text[0]}}\n- {{choices.text[1]}}\n- {{choices.text[2]}}\n- {{choices.text[3]}}\n- {{choices.text[4]}}\n\nSvar:" +doc_to_choice: "{{choices.label}}" diff --git a/lm_eval/tasks/noreval/norcommonsenseqa/nno/norcommonsenseqa_nno_p3.yaml b/lm_eval/tasks/noreval/norcommonsenseqa/nno/norcommonsenseqa_nno_p3.yaml new file mode 100644 index 00000000..f70bd109 --- /dev/null +++ b/lm_eval/tasks/noreval/norcommonsenseqa/nno/norcommonsenseqa_nno_p3.yaml @@ -0,0 +1,6 @@ +tag: norcommonsenseqa_nno +dataset_name: nn +task: norcommonsenseqa_nno_p3 +include: ../_norcommonsenseqa_yaml +doc_to_text: "{{question}}\nVel rett svar blant desse alternativa:\n– {{choices.text[0]}}\n– {{choices.text[1]}}\n– {{choices.text[2]}}\n– {{choices.text[3]}}\n– {{choices.text[4]}}\n\nSvar:" +doc_to_choice: "{{choices.text}}" diff --git a/lm_eval/tasks/noreval/norcommonsenseqa/nno/norcommonsenseqa_nno_p4.yaml b/lm_eval/tasks/noreval/norcommonsenseqa/nno/norcommonsenseqa_nno_p4.yaml new file mode 100644 index 00000000..fa8711c1 --- /dev/null +++ b/lm_eval/tasks/noreval/norcommonsenseqa/nno/norcommonsenseqa_nno_p4.yaml @@ -0,0 +1,6 @@ +tag: norcommonsenseqa_nno +dataset_name: nn +task: norcommonsenseqa_nno_p4 +include: ../_norcommonsenseqa_yaml +doc_to_text: "{{question}}\nA: {{choices.text[0]}}\nB: {{choices.text[1]}}\nC: {{choices.text[2]}}\nD: {{choices.text[3]}}\nE: {{choices.text[4]}}\n\nEr det rette svaret A, B, C, D, eller E?\n\nSvar:" +doc_to_choice: "{{choices.label}}" diff --git a/lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p0.yaml b/lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p0.yaml new file mode 100644 index 00000000..50430a86 --- /dev/null +++ b/lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p0.yaml @@ -0,0 +1,6 @@ +tag: norcommonsenseqa_nob +dataset_name: nb +task: norcommonsenseqa_nob_p0 +include: ../_norcommonsenseqa_yaml +doc_to_text: "Spørsmål: {{question}}\n\nSvar:" +doc_to_choice: "{{choices.text}}" diff --git a/lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p1.yaml b/lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p1.yaml new file mode 100644 index 00000000..a1c072d6 --- /dev/null +++ b/lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p1.yaml @@ -0,0 +1,6 @@ +tag: norcommonsenseqa_nob +dataset_name: nb +task: norcommonsenseqa_nob_p1 +include: ../_norcommonsenseqa_yaml +doc_to_text: "{{question}}\nHvilket av følgende mulige svar er det riktige?\nA: {{choices.text[0]}}\nB: {{choices.text[1]}}\nC: {{choices.text[2]}}\nD: {{choices.text[3]}}\nE: {{choices.text[4]}}\n\nSvar:" +doc_to_choice: "{{choices.label}}" diff --git a/lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p2.yaml b/lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p2.yaml new file mode 100644 index 00000000..bda34e86 --- /dev/null +++ b/lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p2.yaml @@ -0,0 +1,6 @@ +tag: norcommonsenseqa_nob +dataset_name: nb +task: norcommonsenseqa_nob_p2 +include: ../_norcommonsenseqa_yaml +doc_to_text: "Gitt alternativene under, hva er svaret på følgende spørsmål: {{question}}\n\nAlternativer:\n- {{choices.text[0]}}\n- {{choices.text[1]}}\n- {{choices.text[2]}}\n- {{choices.text[3]}}\n- {{choices.text[4]}}\n\nSvar:" +doc_to_choice: "{{choices.text}}" diff --git a/lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p3.yaml b/lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p3.yaml new file mode 100644 index 00000000..cc015bc8 --- /dev/null +++ b/lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p3.yaml @@ -0,0 +1,6 @@ +tag: norcommonsenseqa_nob +dataset_name: nb +task: norcommonsenseqa_nob_p3 +include: ../_norcommonsenseqa_yaml +doc_to_text: "{{question}}\nVelg riktig svar blant disse alternativene:\n– {{choices.text[0]}}\n– {{choices.text[1]}}\n– {{choices.text[2]}}\n– {{choices.text[3]}}\n– {{choices.text[4]}}\n\nSvar:" +doc_to_choice: "{{choices.text}}" diff --git a/lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p4.yaml b/lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p4.yaml new file mode 100644 index 00000000..6eda8246 --- /dev/null +++ b/lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p4.yaml @@ -0,0 +1,6 @@ +tag: norcommonsenseqa_nob +dataset_name: nb +task: norcommonsenseqa_nob_p4 +include: ../_norcommonsenseqa_yaml +doc_to_text: "{{question}}\nA: {{choices.text[0]}}\nB: {{choices.text[1]}}\nC: {{choices.text[2]}}\nD: {{choices.text[3]}}\nE: {{choices.text[4]}}\n\nEr det riktige svaret A, B, C, D, eller E?\n\nSvar:" +doc_to_choice: "{{choices.label}}" diff --git a/lm_eval/tasks/noreval/norec/_norec_yaml b/lm_eval/tasks/noreval/norec/_norec_yaml new file mode 100644 index 00000000..9a9cb884 --- /dev/null +++ b/lm_eval/tasks/noreval/norec/_norec_yaml @@ -0,0 +1,14 @@ +dataset_name: binary +output_type: multiple_choice +training_split: train +test_split: test +doc_to_target: sentiment +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: f1 + aggregation: !function utils.multi_f1 + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/noreval/norec/norec_document/norec_document_p0.yaml b/lm_eval/tasks/noreval/norec/norec_document/norec_document_p0.yaml new file mode 100644 index 00000000..8596a6b9 --- /dev/null +++ b/lm_eval/tasks/noreval/norec/norec_document/norec_document_p0.yaml @@ -0,0 +1,6 @@ +tag: norec_document +dataset_path: ltg/norec_document +task: norec_document_p0 +include: ../_norec_yaml +doc_to_text: "Tekst: {{review}}\nSentiment:" +doc_to_choice: ["negativ", "positiv"] diff --git a/lm_eval/tasks/noreval/norec/norec_document/norec_document_p1.yaml b/lm_eval/tasks/noreval/norec/norec_document/norec_document_p1.yaml new file mode 100644 index 00000000..0171cff5 --- /dev/null +++ b/lm_eval/tasks/noreval/norec/norec_document/norec_document_p1.yaml @@ -0,0 +1,6 @@ +tag: norec_document +dataset_path: ltg/norec_document +task: norec_document_p1 +include: ../_norec_yaml +doc_to_text: "Tekst: {{review}}\nEr anmeldelsen \"positiv\" eller \"negativ\"?" +doc_to_choice: ["negativ", "positiv"] diff --git a/lm_eval/tasks/noreval/norec/norec_document/norec_document_p2.yaml b/lm_eval/tasks/noreval/norec/norec_document/norec_document_p2.yaml new file mode 100644 index 00000000..7b90639d --- /dev/null +++ b/lm_eval/tasks/noreval/norec/norec_document/norec_document_p2.yaml @@ -0,0 +1,6 @@ +tag: norec_document +dataset_path: ltg/norec_document +task: norec_document_p2 +include: ../_norec_yaml +doc_to_text: "Er polariteten til følgende anmeldelse positiv eller negativ?\nAnmeldelse: {{review}}\nAnmeldelsen er" +doc_to_choice: ["negativ", "positiv"] diff --git a/lm_eval/tasks/noreval/norec/norec_document/norec_document_p3.yaml b/lm_eval/tasks/noreval/norec/norec_document/norec_document_p3.yaml new file mode 100644 index 00000000..c4053708 --- /dev/null +++ b/lm_eval/tasks/noreval/norec/norec_document/norec_document_p3.yaml @@ -0,0 +1,6 @@ +tag: norec_document +dataset_path: ltg/norec_document +task: norec_document_p3 +include: ../_norec_yaml +doc_to_text: "Anmeldelse: {{review}}\nEr anmelderen positiv eller negativ?" +doc_to_choice: ["negativ", "positiv"] diff --git a/lm_eval/tasks/noreval/norec/norec_document/norec_document_p4.yaml b/lm_eval/tasks/noreval/norec/norec_document/norec_document_p4.yaml new file mode 100644 index 00000000..59d26a1a --- /dev/null +++ b/lm_eval/tasks/noreval/norec/norec_document/norec_document_p4.yaml @@ -0,0 +1,6 @@ +tag: norec_document +dataset_path: ltg/norec_document +task: norec_document_p4 +include: ../_norec_yaml +doc_to_text: "Anmeldelse: {{review}}\nVil du oppsummere anmeldelsen som \"bra\" eller \"dårlig\"?" +doc_to_choice: ["dårlig", "bra"] diff --git a/lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p0.yaml b/lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p0.yaml new file mode 100644 index 00000000..90001d5b --- /dev/null +++ b/lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p0.yaml @@ -0,0 +1,6 @@ +tag: norec_sentence +dataset_path: ltg/norec_sentence +task: norec_sentence_p0 +include: ../_norec_yaml +doc_to_text: "Tekst: {{review}}\nSentiment:" +doc_to_choice: ["negativ", "positiv"] diff --git a/lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p1.yaml b/lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p1.yaml new file mode 100644 index 00000000..4c2a10fd --- /dev/null +++ b/lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p1.yaml @@ -0,0 +1,6 @@ +tag: norec_sentence +dataset_path: ltg/norec_sentence +task: norec_sentence_p1 +include: ../_norec_yaml +doc_to_text: "{{review}}\nEr denne setningen \"positiv\" eller \"negativ\"?" +doc_to_choice: ["negativ", "positiv"] diff --git a/lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p2.yaml b/lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p2.yaml new file mode 100644 index 00000000..8727e4cb --- /dev/null +++ b/lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p2.yaml @@ -0,0 +1,6 @@ +tag: norec_sentence +dataset_path: ltg/norec_sentence +task: norec_sentence_p2 +include: ../_norec_yaml +doc_to_text: "{{review}}\nHva slags sentiment uttrykker anmelderen?" +doc_to_choice: ["negativ", "positiv"] diff --git a/lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p3.yaml b/lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p3.yaml new file mode 100644 index 00000000..ba5e1e0f --- /dev/null +++ b/lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p3.yaml @@ -0,0 +1,6 @@ +tag: norec_sentence +dataset_path: ltg/norec_sentence +task: norec_sentence_p3 +include: ../_norec_yaml +doc_to_text: "{{review}}\nEr anmeldelsen \"positiv\" eller \"negativ\"?" +doc_to_choice: ["negativ", "positiv"] diff --git a/lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p4.yaml b/lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p4.yaml new file mode 100644 index 00000000..8fda6f48 --- /dev/null +++ b/lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p4.yaml @@ -0,0 +1,6 @@ +tag: norec_sentence +dataset_path: ltg/norec_sentence +task: norec_sentence_p4 +include: ../_norec_yaml +doc_to_text: "{{review}}\nEr denne setningen positiv eller negativ?" +doc_to_choice: ["negativ", "positiv"] diff --git a/lm_eval/tasks/noreval/norec/utils.py b/lm_eval/tasks/noreval/norec/utils.py new file mode 100644 index 00000000..41f4bce1 --- /dev/null +++ b/lm_eval/tasks/noreval/norec/utils.py @@ -0,0 +1,13 @@ +import numpy as np +import sklearn + + +def multi_f1(items): + """ + Computes the macro-average F1 score. + """ + preds, golds = zip(*items) + preds = np.array(preds) + golds = np.array(golds) + fscore = sklearn.metrics.f1_score(golds, preds, average="macro") + return fscore diff --git a/lm_eval/tasks/noreval/noreval.jpg b/lm_eval/tasks/noreval/noreval.jpg new file mode 100644 index 0000000000000000000000000000000000000000..69d156f7f8a43a505b10f81cd89c18a90e3d1b48 GIT binary patch literal 699330 zcmeFZc{tR6+c!QH*+NK)n2IbRl`ZSEA$*x+i-{?+PMU;l!<6hoS&Nt=jeQbB34^hG zCCNG&F=iCmGEQQx4-_n=YRM1*Ngn`KKDPKx77}j65at7a^11* zFl4*bwjEO2wz?ot2xOb!zdZ)>KW^K$?+_5&DI_c+Dh57)xEr#4+m0RE1$GDu3J8FY zjsgD<5s(s;K6K*DP8o-5LWhI*oV@qwxv=utx(?Y(LrfK2uiJ4VqI>1`?Uz5Is-~`S zR8QZ)@RX79x$_p5Ru`;o9G#q9Tra!1dtdkQ_45x13<(Xp6CM#6bw55KF)2AEH7h$O zH!uJ3lY)}c7iBN;l=uTO>f>dcapohdwT!udp|rfIyO%EJVB*1X1{)$oBzJB z$olzfZJoUV{NDV_u5FMV|6%&S8TNN}Nr8547ZBJXAoQ1A+qQ>;YloD8;Gq*crO!AB zT?>{ueDa>~p0kgh*L8>}>t15Ydfgrp-K(NUKf?OUw7*&Q-!m-kf6KD}Fzi3=qC$4< z*am)hJER~m2%oQBk_!1hzgh}s!+S`O*g=v+7%gQBA|2-V5(qzkKI2r-kl$EXtNYhe zBMssbeJ|2hsBL(y8+o6e9FE(U6}_~SW@kM7^XHU7R!~EhYtZ*BkIRJ)c8yuqjngSd z-rZO)&}1r-P?CUp_{z25kQy%JQ;QK3N}=c5s-! zyBTSQ)Hc|601mZ=-H}xxJyFSCo6EZMTX|zVl`FahiI?Mx61N~$ZZmMMIBpB_bOp+c z+k$kskiIXOiflpd)o($56_xWpcVGtx`?(40o7i8A1itkaqz~HG;5y^VJxF3Y6E=+p zw_|5+Y(cp5{{QerC->?6{{0*KFeVnq-K^SzWMO|=t=7l_vRe>g9j+{jpHbhA0~BCe z5F;v#F1Q7WtHBP9N62hJcKB^U=1r{FJ!1UWSmakI62Ap8uiS!s#c>~TmMmfgvH!CJ z2LEzECU6EjQ-xd$D;C&-v`=h7et#U|)q#fLaQVN5C<|MV!AjC^wJnHB?B)>i9`Ya9 zT*}{#1ddX8ihlQCOam-Gqyc;5DrtWH8~@C|obV5vVEqX;Q_WX`QX8QSN%t^T!Ki$@ z0|uE9=+i>SBh6I+NBn}{2dL@-A?S&ikafvb6(pAT1U6G@uGIqMdUH(qpOr)!0R+qT zVlf5Iq@=D~&4x1vQ_!m_d#OPgr6}zwURmyV{jHk4xrFc3qWQ$Iv<-vf zZV|uriKZ3Sqc%Ni#|UE{@{T>jPjlRh45qRRuGF;VpA1sn^z7S!d3-NP{bAgh%6-PG zBF&T(Q0|)_wgcb1KUOq}(c4pw&xcdiHJ-g4HPxjno;B-_EGsL@9mo!(e#{}Yj@SNJ z*M^a&<#W#)zu4F~z8DpY`*n5!%5=f)xmO(sYcRZrQ4YSnLs2jz!S}|V2g$o)_X5c3 zfCpY5$%hwRvy40jgHM=y2fKba{wDIaQ~BHG7vs43+5%0a?bF;7ITd}~8cqS!D=BXJ z&2vGj6olvG6&|~Uw4L;i0^+{}9!hfMfgG31CH*g*JNt`*Kqhparf_+U`~SbW1PV(0sALYO)1A4 zjD8qQ=cgoL1dLZ_>`yrV0&o4l?FRpU&sF#T?fLJL(7+lxal6U-P>Za{iW9)Q&ur05!g=i6ZJozOOx1TA9D!-^6tuCXcN3A22D$T z%OuW4t`V=wT+w27aGRZ3?zbG)LYp=-qfEsL!OUwIdSa2gk(>Nu8LjN{c33~!XK!mM+fr^AFRZ>RT(rBeMg!QCbd)W z7wRP~BaDsi!R?q`Knw$#LXuif7;4d>_g~ZQsw|FK=Y9Ej=+)3#+VS@r)W(X?A?jyp z)!~K=pEr5a!Ax&k{bqsbo}1K3c z!!+q{sibs}tM7NuXbXbx16RkF*ek9$NBxeG?6iu?p%oBf@Yt~zW+OJOsmCkyxqT`) zysrK9oqwu`JN!opmibo+{RbuVUtsEAi1Pn(SbY{YQ}xLAJg*WXqI^)Rj#q<=^gNq@ zn_(uTDM51LVO^*D{~3S7Bv=r7hkHa5o!=CZcO2uwOe}nf z2;7@xF6N=ZWVi!TEF=U^7KZ^9+!M*6MsMJ*u7aDN&jLkW@C*{ zEr9Nd^2w`zz9f$+L229mF?!Ip(BxetM4(siq-Ic<+wSZG_hvOU~5S(9p@< zg6te?4xm{1hB^@JUt`jzFb-J_x$HaR3mbN&w^KD4ut^>E7ba{u`&mHV?SdpR!K;xBL&J z1_~KREu{~Tx+Ec2;6d6gOD_R@CpG)L{-6k|KEA zg+B07=E%n;hPEam`gfB`@r^cdW`jY&jeh<3=AyCD?E$WTYww2U`SSJcPq!c=U%@r= z19nVvuo!h8uO{c{TuS^OR1*4cRFdkyQAq)l|4e)MpQSr5t9=_*eYe;Djd*anpumw49_^~lO7><@Gdrs?<&A^Lf%-fCX z!wWGHnq$*-#_HloG@Dqm8{KVAfLKHy4x?SjO(v8_b$5NWNhD$DDf8BW~)JCGV*0Ey!K|*A>dy(u^0>OAlsn)GT|= zvF^tFLYE)0%R`r^6wOb9E zUcoX><<}PMiGf+~(v6)M-;DZm4gI@~OjMT(>+7cSeBLb2pKSNp?0RjA5XTp>@-RY@N}BB(&=!nfnH1PaSjW(eX-OtB=E)c-(J7-7m8%L zYY|%zzF6TrqF<Iq_oo0)&HA_g!&WEkc&&KPsa~0c z4ukMJuX~PrJ(3WQ)Wck*rxLyGK7Cl|x#Zw{Sj+~ZAMn?3WFf-?>`dMv9lo~sm<%C!!|+Ke^(+k^nJ3y{Ecr3+;e z>4DAj8Rl{r7%TC$)^h{4w+qWKeys@Qu=k-}R>R(spX_S_+FFU~jfWn+|6yK=tUMd| z;?O0-ZxzB$>0>x8x=7Db>9?mhn@wtySQ4rUe;iYjN~kWWx)!|efws?n4GU+oDC8>7dS^(01y1<~T-^ z!7&+#rc3rn3Ujr2FIQus+&`G#s#)Dp1vg8}V@KqjyOYvP^^Xh`;U2VTluu9>-L(hC z61W;M^Z_)PI#mcd0ouaj6YJ_Z;q)cgBy-1)Vj;K#Dy}BC~4Y&S~fm)XPQpJN5X>D0wqpUe$}m zMwjNi{K|l)i^u?KUQvudxQiiMZv3@vP^3>}ke2J-CYPUmf?d*9dM8b9R>vywn!!Fc zE{_^vMV8O!%J^WtTF>>!-(pz|W^(m8*8`C9H?>-<(TcU%I`Iu#kR*3XCv;{^05)TpliSkuvQ`1Gt{w6+$OK>+kk_+t34O<+aH!xEbwU_+FxUc~lbWiYl_ zex;>KH|G>;v*rtK@tfCmKwJL{P9+hW=t2>^4n#li<=+k6#yuI^jbtJ+fl~xFo%cFc zi%;eY@Ia_JQ;3x?*Nz<=0g9oSQTT40K}LzzH38V#QAl*Zy&}lWX4RlTRx+X;bGEI4 zJ-Vx+vAqR!#l}hyM}z-~(&UbAhf>jO4DhXd;`D}LBUGqOmwO6$5*-kv@c8SSW+D@r zqW^YfCc{GgXTQ8EeN2Vlnk4B ztt83`PbbY3Dk<^q@l_&p$XE$i$1(mM{&2Ap@Jp=+Cl|qik7CY*Ret80vE=TWt48S1 zM+;)h=&OKbkg3^o;HBB#g_biCjg&SGjv~J>$d9WH<0^rw7Qu_@(?;fy*mj3Lg_Fjx zP6AciL($sB%7s^XPE&LH{tE)T&p=YQAaJk?6VORLTEQ}896g-i%%Y{`I-hoSM(z%T z9KCn!12ET00zK4FeLkF%u6~GBr=xh+JU=|WLjajaaCBgct(*)ha73Y z77SG+hG2!pinCO!W>&CH!ws1`xJV$8-Rwh1ZO{%mo8fUMMrm$t)%sn5nAIy3qCaB` z!j9+RNi!>%ciZ-jVLZxWyP-_mOsb^qkhWjjX!RDv?@x0WP};TR6C{=A7D^BVM%dW^ zc2-j6%hSkP{?js_)~BL=T#_EK!DX3~!3~Dof_xLCy=vk`@`qrX<2WVM`VIsbBC`J2 z#=(2otMAez_g>-pvsRZ^XQ9+UvP1prA!vs?oJ`sYES(3U&D!3U+Yve#Vlh|*g_wdk z+xVI}#(xs*SF&`{bCE5#+kMjGE>;7{JdXrU{bF-L-#h?)V-X7bOk?K)I-Fx%BWA<2 zFi>9;@etf)A*|9)sMH!#24gbpIcFyS72wR~XXve*>Zs_}&uhXXfh&trdY2GdjfTP> zV2x4y08Jy7nXhT*lMlkWwm^yi+J(`*Hca;r6e*#?K`1zWbf zyEgy{7(&Y601ZE`Ob#lEtj6Z^QhHNBiIk7I5{&5D{=8NUr`yLd8ir?@PpkbRg?%l= z>VDu&PknLy>@Goi9)Y(SX|bTEqc5*)B{+DW6fmXCFt4IZ zDn#hoFlos@)}bQ6S!81INv7rVGR%c>{qJ0T$;2?|?!^JR^7h4o*jJ6(SX9jdavs!? zNmXGx9I@Z;@!O#Q3g%DXr-ITjc=Hwn@|HB-zXkDtZb8oKZR2lXcbVE@XOW2{NmHLn zasavlXlkF9W*L<2#2pyAd7Gre1@}0d7L1hw_t4q7x4=3li2srHdVPJYohgx0qLa?G z!;`fdm&$3#bgb)m>EoX7@@ME!5%&eE$Gq;s&!uMkMB|gM()=H;bje=WZ`;AUu;sia zeicc{Rl-N(oQQMx#LrgOoar}O!tNN$TtIW}yDzCb*Z~HCNMNiz$*P1tScy80C zEV&inq?dnAr>?!^PMEe6=O!z)w-~v+zVsMbu7ty7+FRI zI?PO8|DB34A#ttM`^vEXl!V@8bguS%bMvsiqJfE``|R=(-hN}Vs7v=s?9&$I?&hI( zbM-&>1{-IJA0SQm)@Ju=3539wL4ry!(J4=xu3EMa3 zT}*se=gaGLMkl*Q-Us@QoxJQPsTq+Or1frTbh5(b)cn!Q)+4=krkX`I*xL?BQ;ab% zSm_Z_Or!gC*U0yFLS=61HH>3S!po)=R=0BOn4NJH2>dH1RtMGR<9C;Lf@{7^wa%U@$Smau_0x%8?a}~%f%kl8;G~dROD1C04 zd?XdN+bkz9N}&FHM3ykIuxh{BgIUbcrAJRQi$IW)Py6Xb+YZ8JTo;yyC^oMp^F_g2 zpn8pSdJFP7AIistMsdc}miG`=qm@v~tpuFdf{6J!oGStd*>h=Qc=Z?h zhrU=y)G#r@`PK^~?-IkTLJpqwH=HZ$K zS6DNW_n=~?@E1T*C#gY}6psm_Ye%1^wCQp84Vj*(L}3NjTCslw>6;&(j)(@69V;O{ zqi}CYb7C%I2`&(kNshWxDj>Dk<+)F7jSWYD44rJ{4uG2dbzXE@s-HRszt(wL|w z6VDg%;9QIyBkjRBZ9&A>mo#BiW%jipj84^+@5si4M51hNaZSw`)qE^{V4I%Z`Tj~; zpnP#=wSougZ0f2jRsx~k*7ABD!#Hl{5EU?{7@^D@EwX%4EqomQuQ&5@t{%KCh z^4I3TBF3s*<-8Wz(ack;+$kE_uki#->&{V&$` z-NFa&o)Oq>c}6Hs*mSp*fD-X{Dbsx#Hj%^^jloAKci!jd&{tQa=$idigOZcmo5Rw( zdHKmw!PpuBxXnTupt>(ij$q-_7npvs1Vp~_d>>)D(Uf8GXxED2~NfI6=pG!@VG;^lm6 zecQwg8*6~=#yAI|v>x(WiDtGQuiqRySIKL#VUptYyN2=3164rk!Y>W1HY{X`S;_{B zZ%!S55JZO7HQ2SJS=ku-YyV!yV)o$X^1nny%#H2Lrxu+e>3y9xm7VYx^*Zy1f)V z&9}-lD%ZwrmJvDpq!2llt7Kyg_-hz0H{Dp1 zm3an4m)-oXHUpMiB6t6@4%S^%?6L7R!ma@A=owoS;zVUc9DDx4$iUcp?&*Uaf&z_WMd$W@adTO`MKXM1K+j0}m~V#Z<@DBPVvaLt z;?oETt-fD>6f~mNpIj}Uix9c$mt2Tlg~nKgsN=YVZq^VDE_aAgti*_l zNNq8D>hrr>OJ!lW+o|1a@8MHXxmOo2;hig;-uf5^=P;^Q_cevpwSqf&-mmRH>BN7h z75~>iZ;_<=iU5kybg#u~Msn|jRdflEXdGvLXeB$ao7vnOgpk8e^A6v7_bs@!*>f1a z`kM2oO_rE~M>pW*=H4naO=rVyqJA>~@gV4mCg&D^7zxGrmbx|qevCXOh1@STc9Obq ztGfba4rLO*wKi0zzBxUtGUZf7j!crpkLub0$5$c-3rpa0f?iH2>tEAhXlO&fLut z(Mhcy>#Xy5uU8Zl*C_m@EUpNze9+(&L3Ma-MfCy$HD%kmD$~KK_q$VKS$Ht+CF+Un z>H39e|nkoO~UQ$rP#%^*3m3VAzE4VRDerb)>E&Xr#D6m zhty)s;5Gh)J}5{sc6h4Zr?V?&Y4@=*#KCP5oQ_WMXzNi^ZAL`{U9a&_ZFxm((tHQCuz?!`cfz@(-?+2pvsD7(uv9h4=FB~Jp z8uC9w>*32aA>qN$xI-h^z0>OTjd_=04afAwaJ$=HWSwljAMQ<;6!PpS^C+izO)HPe zpPX}3CQR9IkMLf=z8!;sn)&4%68|}EZ`Gpiew%&9!)f*3qBb>P13GV&GK`34`&aCx zVgpx*zh0YjNAw0vEJmvJ(ht9<*bO(lvyB!vfxnJ3L76CMUdR$d*diiwy%L2FoljLS zbv|z)`fQtPc}d%zpPi|iza>CI0i-H+@oGsk&ydW^u=}t&SFVg7qqd$oOLtu!m=!T~ z<5*%mKPA-G(9LYmFvobr%&v2c^rPqB4z;+^%TgC%f&iKvg{RgK2@0qvRd|wG{pdPE z{QI@}q&VLr zibKz_^rr?|ZanG_tBvsMDq{|{oOAoMvi`(jeP_T|e$vwK1u(-u#Quq8Uk16(qL?j6 zS(`ZFyL%h?NKRRNL*bn}uHUn>ka4zOGtWV$s;o+KxB&>*KAzszVDYB9o}@qO^9IwVl;+BxJRr8{J9QNCu@E+$%2+CB2iF=Z#`-*CPt zc)3Rm*?YljO`1jXE|`l^ru4tf*JpQ~mBUmTHrbXb#C#cC^)t|DN~yrUelGj zo33%(^N#@JIrAf9E3O``jqh5r6bk<=X0IR}nlHa-Y?ScL^>LI(ASQcNX?ep_$bKUF;6t-0V{j9YtX?jOfs_sbnB3TA_e%*R($0$sc4uSP38TcbpwZWU=SU%A#8KQ6Ox znYAoBbq=>s*P>W(H6r>5L*wT-d7gOGN-&8$Qsdt8+&6L&vlTuG%}gjDJ^e`huFel`J#jm!-5A3*-d%* zpVHDatD^hw_*oQ>`_IR>*bn^~cj3vWmfM3px_VoevetCsWR^i34$bz)rEtX>Y6CCn1MN-H;cfRIE$p%=u zUg@)RjJtnRqtB`co@;blrO*XD%eZ0`m#Y43`#-bZNTwpix%#Y-cF!Z=S%Y^MY8x`8 z0!FC&l~l5V3^)NXCYS59Gc_*`69f#$Z*M=LcL*8^KWvD#(4z*t9I?|IU%-b_K1k-k zW-^hxP0y5LjsSS3cUL$(JcMb>5c=HE`QxXUIJ6r{%`(s3apE9#5=cAEQd~O-xwvTm zSgJaZL|6wNfg~8%j6CA7r@-BoHbdi?VXhD+qc~iZr?1uu$A0qR=O`UbTKcL}R69(m z#*qz`hYl99pF$J<#;53jc@awj!Mz-q)i8gZ5vfTVE=btrB1L?>Kk8@ZP z%PKGTrT3Qe-QU7=TMU#{%RCe1YlAFbBH&UfS&2#sf86ktm#rFjQ|^F2T9@ng_Q*aB z>xSIHXY^w#WgTCf$yYp(LdfH#oxl0;F4_iHBm4%eNx`oTYGYM-}iI0H5udlS?yRPi3|s66!6nx_7a3 z$A}LZrv?w)HYwm%jKa`nLgeQAlGJFvaNjjLe4k9ZjgC+Rg`L97S^NBTSEpqo^8o(< zOFX0KZO88%nO|2M-imvW?!jhWfbzl?1P8@62flGkV!L<#a%N(ei(x)sfgYfOjc^w< zlNE@Q)CvlVIT_>VAP)uSOS6*F)Y|k)MdP0zQTwwZg1+Wo8mmnoEiqpeiOp<@>4^Af zR8`P_h;VQ4jf~l!C%LhbCLQBsQ{NNAp+{YBmazAQD+SopKYcDJG^mc>4_nAALyo{Z z2@0>_Lglx5?0TN~qS~j`za0oD{x&h~7>^&wryu_;A9m|)tgc{cmz7*v*sV)=VW$qd zxB3{~F_&7VzbtR@G+$h>L8^h`x=UL4kYyet;K1FwZqXeoStkw(TKKBnHQL_xh#9^d z4W8rqBOW{nE6bE*yd((J<%UVArYYQmJGqn9`E{>_-Q!+l}VAsovkc~C+5t*bY5 zGpx$9+&S9T=V=92lhwNnybrx9l(o#qVY8xzLI-a8; zfG=Du!twm=7e5+5VOmXvCJ$6gX#PsYTHb{AIRysO9+*P2w5Ju05zaxAbKBH=kZw7y z+268<-|K{I+;Ck{ot@UMa21=GAIeXCllc1P@orzD?sA_a`m#-hS;|2S>m5p zU&#Nq>ZUR6S0|O}H=oojO)CQm3^S{9_w8DXJzU#2lagW&pWAifoDR;c8-I_OxjQhg z@soy*+0-C3sWrQ;fxzZ%Q$UItWg)s;m%^=jG9QV8bTm=zu9ky$!3TFIA z(&5e39>bL=ss0m1emi}X1CQBCngL+R6kss%Xtpj`O-T_}aF${tE;<}t3N)Pt zh#Xbx3J=8Oz|jS7Pq_3_8rGn6AL_!@phGnTpBy*)nq5DG7Cj{*cBG)ha{-!#Z!qd} zzid*;E25^_j~>^r4SuJ3%$G}sb2-XAHEZpg!9Ht&bEfb$k8%=9-2XPb37MEVBv8sz3_5=uAvs44wzsr}L{5ff_*|T7XS_6k%lTrvp;VM2+XU z@Pewr>~o< zj9qQc*i^JdrnOV0QS$MIuS(8E%FdNE8J?IrpQ6}#K{Z|W%C;3}cHN8e9zE|0o6g9%xJkE+KmWjQO}+#;1BwIFFsL9<@Bn{JO-3U1)c4f-<(UQ3wutk z7DJ^WhUl z!F7twXj{#qJ4q8rB*pZQEPYg~0VWe5e_&dMWzm=&^>r}hxiv#aaxvU^SJOT8$27Af zcAORFm{ctIf!(La9t1MbpdNq;Urjj6n9~snm=*yRSOQc+pC6WChVlTg^zJu>LcGHu z9iq24gAxI|o`!h$^H0u=w-CUPFN~bQ%40hsU^A~gRRALR!E3PgzU9CpkTa{DI`Ulf zSc~H9bB$i()^g}bajQ?| zW!lo-X53;XSMjfmInDzMq~u*(J#J;4Bg0i@CN1Mxa5}9wU5K8P76zRlkT=6SP#O)$ zJpiKH^Dr|!!@iLDgGQ#_OlOdGMpSjUG@CzR6iH9oQEE$BS(A(5O{{<@L$==liFtkJ z)7Ricvnj9kG|iidpznQ(PoE4!9smly-Cd0f4V}t3DQ8yC69B`Ix_Kj<#@3^M*mlyErDi)Ud^*ziC(+G%N722()BZc=GXaAge6h^gK|mcD$MxVGj2+`2 z&g{Yx4ck{mCL`W;LZxqZx7Jp%-p#C+O&2~xP6JPrucvN77*=<~VyrGSuq5iyf-P;w zK7~ywBi<3;P(lfNQSSNeSY{-gD>Kjb0wS4cdM0;|OuN!{xJc{}P8JqG6UQ6^S{X6A zuH)0a443p8%zhxLcER{%>oZ!QT-1@;R(cs=Q5H}TTk-tzg3NRPHT|pnAiMO;%f4&e zacoBuYA zcWSgCj^_tOky?mnVXs#KGjF~_ILCYoGQq{@Z9&E(dFARmfLCCE?B*hPuejSdei3+j z__E}4|0XOEy@Fy2^+N0A$dU&lvR1S)CQM5mX`q+h+l4!Txi#)$XXui#NTXB8w1ncL zqczIWV!u$81Ng;Y|7rC%BMfG7s~T5UfiL_H+;|GSgjkqI08O5&JWHaw>!gOC>y|fM zIl}C>0Pe9&$T*o6J3_zgV6rxG;Lb1Ww=DgnE_EVjVrAV!xtoPpis~9vYIuy2L%CSHK4cmp~Kuy76 ze1$|Qay1e;ll1rZo`!xG2d@BZq3tK z%pjR?ALLN<3A?*P7&xj7V?NW=$j67li?(F;cB0;b8j^^@v!gDT`G;fj{-mXuSw`D( z5R(Y3CirbS-4!|-?J7K%| zQaEuScHU`syS_MB>@JS356VRp{7}DEpWkh+HvaJm8da92c>xv-o0)>&!wUX&h8!H< zcx4K40|5)>AcK|=mafT8uLn^eORc?aKdUhv8>F*>WG%|2&cj(sPs*u(nwB_U?rk&Q zb)&k{YJOw+FEP`i>s2V#3~NGT27~ZQ zlaAx=fwNBmcowR2dOLzPd5(<*yrUWLdu=C2YS)8nP}7P`Gj`c5QW_&)c%iY?Lk-c$ z>d6mbpy0c@!>^*jA=3fmch`r325e=Ah8F(47HbAt956gHhH(T3a=ZPD_b_`B0#u*Z z0GV9{#KeSW-`Tro2U^F%VlFW#@R60nfBrgl^T(ltlZL1Mtg%H@A(jxRNzKPPT7-j% zi^D(0{eNiRthuS|R`VmM_K;s{gj&k*W#1I4k+)B%uZzH4Y+O@Et1bOwIxCv)LSQA4 zDY3@@D{#QiG$>=)cIL@#Y=Uu+!awjCz}YslNJ-U1v!io^PM~pwUELK6jg-xxZ#BFZ z6pdiN5W%kC_oS8kGo;VTY~gX3mxm$y6cgS~N-AhtJ~JC5cY(FHkXh2R>N$#$WOyjM zvJCQB2rEkInA-SDmLMlkNuz1F#SdL&-msCqsDX~Uxi3rdH@L?-+DWf+uuS+C1UL#^ z*}?257pM=#N^>ucdhwpM33muU{YGl|N*L#s(J)P%_)HKGv`bsAhH)Np#LMJR+N>=0&it z#NOl&V8JaML^!}T12^352)|1#os=+fu?y&Ul9Uh26UR!V{$BSP<$6mdYW3dLBP6+c z9v*EugSUv@f`}lNkdtkPhpNL6(DqC~3;ZUY(q394lq-wqa+h2&;2W`$mL)mIW5@CB zb^Jb0d6s3x%+!ui(M&q>x}n`lWy{+%9bA8E8_d zIN}1xFP@XzL}NQnVZB#&u!VU^SYN_~?iqwyAzhMe2DN3ymR)@wggM93aSz}+uxz`S zMU``sQkb*jT-&ON7cZ@~mupvQc4ag!jsg3@RB#To(;nOL4hv~BT*5W2u5cK(AY?Ue z>dM@=h}?}i!;FRlMKxqPE@Rp(j}?ZMS#(Es5JbLHN!_sB9zc>y{z}`4Bs#yMG~&jh ze8a}(=0acPl_{HT>#HkuU^s&_#5kwe5iqG`ri3GfvEGDfAV9T}$v9eH12?x;e>R z0|XLA zqL~cY0K2Dww(p!*^G#3^`BQt@WoQdh0J}^28pTBN(u)s(M+3WyAlNn^?A&=$z8AHDpz}R=jU)g!myoN zkhPSjmse*9n;bA?Z<;e$wAAGm@fjL8fpGA(g-ZBSsKowiFRw?`)5B>02M<@zUzI;I z*UNeJ`D!GgfQy{bNx|;qZwKnxS>tUQf$l=dM9HLa&toj?-Ox;N|GbpSK*>_;;A?bk zRG0m0lzfi76E(L>I`1c@uK?)X6BKEh6$&K|{P_ydWfa+79GM@(==!|W1q$JD43uLq z-nz>F5%eTO@#Dj;biH5O6e1t?YWbA99yMXMVl2R+q479Yc$ zvEBb@exl4YE&ker_rXTaZVnya_7{gFe>ncQns%b-z=nF4`@RFFx127OKE9YM;G+17_=H*His^m>r?nj0NRq7pNKh9?rEML$$&}+WVwBh z5>FEok7W_MxmAa!uavEgz;ADyv^HVIity4g=vmwZjXs!0AO+AeP(GDh6{gh#ZLWeh zV_~^P_odn3{j~w#Nt@xdH-`|K+Ay163AIlIb|0l8&h5D+Ao_+4OJT-R9>xx~nUA6M zSL1dAM<-`iQ;FjC!|lM!Sp-_FW2TCewuF!?Bpc+>r~_qMbqLcY4dIs0!+%sX+suJx zoMZoJ24tBez7#YS2?@mt_A|3Pl*yZSh%+?l#i%h{BD90*$968phlwH(R{|dfSPJXB zj89g1=XO2$w-0Is)M5Xk#AO^c_35a5$1p1M818&?JI1o;GCkL-^H^zb1u0;Ku)<5T zT^i>N|D1iuYzT3?dW23&Mv1<$DTp}Ys<`6lR(8X*&OXQVEE_pdTl76z;9J7x2ZWm! zU#|@V=yTM8`?GQ#-&-QRStLfyk5(qOTM0JYrU=M6v350QWYS}7+ndXll_ZAN6sFEk z4V1V~N8F~7FT&Xop2Fka3X^td>9et)0A-d8e=nhvBp5Mwncg%?EK-#7e4S_!`U-?L z!6Bs)8GW~8^GrOV?M7=D(pCz~$wKJ|PU`f5G)`s$>_(Ao2TmH@roeRvjGje!!PCJ> zqr?3!q@;e$rFjR+a_Fk~!e&p!i$00hEUJILPQV_Q#!IGIy}{^6%~x*YdTYqf{7yEx zlYh)@RhiUauEjDvK27~>Q25eg@eXOXDONJMHnd7NJnG=c+_=Suyhp#g$0wehtiHWB z6_(2K-nDdbpvtxT+vSw6T)SNcNcc8+J=UpR5-2N$@u_LTP~DkrwjKej5qRc13$c4PQ*>YFPR#cZecYs`s84_B!tzS0ht z-E)sS&+fbWy(fn`T0V5~xy^}17z!O9h3sCvN7@})H7L)$2E*%Dt7+GL~ zSik+DV z0>9x;v4Y=0Qbe+ifi%tm;9Mt;IoftC$#Db}j19(P_j7|8Z!!Mnl3e)!^cqeYl%Dm@ z6iM1MXBXhlCp++gN}?2fu^1bgb8Frvd;G_U{cA2_*SGZp9(Z9J{Z}pZxT6k^VUfex zWzMy+&Y`(Q#3tmI0NUrV`UmbsnnnoQUmj}xsstZA0vf?R3V!d}4HBS6;qEP@Be~MQ zX4O`NW_2dz*w=Y4F%Z*mCbkDAx4b3}pQVuZ5z}D|-31f_su+Q_9;)SfH5O|DZVbg8 zS3IILv?j@ho9L9aXsqO*Pjo3Q21Euz5UvjJaSKupO=wf*zsDZH24vpHHIP!V{^*H~ z8A2S6uCrpnnxrS)=UVA>llFIjqIh!D;N2Eo)((3hWIHWzlb;=_lU&2qMlpncEPG`I zc_OUQE45n?ZS(DzKgO}%Xh1tDVHJdSFv*+TAdWx(Q$bi#9LZ;JQl_16S~rdbOQJA< z>^3yyZ(|bT&E>g=kPtw}6$nO!2S-)_k_eVnGBHyo0&h3QwF|C9MeWRf5kc`}{ixz? z12L!@>1#H>568~N0%~e}q93_=obaRWvPRGDn@-m>Mju>TKB1dhJ%6g(D!2P*r%a2+ z`%4cB;}tAyr>ZQ<6tC_5n)S7uqEB_HcULFf(Qcp}{q)<4FFwp8aUD1}_+;Ce)d#Wa zlomUVO>DopYChetd%(6QD(`s(FyGaz-|x8y6X%97s#?8JMS;Iu#`L0N;2oPE@ikut zs~7p|Ovz-Thuy%yg5O_!MO-!)%fhwmO!#%BFcb2SRLxm@im5h7FZM%J#G{ge`B_pA z?=)c5Y5d-}G!0l;6+g2y1iAW@6kUpLjo@{6fwC0;eD#qjx{W)Qlw#V-)f- zK&(_@KrzOqDDcfNx?s92KEaZWS33OmVN>q4ppY3_dcLO;tBChILlZ6zYPDM{FLxX+ z`I5?`k0@_J5*xv5D_JbyNUli6k(mDyKu@@$efOY5EHUw znoW)pgb7 zQkmE5zVGMr`Pl9YG%08H*?mK+CQT&4r1+u8002*O|5T&QhA%#YRfSgQu+Br5ch7$X z_&2*@08tQM`$1=PpJQqNi3`VinmqFd?aVeWH5Wdzyo|}DnyZ~1(${FhlwYZRtkGRw z#dBLsoS}C>XXoOc!}q~VPs1WrX&Bvqw_s73AYX##F7zWLK**P)nG$I}f-W}aAi&pt z$5w2eI7)VgZV%EPF-2A{U|>Tbcvw>nnoDV+P;C*%@~oMGg+i_Gte^AEs3>s;0|L)! zC!n0~g7X3pxvh8j>5@{bQ1pA#(0l)G>0vl%RI;iLJ{r`uh?&pNxl-KJ-Eqx3kLw(k z-{PQ=zjDq36KII3)`tif@n1xwgrTyT-lYtXN;H)Lz{HugL0Vhg zATyh}lg8cyyc50v0)p<;lJ^kex#YcSHoYZ$k?VLZM7hliEpXv?q+g>NDdK&McH2C7 z6(1~q!1#o|6A^)~xxEBRA0W!4c_a&agM8;gw+x)5Lg`oY(4?hWMt zAEnUcZlh;%WAM$L#>R7_QeHD2E(LqMLP^)>?|w?J>v?SG((j`DvGrVpI@CX z%+D#ycBsww2`9&gG)_zkaIp3$Z;l@-+3g|JhusCqu=`EJic((r!Xfw`%XV5bZveSl za%}efODhzs8+CC)LyYwt1Ld%3ueyCdS@;lq+?CTKvb8wl4niXaiiw@}$ISR-$U;6$!_u zC^14zC4)*?Ak82+Wcq3@m=WDaXX%{HKC&Vvz#NW(R0X@hJ0QUwy%qXnA;y^5>C{I2 zvY?~0r87HW=xkT7UvFaIP1$ZqS(4&g@*`g#rmvsxz?fTpmCDo;-mpvt{q3ddNx%A6 zmn!S*A_o1am}p0H7sHF;4sA14dB@5ZUrgLL&APtuSIzEBXod78GC|tB z3E_U?EO}3liR2{`0k%eNB7H#ymgO4FE^|pMKe*^wcP?&V6?jw92)b|+Oa2%9kUJ{P z+7cSgkbB4)hKV|h?s!I@p!HQ9-k09)QLhKnqnnZgf1aKeN@Q&K7Ow@fGQ;|#u$DZ}9UG5{-4hRdW*vyOFDq5D1JseO z*AQwuWI14Rs>=SvwSSgg_anZf3i9D#$VejlTuf`ET^pTQrk8#J_No5?I5M+QWCStU za%t*V^zDQZ-qhzx6SY&*m-qLLuhA*i6b?Drr+czE)4I^ggP-TwJ;AHZ4w!DTJJ@M* zU62!X{h>;scG%FebZ2vn)_7K|!4ADs3GL;o@iq<~w*o64Kd6xod%jV`)(&0`D_32Z zz9zm7%L-2}3rKwL(OF8L4~2&mUW4;ksJ^_JWw2#>`4g;kfX+wIX32hh?Q~HrgzYC< zj%_3k3nT0~Z+_CUO`NcKsk7Zvlv`cF4RC7T03on`W+7}dA)+}wxSH4{VIaN(yi1j| zLS82S)bRSUx{?RL^G$G$z`9>}fdKMBWwlXM!7-gH+)nC6ihNn4ieFm~wehg$rmbRy zzVny;ZM%ai7XEDhv-?R`+5!MmdI01$Y0!jgYBu1Aw~CvQA7>Bkn880eeR(4 zrMr&}9=tS>mF+V2xzjbT>m%71dym@i{I=&eN@tzgiQ=y2O;4YBY!dGI3UDT^{y2$C zCT)yzkdmfRgcm3`JHU1*_*MLXwBiItSXAd(05$&q`o5&c|89vxZ*c$J64g41{)PqZ z)uw{A!~au(4fwS8IEgbTyW-?nveXtf36%+<+1oYe?8_%5E+Q=St;nFKz6t;OD%o$!lietI zD`0ebbn%CA?R^~Sqt9DH+VAmd%)SrwEI345mEGQ)*AKdvb2L_639@x{Afsm|0S!B2 zIC(RcoXuN20*P-VxWiwG4wr0)ZUd|JANN|~Y`ttZdF0ts zzFiIL{RUIMqa=}&%>=s}a5JPfZC(Vlb(OP`m)dif5C$LO_axc(#WQ7kj6|1!`M>#{Ib8;mV`@@iT!fE&he}0BoUl=kI=}Mn` zGRbT$N^ohnR9q=nbDYxthhI?sy!!{%#Vb3j%IW?0I=u{N=_3_72!Hm@YU?i7`@LAH zaH-xRV7XvG@4BOBjpp#d08u zI$N%)3V%0<(SyiS_r)xzyo)fE-&&g|F0D8Om1Gf&OK2d)18o}po)KE!W9ZBh?F3p z(c9~}F0W!gl{r3zD#OjpET)?pX7NG&`<9g2(P0G+6V$7czw}vfs=NyR@#zdZy@Mi) zfx+tq*GtPm9zk&9*@Zti>pj!q=LP;grW#X*@hiODcPiUzT=g9T2-7d(ydni&@%nrw zD79c89fqH?D6zPGlY*fhsqV3!BXA;pmXs=y``Vs{ z+=egf>#TiC;}&aPyf(%qQNmM{Al&_M^&){2~Vzlf^Y#lTQa<`;F(0 zm+0k`X5Xdi46MENbx$`f?Yt3TXYS?Y@PZnfsImTBY$yRk#uNG8)fCJ_eCtptC5IGl zG`n@s*BR|Cu0mqeZftVm`yB}G$|Wae>;B?d*+)HsqgikBcJyt32ey8L6MyhtHHJUy z>-e+iqj7jVI>zh!aUsfE1m?GtqhKGM|94BE(SE{imi3sY+@-el3qeQ$tO~vZLZMH?%EG?Gw|vKsJSRsTpz`;9I@}6-Av!0?DfZ^ z{W~U3keZiroX^i4P9tfjfWN!Dnn!B`WG&>0NBGiyu@PX*f;2 zUw$(#muy?pI0ktQFz4v3_&)S3L-40K%WwnVWeaTz$|pq;h9zuS70I|0^A|kgC38<# zG$gk5HcD>$-#MS#w}{MJf|b04UtZ6I~JC(+8?3sHXeD6 zfN2a{1lXxo$s;&R*6v%c6|z#Zam=vf>Jf2-_yz51gp+TZILE|jCm7%Lg?gl3OMqdm zf>ByW+|aW-+>O>pNvA3F9LA*57x`=KE+=;D1Naa8el>s75p!>#qO;^7dJ`rWj*U}g z#K?ZH1f}~AM{67pxb9FktoK1`!!OzaS1swutok ztLa^O$KbkZAN&oVTW6fGGUdoC6Mmak`7%^tGS=oqQ(p8c)yh-O{78l40p{%FthC!L z%JoKxwd(T8lq^AuV^y7w-Cs`Yj#uBH}W#YXi5J)aJAC%4fXAL^ZCbg>0O z(JA}zq5k-5%?p;VVz3QA)`3g}A2@BYEcEylF$i4wK-Ly;1PT@HZ|Lhe2DvKDV;R_ACQp@X$9LE)ubgz%YJw{LkP!`s?PP6S83;QxQ+o z2b`tYi2gZ5HhL$)^WQCq9&I3c3AxABCudk(L&{ofTGN2-0!Ckeir&^V-cPjUCxio7 zhQG0r&{LgG{~1!EI{zx_@DlBHo@k$pppuUmAipnRjf(bf9Y(*iMi71%*HBTj%7Y7L za4LlCy}LAf7`>YX1*UxIBW*PyDK08kfb|Y-_MiOv^Anh54L52!`6&8%E8_Wj^-$+&;%QQ4#Rk7J~a~TFFct=W?zI|H?oBc4Ng}ibWnDYA?0jI5?f_Uc}Yk7?6To|U_j z5Yrslh2bv0|7PXOnVXwyaKbj+qjvjzGtFO^*7uHRI_?#WY7N z>~rQ=7TMY_%h~o--%s;{gU)E8hpw$c8HAoIW6^k1 zp%7Rnq+f7z{S{kaWk~2JCUc;T?n&hU1f1Cf2fZZAi~K40s=oa%Pa(!~!k+o+${!*VCz7r0QE{$QEJ2;*! zsCPT?(0!)1w(;}xa{nIvUVB#0cntyzEI=iCg!<|aH3#c&e4~a7lGfW9l3pFk5M|b$ zRo6$Jlb7sH75)m#fN~_;CBu!75op1-9HZG(P9&9r_xgj;{28hnQ1f+zF~XCg)3hnW zYqc-(9H`Ivn}ee{?)V#@d5xM3D`i$vNW>ZXITKpmCbRX^mwqICMw#oiG9N4ajX0@- z?43soG|L7{-`>W67srWg<(#X+moBq!^_&0%mQhB45roLJissMtsq0O1+b2yu;G||a z+N*86hsj3EEhWgMK04cW#+`0g88LPJI^9XZWbwI=9tKEMYd=&@PK}S%On)3K%_nY@ z)-IV2Cu*z|=fzPOog2s}KKk&WxRDehM8HRtMF;EwbNK*yg{CO7x|ebqdN;&j{lO>dOVn8+(ERcO5fY)k!8=eK0e=#Jw)%L-u1 zoHQvT(X7oSV?<4H1CmuwZu446ZX}K%0UrkPkzWXm;4}O#xQu1ryQ)Tz=7*+rML}e4 zNJ$+$(iIEU4s&rlDIszOUjVkBZ~sQfo837i*9#{I7Utv-UwKmV6&g4o>W= zG|Eb_txGpr=iQ*w!O7p96$X6f?RD^_C7)M}Hndc2O%$6VA28Igbe@2$#YA;~DB1T``IWT> za1yyAuaboa+D63|1dpauAFJ|ixkqSglx1fpOe`3AcTb=~N{t<-FxB6Rd=h4vk!RHx zu${=H|EK?wxXfVN^FWKP8zcrp#7Y5B4=&0N>ix_a6~CpxXi0(b_z>wA5@VKBp7$;R zuIop)cdRh^dHo6Le@Tvr&PPZMB$!r|&)zAtI>}9PUm0$|M~=EaK}L*qN!Me>#g%V= zj$brb4f)>1)Qh|0{b@!(C268N1{Ohu?=L=*VltZkTl(6zD6mC;LokhB&3)RDRU~}q z9)`$=J1dk}Wdx${OTEuQce_qs*!@C~4jauoY9`%-})40J+DFj{|jeUQHDq z&@|OC<8VdY>-VA1Oe?WDP1^FpJ;E7IhuwhhxdAIMlMp@qN z0YS@^XJ$U+^lKY35is(_3u=n1$hGP>&e1Xa;!GL{I5?r~ z;)LA2wC{p|LAh?IMcq4VW%oHON#XyF{QDv`aaSxZ!$-c4vP0y~A7IdPmFYj*pPJ`g zi%E1!d}g}*-em6l`XGPwPM53x$bp}l!+tW)nvU;#^-Gq6U)y!b9K9S@_!;k_+{azp zBP*xz_WPa!)B2)u!IOf9d!G(od+m5D)UolX-Kc_vYkdw#aww|PHeCf0+v`SY3;Ixc z$|3VR!zsoV<rokx>5k_R z>zlrU6c^b=izNrcWb=OVp`m)e%i%sB_d0LiEDgOaI`a@N8(3SsKC*2Y^}xQ8x!SS z2>A`<49;`1PJFKoou;nIXlfQ6ty}cb?n^TE=6}nM4#dw+SJivZqaJkt`+{)Se@5^B zi}$f2c<|}0w`BEhD4tycIx0d8e3PwRIy;V1Sz3@E?{Z}~h4!^8Gdj%L;}YK1u$Zdw zy>*4yBON{9sxh$^#sGKJHfdE5y z81K|iJPSIE%r@jsyms$TFim6tSsZQuQe3H7GMgvw)REY2?*WpzfBf0{^l58H?xzY4 z=TffX%`-<3pUDzgT{xbD+DYm)5_xd$-GnB?sw7{bA^ z^)q*z^x+Qnzgx0Qo`3pB*0A8yt({sokyz06G35zTkdZ)9++>A>lzWLv&_Zgk(3*gT zoGY@qvfG>>^&%_>EiJN!lxS?yUf;!^-A(h27L0V$IT^K3n@Mga&ZNonE#ugj5pIfnHo?2mnFjmj=paQrqhbbr8KTv#?%;C2$6iq#%29ue2de9_cd#?vtg zfbghHI&-c;5YHb468K8G*Hbu(OHvehzUxxcdXAXcFbK7Qv5?mwX-}ib-JrmJ&+Jue z1Jo{3pHp!4f*lBa2!1@b_%lNoOC3n^M~{3Fsh$y@5pA6S5dHtit;#&bTEc$Vp-%EH zK}2d(y4^^K>VRX}=X>;*X8%C8A^Lf8OvkC7Bj;+*apvuv((T9El5}H)xmV|MjU6Xe zT-_Gl&L8`-{_mFeqc3}Qi}Oj>DBC4Wb$vKg82=ys7~5^mr40DvIPu9{XBzQB4yiX+ z+nb*Y;ITO7IViFltJEMoxu||H)d9C3R-ZPoXNTTudu6*&#CqCH3N{tgdHlLwe0-ma zabb{SM@x?1rP_%{tWTYJnkEH>H9D`>G^*mOicUW+$SQC&{^ZCg*et~*`dFCFInQ54 zOsFY}07utrYJEn+0J}z{I$uSwDq;ShgbbP6+%kupTc1Xsb`?9=hK2J8sR zy(_FTFrsd;J>F}x^+0=Tdj;n_2SVIEYigY$9FLlj|cr{}?k-O+t(3z7|8KJ-Q z`FYTUSLll|w10slC9F{%*unZ1q~4h@cjNWxK2oV7gR0*JKapu&2342^#5E-hRAK)1j7phOyoo8BU2mzWCw$+R>YU|){Zod-Y=%O2f>@B1+P2%n%i z-Q4D_>}L@yArCkfG`PBFQzxrG8MW@Rb1yITf&+K1G|#p8&IcWQK3sJ3MNg)-*8B^N z`I1T`Mu*Z*o|BR6`ozW^4rA^V1qrtRL74a!NIP3i1;L?RmYl2bCR>(~M8WOL{hk2_rVOoizcRW_{53#UYx#Hd zcD#*m>VV2ZSu;LC0Oq4pn{LqAZxn20jiX-|wIkPFr}Ks&>k@DH;G^euF5r-kU)8@c zJbUANo%XOY78R>sr~IZo&OyKSiy(G9(9|Sa&Xwp{e++U@+q^s&jpeL#AK;PwP0~drgVy>egcE1`Sg|+7#6ZTOwP&hn z+(_XoMVl+Na+Z7AS`M5VL?jkHW5cRTk7eFswom{k6@-%!5hkp+^<>7gNZUZ5#+|#( zBE5GS5F@!q@k7~p;z1iulYW1|(TGHw4*r*08?sk^E-@}LOwM0@Iw26v@z(@4;zCgp zKqjn+?`hJaSfMGXF(rMFa36+tK#EQ|AkyM%(O6NCegNo%L-j{+94XPU+2-4^m83pQ zV*tLaKi;Pxl5O7VJyr@&+NXzIcUPZNg_hiD;aSd%eG z`Q^hRO_jJy=fSzYbT!lrYguKjk+z#Ilw6wdC z1C;D8?4Lh(^kmkC8DJ2X+Enj*_YnuYa;`2udVR_iLCW2b+UdjsuSg6$x9!RU`~Lh13FEQ>JjJpt~J*;$sJid%OAC zbfh9Bi=fNK(snvL6#4U&2f7h`fT1JSD`(VLtMbp1Q%s+kfTX>zN*0EcVD_r)jL(rL z70X}hUm+V#__)Yjj47iQR<4eQ)IB2O^^zv zTwZ|Vmg&o!5g-7K&vX@`cRp9IogGb&Da*(RJX~j&O*RN$=em{q;p7+WZq!JRmM(c5 zFp0{klG4$Q{9)_({g!Im7e$>0%&Hcrta~Egw|cyj=1cOV9_|a;Q#oJd@Fw!e&AZjN zL-Yvk#U}aM`KC!VW16}5&UxIG-h;eLnHxnh2w*DNF$8{c+Y$Z*7dM35gFh}P;Vb7_ zYoNdSX%C@j`h$W^NfvE1U|2S;fPX#68SM!*&;4av8&Zxe)pwC zm)O!GAJ_VWdTw#zl|`3RB^Iq&x2hXUi=HPjLiX1k95fhNaCLLArL}P;2=gaqSKHI; zNWE5F2H+!0@k>BCJA?s2rSh31DWWbRxQxThrVy0DuQBeJ=xivz6zjYnDT^_O&asWQ z(=|7GD9RaMo}Bs`9sUlyINNOWG}S)>SIS&>AxXLa0kulq!})Ynb9+l*u1o@U~<02JG(aD)_zl-z`I(JL@^Jq5baRmY#hy zT*jBewCIKFP+YxPu?ooM|`t?W`_6=&`n%orj#=J^l67jpx^0kd}ZD?$ebJ6%T&aZeBlDp zBZ6xW7I#n-PKCSUAHbFJ2zc9I!gMRwQpZ&65)s-bx1csAI(J#@D*5p^t6FF3(kBaP zkF4zM{O0lelBWswrFxzvrwaUf)`dp7Yq~m4MX%MrJ2Kqk;y=##2fW>+QYGE`Y#gjX z78Nn)h9s{MDgSQS(1vHwF^oPY-vRkUpVpBlF8nVlX9)$hyl6pg){{V!QlPFX=9Dx7 zFTsbrxl&02=rhaz-SXXBGEEj-I)PkYXaAieE85>m7`r1k0W;pH(NF-7Xe!e=**Y?<==E?r} z15+A^7kAk?C;xyenwt*XS=1{aFSoUr%DUg@6>Dt1%k{;mJNxaT+n*Qt-_Zzu_EQdD2cqTM$734>4c zzX~I0>*;2KF2^bKITi2g+?||)-nsveY^y(8KwiQ~Zb5wMvzuG)&ZA0i_n$|aNDh*E zFIcGx&){PK=cP{(wYF= zr~Ljsrs9~&i8al(gnpUo@Pz5idY@kWV%?I$qDs^F?1xvu2A|qX#0|kAG>q4$)s+U} zF1)>f2$!)bnX^h&P1ddAyBvA>rM*jAf4(2CTPi*=K47JM*Ty7UAav3%x_o8V__;kx zZ(I)&AA0qF{-1Z?NtXl7^>6nz9PGY%ro134=}I};;M=5+^9wD}HD^u7h8&_&VSd6a zueMvk2kjd<-%N}l&gcp~M3K;F8tp1`#*32!1fG|C>QX+#_aLF5I<-%!;uK``mz7jf zV*p^PKzm8t7uMO|-YFw;hU=TcB)?X02TE^G7}3eQ;Nu>Xyc1vkg9$SKl;BC=T13fi zS8x(4z|VzN#4(F6yhSKaz!zr`4Mgbio&$tG`9?hX*hk9}1OC+K=dyyXYi-Upm!}En z_&`!38^FL8%v&J)S0+z>?lRW;aD)_^KUh`89N1YS2J*yE-WWeoqMj-YAif0x^ths} zOX!v_D<~+N4$yxZ?=JT@C4U(onD#JC`H4tHToR-DfTBd#RqxE1CQZE){071 zxJn$2#1bcR;>ZVn+GkiDdSw5TCk%j>-C-sqe;!Qehu{{Rj32Ntd-j-$nQ+3DD_Q`6^}Yk zv zZ_cF+pprp=+Fp$Mf$5kC8+L77B}Td#9DFvAB^!j$n(8C>2Kaw9mmIyLW1@!k&T}ZB z`)sCbwM`loOQ4OTn1~3(46uXBE8@$B(Er&&--ixLOd&Zchj|D;B(N0UBY1F_ig1B@ z_q52C1CsC3fyW>=DY4%|()ws5`ce4LY3kzz|0SR1I#SeGo_OQUrkA^uRf{E24?Lk8 zpw?7??LzMTuWs=yxrk&?s$&57v0%p6PLmuSjpw4a1}t^jXGwGx8Av6G8b)O>8)Ec3 z5Mya~JQLcK?LG2ISqbS`6rR=Pz$&<9eCce^g{)AaJ0P3E5M#teiS}j-{)D}R+W+nI zt)fJTCQ?`y{5EDK4~1SXyWB$M=NN;OCs*xk?No0*ul6AYAAf1GOBpD$A`k;A2D#Hs z>?SYQR))zy+Rwg)CAp#9H+nBRI%waimvW1l^tCA@jZGV-UZ<7UQhe`S8-3zI0l*w{|Q4Tc%tM#~J;4Jm$qK_aBcy{dk5UZKYk6 zhqci!^JY~AC$i~xC6lC9+yFVbG@woxCX(fg2g|?`Q&^m~08uuYm*G|JZR%?WX!ruO zn%QK!6QZP=A`XfiOV9eKmKmwq1fMW>*6t?tzPFa)D`%2erD%DH5wKGBF_8OWNO#D4 zVo-bFg|*+uzjXz%X?x%yde5mTD_eHAYJbaYJEivP?EFGO)d#FaM9~K&pPIpfI@HZQ z{!2HBrq?QjN=*tehB+a$c}{wOv=+!V6a%U8r76!t#{`wWxZwj&-%>o&@qf)J=^8NNeL_dt@^KD^i;S1Wt_wE$P&GS*~0z)aI`=T zSj~|V>HBq}z2e(FCkg(1!Cx_y475C9M=inV-MZM7B0@@KneyzG0SxwV3M?}PG`y1; z!2g`d^;I*z$Pk@OA_X-pM#xUzkBGv-il8aKJ}%io{0G5zp(P{Q%vUZerym!CCm+Jh zMs73dMgdGsl`YIQlh3f$o@k7fmlHX*Ld+oD1_KqyQ5EjS{xOhv?RIezh(EauYoa$h#58-J}%{SZM-O9pI#joE6Iy+Zauj<~Y zo}TqTgS}cdcDdt4d%D2EmUM>3_eIXi45C}V^dn&HI?!mM*+#k6|Bd=)B6HByGsx{o zI&GJCEv?3hgHnX;{}{L41sb>mRtB+W+p{^V(s(1^DO1Y+Fl#l)FSXZ#meuzD+D58! zW+>kTN-7X;nIra-_gf`F@`6!uN{@~RIZ6Blpar(9|1L3Ld(mGRiq5jh>7=~``6f6! zK>aA0eVkrr%P#<95x5qf*A`8m=6c2lduWh!5G~sMmH-`2 z??Y#jspdQY2W+1cR{@_aYL6%Y?uC$!@EP?Iz&D(qkjN91U~{%?ByJbp{3T7+N)`$d z48!J}JZb#T*J>7PK7btQUxmFo<#WO9eN8@2FOUlUF!ln!(G1~3>QzC8Q0DU3G&*vY zDF80ulv^TY$oi>JbA+n|rX|-2ph3$(Z@0FD3%p=`PWGxY=I3f3GUF#M8K|rLo~Q(5 zv&G1l9k1B)J{&-!UJtiE0wh;e;pRUtnEh=N`)gKVc=S|Oz1>Ck+Ih?2do`wxQ^C59 zWyJN+UH;lSJ(s?Hasbn}-#g4{VduiD@4}G|r45&pOVb(RKg7ADNE|5l9Uf%R2J92< zpyBr8x3SPeyl4R_duV#|qCcL*(E_N?R&ZrdTsJ3gm(!MhiDI@J)GtvVH$|RGB<;0- zh|FkvRWr5`ab|91Nn(p+F;FvPK@5nASe-?2%`5>kUlAsux?^I+A?d!mijP-njY}_7(a` zUNhwZ>dBl-7hD97a!O=5Au}R*wx^y0 z`sAlp`L7;_tF&j#CM`UCBwd&FUEuKTNCjTwj@OTZLFD%$fC6D9{il)ZUW0l!!<`V$ z+HVT03md1#3*F)p8mjMCJQgHWonG`Tby@VR3#BqFrsoZbKfQpZ?M8QUYudD%6 zpBzj@zwm&agy%`G{fI!0HKR)s+w>TCO?)PO^PqvkxPGr9%@Wp^^Z-_BF7MsrkFleN zFbS3;!CR8Nyfbf3oVbUYnJ?DL6S{4*VrvH7@O2=a@+a19?Y%eiz)3ZEK-Yvxlp!rj z$EecB>v;LRTmTN}6a({#71-};mBDU=rsMRP#zLf#I-DOiF1NaO(l z8QJ%*KMn6cPwtS#L?t!hOT0ScydShYy`vk6+fcLPEWH2l!7g%D$)S)2G4doPpDUCm z*j***r@G{jnfO8c?3p-y(*Za7r+b&KPw50osM)#Ug<;PU9XGH}`*JC$UvAS`QHof) zvip0sKCyspp9T}p4t(}l+Y@}v(5t|%=)$F6*FP>_x%7xJ;E@wPsHo{;@+eUJ^D$9SbJ-vUZ1WYp@v8~TaW}BJPunu5`gIu+QE3`cvR89I`zP0M@8${I#e~z=2x?HV!ztyTmg|SwB{~3YVrKPFf>)5K4 zmmb46D~_Q%#(Y6 zZKcauabWFzUtY3WaTP8W`{)Grp!c60ozGshw%zu$#D-nkf$$Oi3a8%WHuyUU?Jsmp z;MzV-d=A81-&C5I3qJ-0#;&Iw>+*0>0Tv|OiOFK=3 z8t)1*8&>$RtJ-YQvg>+VuY6&vgNoC!KgJ{WTi73ey#Mu`-`qtLFjJ5xekKAEIjOx8 zu=ZNy`zf=QAR24s-!0tL87{|uNQbBd48Td|eH1B~GDosDi0WkgnxQC_UCDn?0l9<%OjeR+LU4%6~azO8x#C&K$Zp5%!6 z`=zOxl0`u;9P*XAgbUARr2-9;AmVz2M z=Do;t02jp58*)Rv#wCi26|#Kon0URTs49I#`Ysx%{_$nb02Xld@+GMgMO3 zaPwBubMXC5{%1+7dtf+9u01?QqAN+>G=9kuQ+s6LGgK??U(8v%)$6b5p&AuoS(o^Z zZ4o@ZEB6htTHM{2g4~Pz9Sm>2{XWh+XD8}#t{Jw!PjG7(PEsrUvnDu(_hz9pJm1$J zIl_DoQj4Kx8=c>#fM3kKU{4LUHY}2MuaEA86<&4n?8Vo@+5NCt#=7A`~)mwD`K!AthLDe zi(nK!qkqqO^t-&_qLugBy)%+w|BL@vj|y6log`Q@d=a#&+lRsNOOXBb^PJ>hhafoA zDwNm6$MufXe>uDp00)U_E~;lV*8K6E7wKby$-*FujYr)FNU_SPH5rA+O7n9LjWRiD9s8?WLv~Ks zIJxD~lh#Lzy{}D(PaVs@ESz+`QC?ZrlI_HHoX|7$aJv;eY!vd5)BKQk{18&3XwG4! z>#jHx^YeCEQ(*98OniH3@x6Kk>8r+yM3Og5{`ETGJ?l?Etcdx@^Wq00+h|DJA3%E|-3b#jSQBdAr40?% zhdzMqh8-gUNWa)u+Z^azhc4fD#J*pDKbAXL;uOB8>gUd8mA3G)5(A%D-33w4xSXMb zS#vYhH@t1K?|PwH%^|zZj#Jd}xK7d5EQte+Rg2zZ zh|z~sj2%^&3nL%2!g6E~yO&ks)2wwCXL~w8=4+4D6i9dI-!%7txb#cgWBXE>qS?)S z*Mgpss20rGfcoRYbCtBa=(!;mNInxFInS8<^YuhE0v6`7QBuV5g z>!uYt;t>qkC3GjhxqzEkFS6k>_rk+I-4H@_G+u=&sR?9fn9TJ*2}gUSG$cIx-|v?m zu6!(-zdT+ZY8^lPPyO3?*OF7&4%*lnJ5i<+^HD+0O@GQ;&gA{|4*mGn?xx^};>%;` zy8$}j#hXiBsT+yQeA&Hx(TLE-Jh>1vtQ3D1k1CnLXE z`FZts5LHV0N7&LU@8Ja5-Lg9j+F)yS@NbzC0c&y)$*Lmro%4W?Tbbv;rDQxIM2pL; zFkDh{fG+%;FXYJQ^r!#{iDE@A8y}mu6W_TuMBFkZ(&t=););^A>K-h6T@2_2{MR4W z2cJw?xr&=^Qm~m(B~NndU79TJxYoFp>#f-oJDRHI=nd#vhjl2G`{rddJ+%k|R<1T7r~cMTViWK$X~}$6I{qPXx%D@etym7If*~Eh<#RcD#e( zh7N~6mENyrhEPBmmPK=xbFYO7-kqmi9N6*_S(DWn!Kepa8s)_B57siT;fQa-G8%>&pqJYU!1Ox<>7OAlzO^66c zgpeo*NdGGWM-Xz5UL(?^M*2a;NQY3;6r^WV;z)|`KJWc>*SgLXMwwUeTNo9@B9*5@BEWi}V(pKu}GDY;s#LC>K;UT_YEPJkA zacn5a)|rD%-#9DXGojc!{-`8iU7s&C!bJ))+C0Gh`)_1GgNePUI^%b!{eCDB3aE%4 z7czpAYk;e*Y;`eSc{dx8UUv}2T#x>T8pt^N29EeElgN}$ww+c_EPC7R&yCl{*<3Dl zFDy6t-)6(HpBnOIv(7bT1NQqf>X~YqZ_7_Ff6T>~PrOM=`SRsU`%B-Q&(QOvBlC9O zM(hlu!dr(BlXfzEz$EQ-LIs?a{!cX#^Aax##{zQN5eU(X0rnP2fQKMPO~@B-M@pjh zcS!N|1Z5i-gx8YyK{x@DWcGfPqgf^5T|^%md=MA23KVF&KoT#Qb{XW{kY?>a1d1=k%l*gw3*{i84mmH$Y8kDSn9)wmg~wl z;9%qK*p;EWz)1xuZj}}G- z+uNTknlJht(5dHdX=6$=hVz4SQw~hue`aSumR%c3KN)GGnQX(#$<9K&0lJ(90h&)J#F<-akN(&+MIASwc;|qPYoc9;N zf8UYh>RVBdF?(5Xo^gWNDdY=C;yqE(FMogj_-1J7vQRp~GTp05I5_5_6CUlke9|jW z@lw`3&q)h^@gLD;<~;$JdQ=K>pC4r#^7x{n_(G4qBK%7Dz$}W96DNLoUQ@N^)0f1)o|2_Bz2 zvt>ZG#j-l8_^MSOam?+iM}NK$qa5}-sxPv%$043nH=ErrDau!at08KCq#8YzFIoZ` zTBmaN?rJcAh7-EcO(vF=|g~S3Fhbt39N#iCi#)V_ViCvfyi?fXlg}Q@9m@b43 zUE&SsVR3RPZ;u=+HSd*_I$eIP7B^!VVWujrfKGsKrv=9Mox81&CVK3B$lLIk?sM~( zb4j(uCV}Fv9mmMys4~1PHAKFx&S2oj5>(x-eCAnK`>}oREF0DJjXnl2E3W5VGE%KE zmP)A9@v@E*Kj5=6T~R(9lK@KSI0~acbsJP&57HbuGStgDU6;EzQ_72Q2@a%3Y9+=0 zEID8+ug;zSo#ydiW$bCEa*A|2Go7|)q6cX-q_4}YYl!VJ$*V~%@4xfmfbqRqZ@sKK zuYG%o*b%Jv*xE{-@vX%qOj=hduV+Ep)r~fw8vN-=E6^P9lLN&a}=Z<4A%9&%8Xy~Bo zN3*Xw{9^+V4JBw+A7OwZ+kpVq48k}7 zA2bVI84`EV03m{&+L%*P$|8%F_Cyi)!b>cmSLv;eVM>s@L>X7+nys44Yp$~_4wtUKE z0mux|pQG_`V`}YATToO8D1CMoGRxbXlw0nn2js98?@ReFK(@kN;w8fA&4g8Pa9Utd zwe%Eg)BafTb+At(yF=90iaHXKmA ztDklq{U%uJ`};h((>z1-r$3slrzj8Juh!(f!K@y~f42GAJ}X_9qU70@^(!|ryklWS zgD*wuGI{x5n?DuHS^!)VegsT`OxdJP{RFxc;V|6KPUt76pcKgZ4eR?BN0zHvByu6` zz7F-zc$u5GTRKNvCpshTALgXyrR8rH4tL?O zeL2b~&ZU9yO0QErn|h>b+STP8t9NH_tXodhe``>RiW}LOm4#e?S&rZ7QOwGWd^$Vn zFv3?v?az4R7n-(gV#m9Wt#l zb*siwG?aMOjX$dx!?wPkMm%&61O=)!mj1M|AnH;$0i=YgHaAe+WkUV&4NC3z`?@^^ zrGyjyP1(ScM75nMxz7VgjkTH1_XY~#(k@!sQWLUjwonmu>Zg4>8@6>}V~bubm7diyAhlN7(H1dQU+5o&{hDMpMY#JZ&-kh& ztm(^_(HHIxjQ}g*7@-pcpt~`tPlUS#q@UO%km0aVY0LgM=oU!51Uu(hNso~Z&3?`= zyOHq6v1rcyOty+qf4?!!DER^V1>XFCNqb528sfD>EC>&`5{fPJt_ZH*0gb;_KD zUID#mdX8LYyH#(87UY{?CI%t?77D zB=e1{`Br1~4vPL+COZqDz}`r!cTuStUY|P-!d+3VXR()ad-Ih!V`}!{FI59qA*4K$RYvs?MTILHtGP*$_rL2;JXr& zBw6%bc;p0#6T|`S^ho!2gajP9&@EKu5ZYg*i?*Q?9FlED*ReyPSHl+VivmKUOu{NY zea1JI8`@t^4mxB1yjYbO@wBz<)3O`azt7Q;qH|ND1T-%0_KQujV?P@g&q^*fCJDjZ zitz*th2`QYK?q4=r;Sj7Fh-X_n(+hSGYsnV&AtxtX4t<5e;ZDHrO{*OVDZ{Z%mMQ4 zB|Ra?K}M+%hP8LoHSyJAV+$f~gQ%<@Q{5msYGz&u6TK%s#g*ZrRqf&_+ zwB#fbr2seeQo!zZgHCI*`Apa{1yjA+A=PrGMbx#W_7Kw^{4@ zA;+Ru74G;?B<;Y=BDS7$PEfnkvzEqb(-~oI8Tm05#V7-lEITjc+7f`axPL?hDJ$fv zVBfm-x)wokhor}>yVtbM=b35GPW|NZtG+R}a3x6=ZbzOfsF-VQW0Gk+ZsIs##KKw$ zE1*K%-p?WShzE8NxAPwWSp%tIVpWADnSj*l;4CB}_4%h8u%NFHM@oECHRFsImM(W7 zIK;#as^lyvcce-=CB@h(xU#cu^4oY$&5NR$rNOf6d(T?i45us>?;iax7)#SB-2k`@ zoymLYcqOvga#g!w&&%98do3;NCI$Wb$5tzg30bz49=Z;*AV{^66~W(skJ?RaAcKzB zfzah`%2`?*5(RXs%Bd|pt zibyh;MZ0Y4Y0Os_4x22*)NjxIVJh5X#URQ}f@K^{WMt(_aj<>#)DHR2NCQ8LG?*QE z?Ob4$rUQq5ti-b7?F8s>x9fyZ31{4!e|X&N2e>Y39@ zw&LVue5E(}t{|Y~e){e(gbcX;v*6wN8#MREEB(9({27QCi??Oy(}-?@BCxpIQN6-r zjUx^6y8N3Zd|x=bR7gi6A<*Wp3M2vtudMOW#kiVvM@S_uE>FzJre5k=jkD5P}5pdF9_gU++P&*6j@3Wz2 zl^W`aH4Zd@BVV7rQYTA+Q;+F3t`nR z=e?;Jc~e@}M}J-^9ekDD_IvDEi)Ug(#H(M;7+rEn^;H26Zspqt%C>^n*dQVkwp(xvz& zoUA6WUqiC7Kr{5%PSOibEe>$E8%$3%OjR+{XBG`_Af(Qf%}#10{JhYRqc*(q7!ghE zyn-~L3kM^A@ufiuH}gNQ56CLP%#`PhcYu(wWc*pd--eP2MA0^^ib2W+V5AnHotgin?{~xf>U7UfX9l8VXt0B0J!Di^ElL#ig--0VWgeuozvhZ?{p=e2}980}%lfnrzIpU`CuL2)4mtXe$5 z=ZC|?zr2PhKE%l%Tz}xk$G;C-T(9w5%%ZDy7{SXznf8UpfL>$2-W^#a&LYNj?BGy( zRV2M~J3|8G2-1*ba((M0-ou}DRxe0z+CKZSomO9!r_8a;0CLD^QMt{DfXk(>MVR`~ zn4;0)OuDiD(V8f=Tca5^C!aUoOG>tNQLsk(s=Z&Yo?X2ig5kKKl8E)Fca&+Ka2H*) zi%P?=BVvUrgkfUiL^m~oBukL|=B-itnSfaCQ25-TC8TC;v+={57Ji#MG-^QJHwhQ^ z%61$d;$I=d8FtW)j+#5E4f)wv0ZIR7DW;#wm(}3909P%btmrfcyGX1%`Ca<1DZDy| zoJ_=0MHV;vyNFv8z#c6Y^8mO?m0t|PWt7Xn>kjsx#L26-p-c3$SL*J zE7d`*wzjk1@L;w7KXWJf;GxHu6aN+Pf&rRm!NCgiJM_@2?a^tH0$Msubo$&$Yz4BL zCO`~n-f`fHm(}I}5`@w01il;zJytIXtM~VcW6zHns>rzw3yIl^iD z)!ULC8(xj-EA8IR)(mSm?@!c>XJQ)pqw zg4+jM&G$HR!VjIXHq`h8H(1Q5>qf{#7jQ6gOYRz}DozNis2F-)ZW8MBT#avwKMw9x z24RLE9;0L@%}s?~)^7c0fTbk48#lamdFPf~Cn!51Tws5*FI0_vE45$88dcCC1HS+Q zaj*a!g!}OB<9zF=a7Y?76){=?9i|k>>y3whWGt>@ow=U6@w?%})%x5hC0(PH!2O0B zZIM33c?~Z+cNDit^*w151(eGYZk>r?rl{@hXgo9z6+iWP+IzjgJ6}6jR4p;3 z(#HS#!Yab}3pt6Pfj5U=+{*!k4^^1T+F6J=;SEpsnye=UA2i#-^>6sFywGTIL`+#+ zQIqM6Mm)Gnpz2-ZmB#%MSWZ{ooHJ^Io1ls{uigzJf`56u z%@CK;YwZz*jTypgj5r=W_#`Vz`^EX%{nS%CLVoW+?1HmS?ytQA_K_O}2 z?(;*k%`Y{-juz_OVijxG!T-PscUy}Uyvh0Ow;~%$Cq94q!ahw_`21CeR<@9<^olrp zRbBXYq2vT+uA>mV`=E@8xgPN8)+owX_cn9-Xa?`RaAYVglK-0Tn`#-w2z`_vtBg%T zF=7C;6ol!H>_}uv#vx5$kII1i!ycV%{ayGVxU~isYvb$K5uTae3%J1-I4Ogl_5={* zDJd~`D<;F&(__ufS@#u%>bxo{(EN}=QjiFCv-GTR>^@yG?0?41d&1xIcM_%Es;lbO zF=haVh`3&kE_F?=`F2-8^jYF77}b6scFI;h)^KUfo9I?>-CS4aM~v@g@-fY$q&(QiYKtJazeET60Sj)k6g zjdme9@#8-(kMz#FWgSaDRhaZKRM|yC^ZUSX&%8X}PD8VC=GNj3t!9fxww_<(Oz?Zn zlD9Wfq+-sv?)n-LcVm~UcIM0hM&`_^lv}=rR0I7W4jytKcm` zRQ`zkNg%Ibpi&jL>P4+6{F-IrvI_svdWJkCAvjldrM z00>Q?`vGkJQ_4oTWs zuJU}?qiJ!SFKs2=c76>-YNH zwJdKLJ8gD7NwJN{2Bu1wJ-Z6xySrOf2nS!MUyPxf57RA2R;!=wlca%T(SQz{fT0@z z_xAR(9rh7BEx>U=6@7rHBlVD^bo*DuIqThxN_PlgKnAJ7sb=A?Cf%*->|i$t;ns4M zth@j8Pq4Oe-+vn{TP2(Wvjo7O6X;ZC0u4Cu&@=5M=)j?nWVeZa3SX2>mRMHZ52ZhD zv%0>s{2ve=jy(99WIJAe_NIM{weEV@s)Iwz+xpg~Oa--gw9~}FZTz1UMnCm8U`;fw z3!XW|pf>-8mnc>g#t(h{Gimh3)$_@j#_nPvR}V$%RzkBhmSPz_bs@C_tSQ4 z;X?|8LqaSfrNEW`H4EPsezPdUI|+(~d*D+S8XZe)E87q2%i*nBXL4hcaR$__-ews) zG&35i>|ILJk!LH3oo`6e#G8m|eM>c>Sd1SFMPQ0`#{@#Cu*`kVnMgDX;Y9z` zm$3>7ViC8dE8*N(>wRd9c!QEcVPid~ubnv<(W`iNtT?Nwm~YiE9yWXUhux&0mcn>K z;n=9iv+}7|aenf_K zet+-8reTG@iNlkIn`~XBuB>NsRz)~0~Ok$2vbjm>*%Jp&qZTmj!&_5IWL-3|_W`~f%J zS+Oxc8#C0zSAD&NVgQ7hC3|_57lk(pKW{njEXb$$GgjovT}~dnv2#nfs8T$!m^?}0 z9uwrDu1P-hJ|M^iPIX&oupBT?-P<&w^Bq_b&7Feme#*XHCYV4APj01YQ>1bI-aM7E zQA3Z+G^Hq0JJf}ts-I*@e?{ce>;L69?7)MkCm{sXU=A9kO4CRO(`^(747z>6SSJyG zg_F+3j-YPl$@S189yJW`jRXG)$eE`=Tl=}O9ApC|-6LnkyZ&?4!FxF9I)|HEJ-4W+ zNAhbj$(t|R!jC6*=4J2u@BT;SpSk56MVUggUD{h}J9B)-h03!Aex|RR8uJ}d&C>Z} z!V2IwNO(`@_(k+nCQI4CvBVElu%$w=-O)h4`|F|lsM24MTIq^@(XuSnw`#Eo=tT>c zO(ZBhmw+6J_E&)>b{n(t1;tC7;j>kV$7U}4Z*JP%>zv?!{QVqiFLozze)WwbIvCegHh!2aXFa8;G)!$uY`A#)ZGG17 zqfH*QzQJ3TGKHcPX}yE3z_oPi63scEw$L}H{kADzT~hh89T_ulp!V&*x1JfHEk*Yh zNp+IBSa5?A2z{icU#nO(YCP&Tm}`8Yyry)CM=@DARA0U%hq98*)KAVwK|7nn%LCVI zaJjZHwSo^E>XYzN*a||xfj~Tg4(VeS;qQb=T{PY_7XRqWB9}1 zYqXO@gpWX9GBI#7Te!=MxA*tGqxoJRT$;IjFZ+~R@v@+0r zXL`BkbXzcC?E2oPZ*KqfM`RGOavf~j`@xmIBzB^ZN&}owt_%3>`!aG1>`3b?Bv12! zmpvm7vqz|o3$-3=cp%8= zXnd@o1Z-c@+&AzehMl{vRkxNA83G0ghn|444G4%cLM?gh((hf?6%HjnrQqQnC_ z(KACVnIMLNd;3mc#=2og|tz>4Q_8zIqEI-nA@bc0gjC^SpyM!-JB@B=X0 zQf=}r2$Ur#WDBKb1+Uuf8(>!?_j@lQ)n?4JD;n0h#@vADxOKTJb;6H++h8CEjvXLg zC&FHcT8D1bG`hObuwvv5#kJ41wzT?X*652qMygH2>QTGA z_9}X3vncXQK}V&ci$3s&LDN%)scmfHp^2T=^9ut8??1F;^`Om7#8?5q=!0pDaz`&^|l!yFX-jA`Plv$6kH>2BB|P3f!R>Kh_C-A;}5 z?M=9a7-P)8MEIZ;@WJ@63o|EtK)eNM)maE>&@{LldGT{#@2x^{{JvGrIN=6@Wa1>!LCj z9^ymTg7f37&F*(|l8}CiepNM!z4V>hZKAu@5E>Zt_|EG3O3Wy?PzZQ;L_)<~RL zLERsbJH#|n6KPcrlIFe;6yPysaAki!ay)pgx>`?P&-XK5iW%P8oZC&8Y?CRa9_cNB zjl1rn5mTE+*#Tura*CO}*=5DFVq%1F5$)PIG2Uc^Zsd4P&7Cu>o-O`(g&Gtd6ryBt zw!LS6rG~iHi`B|c>yDH=p8F$$Buz&M_tpbDZB(jI4@zHA0ZeCk4&dU*!7)6{ zFhLd%4uH7Wf0ig>)!{+s=huk4;HX~0F5rrZg)g<8Ehs9-GFz*$mp&UlA91B$8V}yE zoz`kIuXspwB5WgZZ3U?)Mj}$FjT4bfK;TUUfAf#<#HIrj5vgBBb;9W4%inRonAh8r zd#WSVUo?Oqu`3WM(c{Vqj{#y0uCfoQ&eJVIZ#EY01-nOT>BE0j&v33E6){6IhpkVG zhW$pIM|6j*(NuUO6?J*8#`A%;nl|CdEX0%vVt-XJe^l5EH=E{NnA3t8)Q?W&-=b^~a zmuPsW&AF`e66YcgrahKDis++EDFRxq56_+t_$@K`8A3m53z1PuQ3ly5OvY3kO0a*8iL z0#t{<20D;QmROA3`G~Q;|6B?G0-Pi?8Lh7$d6ChVQ&SvR0|vX>72l1L86?*>-us7p zG+#a3!$0&N!$9i+2l&E|{}CCBV80j4k-HFS;4JYbAQn>TVjjV`Sz!OQ-rs&|Vy(%+ z{97P(5H+&|RxYr3l|Efd@5F3yNy3@vZZ>_FzYxX2B$>%oY~doL{c_^dO`o<77-y|3 z8*e$MltT+u>C(IFeo#^d675-6;h%%+c&2=7L55D7R3%LA0oyKq2XoN0hgSV%PNf!7 z9{1jxVK)j+2^kBpV>EvHBjQm%rW>aD@t06(XyYkry71f{FZwh%1`9ITC~2gmJ8s%> z*esA)eUJAJA;Z^Frf?GR=Z2&dzYpD-_tN%!{l8>~eeivbT+A=)Q5m+M#@mbThWTOH zl@H(6?e3oJwK~9d|!h1NK^0Wpd0c$+o>lzZ{7u? zm|8>CQ(H_Z&Z*Y6(vR{@cPM-5+?IZq0NijqScsKGcA(^5y#9C_nu&m&08|{`%C6Oq zyK}A9gScynX8go3s?rD={8YZz{ekUtZLmSPp7-30&c4|*PU?B3tL8jg{80hPOd}+M zr~RCxA5T!knOoIEeU*?$mx)X9cs7F?-r2Gj$7-fBa5-KdbDSMJ7%^_6)C;HGXxr*33vA$@Xc zZf7fJrqNh2T+o;o2zcdF27cdv%^44OoW=XShJ6`z3>6@f6-O$1!0yvLxjy&WH;qtm zDYH_{)-(vybKl*sd9?I%kWZt>G)7`tN6sXXrXMbT7-`4c^D?B(*WNhEG$7f-XwVeElQdhhbh(;HVZud?xWBpIZ#+y2fP zg%7U#uWs}h9xT)eyN9jRdGvMIZMdRN?(de7tljMBB2SVeCml#0uW;lT3lD{I9(Zxi z{T#Y)aee8z1kR^Rkap zs6l?B+~uF(*ipJkA~>hc%aL8_ne6C^7{7R<$i+U_!o%W)9%`9z@tmecvR=-75YEo(_t&T+^b4!ER!P9d+@KI z>Z!GjZQG3%i^pc)ztA4l=8*QCZWZmhN_&2<>a3#t8FuCTgEaNWCkzU5=QrA`)sjp8M zs+A8I)Z{tjXajaTOtt1L#3Ay2D%N5zNBj2Ju~Ldt+){Dk}QF}*o4rJ$QV z$%VK`rNoKVmi8)Zo!UPlIuuSYwb$VZ1-KzhDX~$V6k0M@1MGq+ux($ynxLS8l0;BR zI45{S2|y|MZ&Knni1C!Yc#XX$FmMF3dgXp(kz#3VxOqrWg>DMMC+pKyy%K+)G{j0v z5_}Bzq$|#Se=$|DuJnv8&3?aqw(;Sl6p&c=bug-_WE^!N)k5Az!q_n+=!bz|nl^41 z9wCtYA_nAOFDQZfur8K=pqpz6HrIlL^hOGx@cy_q6f<=zYi*l9b5+eST`n*uu7>=e6xRA9a*4>OCn@mV z3SkuY9tYTtPxW(a1pnZ%4Dz%S*9QK<8bmROD->viWtC8mjRyrZL1Phdpv|Ie0NU1_ zzqK8Xeo#S0B&_`OKuP#_XtqrNGZaHL-t9d9puh)asOIM`Z62Zj%xdy<4Ak~GJI`mg z&42l7dgi_3^XcHBsiV)_DS;7^;rB-DEamQxuQus=1U1gh>fRy%Q;@sBHEj|g|A2e6 z8rlQA7p0+Xm*GK9BqTKE?*&bEqEHrZ0U`)&-P$;`*+p+NI1;H0rkPD~$fHvMs;4-K zV`-0=(-d*q44l#1f{M|29rIoE_1laU%;Ya-1+nuC>e_k&4c)h^LkI5&*M$a_8o#^-o4^isZLdZSy*);}ZatvAOtzg4 zZVg_)3I71q`n51rDyA-+z8cP@QotH*t5o|Llz&KC1%lI*l%+Ofckn5VTk)Q2WGu#3SH1AKnSV^k)A%;vPaD22?}7 z>$3ddVaoM&8sL%uVSf+#2|^C3f4E<>^v1D@#wW6#z3GoC zT&5ZRbj|6Dt)%5fqyL(?qhsi?JE%cSUM>AM3tSVBMs^w#42hivs5@j>6Wf`tIs+&7 z0-ACS@U=qG37=3hSXOh8WFKPh0+i5S?PXB;K*Yp58AQ4)wG4iTe|6nQ-S*Iz3NThZ z(omma0okWlERjaO$~Z$ZT0_<6Oh;-82j$*8_fgR5udh}`4_~<-b*QhXaCbLuP`h-f zBqxw+i(HO=3h#hKx&Mr=fSg!_sPK>*;UlI|Sj#~G4nuYyYKqrOQK~0TMsZ(LIF{*< zHcPaZnt<6?zbua&vK<}omR>AR=k36K+{FYX0|P6U-v)-&0+uiKn+Xy1ojudathM_G0xja>a49_56s z_@=lT7hic@x3S>n_S#Lsfl?Ri;k(JD$1bNFc>Yy=Mor5yvh4f78RNB9zv{pM8}wKI zdvA*}9D`boMyA|me}4b{N8~t>bDj>XuY=$n^@P<-u@DIA3<(;rLp<*iTk zv#R-M-2%9q`BHcs-l#0e#2ys29m0Ds+tF?v7**K5f8~P@OV^U!bHDMKUx(HZaL51k zf%_I%viZW(U^4kf1OVzohdYwUdBngN(G@CGcT^H?+MHZSyL3!s^VwgjMXdU~2dopv zs~u;)XUeE&K7V=U&|pwTW^<0aS1a~ax$%`|n~4zX>B=h&07u$Vy6qsN1}T22g)((fGV~Vw(KR=Z}at>X*`47vVf*MG2$>p8AqL z`G1H1Z8)SOcD8Br&hZaH^iHyp#R81!ryXrkuibXUZabH%gWf7O?UOZIv!b50i$$sT zNcIobRrOn4TUk?7dTU1JDjpQR$B5nqjclS`{pp3LtU4H9)w!X?<8 zJdNKvx-$+6bcr^a?($Y^hcOj+kI(hnRI5uJs%DnGC5QP|Oxxt*md{R3BR@aHkIQO{ zdoNM1G`}7T(y0ksCMA_rn&DklZDfS?d0Ou;KfZp&rA*rm z0nrPbUtB}#m8JEWLrx65Xosc*v$5tVXEc(AD1~<)m6BA(dPEqtBpBn zWajd^qT3Rhiw<3U7T`MQ9}@MO4(KhBf>sYUB1Nb#9FeRcjL^~wics+cRh)jGqC;P_ zG-Kh>&*aka)s1R<;^Ro;q?;V|CKh(zr02K&M1O|@_jIwPNHY+3f82A{-P6Af5FKu! zXk<)>T;(g3W25J|awB(uKH08XefV+BXTM)rwA&vkN}5_ohm;87+aD{a*dZ~ zy#0O!-*{A%>+trOE&6Ho>YTakuck+T_vDMp4c+a%UToQjL2q%8bP$UHc^(P9?^FDp zQYy~ZYmMezH9XpU%a&*>Jm+=ostTua1#@Y|3dcO_8^_1|qU`-mZS(@O>}H!S z^=G3j&h_eL0U?fpa1f*LN93A5tc?XqHBEFE_VJtGPx!4v2 zp&9+XC`s_w#B2DgLqd7~C_V;~>{4vSg{zOR;>Cl=xMcfQ2$O~_;;eSN!`>qPt#JB>f4Jt`0c82qS_5!4j8hq3@XFf z|6&5YIw^s)yFUlOi1df^|zl*{cW=s_AWcofL83(UTsvy&WMHSIye5YIVZv9PHr z^6z?oyiiUt#I7~e`qRC8_wPL~TAO`lZK-v%K}|zVCeU(sS@o}oppI&a-7}m1AJ$-t zzYP2E{^B14hio&k^CTERzd}e-ZvU4vTwmI`9yzqPXejM*CPJNkW+#|XB1FKr2 ze{)BELkQDD;ow{0U`{`)_){siY=#7eMBnsD0#yvsinMFVG|5Bs(KEn-bOuhx}D{ zkG7p+QN*H2P7{D)#Oqg8n!szJdQ=RnXM4vs1bGqoa|RX?I$G=aiA&RI_ilMg4QAXPq_x#)LMV*|^HUGGM) zKTY)y;Ju%HQbC!IQim!gsa>=~<_Wjzh=Wue?cl))$qY}lV=);~* zza~3A3!J-WclOR0jpEXtzHmgiI~kaUgIP{Y1fdJlSVLl`@5197K;)4zo8C{A6%HUo zbUP4uD`1#F=_GD%GcWM;qwlS1a{yfkGwl(ZbO6To3ePkQuZEu`{*^y?khAe~JUd@M z_X31$uDLo*ups=4;@Hs@`1in<90xFh2esi4E*yOi+H~2m>H}9iGkp|3ImH>; zoD8Z6d0s3(J6qPQrlvYD_rbRQ>C2pZK^7Lq1m|hn;mzqci!(wu6e9ttDkujAmJV_R zFdPI(517zRR&rBX+9AwpHCsQKAT9VOyE`&Ck!Rq|KMtMEe$3adz|}Y;n~WC*23B~v z!BL(2aLL_svYh+rEb{)q&G`0GT$JMoUx&N7R{DwnCVHJ$KqI|VI8Az;y($e52`i$i=3y>?cRLziOvBp5Nzgjc-jxq+-3 zw3R$<67H+vXq$66Y$WsafU#D?_Az-k)B3!knXvirV^OwUx#>DZGx^o`W_wL%7dbV& z<C`{eb0=PHJ#{#6Ubjs!wcOO$sPvMm!QK!yZiimET2n zTAlf4g!^HAb4c)WC0Q3t*$Ibnbyr*bvEd#siuj)(a<82OV}Y+C2U+!Hk;IT{_^ZR$ z1&{D&OW`_e*tVdX1u}TB-4S*0wm~R0;SswHKwPP-26?W!hpEn-UoooRE`u*g<3`Da{z&DapCm~?lkoiXf^Q~1@u{yCO~RyTi<{mV#kAg=rT*hawa0pc ze0_Yk8;yUyP^o;p=jp4s%i6yI$#XYh)&#|uOX#O?E}x-Xqoy?|TE>_^;T$xsB zxU1rAz|!&1RKo|KNNSqKnoEUB78d>=8uIea+c;Kh+$y&Bw-~SpV1JOQn9|n1e>RuB z-bs0a;J8qywYfSIxs3H*(qm$+)rvW!xXz7-uqj9sprAVdX*3@X50yF;Qo7ScVBiDL zqpc#!a!H6aI~7`(=Kv69kgmfwEm_XFPbU837vkx8KHC^`LO4y?1rkzV4GLc-${TDb zMFc>*B&87Vpa85_-w)baF}$ImTque+v`wZyG4ZZu2oT*!g%MDr;INjj$HPCopuFKr zGIfdFLbVZ)gnF7en%xgN{iE*U*@*`2-s_v0smZTQEum9~>nskPhq6Tep6U5+>9w2I zcjoB%bCWA?Z!Go6H=hIf(3vteVJZq|PQTByuE?0rssH`;ie*5+Cq1bv>Bs8gTuQQ! z-Mw=6OGI4?=~U~`?0`mipg~nt6~S5fiC90en#~KqHFeUefUHHomp*CEjpCG4`?9dA zf%JOMP1Q^Q=FJrrvDR&14+b_a3v-9UF^&xm4=JUC8GZuWIOc~3XRv|(s7;Oejl8jH z+l!R>3iy9sN^FNG0EHt>0Y?of-a-CV_yuR11N^oLv!(4Sy-uztJq;-W619P1N%XPW z2|pC46fe#OPKv&L9or$ms7Ss9d3_=kEJ^xAm3YGFp1LKIa8AA8;j23CkYlhx;SQgF z3rv#2XwR}uD%OpSIZfJnUi$%YI6SJXsJ@~2C~KsBvN7=qFtk<{2&9|x1wZHhc)(td2N{w4ad*r^YU+yOJjB;#OUn7@+)-Sk zFn0&raWn7UY@b-I2dG2v%iP*+s1b#14tOMUi=X$bx3O1bim{y_CEltqC_}w z@SfGb9?wkJu~d7>f|?m|H`k9hUGLB522E4e3YVs@*fmuZebuP^JRe%U z`|MZOTq1k`y;wQh+SJTj$HXBT(EQ?bE)hHqY6_8nQi0b zjBMvf+!3D8?2G)`aV4!Zt>I@giI<1rPz36}g=xaAe?-XO-#&x*XM_KcFlYQgP>DRc zPXuudCLqnFyphWAW$C|VZc;rT7Cid*e|2jbn}0<9h@cimAo5pj;82+h+7da!u=iQO zNa)zkcZc-ii2jU!R~qu<-gT&t8B7=6WMCz(dg;i$Ui@yQ)!5pYKknkTD+zZiS>uDe z&>#A+BkAMmt@%o#@jW*IA->w}EcSX__e9S)*ZKPl_f+M|>z{$6&dg)~ODS3WZt3*c zGvm24B}tg8pc+Ji9iCtNey8%}Aat4ivQWy*2%XIi)w$qjuF2YEUtf0ZIF2?{tT0-t z;F76kfAZxKt#bQ4o>qLBV-RTY0Uc~WDqZs_6v!NSKj1sR52?zJM4YtPKl3mmbpGFW z%Gd$F_rKTqO11Y70Z?((_N_%MgBbt!{tvffkMztP-v%Ma0bK6!HW|d#~{lDEgi7#d9pcm_pQPte)fO_Yvz2*L#T%iElH@w~#RAdc~A zUpGtpQ{L6f-QxvaGMX;M-c<$iS1N5Pok9EJ_RZ-eQ@4~@3}3+$tb&8C4SWOmAl-@D#fnk2VXKeZ08>5Ys=v=T;5^Q zNIUG$_4iT|wy#ZY=})QmZXGM!WAbe2GEwpA|A<2$-svDZJ5=r-5FL4S?LeRNPTzxC zPjoCSaVIMJtTeYW&>V9>SWyr(hNY*Vb~i*ja4!q~37b7-17|bN+B%MskuvZn+NZ9r z?p_m@v9DjpKKSox93O9ltj&HjZ`CY}IJ#qy7&W&Ar0@cY(JGmdGtN~xHl z{?7izW0xvVzSWrj;f#E6ZMUH?OK|3G!co>is_P`Iv22ZYhGEj3; zdGuVyH~S_uRH#)p(CHHP^>hiVC?k-M*);xoCd(x=t7vDWL$avPqF+!5iP@-(yw}!gX-4byM#1^Q`rbHSgdo?YRa;bMN$Z) zVdC3Rr41Wj6vH1!AfQZ< zNoJ5)rbGxs10q5gBnTlvWQ-&V;g$@Z?fX98uk)U&^Zhtg=bU=1vPynP?(BQ-z1F(c zwXUmjQJxC2u$2fFB9^!dXgnYePERZ{>q9w%yi;#-R3Rcu1YocF5u=Po&_y73pp0s` z^9cZB7N3YfuM9-=xGK6d(r|T;nugpujgz9Ow<7`tKekv9C5B5&tnJR(6nM>iTHR@l zHMfoUnlNTM_{3UcRwEfvn6KydHuV!lj*rZ5bZvCI+m@Y8QiCJ`qS^>{zb^|2vpiJX zhEjn1x%nOL$S70Q{O%6xJbG~PHf%)k)V8E`scXc4;JUvWyDTZ#f6KYwy0 zK+*?f!RiGFU?u35XBQY)0aU(Cct$M#liOj3uV21zO!y?<$7o47`UL6^`bdC~(K$0T5!Oemi_y@RWadb5AEx8E?RY0TOnJ zpL{sbS-pq@>PKKM*DxkZXjFo8l~S@BZFLHrRcj<&9IjuL2QSERiP`9Ok9d_xf^(5n zWn#NA0#hN+FXrvZ6 zCnex%#)G?$IlJIswIf!D;t%@cbgaWUO2AE>lN7Wn0)#!#( z#7kUMDoYKmz6RUtUvb+sBLHMT>3l~h5Y&ioo0KJ=}-nJHAe>;4;a zcZL2JeW2_~EY3K(`6)H*+F`r%1*LrzvyO#ThM4o#!r(%k-VRsy*TQ3FweunS9aB=$ zlG9zv(cdiE+zFZReuVJgE*3d~yw_A8y!UUA2X}U6kYgpVz|GDeIab^v_I%f>3|>Q& zE;bRKe}9!re%N}tA!9!7GcPoXpyHRan>xDgKRFw0dN$Xe5na8F{dJn%F~R9m(FY`1 z-)=ySVG8%lzaeuS(2!>6$|`8W2Z_iCEVa>HJc+K+r=cgPv&~R6(~VyBMFVkQZEz<_ zXi}a2hM-4nYZ^7ZP_&3Ls^-3{oYqX~bm-==;PH)(#<2D07JIv2I~CbZSYAr>KVP7J z_Rd&Rm{za#Q9-3lqObj>u|3*@Zg&jJ^HrSLPe)P$=uY@|i@LgMS?;3-VS^pyI0n2M zwsk>l#^p!xlxW6xt9M^C(2r+IQVt81dEL(uDZkZTSytHrky%x{p(13)#LKTLp~I8N z*BNIi?Ga(W-Ee{`z#E+q0ip*6&A8BksM+8+2=*T{PUm0++l7}~pbG9}`!`R_SWJ%! zd@R6Js|1&6^$KghY!7r)%XHn|m`jgff#&7XU02kR_ikP8zv-G+lw#6C(QRvPX4=E~ zu7;RGT;t6_WpRhd@Zn ziWB(~L*8BSleVis^`^#!zM90q_x;pFlLaI{t1MLSUZEzQk*1>=$5{~0y zk2e#4#hcaCrCS)+cRv;D^RjXjYQ_wAiGjETlJi^uEDvuJ_7g^@)kLWX01iwB@vUk_ z8L$rcLn*FI$8jNYoU#jIb{a<}#U!XLFr*X`KwJF~> zgfm}{2`xbSx3eD#r*A&aYj~`&H*4-U!=oL+OHQeV##J_PT81u-j=FaWWtG%bZr1Bz zjQsNK>g}~pCg#nCMV5J$nDm+%G{kxN-yNLu_??cxB4Vgq^k-33tG{47KAZ|bGD|{_ zqEQE2%7_Leu#|ypWT2kuxtWQ-&?KS^+Ved@8d60Med5aUb7#$97QCYPcbPAjjXepw z0Dtz3b)Yddfv{JUN4<`O&ggf+8*I)XGA$+{#S$q@9hjdobNFGXl1_8DCX-njZWY zy+h=9+p+j*N!gv2sn>m{Ni>4Ld0>JShuWkRp?`gJt~?ak?APx<&G;Z$?@@0BPLNrIV*Hg^>UJEJX0FIhyFUl{!S*a{ zWRPPEhAV`!tRySi^x5}9c*VrWCD&==_v@2r7-)~bRdGCK)x_cF%f!<1ZR0Iq9p+(q^6&fF5bb#zN<6<@iG>%S9pVq z@+T-Xv~Z0cwwlR`+e&an9n{hhXQXuL`Q5=<#(%#ov8-po87&!6I$PsX8~0>ebnK zbc%JWspKxkKsz@dlVJ-bBV;z=DGB`1<9rX*yO~7L? ztVsA1tiXLso1xLi#D|xsk9kQ6&uX-=D&z#0Qe2j@O0zn;3Wo+p~VvfG~KUzz5A_Mb4|r$AnLj6mC$gmCc(V)Y;gCO8g0 z$>XQX5sG+&7)?0}eFGMyIXk0f<~e0t57t{biG+y>4;t{Joitrrg#jBz$(|tXCULMR!vV#f?sMyQN%5aC1qxQzQO8^4n#Ekpjb(` zr3o6EVa7A-^?Rs~i8}}yguxRa8(h6#zJ5iXMFx(Ccz^EreIQ9Q&qgM!o4Kp{mSa?; zC|rqXVs+ZQ8&4_oA8*p;j=UyUJjopKpn`} zsBDGGc#^F7l_-G??&k}BklD$n=yDqpuzAbj@*M&JQsE{;y`G;QNi2gDgx2{z1p`VG z%xTXHL4gu$U0^fD1E~-+t;t+t$=#?k+v%K__M{uhl1u7<$LE}e;!Ne!$o>s=Bo_Li zP`Mm$O>()eBlRvNh~*J^!L9{yeH6Rmdqov#otIY;;%ab&<#V1~+nkk|6*lM4h!fP@ z$*Kn=-=wmpr{AB-BsHn_w~TuAdsHY-GJb5C7p4>2w}ABNpg%y6;ghyNU%Z&GgD_03 z1@$QcZ!OA2;V8!dnsR^wUtHz(&!XGmF~mJio#C*AR_siY(wlL{=gi{8>BE;>5r9kn z>+@qcIdA|^Wc$y_LJy%?lz7*&zzV$XlL$W;BZdY}L&)Gh4Ez_0 z8DT*23@;`sWN@3`CO@})G2EWYPFQg2a3{oU|6CXn`ogbyK0a8A2(0_ss<{;E%#`sw zZH~FQlGm7H#IW7Fbg9S~7yqP8J0L_}{c_I}`|OJhKi-C?JEq9D#&0UX*ZihMcfr%F zK=ppL3S94%3Ikw+MYsyav(em3lCTf6{AbYj5=f4DPE=CAx@`&@?zqCGQMWY zsWs^|BBWLIhPnfnGE=k?%3EbM3ingM{X^V!11J`4?L_=SIVzMNG2d^8VGqK3XNv)g zW=P&#ZiuiSis^vET9xHvztowu!FzHH0a(pat$BX~a={R9M+Er?iT)V6;@H{jOZT(Xx`^AtQy;pT0L0e9v@Je0l^(ohVrKvIU=jTwGtU(^^kv{g2r9=|lE5q(@W+ROfnR@L>mEf=Tn z#BcePVgO#I^^YyRUZPgwO72hB07qg{>V_R2pt#$T=h^cdHZ2u|68Q&bCCoLT50)_e zWB;Ller%a8__3u0{Ks_A_z5dqMu?15`JbsdR7evc1Z}YTB2lU&Ok|8(qUPHCmyuld zLEsWp??lt@lKideHZWX4DnGVz_(ydKRv7+#UQh}wa(`?Yei7^qY~p$eUq z;W(e$WT+(f%zAfHh3M?sM!A$xJADANvp#{=crs7tUOGYc_Py&L+w-8m-bM{X&ct%n zp1fh~P+P=bPkC0i>qT8lt6a|E_=tzKb+22K&md|-aDP_Fw%)Oxi}Q_>e574<q`C{;FQqpYbBzB^ zsC8c9XlVwTTrO9>X!-PM(E-X`j+PpyU-{O=@3=jJn{Ka$u5US`PNrQUp1dL%=mOm+ zNvf=w(Nny9&C*lBC$(ZmhaaRxUqc)(>D6cDH@i+9PJJdkq2K1~dpHgs_ws3RP<2Zo z&n%<#MfKR@QOTw{GoP#(?_=FP$9fdhzS>(C5j&hfkTh595$_BkiO+$5t}l!HrG9KV zw;#a`(HH-@%pal7a}dHkh|ey`1Dwj>u4GI=!G%K+ab2%7ERWlMLrmy$uwPHr9pJD$ ze|b7+SaL<$f1jT`-{FF9sa4F==K%)UHe=g<-Gv-9>ngwW64|%)%*QTgpI8YL#ls{^ z`x;9-=;3hgwvXxxI^Ra_wtBeLx}!Gp2fA%emdPt<>HSe9mq9hCYev8U{6_>)Z@WMr zvh6@iApkjK;sU=_^n#!+bht_Gr0zxdA|`dtmMRX>Ldu34i;=lDll>BNkGH5QaYn_Hl>sI<${FZVO?!B%tgDt|~ja zf!D80hn$XaXUL3a?9T*^9@mzp(RHdcMDH4BN*PuV(LA?A(+d$1RnTB}^M{|-RZ^i& zBlR2<*4D8ldrn+%v{Er-p*y3<^q8N)$3jh1>%qC9*X>6cIv2|(G^W4a>O1uyKB1tn zVsbvCWidqAz+I2AjwL5e8-qtOAanvL89E?Mf&%5i1JL!=>_jleyQq7FGO(@zUm9AD zVvd8;!clELdF`KZCNu{8p7^B2p_jrQ3ggP7oN3Ru}9MS~rV;APNSpR46JB0pe2Wd@?WQo`GOWzQg?yla@P6 zir-K%9b2Uw=sn${T#7k9f$Qn;MmK5sX#43L20ejVKkX9lYJ<$VGN+W}0#zhq#CfJj z!BTIZqo4U;W=6&WyMF4SzjMN{V|~HXu!6kG3|H&L{)nF1i(k?A;lGMy#cx3%XqDuu z&X68aS-T-8F5Svl;sIVq)W}9i6R$1mMuZeAAvuF`OPd>UhR3E+Q+D%9j4sjODxPZ6 zcX0?IPn>B5oxCEBlE4NmoTI3O=pA`-vI|}hUy=PXcjOsdsfSz7}oOf**k}fjM0~ql#(e!HNn(rvKMj==BLk#E)9_S14L*YyHbl;69IJmmD-pat1 zwYXsW^6j-Oi|PPp#zWngkmglcw*niAH@g(&d zL7F~Q2ZKQmz8jiEN-l#z-1H882JmJ434r`XFMzxR`a*v$x(l?S=2=#q3shf{%AyLK zG<&xQq-DD%mQ)mZzcg~ukMI`kDjnB9c)EN*X10m6L?}!xxStasV^&NpLgk~=H@)&6 zCH*Vd^Js@TZJ((Hf>2&x(m%NCtVyqt(l--pRDX-?ji$3#WqTb)9@h|ME3-G+8kvpc z|6+L@11;oP@V&Q#?`?s%sS!OV`j8m@d?K%nDbzOP`O;u*DXqDMHE6Bk!CCMT|^CHyR9Da796NA3HQ}n zL2ler;$0*#>L!WU`9-|Vj!ldPq*qmMR-iHp_R9r{XuVcV!CBz({c-vb&=>)O4|hD4 z?atc~y9n~oO>b^m4(A2n0Lb^2OfH~5gzR_f)&??5FyzLW7*y)K>te5kjdUn#$X(0Q zX(KnZuOeU7BR%assmbWfvabGhx)>{$wA8E@Guxdc@We-qf(qVnfG5vC4CDe}6CShr zE6Y5Ib5IvcUq+}xCmhDnSZLB3$L&p8&Tk_W4g4ACb_h?tGl^Ao@a9Q?QE76MvJ}VT zbyw8hhX5^6qd(%-{k?#w#|Pkb&0{ei!NE9jBDx;2@`#(>!>o;2F=Gc#L?gC>kfD0C$5P%)?M40;`nKf4kUd8Pj|aI zdT0yMO1&ncVgvDb^~0b8c+)Y&_T2WGZ&hUlHfu=qgy{$`29W2P3B3!yn0?}fI3c;*mw zB_GrWjK*2Z{0d=!D3@|zKyVU=aC$!jYQ7G{L>kB>Gc*%TBfX0}r_Yu8BV@#9a8bWo z8|c6km-Fb>!$7lome%Oz05n|lcw2Fq9^-f`c5m>YL}a~%F_en>nyPra!XVXVC@27{ zmer)h|5sL%n`vI<#+@aA;e`4+>0#8DS<~XNRzD%EA-3pLf>dtWBT`0Bjb*Tt@F2Aq{y24qXLjJKy>*8={`NN`T1P$Rnu)w*1T3%-r zZ;F|r#!?Tqn$4IYtwgCH3kNI-p4G#Y9+Y+zOS5zJ0eqkNejMD3E>^~+xN)^1w?A6~ z_brQe=$C!?h-AkWac|^}ha%d7opN+?%F7}QcP9sbnzt{!Xl=hs^JqWH#c+S&wk$Wx z9_`lQpg9}6uIZD>em1>z>-C({4?-}*{y>NDmI|j{*M~Ia-{{{**EQq@X?DS-F0eWF z^E5a2(Z<-9PzZ}DgQH#<7wU4C?EL!JUpu}Mf4kYpe}$S{Y#e8plZa;1QMpXiO$Lg5 zTl4BH{JRZke*k5yYX{+*Ie?w;pYkg9&mC<@UZ|(@TCh!ycPe(7pj=)|)wu;BFKi4`cM&nL@3y3N zqCg$ol*OT4r6NXE2by6u2Y&x+AV77Gu)kr~%s~jj9!`wBj8*w}e>FAmfAv>)|95|t zKf40n^hPy?x~COe{IOrQEj1=OBw@;)9p)Xa6nSK<53`;S=aZUd@K{REr0L4@dmRT4 zvZ=7vpQsN`6a^@=ReOc!xj9r7A#JTu1}$e(V;m8Q8ag#xcWv4@4)a%2-eMl9;j#Nk4Pil zfI&w>SF8mm#r+f&(eq&DZa<3Q17`PWgmh~V58+A0nt<|uulSHfkrXgkxHKAAm$Dl} zk!u=KnWc@X?4%F!EmQK|jqmFXVfO}_x`hc-XVLf08YO(~t8M;xF6+AR0u8chy6a-K zT|^dTqIbphloJfua|v(QyaT45b6O4^pAV?JQWW=QU5(DHGOnZAa2q=rrV^&Ek|+9R zpM7e2c{)w?IA5j;c_OcPW?E3dviuhd($?>Z1Ni~W~|HtqCYK|7SKiQBJMOV3W zz?tJl_2j73Q(FO_CVD*ub1)+(rqZ*gi21?u){dL*Ix!Z{Ra5R9;}0B@zNe_Ae9>^D z9+;tC;G0DYE;pyMfY2MZc&aDlCFV9X-<@Gbdg!cB@Vc5?Fs(Fh;N?pDvn}VUVRvR& ziJ#m}EJt^RU02f(J&LnKig#@VAzU!Jn#5tg5!5{992?W&nXsE`GOV!gR%&nz+h>i{ zY8CZw_a$b>)(gj>?(M(A#)x3j>uI+8d(aG1Jo{OG$uT8g z@j!gwF9-K8x>TWIj=(R=P#e@a)PnSweGZAZz_^3^|N2hIe=>B5T$8V5DU<)KIoYPJ z%rT%QBRPLJcg{tWs{f935G?39!Y)tT6N79-*O>9cSH^0?i-Ofdk1Ye-BQirGaL2=Z z!nP*>6z z4m@*-sVO>t;Iil%s@JKsRB+LfcRL7Sz%%MeEPMRgO15Jn^f&K2O=E9QZgP9rCm>_) z=1a+aPWz05Xu}0drK58z`+XEYJ{|E33NqTi+5K1i+t9tGmmj&_J@i|4@%`+~t&d!e z^*q}4x6XGfXrlu9)&*3%S64y>t_#2kk&|9b7$EON_+&@SkdwqGJqX?KdQ*xF0A{Ne zh;{2zPybaaR27d?hK9efi=x&d+V1q&HVv<=R(_d#-TatlM{R8sSKh2z*_3TdDE_`* zRH*ngBqb@|Rr{phP$)*-e<5d)Jn%PJ^Hlx!+@k@bPg668Oej;Z9AFK3U%W$9O}wF> z0RIJlvpJjmz}#G@J&r#d8WNVrTtpq>3Q@;~-~xCj7$!Q@KsPkg@63@Mxj|oFw$t>i zscxgMSLi8uIlh~|>D!3z2T>vhRC7t>SD~Q0T?I4^D}HLQz+{C!qV)1sTM)%r^Rble zLO}lIRk3~;FeX%j09KQ>nUo&yG($;Z zw~%C+K*Pf|rDF0=FtTfE!L7Z*d)&z8m0*%J?lsR%J&w6PzUdoNvF>FBK%NIUJQBzz zOAvY}iXsxik2$f-{~K7npeMrnJE<_S3NB#=-42UGD~b;ihNye$+R(04$gLcbU?c9E znv?;bFgJZx<^a5X9F*`(Hagn_hp4}uES-A`W^a;H>s@bqmJXo$Q;$xTT2F71ft|S(`2&I- zvr@}qGJ^PtqL;_k+Kl-1qE>=2E@pPHTVLwtW?NP4EpvsNq0P*=5MXbrLFznx()Oi) zBSj{qTSbd98e*o9Zs}1GZYKGs|1u2b?#GBhgIG5V-R8mA^FrwaJ{1b#=D$H1K-pk4 zd|Dog1mmv1;!S~$&n!{__ki6$t53^z{3<0wQ&9rV#Wr_;yOZ4;4o&eL-K?4F+UAy2 z$LVQ|%z9otx!jJe8kcN;yTsnlxR^oKRFkb7@jhU=d!R0jGgZ%l*SRQXIwn@sY+$dq zsO2Sp^%^`5sTVO95j)KHVZsYjr!z|yslS+&G@cF31IP}-NBlaHNhcZ%20Nq}8k>amTXz!~Te?dZfSJx0?;Fh2$>Dhc^Jd*Yr)f zGFAfT0^tez2q*1ipozzh8%C3u-UuSBdgfN zQ0Z_(9qmhR-tCqyv)wbCTwJi|9O7>ssa%DRk7fnm%Q#zsA9u2#JJSJUgj7Y8tcjk-u%2ydD7jR}P9ggzBGFa3e}G2sQ! zVuOo^QAZ)-OmC}OAba&U%-);L@9uM(`TgCBrxUh|pA#d?aC;$=MS^wC=^9E)j#2^+ z0qUEXD&i|!V!I>puRv*`Ecb44jsg_7r-&7p1m6|{Q(SbPk1pp>4&efkqlf5q!~+YP zl;;>W8YIQY-B4!d^u7_x+=6pukn4u0ozcUf+ zo$v8N%v9ZL4dVqlpkoOpLyqHiz>bWx2LkaxgXbH&fvVN9@m%APD@Afz*9$~Z|Dm-g#p=DAZW_aL5h>?x9wxeUR z-SQ2FjlkkG!+mXZ0zP0RRKgTN8+<=-vEG5wS#F%|ToZm8bQaL$;qnv>=$k9Is1r63 z!+*dPR}@1|o{e0;#ci#Z zQQENn+jCXqNAzzAElRxnG6>|n#f-GvQ;Z&wrGvXOu*66RNJh#ZwQ(&$-0r|Hr}0M5 z1`g0IKHe!C;6#oJ#xgf@((nvSW!omazfmaNp&y$Aj@|=s+S`Gx4@3u(W~E}dP+wFe zM9#F3AE2cPhsAx&I-~n|7#lK`DO>tcX$8e=yp-CUt_u=A-x36z_@L4 zrjj1b%~i$~S{Kqv{f*?tzS`|cec4~J}hX zh%;I6Ul9SIj^sM6LPuAD)oB!81uFU31_dDSU{b4Q2_C^ahLA^G2ff{#-E^TA+b&$m zWHpwe#NFU17cY(djaopdVy}tyB`ju5mUpdvubVW{b-jX}OUaX+bwAMCcRY#vEd0dF zn4m&ZrO7z3} z8K|IuKKy{WbN($+8IaEJYCmVm0Wze$Q+`atNy(4`Xd;kPC zVia7aTC>nOkC_{BB_5FyLa{E>WQ`LUOS4F!MGVD(Ve<~!%wnwu<5hW zc5fi-ATv`J3RWt|Z71&%z%W+(MZ_nB;23ej9&{^*nLEbUbkT!gj5Lb8SKPOB)|+Q6 z*nL)0WYV60f8^`$vrQ^_)Lg&KTO`rwf1yE(PR3mL*B@KP$l?@Lo|1U?;*TxrMtnKI z@{XAe6kRlz9G^1eHVS3wq&V`c7M1<@3%mv-K$2u7+M{|G&TRYk0e%6vcBSVKTYH51 zU+)h9hzVD=3=|)^wUg)gvJ^UZ;jCI0c=%22G_a1B#8Y_Shqss?oZ}c~t zk@8j5L)Q!XC*wKP8_fNr1dLqH;MIHze@LtoLlWk zU|msb+o;2rt99dscWbG}!EVjJg))&~#u@BXJ8yPmA+?+Q$lYrI0B?bNCMQtssa&@`HG3ifp)roQEq7Y-Jhf5YBqkQ}g9zfw` z4XJyU2og{_X2#Y(qUo5aL&-@aW5sP|6ZF#tUc3aX@ zKaILEyktlb?y*1a13u$!87q7Du!7!$gMp)9WXR*eJ=1?yzLG!v*z&9RA@0jBvz|sO zyzZdCuYKDim%96)R_ec|y<>sJR_d#|+Xq5A&(`1i8q<)Es!*h^G@X_{{#dSV4)yY^ zbH3$Ckw+(4>G*{B96APa+vS=AKTC5i%4UDNe0Aa4O~*2AnKX2|Q#;#5t5F+GeXBFj z{5PsEN5!VXx@T?oYji9Cn%49mG z8RR1XGNdONUPJ$CNe@;k<~;r0=y+}?X2)uD$SBV!W-T%@xXu3$#rlWx75PwtZSzq5 z)Jf}03b#V+PV&!u(bd;2klJ6|=Y5BAOWJm8m(G{*4c&5%dTp&+?NHt5xM73el4x}E z#}@sN@y`ulR$@eamOAs%1D(jM-QfHrvF)lCugn>Y)hX=agW`O@KIn%hZ$g5?)Ot_kd^I*@%wEC7W5f1Ts z?h7R=Af*c#V}tGTX+7sg@K&(6X)jx=CcAxgQ|EPf0+GdCq{3iAwC-^HQy+WOfHPSq z^=S9;(Vpiz=Z$POLPtF(M%!H6o@ivD^-tB$B)c>CHE!rQs1`g7W4L}W5zW0=^$VeQ zS_2B<(Pn(RNK%AbP zHe@FILML%zGjb|1ywm+|?wvG4uDozx5j&7GBsfIq4~Ge5yt~m{J0ioCIh)>rR>a%W zP4#(YopOK|#9Rprd4Xg0P}F$UJaWAFEZSdReKaO!U4P$FEMOQ@cjN8GT5St1F9zm$ zfTdrmxdK9mbFj0iqhG7n0UOz+QekBXBGe_q*6`@pCd*N^EcNT7ZJ+4a2-}jPZ6kAo zH|DTW6q{(;U1Qy<)Xey-@5_@wYi!!Si3nq^a?O`lp^lmHt7U9aSa_MR@%1@xAQ&uB{a-ZuahvM!M;YGrO&)KUCYtYpsGRE{LR$WCYhoN2GygcHBDnF$G ziqP*gxrskj+JB3|8SGfdBc1zJ*D|1iko_ltRZ_omh&_txxLk$aOv%gaMmeQd+ma;@ zs-lMPIwT(6Zty)|&hPe+pphD(u{72DwJJOr5I>}?(O8b`T$8W@f10PsX%ieFwu71> zfEY(r0Nn)`qn7`gYsAsd$Nl_@HOy|}{b~94z(^V$O_W+QJ6CTf$!Z%+zz4FaJMddHBEtn~u)`vs_p#2!r z&a8jj$fdGflXz&B<|9Ky9CKT%5`eBGuRtv*N^NEj#`}doV(+>@SF=ZsAp?1{&<|R# z&SEcgH(Tbuq|FHI)T65<*3VhM6b2N!-WLhpnjJG`WLtz-w9~U3@vr=!1X*fz+clYZ zy)+)TyJIz7qze8j(~^DcSS2weFqsuGoyoD zRg5Zt=hwk#bRL55D#{fqd$UQ=8(hCeOi7@0r66u zW6r5;Ho3-j`BLF7_X4J7P4K6zK;?kN?j?8k#lYY~S22CcmT|Ob)$FXNdqV~@6(#oV zbORpmgOJ9$Xlg(NCL?54`A6C)ZLc%FeEqu0cYt6)p`oEcs&==7f^Hk#%fWond_H~U zW$Lu`^}C@rZm9hv0|BpaZ|bBZi^}m5905N%Q)jY3>zzZ;#v?`Pt@{w|fpLi1u_u9n z{+bDtT?u5F!Iu*_Us{9oWlw)s{~~C)JLkZt0d|`g6k{!q`VF|2<)MfjW|4gK;*I!s zX&M#U@d5Uw4q58EYqD6{A*#J+3_dd)lTz7ix&vk*E(utMyL}#dSukmT(lvRs#p`bE z&|QQA@kSuv#(ohs=?hQ1=n`xZ`ibtfN_m$qW{|tl3B9N(7Q!HG z1rB+Ehls~tm`%O6kPeqNwR?e!prN*Lg`1RGf8R2wOuVnn7(ED#j6m=qt&N#;DD#ywU^D=p|Ym4^$nq?{B%x@ z+!A4V+TARy#MvRmZlxlifbc?N zh{^;7(C~Z%2K3%U==$kUw}9Q6XsTMc+Fh>wr&NrA_*hLmi1 zg=ta#Y_}ooHEL`1_i;X;e?0*1c6R0(I9m|FcH(b?I2R%+SPNw2^mPEW|HDw+Y}{4o zoInHR&;Pq3CzmJV1d-Sk3ITPiQms$=tEPcIax<_vv&Tg(P-I6!GC@imxDX$IsZ-s; zp~N!#a>t!)O)cldlBUUQR3^z(ZB~w@qf=I*Q>X55^p4HGWV=4yCQUg`$d~%yl}u#m z6ddq>gz_*jx3XFW#CEGqseABeXOWY}Ju@2|Spe`68)7>2qZ!Jd+=wcix&c&A_vuYD z5-sweA22H65@?+#a4|GR-+Re01wR$|kRf6l(~pOX50pefkC|_XGv($-MAcwWN=j3_ zt3==|JOS1qh%lxoPeTF;g11m84+$`b-K`ERJ4FG%bPQ`SU`xw6)HHGkg3T!Vb~{2B z-hdNy!SQ{1r^*?geN?$^rC+#nbJe3(Lsoa=c)k8|c>HkA3A!^e$lftud%)&fjDdfN z_LJL>@*}K^-+dg4cd#y~aH#0-{bKR@Y~7d9k=3^u?jb{8QlAK4gM)SfZz!-`fODM& zA$5J>egO?#UBx*8keT#*<|@F^BoY+Cnl$(hkX1`l&Iq-MN*MIsU@vXt0q`zzj_~%x z1mX;c@l~ZL+ajpUfjdSfva&rcPv;%WcI?5K8D}N*a=O z5CE}Tvnw)?y#Z9GK+3+XBHPL%0Dt%{#j2R9z%?m?osvi3ZzyQw3Le3A&1WN`HvPfO8#@(9uwicRq$HC4_-3Sp^9M+Culy zid?!WhL_JTz7NPO2;t$A+8%!5Ge8!;axq69isE%g>P~#Yp8}nf-`}svuqDTUX}GCN z-p$)f;T&oYu^zoLCAbQf*n~J!HhloR_@>;|AHW;)I^{D|{!U8=?2J|yR(AModowb2E$RBt zbj$Ivw!yq65@usCzl&aCn^EwT9+&NQ(X?f-1>@im)tDvc1`4!*i z+Wa~2xqEN-q+ulT6OeB)n}ton`C?H42v;dg(023%BUmI(ZlTajl!ov-W3CS+*h8!t z@@}CbsDC9%lF;}}QDR8j5%g}TtlMD`4l_6tT(vXX=mIaW3nmMl@98!_Oc~L|8(~8V zAhgR^q4>4Xtc298E03nbL6Lqjj}~h2K*7QM&W2y}|xjNt+<7Gr7*sKB8W;*?MjA zysx>>&f?-?Ep=ZFj#iLKBc<&YDH<{JH}flkFiGV*;6abNs45{<5QvV0b1w$q!vq1x z=a8LfiK{V(Kv_M{X3eab?!~Q~`=Z02rc6qRIaWeM$>I!kssv0g{n)bU`5^viWq$F| zy!b8uPyE&SX}}sWv$`Lz3XO6#`Qt#M#Jr*aceWfa#oIkSdR~+GqZ*8i=c$XJxo{grvX;m$YouKAZ!vRxHZb$z{Rj;bp8caFPRy186U4R@%r2`;Qlr6OpqWDhgLi+yeC?pZIU zGHVk!M_al!fjR30H=R}$J1qrOu*~$kr?-u945zlbKDr*yo*Eyld;NiAo?XKDatgyJ z%slh>YsZ`;PEF9~@MQ9J6-e&D>B=0#Yq7CbU7+K^*~%tIm6TlkYRi&xUG^l~$I5?G zt%>Pu*2v#c6jU%N?Ks_o9yUD}Ebosj`FOD~Gxf)o=w09-#RGLY78DB1=*$g%Z29a+ zaerwY=jgZRWYSdT=;X_i2~q!mY99{&hgVxlde%WtbVdtu)tXFjK>zY%3vC-XlNQ7U zt`O7$QDjBpH!UmSaR4y6iAgXT6(HM1Ken7qg@1VcZb(4>p0aWR{*6uiuK#0;TI6>N zyacd=kpY*Q?2j$^G+VG4caw#E4x)ra4!r5dmS+*a zSqUMs*cLF*;r}}i@sxON*2L-y@v{nT-;XWknm@KA|Jd@?iuu&-4e@{Yy#I?Q`}Y@# zdDTfAjQg?W6IJ+RV@{R}|381}r^NqharkIn1VtN&Su|7ju68Hh$^_hjS+S_8FK zg60V|vZ-BW-sj&}t?5c7fBklm{(1D+7v11~l5WNs!>WDEE+3yA|OfM_Z<~EcC9*nnX z(!N*)A}a_0kY-Oipr;3X7kbN1=krwAT_bM$J_1bhkG(X#Ed5I_^K#^eywko&!35{r zEm|qB+eX(X@^U`EUp~^Y6k)a4@u@!b^1E8#PUvb|IvOx}cS8Y@5nOI7CjTbVH=nH$e%sn;Z*0K6f9PB2KlH{h5i7ZHY*6E~8LS%}a~h~H$N;%1bfhuk-0a{ETujF@zW1|~@vE+`}>`#v@ZNM4rOrfC~tVY;Gelt*OG zw^6|kp(g@n6^pozOvuHf819!zs<309?I`{#T!eqrth&x?P^hd-*UxE)c7 znRb5H{5h%2!wVxB6zFSEv@Tan7=yn;w+}K}RyO&sL2H?J%C7z~-4;q>#m*Qfri}@b zDoC+wy@_`zM_$$4h53KB^DdisborQJrmbtnbl^iPp-dH@!~LL$i~`A?AYx$0$j%kD z@eueycT~a7Uwh;y)Eap<3=75WmpQD_m(l9Kc>nTyRYupRQ6E>QJbL0J)_gTI-ws_m zisG1<=xg9?MkXDiN+bm}32u!RYK&T2+p;3+6Ofbs-#)gU)PJA*=Ys#-KL5Ff|9>8j zy4d>r7R>hz8R|>p$ZqsCWeKbLu-mjU zmYD;%!%2@z1RjJbl%}ZUDm_q_AcH?J93VV3lQ6j)q_`AbKmCN5dy|jy>?^E9^?35& zN&2}YJY)YxhHT+`SSZ#YyBknPwiQAZJS8{!D#9<2kd3qbwY(gKQ?cOa>Wr;JT=D+gL_ki%A4u60f z@LZ#uEU8~^sx#Yy>)xh#WI4qzM>%=v;?SpOKmQ0L&@^cSa(GLvuiNO3T`CJe-Xd7w&`@Y1- zi7xkdv}KIsyFXp*$7oQ(C%<978)0vSjun(SrJm+Q7znIQxgqQw+avUlwVb@?!LZYt+pV^F~#3W;$(K5RQxr-rd~tGKT+$njgIcKYwS zs-DYUGY$Ojl&Fyt$NG5pUG40jVg%0y{+Uq!b@b$)Hkp_F94JXkoTUuvM{-C|AMV!A z{4%w7v;R)D|C6EqCsqG{x*gFhiDvGq%8+DX{qk#AQ2uBE*B2T0#}2ns^@WED{t!;xUYCze_K z5;oRv^Br1x>dKS{q~Gz`$2cZ7geId~?BU{VjFAhf3tCou;t+`OGkvuh8RX+1gb+$c z%)|i4D!jxvMIFaru<2c76;mr%qA=l~hel*>+?7c?c(h)X>KmWT%VfuK{JAYA&Wjn8 zEcTgCVdvVR!ytGSaYJKFef)Vw{3M7}DIQWWtjkgA$0AUV`H0=>Q!V8U!~UwSBkS4z zT@Qwxza44-v@3%vF4IyME>IGFrMkPg09PpO;@zb$v`dTMxNP0>Uw^j#+E(7EhqK3w zgqG+{88n9aYphgGVjHph#vjq_W%%`u3D(8u7waPr7L27g#A$HuGXKyenuKm72zHBs zKa#8g;$Yb#d$A{4>f`y=8>X&|&kR0e`s+KkWQ*S>06bb`f` zM0#s3u*ZA9{qG?S)au*W`5TCD^8X1&{|QF_jcEP9^bQULyk4;ikb^9N$X)L5$Y7B4 z-?_d?Eaw*IP~x3I)hEr?2g)yS z2xHS{^`UTkNW>l*v5a}MQALG*tHb7az>Y^{xR|c+RtJNco9WHn3I2zVEbSus=MvkG zihVv%A|fJ2MM2t>qC$w6Qa~bvqyz!!ObG&2AVftv5h8s7ArV4TibxY65`>UY zigdD~5SAoaey?ZW^Nq9b_I#uE{_Y*)-gErnaJb?MthMHP-#MS>na{kr?#Awl@>gE? z9c`fz^sBp{2GI*d_)8J=kZsl0q1yQMe?-{UZA~-IuX@B!hL4X4>cVrc$Ty@-j`73+ zK`ptb-XG9#w6A!*#f){n+i{|P;NVn|bT_grWtgr5y6u_JdKn+OnQmT8gvQ_S8nUrf zKVQ(BKTguGjfpmUeyfOW6*d_L^mFVTp@oOCy;FPpqAxB__Sw5iS^6l6{zDhwH3B$> z(oLzSOFmB_=}Uj~R+#(9t@cWOzE4x9cfXJD2y40QL*r!ff$_bnJ>Ts4TcYnoG^Nfe zEQufYSe6cYBo`k!F;ITxQbevkg=-n1$i%9_M5%QHUEg+XE-ZSI(OXq{Z~T{>cX6FQ zL_xULJBwnU&fQM=0ab@3yNh-`3Eq7+v5mZKhvSf-#m5%lz|H}nvHHO0M%8}cXdaJM z=j5z^YJELKJ?r9#)4zeql~VJhLeKH(2R}5YnQDYPOEz_ zDCBKyZf}2HxVXNay}8@j_QRpJHQzNGlt0~;SZH9nI+uF9t7PLc(MGJ#%y28qfBa=UiUXgfB*B> zs%~l_!ObsUw1Fdigk4(~z8YgCNpJsP?5I1A3=*o{8X3#s71&Z>wM*$)FAa^So?Lb{ z1AA{%R?*meRY9lM_J??1iMQMSPTTbl9)-WDk2G@gF5NzMV{XMKmHm|xw9033Ry4ib zEpi=X(#S$3nc}Y=|KQ%sI;TIhPHI(9y43Tebjic6mbrBM1uGhsyVhQ08d`GFPy1e= z-kO@7d1-6$(`{eJ-Sq<>l%aw3=0ovMjk67+a+W;c-w{~|`-$gt2kk?~2O=143>dVZ zKKaZs|!{EV5lCxv#FXTcFH^v+pMqWnADE_MP_4t<9x$uWmXPB zvgQ2y6MlfUuE1R9`jca82{&dE3xDKpx}J?{Nb~cKcD;YWa`gbpB-5GEwp#d3d;H@LJDpTeK-}XF11c9 z;h4}~aJ51@`b8~sLG4$AeD6A3HHqF(_xr8fnsf(eQ(Fnrx!Aj2RFd|-mS|d(m*Gq}aXOxNBuG|~oI>2vIA$G~t2a|$K@`fI zv9aAVpQ*DaBIp<(*y-5cCM{-UF2Njc?8d+IjT}H{I-W33%{4 z!-~1vA?Q(nW)XRDZA6x_lf1~(lc9lo4AfBvj{2s{S~`!L7@nnsunf07OyXIbv)!Q{ zyD-(%;sOA8-tWm7_EW4ZYVgsOrm~uVmKQDAR3qO{dDAU19Pq@g04_h#XX>j%(u5H{ zW^OKhAxynJcM6P$jE>BA*x&M&a5`Bl)*5s@{a*8utYoWN-kh7`)D_wE*@1qKxsz+G z)*TsY=dtr7$=;Kh(D?HM?mSAdYaVDGj2@l4!D~FSowaZKkr&&G4{gVnCqCSi zU34I&ca@WF^z}O?alFP$S8n_HB%agHOHO>);y7MG+~bR2)^5 zzv7_Lw@k$q1{|TUSQ(2XFOYvn7k6~w;3-V;Y3^GSApeR2O1Jmjv^qfK-eqGD9YAq> zyfhzVPxGbo-g1X;<7R$A^@9&JX+TTS>JR3ncY&&qKK3oTToqZP1i8q~uGvS{%0=Kr zC<2XmpuHPDsdV~))`ew10kz@p%3b%l-*d$?1n3ZY{-Ek4_%s%`|C7o{#TS3Te4il_ z2~LEuAcrwSl?;DUd5cj8ZTdUuh!^cn7vWbCG>?HdqLdpJ%6Ti^7ndhsDt7NKCZY=(u3N)T#)YXu3jM zSJnw8S=dDo^T%D>%gri#n}4$d&8#@uE_%a3FtoIzpV>OcT!!F~KEfib z7S-V~A9NqgHPCFNoZs$IA_a&00+HTaf^+_$&GR&YOvI`DCD94}obC3@m$C@bt~1|B zFK%x-nwV0aSaduw#WOE!Ki}KM@#CsCeUL%<#q$mv3q17sKiuydKbCsp^JgUm|6Hm< z{QLYO;cHs(w`}lhTJSaB{C{hiuleTJLgs5B^R+_$tGLA13i&Ja@*jbnuh7eX1L(yI z$0Njn`AV(BICbh#*j+M(YGtBq$qzgyhP0%N7}ous8I+d4z&^SvKbO#W*v&Oahi>lw z)Kz$?yDq-pqMHb4Fblr3Wia3E^WRn(p@-i(J8rT6+hL0gVjlnVaWD$5qA?!UmM|2$ zc8FR#gl|PQ!zZOA*$X8TF;+N$zFTFAM6?*rZ=FAqET`&e>db z$7fySzkQbS`FGlsL445hPeFg_e}~p_z5;At0k*Hm?N>_R3gas!@XrA_{$IJKuUykt z67wsG`HvKQ&Q}uiD?9%0^!wlC+h0k{|HDbl`2TOnpsiK0MgvM0&$W_a%=Dxhlp~2!bQtM{-sCF=I4d2n<_TxV6KsuQU z8PHb#&xcHB_q`{V3%$L9-E{c;np#mKt`vut82`P+;y;@udeo z-VV9!jM$fXHMJjZIyPR%y)j-ncg||$NLJ1&kGVHhCG#7v%)XR6AZvi$WD$_|b&@W1lf2862T_#u%tN9n4Q(w{6^lU2279CTj zE8}cKq-crXBBVk_6Q0nZ0N7=KUa6~${>D#v5BG8M*q08v z^`fiDQ`ews*=BgX1e1tfS*d>Y;*XIdcnA|IVaJQS;pmYqW`i@_cNOHU*A$(RnGF_rePa_W7QJ1Sj1)p&8Lr&wjWS_b&ezi$v{7=+!%BS?=@u4+P~m zZ(V#^Z&BAz4!nW|zFGfsM1CCi{pF$#V3d)yZAru_i|6FqN*R{%4*3X@MN zd*X3lI0om-y1;xG66)EZaCE+U_opwbkLc_3xQq3fM14ac}%)i+|~|l&1>Jlm4izv2s!$TDr+9?((!z-K!ff zpH#0rz*@JjdvYb)|6ReUhvh0ujT;$?kGUXdMi7VBp^9CGh+5Pql><*dsr<&2T?Z4T zOC7#+(1Qs%GeKlF98??>8RNvmKC8QIaT>AlGyb%d_0h4B6>YI`15IcsZA=(UQ>&?o zb%8GadJFaiK8x1NcC^n9?J&?`BHKmb{mM@^RdgVZOGR{cFT(mc|&WuyI+Zj zeLs!6sp8o^AxB$YhPx=6PCTsHyzfNL>hS8oikbMb`hmXjKjzVY{W1Kg{8@pQUzory zl$IpOFOP{d0+DqTzveTojkVmLD?642*u=)MO*WP;beCe&hubFVRS#u5?^_*{zCkbB zxTe0s&uLnNn6Bq+HYd_CIFPE=?07Vi>*x}n{664C$LL4qZ6Ip=hxhwGdF6fn;{(He z>Dheofq65hGJ(IA7KEQ{ml9-Gl-X2sAu0iw4_<-WAZCMrV^EtNoLLs@=K;J2F^O$9pTiLLfEWJK^M(uH>LR0#>fwLnOh9H0R>@XG9^kTfkaQd%|EHg z)VcyKnekI2N=+0P<=j-)=<%ZZ^sqI5HilaZA-6`7kTvx+Dc;WCbQ~p4u~>n@2}J{$ zru+S0_XH2Ow=Pe0I(B`~H0Xd%j>(mhWY^vS*V>TDtAxLNm4y2^)r+)^K2$JMDO8%q zbL>XUy$**8P_gGesjOd4R$lJ~lKWfIml?sMH=o}bYs62$ z(O2!QNj#^@(tjpf8*5Flb$$SwHgYyf)9QB&yrr##ka9`jc}lPMaCfkg&FIOV^ZP@S z+(*hQe(Uwv@49uNP2?$>GmVEf=Io|7*n{ym(<93KDb+&i!Y&ty+YN=$FfJOcrZ9r& zP2Ch1`%bf&k23>|cwuWZZXsyJo&^>}6ORMFCwO~O0!E&Ag!M1=;~yr4EoN$6FBFe# z=Dp-!PEI*?G_u>OuC8o6-Bsk=-}e_6x&fs1^AI1B(n)q(SkSiJn;pPKA*+$}#*VLVPcRUkn%f~AOC;fWxL)FkSBKX(8c@^I1u30|H8gAb97C=5xkM`&ztjd^~86Eqit z-2~S)X0NWUudDk#Yr2v-siEy0aRpgiUdrh+kWJwE!F5}HraVL>9>SE?^Dd+=fC5imHnAgrdS^JGJsQ0`!{ za4gh-5c`N~_b@-cQa|dbHRs!8(P4fbKb<$9FtMg%YPW;|Z#n;)oQ$-GnM{s~$*2HA znehYL@4>u@d>3{Z6xcYG-W+zFup+a`Bd^kiQoS$T9@;ln8yltHNctuq?0MPVor2dN zD#DMprE5HUZSY;j$vv5W=B^GG93zus%qGlInP<}fdPPyBbCmQoE&rO9|5s1TzjS@N zkefp11ENNdF9CSW?x)`OO5ejY|xR3|Q8 z5>n80;$h{XjDS1B04--r{qtEB{>nLkF|4IO+lc#-Ah)J3rN0gBoxev&1(3;p%*Xxt zlb~z!{InRQ)STy@Fr8g)A}=vS8s5^!G92E*n7M6eW#nn@tZ^I??6X6eiv5NrAV5Fm zD|9BrpH!wafWg2itXAhXOU9}(0|&)M;~fyQ-`0^B0pa&o_s~ScbS(B_qI&r}6(=b9Mm6fjC?Wlsf0RpBJXH{w^k z+8nXF1wnP&E-o1thBOgPZZmShY)pERVn=v>o9O}a-IN12=TG^5?;a2qTyjzMJzu_< z-rzzHG{p_zhdUZr15T2x>O`8MA{-(_qoR!o3#i+ntm2z2sUOb+7mwA2lXg6665_w3>cb8r+_=lc>0%$4v_DAPp}9|N&UPPh_IV@P2VtTvoGQCk z0JpYyWJil5c5jO(SCW2uk&uRN&sS*2%9=swTtj*Y)p!Q|{B)wIGOf~AOpo?(XsRbS zeGjXPF_Y4~4wguPm6pT~_jWz1bS4bHVFmf#A0m)&#O&e5`HGB?!m3Ew3)wK?$zuE1 ze4F6GrMpgCNh)l7`mvhi(>~jUTR*$wOM&p8Lb_J}QkC$(38|)CNBX(YG4vwjxkwk7 zr;z(c8>jBVbt8;cav~QLh~eU*!qMNWArz0k3Lf3xY57TI6&#~s^RDlW=tPgj#6%H# z;g~?*N-}+%XgN_pu1H!r;@oSe1992)2A5_yEq#aA(ICxU4Vn zTlFr;GDit&Ac03;5M1|3<)w*zMIX(ynlZD}7Wn*495N}!uJ^<6TEqco7V7U z*&F8&3-vvF^Q$=ZmEGp7ix-Nua?+3SIrZ@FAnDc*Vhqoen^*OD?IL0iE^d>*}hs0Ppd-=bPPe3+)d@-MKfp_Fc2n zZ{3XbZ^-th0lk&ps29_J@rt;vrcxZDPL>~}b<@in8Zgn!)i^&wk_VTUp!AgFGI^L4 zus>zJ7@tsK6wu)gr6;mal1pr}!R*br$|4Pdb7rH{X#=9LxGFu!uiy}WY$|k7=my<` zr^oi4+;|}Km>bb0dMdc0KK*157=iSUl;M|-aR!vPseuxD9HC4!*)KJ05i(^Ufulp} z!_!G@&qvAxJfB<1#smJ6wnh*#g_q+TD;{@0!ipY>1_vV&4wshkr>LTJ&$@?%JH_~j zySWFhCnx6|Y2U;RHOg=fp6vQ(*0k|v9n?!eSuriRkQe=rzgo1bU*RM8;=KJQ6^Gt!7SpTVVojjt$-eh0&Q+Cn$6bRCcpIp^ z-pGK0kfrW>4!-Y`K-okZ!a#SL}=76(H)?Q4Wf=bQE>1@+Q?oq^El02vt>#Q_r%K z|3L8rCIi2)2BPy{ayx15vP^EplFKp1mRtD2Ys2omDt3vkZw{puIam1geF4z_!lBQ! zUjDyI>%AqkWHRGqgr(Tc-U^+^-64cTJ1NQsuvrE=vMUjyb1R_ zM#7d=UHSuKJ~=7+$*HZ)>3o_5lR_WP6oS36PJ-)4=8h`sElAF2$v7-0R71dN!HI2> zLvlkF+~nB@ZvW8afs3@FTy9i`2&B4Mzf@H{uBideo&!Z~J2n*23l`O$(COK8eb3vP zpsmOLbS1h!6Iw^;yFBH7fr|SQJqIwA$ogpX#}D9zU)-u-Jfr``sOGwEzPsnY`iU;n zm~Ar?yX)b`!{J1Ln{rvoUO7Vrs$*D6}-(4(u=NB9^BplMYnxE}=@Lrs;qtP)( zbQZHzq|niB^_HDGNIiT%3Ob+_svWBIaxk?B3X! zg?lXEdA>_Ov|^D|o0i+DmAlVN^!xRzXGf-w$JZA>3hpzX$L7EO>+Bc-RC#Xy>PrXK zEY+{)f>i4-=nnpW-K(Uo8olaGPfv$8ENba8_a3PZzh$^ePjayL&x0P`k#0#wz3RHI zKQ2%J%sBD_m<+VcL2>|=f=1zQpcips_=2rljvbUbxy@EL&&&465FiV#S0CZ$K+f&Y zZJ1HPO$;6yN}33pctx@<4k=KxOd|STDOC$D*m3~dh+<}*JY@d;Yqz@G{fa!+E=RgR4mA2(Vi-)4{`xf&LncFTG6;_r?=JAYd? zPe(LQgCS;da=?bZOqaR=@-tigepFNQf$y76Bp#1(~58F=6)z?$2`>^p_+6 zV(k2h)eaoAOu332y6hpC`tsRSs%N?}WPUH<4ih?oSxWs0ZjhTOj>3x}lVQJjWE+Jm zX^G7xTFv({CEyZ$v>;}*Z>jWrbP>T(qRzcR&|;gEI0u<$=6PjVp-?d0vnpe3_~!f^ z-flzO=$c-7^sx;CKV>*>8n|e-A=!DnA7UsBBR;8Ya+3NYFTnKJ`9hkS;Q7#EWj+-x zccVABafwNWvOAFICdW*n3Tufw;<6XO){qm}BbL%OD`-$v<|BBwL^TGB9;6=X!5RRx z!!Icp>FhmC&oX`{4_{9`Ut7LGrVbL&dN>#p# zQr-~NOwhCB#$~uQfvMQ&Zxn7~!`4H^xjMtsgW!p*J#s7b7!+ev@Ae)QrSF|VwqKyQb?iA^YI!bu?9)@cFS&` z^LGqCb=lg~JltCK;@Qk2!u*^*;ZKCJ*2yBNpX|BvHl2ifDVJr!K~j2S6%!=L%7&`9e$aADV)B4a8sdbjN+12YLHQB zb!QI$^9aGnK;bFsae_X*VJ+Ps6NA%K7#7+~^N+((BLrR2JoXl<$=ePO|9CdDnX9fa z4KdCTSfjPbC6l8Som5%|YoG#kJ*>KkJ88(;KEMMJ)`dyHf?d18euf;0wUTl&Ayi}b zW_WglwAq3l&0R=U8)3z|883r3{TTDsX#^cl&_>q7nc_*&ijaY)m8gogyYTuWDZFu8 zT+Z7xNl$=>AOQ^$1lbVyWzW*OuW)s6=Q@N7Wh>#`(v65U#f?|xvlVU~!LL%-!KnKa zVccd<^a|990ZMg?TP>qyNmLVhbw?=;8#iT^&Envl+QW=SR4nuHXoDJC4H7hDqf(v_ z*4XYCMlS|9>#n=R`F*foL&jVrZS#b1I@ZE}@^s8@vmG7+i$IKfuQ+Z&Po?K7OZC3& zsl)j*pHzT+42JRF$zepO1~O~(%DLZNZkt%`*}Ek}%7t@ZNn~T7ARe+BVjb za*0z>u)C%658fn?3dx&4*^%D+rpkZI#O(z4*?|l6r*5)OD&Y1+z$l&-TEh%$fOO8A zB(aU8Kwi`h&8QdoX--=sJ!8KuxyV%5glvp3=LZ5&OrL;+K>LMRF{v|%g;T(x& zr0tpZ!MS|i3ADwEQNp?>%<|9PdJAZLV|9qoI0L&w9S!JZY?EQ8CM6v&<=(J733(@G zc(>kUdrQm=Jeitf$>X&rIdS=8**cw;Mi+NGUZ_kW9P2w5;nC{YW#_-|Wg))5a`myi zd*)6XdOwy-)_5ePcv#;rugW$BHROF|3Flkn4{jMs*4h|xAaJ1s(@ZcLqpJtw^*o~W z5fq;_gIfW|HW{z&J{xj2*291tJ8+=;U8FL{xu)+x>G`f9;^TwNZFWG1JS^olcl5i+ zzJYmQkvrT;TY+@jgvKIks08?)^h1+AvN62{Xpqg2jWEVuN=`6mJ4@QPwz)}77;nm@ zi+lWjfS8FGx_C{>-kKhZ)tHJpwb$_@Gf^OMmFqRguOQFEMjP7p1hnyJR^66LQ;8~X zPxuM>-mVK&@?Bv-MqoXMy=}Or-oU)_-bC%Y*sGU_ zR_MkK=QlDnkY~WHn5Lpw_ZSrNKTqXB+-tyD5~Hm++c{f=(WGow+kiZeAxCpG)4ima zCPOZ2;Ez6-PiO6+aSRUA0xmW-nxd(pnBhpG~w-I6Q$);FdVjw4eJdwU)=Qt-ov-p-d(!R)|GBfIO3 zC2hwlx)ggu0>vhohbwLx?KAasrv)kjN_dtzfFIFU=!^o1V*%EX)^5B3tY_T*p}Aqp zl6xiAO&vcQqkGs9s$c_%7jm1JE5?TmJ3~4W49?#>z2qLb#3^^ob!cr>*TwAj?XPPp z%PZ4Nb`vTvlO#Ef&J!pM=))S&eNSt0`nzuyiA363VaO68jTnhXBPIBFQthHfDtbEEDH|Ea(Y5ou5cD(;%qQU zXF1Z2Xk?=Y#bs}22{l~od9WnV*7DwXjwc*dFp<5ITZR0=*DSv4Z-a}Q{;{gJBxG9c zl<07++hbZM4rt7$Hp)-ZT6|W(-^=$Qt;R-D?$o6`*p%n93dHf$TB6iJd9 z*BUf}(}T-Kn{3FGX$zx)Yp zHPeZo>jpbZgCrYS(IHIrA-L^Cd4{uJqX$x{8siI*lc;KdVp{F^$iKRl$Z^&`8t>U2 z?pmhYp}ZDMf60(B_;e!680nbr<=(`qD}p;Ydazq7=;R1xSY+04#YY2t46ESSO}XNJ zTQiA0bMLh4BXN)$YNC{s#gc zrX-0seH_`z2{>ZWL)bR!k~j72{&YC^%3R84C(x+N3f19`xS>@_wn9~^&a<&jWUrQZ z<7AggrK&NucD|aqIV3$&h?FxHSgCFpO}ThlUk`e|{??(7x7Pb^HEqWYUt^Z*Pb%dH z2a%T^eyxP-9CesU61SnFsyJ(~Tf5J2YoKw;O+rG|d{2hPJ70F!JJv{dMaT9J`0Syn z4gASB@&2!?;{@+e@Op>IT={vVAsOL!Qv;M~w!W9pP4wjo+g@AaY%Q26x0%=xP%GkP zGS)oE*1criLfTv*SB7EMG+)Xaw@IuL8nMH$cWcvf{VdY7d;pla=)5w8eu4GU&rO+c zPTNU)j@=e7khn!tR~NxY@ahU@CDT&S6EYopobqM_(OQy(#XPxCiPL7R;^`$62AC1W z6U0a6^lHn9O{>4n{-H6|`hr2$uJD_+h3(FDCeKIODfNb-Zk*a22^n5XZ{AGzn~r8e zF4Kvo`AJ^X4Y2bt-q3w)d)@tUfsZzyvnwF%%5Sz^fs~vcvYFYV(JZT9eSdqis-NJ1 zG;kM&bii9$CmrS9)F0s*NhW$V(o5H(ZKY&3&^0()D6gk6>tIZX7^6;6ZNaYObI=JC z%sZ_9+vEw_26n7FNc5MqypYf}y$0N$k_vL~3)=+rh7H_dCcUDgVS(I7u~}J6_iGPo z#A(n2=rL8s8f+;xRcZnr|F#PGZW88(!I8S0p#j)UvZ+quNox0Xl1@h1_dKHF3Os~^ zWACl*$__yX1t4ARCm_qf#3a(N0N63ra1F8kay4|);BZ24pTHU24Q5MeTk~ZEDro7MaNH3LAbtQDpO5d8X z@vdTASN?h!Z%S1;iKa|TT&db8r)8nZRo=>E9|yd@lX=+5hcJms7c?( zs=9zv99}YJyrx?ssMJt*lA;FkWuT~xOhkHoY(RBn$%CAR>tl+c?6!klmW=CQIzn!) z=icE=`f@nRUNGEa$oc9*Je}=rykv@2p_fA;m7fEmMqUlnt$?!IlN#gzLNVbg--=c=|a}ved zgcBcIvKSX|`j#LUJr_IaPN!55i9^en2s7B#S#xWAk1as!?pWVR9nvJE?w5@YCYEj>Q$up3OGr z7&kB$fBxd}wCD#6_1^B?)pzL)N4TdM1LzSqz?iK=1fp-SM(~bSjtOjcm%1aig`l|} zPPp&i7v>PpF`_#5a5f#d0Meba&~Y$)Eq$(b>P4cZX}G#ka6?}41ROBP(i0lzBg-n~ zHpnY(r3mz$(ZwdQoTX#PPPovdWiw|7EW|H{=fpRopbz4zTlp|af{F>o$E0O+G|>$x zrcFZ@f=AmFq?1524s5o7Tt>=1qFJYU9Kw6@6I1o0~Im<@yeH zETL|Ki5C#Vu1co^@gI(mbpm`gCRIAg)e}g6%(Soc#ylrEUz{@Kf;cIHFJs_MG3jzW z>P6+PZ>wO(kwDc`eYR@SZ}6H98)LGhMMt`WF`5ClFfF^)uZ}Pzx`UH41A#29RD!9V z5Ns!<>&Ln|?`u0$S2o8LS|e%+sg-K%3BBC)`+EOp} z_S-LAZew!Y)_Pc^=Raq>5e_MV`HmHQwx>x{1#Ttmc2a(_{RPwPQJ7CnE|DM%oeQIZ zMB|qHTcn2$9hbBEUBshJ+-Ns$xgU=ky@LR!jBpba&LhkiAG02+6Kgz-9$V3-TtECS z1sZM6wy0rOX%U4Y*_HZ{m$s|+hc#CY4n?LA@p%D1yk1s(V(#qgt*1iv_Ln&5YKG7s z_oQ6rn)NB#>^=w(P3@Eh<2{hcq>=oGw^;4&8G@GLM+l!365-@=v!xreVjOEw2aPro zv|)mspz_p@F{7@rJijL4{V^h?dsFs&cA?DT8o5OG189gWr-O7-%%Hl<*C2etV(hon zf{pmGrQ)VS(=o*K(~+aUnNWzn!dI zS#7H=-{4PIrJob&mfV9xjYHe|Xy4|?VNH;Zu>C%0fx?qGJh&CT)bp80|BW~4Xg9dP z6&!9)h_m&JPJ=d=t|iPu5yA1%skaT5g-_WoM^jQxtl7BA$-U*-<>=(Txw@$yB+0^Q zt~c0w{fT7DcxNZ4$6U%UX$Q04cN@ket>&CRL0nd7`^y5`zTBUf>eH4W)?A#NulMRP%d4hjJ&5-; zWA7PMeNgH65%YAh&aodF<2T>^=AZb3dVvFr2yzfVH3ERcEn=q9jzyBMVT~}5 z)|OMi)UIa~zfUShCxl3=>5NwKOx&9<+xD3enQwxL9a%&A!Fj@c=UnGm+ptpm@iWqC8G2oNR*ujqmpH!y% z)0K(eqqBCTN?$@(AzWf?lW98|52-=%p@)K*oOmN~vV|9i~N412_m1HQlEdr%G1A?J=8^P*PVOYK&y1-K+$@rNN> zAJz95+P6f+I8x%&MnEa{!`?xiG<6P4 zXrnqSfl7=>j~QCZUs5>&BC&j~g|HBIaImhWWmuARDCGbSv@lwD*yd(&h;QyGb)_rP zEtsm!n$2dC-h7-(ca?QL-A?7ncfqYPF}xdMGyoAQ7KgNZ#J1p#DC>Dba?Cc)R)r>9 z1jW;r!7XhE%Wt+F>QQ+6O2b>|YE<{Tq{tVBq`q;C6<||%sBnZ-^fju z-SQ2RZSJmt*wlr<@>U%xsGU(xGDLN?_<-!|P6}n641K~W878@gtZJgJSjW61t7^(S6~y6R-(k!Q3Dtb^Id)I*9tU1+PCe^Jg*(~&(fXRc3$sS6?=p_D zyMvj}a4*3y1VJ?S;) zkI`l{ifSpC@ixTwhU{@_m?$B@8InGKTl?XHS!evg6ql|FkuH(*ZTIZ%Ozvbm?r5!3 zpw4%;*ZCto;=?9CdGLt|IS!m$;XiKkxiL%se%@q5tmFDNEq%=XfGTVHHhN% zn5z*@MW{5r86SsR&SGDZ8vY>7zed|ei5gDlu~4u{lk0p8vyhVAdPtBRabQm3dTY2) z;p>YT80FipEqK`xip`}L5Zw48$fYC3BXn}D{2Qd5woUn4r5Q<#YDTS~r0{Ud!5GBD zkRF)o5!yn~tza1{zLmyQk2o=IguTlOF`7#tSK_rOSK3bT$(0ESZOr8_jBGJhx73zm z0T3SfwP^Y0jFmH9xgkR4KeUJ+Ut`h@mUuB^V3Ye|6%G|7^x%e}6{L_5u9x*9l*9;4EPs64+xpD%CnTFjUzhy3` z@jt07q+%eN{0UP}p&@oAxKzWrcPVmj@V>^+5Uz`~EQ&BIiM9BsyWCFd_ji4Wt_FEI zMFRoY%{?K$j{;ov(yT7Mboci5LjP%h@WqPa_Nfi2%K{Sr%<9c}Q;`KFJaS6LI3f;N}V@*1|VHD^BC3Y0T54aU(3NyI{ybc3av$HQ5 zI+!RH?tF(fNnosRGyHP^{w}uKH?YV&dCtt&_cg#iHsAVfQr8z?CIcZxU8jm$Y78&>2iXd11>wsr(VL z<*7W}<#v6ZL1a|o#XC7qBV6MhL(3;$EJT*=mU{q98^}}_y#janf&v8xN(OzOgrbeP zeyr$FpFPsh7#10z_$&G;>J2zGMBl@ULVGcRS1%Y4QkX}j7lOu*6U*(dhMpGJWv#?p zrqjjUz5o52h3|5@> z{xaY+(2i>nX;4ja1AI0>%9!7&ZiQ4Z>UTDltf?>6E4O?!_mlS(T%|~{vSWlXbyBX1 zwBa-@F|eDljxvvamb++dXbCi%mO7&0=>wTC*jrAEQR);I*Ye+`8ld>JwG6pSNdUd* z*Wn;9{(Yu%PldA{qCRC1cqV+0`)s@QB%$B4i2$8I>y9x|3RSrSD3DJ=)`y|(Z>Q(9PTtow9?}lM;s3#D^p-fLC2=Awl%ytb6b$ zmC5TK(GDj+Y`t}4XXue*vDTP%3=6}{hq7L`cSLN+uC6!+)1@de`g#hY`E9J2dwu1c zI5w7jvS45+fDASIQ}qi6UQx?9vQ+2hGxQNkG*###xJ= zB2BLxuWiXMCX5;HOeA?6DStRPNV4iA^EKqcS3?Oby7V*V;Xh7*+>5Pcf0ijz72cR9 zgC@-$R%zyj62|0|e%!km0Qd>1>*}Ol9_k9;mARRs`qsE@Gc|b$Ah9^OuI6z?-|G~| zNU{O)R8)072mFRve0t!Nwai<3ZW^SB&0*%r8Q1CBT)*F`4u$?UghnkWd%C$V+qe#H z2n^L@THXBs$MX)?lf1(Ss14tIIQ&?!Dc!zm`Ns{J$@vyY-Sdp=f_K?d?i>G@A3N_y z>j0HVE(qs1dV?qEv}VMJ)-?H;E16*35~(RpK8C#^v=3T{)|fT(eFDI6_IjumZ%CA%iY$=x6tvys{jwP${UepdOaq2-tcYjJ0}>K;R+`BueN5_t`UGun_QOlqX$J4ZfyHvfX(Of!})=Pd8$yFeBC$gwZw;I~VrQ8vfPv`p@r;W_*n4+Lj_1gs;x@V2N|QeBAus-`(klYiRJsc9)e$oXNqUA*F+BC8@k!JjX zy-^%!Fx1$#k53-Pn)kk&l^LueEp>kQ@K`}n%gI+;-3uI@EmhlTuE}MI1fHoyY9@}! z(&W@jrokc&a-;-q`Sdkz0LF;kV276Uikg&uBcqXYkd+E?%|52mAWy|XX?EPhx77sW zM;tT(wrk<&bh9n>QvC`k`{)KFtYM(%L2gA&cc!O7YN=8TCxOtxE3?6K>{v=wB?wCu zWT;xgZ01I0tHFg2DY`U6xbcmgXI(oR$z#2>sZThitlQrMsyUBwq5X@Lha1A8%iR&> zb~|(e_s5QCI^;@z$$yeUdpa_Vf5J7O?&8}zvAp4hBitL<@A`!>;faj4uGies%!kBU z{PsTKhR^mc!pUrTlgGtaKpg|LpwcD0Xt3XBB2D(bMM#+Ri^HAAL=to^P)$PQKPWcA zyX8N?6Fk}~K6^Kg!7))7!66c(Gw{F`=FyB6Bex%V&Ok_3!8l(OP;`%Y*&*K@AHKDz z?E9+^R1fItV3vrFHQqUAP$7)G?OE6D#lM{rn$h)kfNb~P4_Ou{B@Lsd?t!&C2`-oK zSL`as>%TQN?p2rsf$*VifCh(d6;pzmt}qR#`olP_af6fDMLMCifxRv11tbr{!TY9O zV`0&+wzjCJYc$atOsBBgw00ciIKdroI|RFnofrumBiK!DKY?8*^{)DDF*BY@&Zn44 za5u>nm}Mjn#rMjZncqrmG2(%R^;s9DCIj>KUVoSak)zqxcl|kwphFY$`V8CM(j`v#N#IX2Ru$%&j)@~vaT`-x?r%%gn>(s6 z=qNpZo5}#!C{(85NN6?WX$E659PC=#Ap(ycDZ}ajC*RBgXX}<)R1!NvYSl5KS^QAc zu_e86&Z9NfRyzmw8EEp^c&g+8tB@p|RerWJ=@)8nV#7inuh|-LX2+!?c+HWi^{7iH z>&p8SRnG3!;r^^r2&4!DPtv&6Zj+w!5E@@pQCHoJxmnv9hnT=Y4Y(DGgJ;6NHD94pX+6#Y%H`r#=@>-g_&2tdN0Eq6x!;@GJY-`Dl?I$ z$I~x-QZa;cPN5gkHdCC$H?yP~6~&&-$Y$x1BP!1fTiP$!7UF+DpP?*5s_*3wg>}0UyNfA=33`@vFx{g2utX zo~?8Z+b!aDD|u`7H>GfdrJ$LKqRh1z8%nSIJg=J_cbDX*$63~e{nnwttF@4rR(-6= zEMzruA1q09dV?8f7412pd@*>03}Zd`oOGm_wp$v(YNE$kAzL9DV}!6k2&d!unhbHf z2`|Gh(IWgtNwMw5Jm&(3b0tH>D;Zfs_ew}}$LxL2g)d=V&_1M_lJo42pTS98qVu5_ z7x5{XCtplYZ_2EobsuRxCPu@X={(#3Zn$F#mrvt?yzB-WFRc@IG%$5YSnY~Fs?kHr zr5tHqrC7Am6IPYukd>yVCM^y7sSbGu%%;w_-}~JunDf2x`8ymF-%pmsDH+)B6#Bg& z!J`X#2c?+F5&4hf9vwF-uHs13S{h*^O`NULz9LD)k$38t#38c`xwEu4wIEzNJgp5| zw^oO87G*lb&r1T`8t1wP@|en9iWABVDh@O`>bGX%UNVM98#|)VYuKluiId$5bP?!( zB>ark0fsQ~#l`Q=LXUeY2hq-*WK4M>m#y^N7k>E!$Ew)Rwq^PPjBQ^Fhg+Qd>uUDhcMKPJke(-(f1H** zQWgYaF(8#ci$#MCLhr7W5zjrxy4Gi{O#)5#iq*q(BZX@@Tsk6WM0qfp{3{9CifwSH zSfm3-wWiH$vWon#tR(8wj0G0n>kmSC<%M1i#JurZWz$fpe2u~zZj ze814MfA26q^xl{&WY|~7kO7n^l0@TYt1C+pecBNHaT$fw>SGIK#YSyYctQg$$lAx9 zn96#)2F|iraZ`F$h-GwvLd5d~Il{G&;YjLpvZrBG<~L#5Q9w@GWxwfPbxj zb$7;xBM>*ncBAgh!P=^v0+`|1;}>@~)**i}9?R4C`^V!P0kyW)fU60<( ze$9Gbk~-iw;-*+>($Gh!?ug|AFh!Un`yCC{ARET9+vy8n?1$!+ue?2|oB3{y`Ayc- zPAj0qj)J>ePiZ+hDFn=V6eCbj_dq@;mAitMZCjibBD9GpyKR8U8 zNxc0=?oV)5!VAPkpk>RN4?i;%Y4rPx)|gENU;VL9_zqp#Ts~#5XSqFqj4j^b_|OX< zYEnsSpqGVC$-x&+2te|Ljsy;Fj*xuBp#jK7u6uk^xrSK!DBL>P+V^3CYZ;#$TrkVj zrI;00GH>WV4bO4Pt*X20U%M##K<-^7+E~VB2)(xy6{SCkcg}5ee|TldOOn5 zdFXEHrd`Dx>(+0V?>c_+#-hup^%)WM^gmu?BxmcbJnll$n!1`Y_nH_m&>gl1%}&1J z`tncu@F4ijD7z&NjiQ0k24ft=xi*4Vg(F47&L!1iruLYX%~0&ay}H#cPJigl9q#$@ z_lm4ya$qqla04YVExoEwkT#{;&fcEC7JScYxETF}+b=?v1xd3e@AV9b2|Q*@Hm%3M~i`kx7IoB%ln55CW|>My3RU5E5EuWJNKoCWGg3pQ^9US8t!M-nY))wfCv& zAN;^7V%A#Ex`*q&uIomSQ1DA+%Hw8x=}bG@fH7ytDcY3S8q)E0Qnj@OjxG^~(X>hK zwz2|=EDqo=1=v-GqT@E~+AOUuE4YYyP5{Ke6_3HEo>KuNvrcf05(e67h5*cJeHFWu zRjf_8k+stOZ&_vl*i!rm)ecOeP;a+0>v|Y84ZNvAU%>IFaB>{m3%k+3KK;r=;XC2} z#~LYzk5(Rduv81C+{d%G?9NF%vOCY-OyoOUmftAogZIcTOMj*@CIBYkNAd+m`0P+j z7Pg`My2=6{BhnyS6E#LLbKgicf*`AjWQHgen4-}Wo7JR|;?Ap|=^CV+SA&LUvZBRS zNG{p9KJh?5BQrd}BlDn&tIlVSRv41^!gO$5*OWI|d&u3iwM@z?;N@V>^;rJGbY~xl zG73)bDw0Qi0_PC?{QfkrGGM0hd;#6pm*j$3MI?0QE&Y-rI^0Jn?`68<4+g|-)q4At zZ8*yTj{vA|5-t!kB_XZ!6wykuF1d%Yyg5$NE;UUZ3wM#osL-2EPX6dqW_J!ZHl?6K z)13=-R!r9C|75xY>Q7B}+P~%92A8rH&ye9uP$)qve(Ko^jTn#@@=yov_Oi6 z>&X@aU}Zq&7+6VPHL}ZN>=!iYkF>s=EIcLPYCE*QW!;}z{60xn8A3=|@ z`gZ1|)bshurWljbfi?Gr*WBx_oO8}hx<55#A@>Y@o&Pb~yo%A&gY>?2SW%{Y{ma-6t4rJv!g11SFu z5i_Qhw+^OBI=RkUJ_I7$GapWx>OeLQ)3~K->fnc{OuEs`$j+RTe8NI1qq(83=L6&Q zl{1X~5bw_v;h5~GvW^V&fUQ>Ov(!OHQ25u*XLNGsQ(~!CtrT(RDoLPZpIz`+!x6h) z{S4AeS;@M3GP0$n#_3!PeNZ%M(D;vMMQ>hl{7knRS@rpQu4gj9+eYTO_^?#^3t0Tk0wnmdmp^DG11`WM+I&B>gZz}D$7-wQ`QpW z{Gs~NxM&mZosm4doEludM}4Jr_4(z=C+AM?c;hq>=4_^K(U~XQHr%pkTzk119`9idM4s`PCRBszMA#40*fOn+AyezCfCg&xcTQb!< z^|gicktnUh39eH0GzI06OQqjC_lLxU5&1Dz&s_N2mt}jMG~}KzT2=!NDxKVGWIwl0 z%r+qfz*^8&4o%N#IYb!EO9q~aUa(ReZtn-hv}>&;p&MZ!%CUjZjq;4$mBo+RSX=r7 zI>?*f{@DE)p#xLgnS867D4u2jcj>O&1kO>dNsj~g$})wevIM23u!8+ZRyMq~q3J1N z`%#vmMQA~DjF_(Fjt}4FfNbNf;8h2>j$w3hb_uV&q&{iQaSw4u@$ZKP>1UvtVjC~w zc|xiaMg9$0h_1yb)|+>r>jksUqw{Kt2q=}?;=&c?WvldvIUSAWS3hT7l*LbK#TdMTwfRAmX? zWJmcLB9BY?##R%c-tT#t&09eUVnpn0>v<<6rnb_wZ138xiq493@0)$m)Lqud4+gF> zbMQ$|)KK_N=L1V)C;KB*!QgN_1!IjF zh%MU!mq=(=wZv5zz2e@5dHpbz*>mWjd$D3)pTP%&;k1!$dDeLFom+| z4WA9i`t>2VLkq{;OL4b+1CGchgp_bXcJ(Md9n5Yp1FoI`(1SjW8m2E`5-5fsm86<( z;*+fuhot4#5fb1Dh%XqXh)wU;O`F=m>@oKM<4l4s_w*>t)Y1u>ZeZ=~S$G|2H;&aJ zlKH%Gtt47~UMnFHr3-tqbN@-VVza9=|BKh15?|z>S^6AWDjk~Rpuv+QLN{2Xtlr!ph~92 zw+csXCr0k?F6&X4ja3yO)`b~%!&;J+Rn?(5e7sE_%9J5Z<;kwlWG5|< zv0VF`_drO{KY&G)0E047ZJ?YZmb*yJXNRN>V*6IaH;SL$l5{FWg~1+BLpxyJx$lQA z2>T~e?nC{b{BZe~lO-04kS(W5W%sSlG8N;>WWeb}OY;;42-zrX^mGRBoV>yjb5;-D z!Dw+=3!wNVd}?hB=@6}Q!H@2+rL5|D2q$tzw7>H^Xjdq<3gCQtT{iW0Qj_R}j;_ST zu)DFdb7N?DhkAQJi*X>RZZt`@%0e5h)l-6;nIcVykj=nm6zv5jPJ2pE35EUxft+Y$ z63Pg6Fn-xl{fE1RrcKNTv%|Wcg+W{3g{$H}??dmToIwwwWjxs9>pg(y<-Ur;$~-3@ zF)fK^5{=Ay>Kmk$fv6i{R>P=Hi8*zFX9UN}+Z*kYr4+B{wBMNBr+b;1ly!xd@BuTU zGF*lKkDqSvPkfm4-*cP%)PfZj;dJBn1;Ngy>-*|Gssn1@_elZVU!G=5y-EREgdx1l z8?Xg46#GG$tloB1ak;6d+3fL*LvCexC>ynQy^(K zhNSO8ZF^b5868xu2W^EBr7eqWTN?O#tq~{awlTP*1 zLlTTlwm3Rp=5f4;8Pme)#qjG!)y>E~?@4OP4wGH6-cV>+l_+CBV&|z z%cl+9!XG3ZvR#iW-zeY9uPHTkbE|kYeeb!kr!}^)xIfF zeO?>e)E8tujrh1S*xU!68^!2`qP1%-*m$_N#^xm|DCL1fk$WxwJ)7C1@T(@F33>ze z!f1~@u6+x`-XCS|!>2}w>{)YANRB*HHUp6|n8gyzmBpydBx|YtHPmi+nvzNxOh?g2 zF`ARPBj3dpUOW#u=hlg-b6Ok18vExyR1wC~ZGZcmH!jJ)_7Z|4oaHW|O_;QW_VnM>GZA$NboX zMTs$XRTBM-8DG8vORuia5oZ~}`@4Azp z#8ESrvO!@EW-I`@fQwW@*@&b-Ul$-ZFu=oGF&D8J!%80qg>Tz@9O{{*)}XiVRe3 zv52+`((*C|&@oiKlu&hvi;~Sy>B!CPr6r_wnTPt(;t+*5pA>F=vog#3^+Cah%}{^W zNMXo&L#a^s`fFiSBB`ksi(pIFIi!3yQJ1oD5?eceeU`jKgrhB2gq)THs=Glsew>mS zZvY>EBfa1)q+hK$W;~2J9IrjKv}>jqJ2WM%xkoR>-IyBrEI*{`Ks5XgN^Qy-l@LhN zotBS^sO+`PXl2TXRyRPS(Js*WtkHNg)TXY{cpr{WZfxh!=W2rhlel97(#3~iMcPn9 z#y`T(ESZGG9pam{Fmx0vm0+|@031hex`B#sY6Ypc&95izgOiq&f^-r=JfC)1WdiZj z@neOCKL?(9{UbC|Vwi^7GGoL}de1CyeUf<$8p$jr)=saowfIb5GZ?1lTAC0R@z}rI zYlkE{Vs}o|yL|x@Y-sq_QJ?GH)x$K<{Ivp>a0v3M$^oSBPO=cYsyP_Sdr@QtpB^=) z3q7%FL_J?fnKhr{gfve^Im0e4(9mS#&)v-G%+Ofp;Wq_kzj~2hhdABfGB;!_U|i~m7knQ9m-v;xzpFn|DQ>5A|J=Zyhmaw(n!I?gGPur2q%t8dG z(Jc8*@!q(qG@BtWo9y6~tQ&Bge631^txHwttILu>QMgXw2;;^usc44UA~)>YhHBe&yw%}N0bwo6v?mJl1TNCRS#6QpUC$dk@ibNT zr}E~PW#QQCBIp=(kS_ZbCdh5!5gD`aDt~3xh+dJdl}WP^S5dvr~rvZ^I0;t^-$Xgy7 zva`T0U3_Y4QM7{5fd||k`*xKh)a(#XO?Uc6+!t9!ba+p~2OAVW+FvXjm|AS~tbBy4 z^-HT>z}c`taUV({oK&PP`?>r`@sVN}_4+k56Gs2!BGVp4@zVY-TQi!_rIT)SFsJKQCUhrRota4>6wf zh4hm=N~atQGi~J^{j?(qAIB}7cV6bZ_^fH>xy50s4_z!QY)NJpL<9wJfg-|ikZ}BC zrv%Z4UTv<}v>$lBtTLPDXlW6vyc(MOMB=2kF!FfAGik_db@XHS&5R4Xr+O@pBP`Ff zo{zr542=89z(L`V(g5&w@A7hD`e^^?oJwO#SJjDyu>05{)CB+e3+-Q#p|1D{bCae$tLZsgCq!Fc+0UwN!JYyKb9cJ$K>^Q2f z-&CZB0n<9+#=*$bxa!y2b5@MlDs$Kux?EKnj%BJLAz`*j@>+Dw-!Nk#(9Cv~%ACym z1*RT*8^uoADs~oSRy#SzNE3t~tWe?mC#3o*cVqdD3oO&aYU6iL*LYZGUfB52&ECut zr;NF4yP7`iEUb0E`MKwJ)vbyL8CpN@c#yH|U;VNCIHuBY|FUeyD`f?G)&v4nNuL;w zztWZdnhIMX6(yFk`(Ktli2+Yc87R^W-CeCPVt@WXS#n3m1D;HfyV#mMaZ*`T@!Tqvq;IjRfWyiwQC>w!Y*HPMF zqiRC6nh8?)xU#NUQ=d!CAnTD#;p3n(>sacz%Q898%kl5Zs(A`6wtHML?POxs!0+i% zO;_$H#wcS2npdq@?^VqzGa{X9x)#bVse$(ZRV5io%vSrjAX-6m@rBl1dlY73{T^~nnPm)=AHJm$Gt)n>I-S?`w1;W2HPrG`r;Gq=O0^`Z3cU`I%t~27UP+{l z%Bspp>*`_0N7J$N43r*mh$HOkuTCB*1fEE(0<0ElGu(6(Jeqay213bZaTL8NQ&WTd zo`BYJ_s1^bRz08YeHPCVJl}k{{XDSnc3;VkIDTzR<_z-w6reg!gZ^SsDBvKHRjXPWsM|+I#w6!6rIr#tmGDcwK*tqoXmxn- zFgO1(0RO;VgwZU~$6pabk&4R`V!;6sC0NP;DJ50Dh4@;vS}8Z4cEF~qOs0!$;5i9T zeS!$TlY$+brXeaNIQj$n|&P8{`j>Z`6m&!^>m$h)@(@E)C z?;Z?gn|A~DE|^t+<&ViAKXMcHQBg~fDg)Pk|>q(H* zD6Elhn4y6^dgH=xO!q*G1>!98r`kq&$W5n)*!XG%)ERtf_dQbid8xgJDD=tBzo#BN>QL6hWeQP07waR$^3f2+Ya!xV@opW z*&|!JYnt~zQs(dFexOH}g~oc6_AA66B7&bZe0q4D{Sg+>K_skHu1d0A=IsdOY=$hU}? z$Uk~Y*#+YI9%a&3PFvcSW%cyg=kjp!?<1PfJQtw3i3EtRrAyk~-yBj*)LoK{i7D+} z0TP3|@divuT$70fBP&*PiuO~VKYu*X$L&^|#*l~kNyAH4>|3fH(US?qdKX!?l0^x@ z4(YQDNNb@x>^2YkQFdA|bWYIW-qeEpxj@Y0YUvV3u*+sxKQN$VeqA-3F?!0Xqe_yc zE*_?(*uc?ptF4DHx;(L58GIsM_0yYsy^rtgtCvPS;&y13tG)iccc`S%f6)mbf*@I0 z0M1In6^ei`2;>F+d7UR;{3r40A1E6sXXl6TVBWEwC;LVD$Z5IU3LiVq!457>jINLE z>M!TgugW5c&lG#$@<)oFr3ZOQR!;8HJ)Bq$?qgXCD!tf40%~bt*1{5H2)R(W-S~Ev zvmS8tmTRc^Xd?=q3bdCk%6!H0V#QBQ@cuG|bE-c~sUh~Z6|L)I%ms)A?MffX`Mb7C z8T;|Tq#)n>;nHlr=Y8S!>dK(O*X3?j)uprc+OK{)UcPS^=EEQJ8KryW*&o)t^0fD_ z3UrKUD*2rfX$9$;G2R{ z*MM?Xl*(RDQhTJZlg@L{8x`NeArj2m=_L`6jqKAJ`J_MkXPJa=;T*2NCjU+i zcuO`>b>;I5r6`fa@Op6nEXkREP)uJr2%~@OJ;Q$h zEhaA@wYEQf@Gi7cT!=Wr&Y1a0)!p3E(r-)C5lzEr$csQ#?E;@an$CV|-+?jpkg<(t zM2Lksz(UHq^~8=@L)2&!G0d;hom_UhZt`xMX?Nwg)rib)UmWDbSG*emI=U8MGI(Tal?hC>*5(&U`}S_*40Q4{4n1leeSLy3|r%e9^{kAmj2c zO=f2UxgKsYvjm|t0%Q+HI6;?q3Tcye22j+lPQ|KMfSc?>`MR~pBwB00Qg%!UGV?h^ zl0T1I3Oh>*?K@bO(lMAP#wJ7^>fhD8Z|pTfXWvulL_0!j#-kGEa7Efoir$bA$G2bd zzFA)UG|&6i*sHXpS9-rZ60fds*EnLeJ#vS@{uv&RLE7e zx(Yn$9vA@)aT`0Yen2^s-_TaqfMLqK=Ew+6wYe5DOd#thb}^no%0m7FCp#!68{L)H(ZS%~U=b%n5|} z1DP^^eHb$WT;v#l_Ov)g)Jyog%sH__#A+8zYT!xJ5*bsqS6QO1D-pCNp?a= z=B%8&Mol6+j7%C7+w6;0zIAdLo%5VH8pZhOSq})98$Y`jDEG`E|4V3qSGy0CRL;l! zbnp8Q_W&jsbhtBLv%cjk4KV6?4qukBM`cNA4MCsQN&rmh=8I54XOV-nH1)d51TOG~ z^3rD?vy6%KV+X2wQEH9al=Gbzx8*QGfsYU;|5-gg?A$70>-pV$%Ei@ z&svEwhHz>tWhKfM4<2f>n=}JZ>0`@E#aS_(tW7Xoid5;~5i8z>EnVQ}w`6A*A$pP9 zpVu;(YIMY^o=Vso$gkLSF&T>E5|aZH-U5ZY1nfy# z8$C=p)lWE!quV+M`}I2^qbsHXwdnVrcOxV3c#%%mdAqXP^A(LC&(wiehP;#9quK%Q z1IxT#`KM%-kYeD?I&@iKQlnXQS(y<(dvkBm)BK4cDkr^Vzv8e60!Y=0zC*|rLq+>jn)i6LNB8edq%IxCf!sA%@yc2n>_2VNFHf>5s&A?=Zk9B``m#U`%9qb2C>U{Rt zZQpR~A8(FkR_%Wj?OYIDYP-X;dm_P`*)mZ7xa2|0Y<>)7ABbSuU-jh(k~Qe8M(;P$ zGi(E~ou)dJa7iJ6hMO<1B4YuMa5!vhQNWI${3uuGTsyAK{*P!rvA>azn<$K z|9fL@xh7pmpFd0cdT`>1x&qm>GLsTP7sko*F2ki#fv*(Z4$=bAT!Khno0mV|=0-Av zGC8Wf&^ytpnzIFH&EOu=!7=^k3B9cohZ69CNY7sSK|bx

;sU27ve#OUhF63SY7S z$csQNG4BU8a0tRwaYv$9gQf>=o(^j*oaf1?174R~LWLi);&Eb3TK<=1>ETFbp}j-{ zm0sx=YADOi^FP}~vAv1+$pe_edE}s;tPN1PL+HW@nFo~GwqH8q4mUg^@_b+uU_~$j zU>GrG9j;--3DT!>rfOKb;5e|_3KC^9)Zac{@L2Buj0t(COTTqKFz)RBGqW*!Grr=D z&Jq19JK_#^wb{&ccwNj1i%@kv8)PR7*L9iKWzUS$2WDj1Qp35gAv{Id)T}0V-GN7V$}bD zTlql}H#turG;>s|pUaOBQ)NfMGuWjXKo~b0Yc{j&-h#nb$N?yLc&IZPcUnR)0_<24 zlybNFUBJ(DLNp>Zz&{6T5OY}8@DX=&;N4d4rB{?gEDcJlgX%kS2OIiXBy=3LPf+|| z-$@X;tMkYLhCTIDZCdCUB!b*FtA1beo# zA7g`N42>?Q1vfEzA2$CTGt<{3zNKrg{g=uO@bibMtD#_(p}fcTPVJU3@y=7d9?&=Q6)JPuf=^k&PkunTD0&=f#XB4`e$oK`7&|WzLrV(O+vtjD z$2qJ+!S-UKfoKYJoB(g9frV`ch$XT(WhH7kv8gS#VGaZBIRq8uNsRFo#n7STF?W$V zTm(4QA?;+%%;PQnHp46BKT0D?tKT#t)1#ZxkPOQv<0~W=*|1J9fB!R&x6KxPS?3rq%-zB-&~o)6x!QnMfzDHg7H=8ZDmaBmICE%kM4>I|3MXh`yoW9x~dD zkt>kRj~TXGVv{JVSQZMOmU_cn>{{|B;usK|792huP=!^4LrzG|xOGP==Cq5fY$dyQ z?)y=m@WTeQCAq7}O35Z~pDr@(XX7YIG!6E7iY9!%=dx5Qg=96BWiM&SR9Vb?!u1XY zq*>c^^f}o>qE-WYAM{XUfD0Wi=PqTJoGzP6ubf)K$R6hB4U9NI5$RIHR?{EgcuDBF zXV64;v!w*F5>6P|xMC0H0g-kX7=!tqxm}f3n%lM{-g=`X>9QvCX@W`AgFf?D@8^$P zqB^bLcPh6#Ysl+Zzj;Kx_fv%sgy$_JlXG$s94DhIAEEKrzzBsV@hllLhY*SLPF_~r z{_?ggThdOF3lLqcp7noU3}3liOQd={G2@dRteeEhtEM{rC2&bR`uc+wXgD`Z+BPC^&D9LA5z99Pp!&Rmk8dyg zG~H{HU)sKSDb8osP;O2Zu_4bEitHL}CZ12CX|Zg8`S;C=gLD`%c5_D;9}+0C9m0+fecjwV9=kHfF`_%ARRp)m z1@cF?!GK@@b|=OtbVldX5gHT%Z3WO*wk@e=fud4_a?P$$ltRKh9GHnUtz6+P+KBuv zV0e#Lf~4m%v!%Yig>h4|+dpV1ASi(0V+Xyum#WYXkuWF#gA1X+7QBv_+SKEOWXd|S z5m6;_FY-w-@n38!S`U9&R*hTwIp>f@TVwU-LRQf^uU}QY&A}V#)$FhB!Bezi{Z@eR zNSDiQftr_=gjmxY4|sapSVpywsFSX%@b6_{*AXvisnLvGO5p58F`tTi1ZGRS1VTdQ z;Jm^#oV0mk|Li}Wq;}Njhqo`iX8~BQd3rvfo=f!t;7dIb+Q`xVmWaS zSN#B-r!1E)Z+jhlAC~`^&*vTKAz3{p26%;gc1rX=C=Zg(g$d2ZbMTS4l~1fUZaz@D zCUg4b@q6by_olt_)ZFpv)VsX+;LG`8GuY#!yul&vZgMZ}N$T7=X>bS12+Z+p$sUl$qIfxioAwqsJC0iy=W*h=p`fI)Hc>S$dJoRHC@hcwt^mg6BkosD@bCn_2CGi zriOQq6*Jt<%22)@K~y%NL$O2K1u_eew=OAyU;$Xa8^}wD($b$6K|lvy2lA&;g|8{v`%lb|Hj)x@|zgUFRN_{J!K(4!l(ZU8ff{x6}O zEk#SJ5wZ~gdeRewg&nD4q)Snh73X&0j<}g)i zDZ4AhQc(cm<+6b3NF_;I3OuX`LEUrFn(i-z@A5mj#(DiD+h_-y{J>AALSujGkPJLb z@o?y_`_%_gg|5|7=zW|Q^X5KF17xk;sx<-R*N8Q?=+t8{q7|{iHLV%(7%yx{|2(+h zPS=Oud|BoOWNc_dczG*oZ0^bt3%9RfxWN*m_DvrqbAo> z#KB|YZ4i=k5D4PvZ9uZXf#|KNPEnM4P&OZ28B-{vO;RE4$kf zB&nFNl8AYMfG*7jZGem_h-2uX#bQ$$37}F5I<{c5_i_E62BijvjN&u=qBl`tVv*eUi>~h87 zL|+&mT}f2$9MOS~v5->NM5N24={CXVDj>5}nFY|aal(6zKlkFi4KCcAj3wQS%9LA> zg?g0Mu&v4_k<3>b&3-WsR9)^K=3u5W-0W_cDv-rvuaejL|wM-P+Dt1)hTnMNIyQL&|sfO&Nw5(2g zM(Z2JE@>mpfJk`p1MV05>sz6k^mFw3QWN&Ih~}+!)enY=XFQf7gZx5;*L4DdC}F5l z>4ZXCBnyGB$d3f6K;B%7G84gLD*bnt(Hg;hPG(}HF=($1g1&q1K;;Hl8BMguQj-Z$ z)p36H(5U?2c;WrP=`L~R+#!jhNPDuePk8zE1xgU*EM>q|!sDQYR2h&mC1fgofgG_4 zA2>69Y=)O8s%uD4xR1^Sw~rWiJGocvN+;A1JADMrj?gs5Brw##&U6DafMQUdd|}S| zbJfgn^f;T@(8Ib6K5Cx|24M9cxIdC8U>b#4|f53rE^?^AB4q}3HH~w)SI^(m1PB6u&%3lo?6%^Hdi~YJv(Op z`iRdT*YiDJHa+9E=pA__+xul1Y;P=~v}+z3l0Aa0BnbT((=8B&Qw4A?UIJiiERBB{ z;`lW7Pf!Hwebk2Qq5txMWme|(-BA0U*;&$AWGy$gTN~eH_lfe1K7NbAC8yCw> zO(t)hR&9S%Wte`;%HfmeTV!>vfpgU%o9I@ z<`Y&4B;{>r*p2)>2{a64CO&PSCUq4jLBN_b8dZFMg1@WzAb#aFI`Ekg{#>-FvBxGL ze)q&N+=Zku#LycwxL!dyPKmgKRcQ=03>Hay#pp)5h5XI3E=}C zct9U?a;|oZ3O_vqaCAe}FbnkJ2?s#f$h1WcJ}_~YP_qFUh$nW$XerLYt`}`{Z`xCx zkYj9ZJZo=WfYdcrfh@uC>qj;k*{4=3EG42%Gx=|-NLcUg<~?QpU=n{Sb?5cjFUw-Q zEKY~Da@#PXW26j%&%W8dcgMr09z2(h-<8`E&ADu7KOQMG_ZK+D#emr4Syh82?Lh!k zKbzMIrj*Z=<=Y`y3+^Foe5B82UL!CF5EcUZ3DH`*Gx_J~l8X6mhFF(bT?7z_S3AIsq8+~EeT(ycLVGkrs0M#s3fee^m z|FQ(v82go6J;Z77Us)|p(8`Q4MN+h>nO~L#H}o#N)^JnUqw>B}iD;93 zn8{fxPqQHy!1--UFJMkPTl;L5@*%q4lG9gF(HrU_NV;!mXRNg+zjR zkz<$?dk?9WCp1$05P!KyA1b4Q=&pwJtmt%W(_KZIHD%kD;~;w8ZL!6hSZvf+3SXUl@Kz}RJ7_$Ba3xi&<&y!a=`J3vJQRrp^~OBp*$q&VIk9U zF>SrV^m%iJ*lAD6Opq4f&TCEy*FDZXdP=lk44#Gt-l#t5TwTg|p!F5T+$p~>>|i3M zQGGUyduHFUrl6hzH}koj`z_#=xtTvJ{h|+C4e4O^|aU)rFtZ zB#Fe8&KwmHUA-E5A3SYI)l+FCj}9CE1Y%+;xdX^74GJCtbP46mfYudP!8Nb<*JjI) zNw?=nkBN>zf;5HeIjd&3ny^Nzd|y9x{?`2IK;+N83vuOkb_Vqj`ByU6$XubHDQ34( zzE!`BK859lG zZpFU-W%Zd~qwzdiMpGMEl zd5bW;)w87v5U&#>026|?X%gE&O;i92XJamTJ-EYi6wdTJNR$>engl^G<=L$#&%-P=*Iq%%o>U zM4$84CSD4>0L!o`-isgI8oF9c(Rqh5bl)`IuC2JnrqST7hO>q8v#K^j1mhDhL(6ME zbsDCM3CqG%+JrU053-2)JUTE^%qViQ66?~SXVtl!6Vjc;5xtUW|J%7_(}r;C1-t|6 z*KKw7-#nh`(1$)~lk%!9m>b=!iJu$qK5p`$|Pk3xmCV)>%t--1_ z1XMHj6b>*g79&>^MeV3nwNka0edL{kp#s#_K9XfzDPGvskg6_3+RS}B?@d^X4YLPR z%+!PYvR+0>@Vp2*fVj@y^q8eh79rL)+gF8+pqt_Sqb-hH(F&)nRquww>~hKI=iRCu z!p@ROevIxr+pnO^UYe6;a+Qsxl-ys<)LKtz^DxOyuN)}|Tq%?d z(6qzyZLgOtIf>wHd9yk4OJwfS;#^2I$ej0R9DgbolX`Pnw2lct^9+OrAxcFw|mvVl%CMvjQM+hVMpF_tDcqXPw!CreLRtg^*w2H zz1NHM?w-JVxZHRu4mJCyyUhLV0;nxitBV0;C8(aIz{uJ2Y-Z<6lIezcnyL&?yy~e#RU%ovu z<$ecI8s&9R{o#7AgUkNE{7|-1phzvnZOouO@c77JFOi_Y^_14%(I36gXGL;d)lT4i z_Ng`MC~*UCF`osFj9dB=fI8G30lT!D=+OeS3{TelvaJ7(JU}qhPd)&^)-d+_vlw_g zPnj=Dq>KI|nK8V9no{SW)2(;uug@{MY;e167-_0tKpRGtR-D*DQs z4^I+Vn~9CYdn>c!2-j$cn(+6BHu`OTmMA9O!007xG|OPtCv?mNM~$i~}}Ig3`* zn&CAC;65UsbZ zHhtH)HO%xo*hN|gEL_*l>TB|!)+qM%GMa48Z~iII=SG=i8@>>CC9Q*P64gmCfV3ih# ztyrZTOpl+YZ2eTW-sid^Kv}F>MOwvVHo{H7*q^M1|GXO~yiBjOoN0V``mRx+B`4Q1 zhCY3BTSoAeD5c%FP2PowSF?{uyC|)oH9MQIn}w3*RQ|D>q^`UlxWV*#4Nb$( zm%dkRAlX5e8`Lwao9#0}>yKFL@O}kI$ zNBa=Uyf^7-t#@xz&`iGI@a(Yls2vb=^+3_(!;l_>u1#XeYqR*zh@P=3(pV9Iqd? zee~u0a{dc$^q3hY-}-Nr9iU*5|6Q>BpGvy_{zSeMCn2N) zq?o>(n92c8jJROv6W5_Db83llovzryvZWqyzqO^gxgMHroKLNuxBVux*SgSb{JzXP zFH5t(Kc#d9G8jD$gza?I>TA)U36X*|HjY5>t9omQ)oHen9^!D`66@N86C6wDb(7Pk zIu4(R_!B)&`_GL9?q67}1615R-;5=Gxb>3ek&$s5^!cI@jie>>f3aFDkA9cBQ}^Ea z3H2S=JD*yREw_4u8ayY4PWZI5At`duT$Rc*qKK1$((F&Z7huEbDsNFHNz!uF2CPu; z%d*0AmNoDHe|$_?hm>{fv%sotkiC}VJPMQw4eczoK821HD9Q9m8WNu7!7|b72e7E^ zNmy>92fyk*-do@g27i<~yx~-IiD=uFkkE74A6BL(-CI;Rxy#(XEGz!hL4!|ILNTi7 z+Z2EapQSA!oB-mMV))h2ZNl%M;Np?ejR!l9CwlnRg!7BsYvoy8$HnS%C&#o_PqS7? zJLhbnmriJjrs*O_INk!y$U@iF$k-H?_*J6q3~y8zA*rDOB)6q((NxU7A#_fa4yNrQi2Eb*g1_sVi>&`zJfYgeShYr7{vF=GB{zk<$CpSr+V@b$S1S#r zCk5b&v1R4P1hv$YaHbfef%A0PU8NJ39P(qk;Zb)0%HaM3lyM_p#-__W$fqc+ILiBt zSf;u%k!1+#^A9i{TVW@#6V&q;b7Qmvg@LX5UDf@Jr@ZjUa3}mOLxWH4y!CHA^d8@I zH@3NS=+z|mu=z*%U+GxS{D)k1nzH2bKUA)&KZ-SAgIUo0XyAL+9VtfAklc){8GcN_ zi?2EJpG#)a#@GBK7qN0AC6ECv7Hh#BZ--qb$- zkREMO)?2xPKXW%Qwlpr*5&dshu>akmozA1cJA7s>De`!Q69lNDm@$CTX^=F;^ZG+X z(b2jdk{V8u6%r`jUQmX2K;>-9Z&^%V_;uE%%2JbZ99g~}i0=CTxN+(=Nq5kS2l|;yjp+?dckhcjHM|)7lll7(s z(0LX~*J)$*esR3D-pazVnU@3!75GrJ^e1g`bo4m%O7K%~FkwYZy`bSi_yNkps(&5! z%&ZwZxr4jo(T!R+WUS2nr<(o+e?+(!Ru{Zo|6G~;Rf-RcLHuvc-QOtYKO@fnIcV+F zd^!surDZg0N)YQvXlW^x!&^UNhb#xXv)IrBnD4yW`f$d_k3*c=8I~!qH&842xiKm) zKXK4M$j08s+vv7-3z&TrARtdRyj@M8(CY+*gn5Of{Kr7jA*h^dtfjSI!qdRh);?Zv zEU6&=G}9%nJ`Q{@^RlAVdq;RG-M}-JGk%PwNwCT3*SE_2IHI-QB<(NY>Hifl{+|$g zq2SV{iX#^b?rkb2`@C6T$=p3+haGAE)@iG0%KIB^|9hzLKVx_zChs6Q0;dPAD21(! z(kC*eEx{-^NJEOgsyHM(ID>0^Gzr+)geWU7#?WUskHU|G*q`|gD=ltWwAtL~+7LMr|VEeQveC`~4<%n5br580~<`rFcNtW)>LR_b!AxV!E5rr0V;(UAUC<9bl=i>!}5@dp}0!s`D9v|$*?89Mf5 z8SMtxY{DM>ceCdILaD*#;&|pklKa10J2?M+mH!9qA+CgdV|BPMw8N;W;Rz8Zy=>Q%b1Wcd+h@g5*7mlz2VXwi8w%}#Y3)bt@&GA$Iq}dygHUl=x}2L|O-pq$6c?Pk{^y%$jCPp!55on~S4Mkp z$v)T+>TY6-fI}DkPd`1?YJjDlZx|IR{jkiI_*A(k*F@Bg8Q6+>=4DkluUM}>>KGW% z@WUw+^8;|EMAPX=i%B>$3*o`e3iu9#)3|Y2P*Onojklpm&>%9h{EVWSQ06vun}_VW zlyJG_o;azjG@_Z+n?LOD(=#Dt*CJ+h#9E0@{S;^5_bYRNcBM%g$gHm#<7J}E>k{zBL7#g)H5;o(ZB*bh$leC(EMk@?uV z5U^{=U)7(FP^{NG3T3xQ&Hdmz7sZ&tdGTT=*j0)VBNLl7io=AmJo^MEjbgZx-mETW zo0`2dIyM~>^F&B2;Tp0|G#)CZRJc8EAG9|j-E^})-_umv9W?iuFdx3)?AKp>b*bJ+ zC&}cBr*>rC|3%xoheMsmfB)@vPEv#rvlR(pD~ENO*|G>bgb-r3B*rvm0eeBST( z>-Bm*A5ZOrkGASLF`O-2O2-`i8(pr)mS&+UV-3De3lQ_pPo%)-nsBpQj1x?-ML_S~ z(1oc(bb4Z`MCIx41Y2LYUq|0|)Dmw`EciG5wehVF0vrh?) z96$N3eJ$fb_6KZvr@ChYm!n9*(?6sM%+swMCymvl=*`^|JE2vnP8({VMR$^a8?mnp zCI|>ddSGsJ7%qr_<~oY>pTYQIIqbsXn1^_HlsZ51RyZMktkSUe&2m}Ebtf}?=qj7- ziUycO-T3=02E3EGhfj1qAI>rDcQ-oq*)9Bm_7Fj>T+iRK-pEl$^w8IJgRDSK! z{C)EU8-a!fJv~1-)#UX)sD`q2EMTr^&&Ud5?+evV-gJu8yR;j5U%=Vw5MjfVBv;1A z;BCgco`zWd9;3dn!C(-*rTgY2Eu4)~@MI}$ACo$YgJ~^6;tAoZa4DlD;t2q%p%kyR z7>%e?dkFjGje0vd`=T&%Jn4xchDrP>?*Ws-BkdD=#r{&6`rx&mjBR%>;RJJIc$S}(OYc4a#sAAi7sM^U^=12=b_K6`C6p6dT!q1S@n!E3FEOZCe#w&I-Zm95kiK{ ze$UBWoO8*S3=Ig69eeA-)={T)WmoRG5a5FR)>_<{oiXv?WpDmW*^0mN_isW3tj`2S z>5jaK&0Lc0IvO^1YK89#IC5}v=g$|hTBb->YD@5ApSb|FA!|SF0M#sGDxH^)ED5YC zEALXXJ`s;E_aB=8j!(Z=wFs8fPpt6*`6J{~Cf4m~Gl6=O$cT`&i?I`BB6N%&m&M=h zi#_$ko+m?x&I+wI@M3$-8pIzx`;65V+B{1=Wj6RsAvMscTp@h=%|V zctaXM1(Bajh4;so;)E8~a!_Yyz|Ug;JgQ;V zp!l(PtL!eW+z!|cm3O&rhM6y97Leqwuzv?!a%L!!wwD+a@U(tt zIm6GTsUa$VMDl9U;*a|~<+OwDuuf@5biF(d!p@wlZH$_2tct(id(^jGO;;;~ZP4Ru zuG*(Xh?=Qt%deh~Hu3%nJi*&KDHZ?O|L2(VV1s-}ied7N|F1t1gNNLl_7$)Q@f!Xb zelt9ogO-w}>%Lnhsg*COp8mg;Sl-3{;^G#aeXi~96nWOow?1nM*(*q?S6s7KdDRQ^+++*lIbm^0Yy?d{w=-}07v z`;nT<9=48U`1FH|qP`s=-#;~^ALmX*XEvF0=hY3STs6q=>2;%>ectWBK^IP8c)`L@0Yw22frZjz<{`H-4$MMZ)!+Do+Mc#HUWrWZ%6FP9oERa315w<+8!%o z7SHBZhpX~fsQs{Cz|#bbzEDFm*s(cf@_F))Uw`Y&a`M8m3fLK#q*sN>Zo+z8I#+aQh477HGI7MtwA;#W7wpx!|g9zLlSI)YUf)F{`T)rvo2j zbKNe6S{6zr63ci--$$rpR&naq)nzm$zG1HX@}JLhZg&iHJLQ5#ru1g&iclL2FzMmZK=8Y%2lE-XCGUfZ9zLS=ARP>E?3A1vAmR3pp-#G2Qgxi7IqE-~TXO=sBy zcYFe}<7L#S&)z2ZxN3YHV?Fje;S+YD;%no&FR99D&qZ-<0lRp7IlGC4f3wI?jrbC3 z@Cl(Jo8Z(E656(55nzWFhsirn+SgU|mfDI>wFk(BX=>(m*T?>Z4x8fIU2kHHvmdm%0n(}uFsBq*A`(E%$#$=eGN~yr+NY#cqbGa7 z8B0+kVCcy*YVT~({-G!<{&t7SUNM!*3X$3t&#ncvv4u9<9S5U!vgUfZOGXb99M41~ zOt|@Cd<=`gx#4f3G5Kcz5|e zsBbc+Mt&hg8HUb|OuYINr9+v_Lb;8T1Zt5X)OJs%mH7LJaWV8M?+&pXGyECQ%HL8cRq(@$ zq1d6+thCUt=$a8{4&C+ZsmhE;F0BWxA2-gOi8{R4==qDCsjh*^(Aa0HK}=mX%agw> zZ$|-&49NcKMv-g}5hv*r(>j;N9C|qHw<*{sux=08ynax&LmK~VsO3;#Bq?5pnyQdw z3S{SP$HkmXQe-`^nwTfs8*l8>l&Y}H2U$9rW|;nzoJa)0cw^vMS`^15SVxIf?KjF= zkRTN;JWk+$syjB9J;_gzsWwtQ#H;{qdZ*=8D!^#8t0P-Q^3Lw{5G5t7z!f6jX9is0 zVa-)z@0msS?XJCBsk(nG-LKj4mGN)9ChUjrV_~K5dfh7R3!Ump&Uh+2>R<12%l}-i zXYZbI_T?Guv2TvuZO%O3r8QGK(&<1T$UCx;yn;wD=dk>l3!0Mz_`Gj)o3-8zp~3Bz z(?QjCLTdebWPk={bHJB+8L|LFcag913xVJOwEeVWD&A{?hql~V52DRR<0Y0y0&aaiaE7h->YU_mdgvMVd#Iul?)YAZCG8Pf zT|<)_#(?CwQBS#Qd~4!ovF=|j!D15yRibVuAIb%o3VVp2NdAdrpC{lXwLC8e>X|el zzP}Twzp8o#^|uYn8=CR1z1{(cIzOwlxSqd#KiZy}fEnw7p z=vL-fDn&ztTh=$kwjIX!8k6vl(N$h~r-B^ZVTf=ZHhSE^ePZhW#|3PIZ);x-O`bcO zXC}FRaA0)8mr%tWZCHuh%sd@2!Pc4QMj5*!g)V*lm?*nu$JWEk*=g_A!cwKP!<@+{ z;MQ~1RA>+Q5l*ZIBr-tE=di30(jK0#l@i1 z3EqU>(fr>s)NZQj**RVt-mrG9~~A&MVF4Gbd!8PSk0AOO;Q%` z0XNOogBuwpRs{LkiJxGOIo0}*Nb_6>02kaBVv;#3QY(nit$x$|Kw2kXJ&Nv?+s#^!&CpcUBJQm3E{t3@r>RVGWkD@68(Ce=wIMP%85?W1Per<#6w4|n8N$dD(2Yp-ja2UE4 zb)E$$>LwdASaZ#Bug7q>8(y|64PEpM%r4|!qCVm_N0AgS;OvBrzKVXY8(bM%U&)t` za~m0IODQ8><#qYj~ef*@V*Ok?{>rXwqiNUmb3C`6&@_VM~7c{zq zt*4K*%rrfuvt&Qq67tMV&@^n_IUFo=$+tsN3KTR(}N!4ZKe8wUtk*PJR67_W80MuTE6t zJSab4@~H&r<8UA8V{gO<&99nNgUO{d6y$+;-#kg^E{B!G&ceJDlCd~s0;{4z-i6vM zL*aZg+2O*SS!1C4bt_cV$&q8@Pj#QenY$Yn<=hZ7FL?EzYgP2&xf;H4?<(pSqIgUY z`w0+HCt(AjrbF*qOcyiVi$Z`S9wHJ@8jJWATkLKXZ{H_z9TIU%%tAXZBi9NK~fn!&$`ml9S$DP_-f#2kb@Td-a7??kS?ygm7C4 zL~qwRIdl|$R6@Kw*W)|g$x$LI33b7(tP!H6#^StV^$wG41+?2*N>E$fLkF9sl*kYk zY?@|e(CQWzOh-2~H*i!+#eaG<9|RoNPF+;!==UdAHf6bp|Y2j zg!zR)p(}`{ae(?75=d^78dz)3_0v)nD5)jZ0lY^VW*6&;Km2N4w$I9hX8W#9pXlrW zMjuLWO!t~iPT5rH2(O0D=zQ|BO245L_U_^tXOK<^`VYWi5*z2vw&U+2X^LMgZ~5%w zf8j0C302sR|7 z{@zsI)R5yUfAF1?xT&bB%Ub|Ia)XOgs}jA92S=OkPvsyrW=2xKp1A6VM=nY_-tKk% zt$|gauw%5iSN+)Ju1^eQwWB$W*?7a6Lgu%&wmj>DQ3u?*PIFUl;XEFE-cedyR%c+deJ5mYcm z6Z0jn%u;Q6uHnW4{Yh9?CR?nQ)fSlUFZNXB|H2&lzWdDe-mCjubNWwJy51)suwF^# zI#c%w0-|Gi{fnFrRuVTJ~M(+n|Ha#7^QL*m|=jo5vvP)l}}i`a;|18bS|p4``7X%i&}@#QlInqynS5 zE&WH*gvRp3MA!OZ6FknTHN873u;PM$uVHInOT`VJy*Yo>9`f0zMoC2M!y+^}lR0#@ z=h6ZiGC$jxIC&Y&RaumF=1#xYm7$i&NkWC4Qa!j&xU?CE+9o?Z@ej=sGfUQj*vSng z`~bQ8i&_(7T|ARj_35M)ZSLWS^Y`!yQD1lyX7vm||CN}=8R4^^UkOgz%9Bh=MegUv zJc{#lO?_%Af^qQ6{8HH9pSF&@gCdCH_nh zuAq?@?-wvaFn89GRL0$}bXqDm*3ZgWGaD^;o|wL3P&pJW38OwuB93?>TGd`TpBFwX zzIX9Kqa?O;h_QIj`y~fJP{Le4L;w&M>BuBTo`lZr?Z)g~z6mme)-ah;Iq?w?({p@5 z{*I#-1L{~8QWB+za?)Au)L$EqCzje(^Ow!d{u+Pr)lctU#J`)AIg^hlwdOCtH^4*8 zuZuCls{k+yW+83RI5oM&yo#g^59rWte-=m0t6#kt{t&M36ny3CG{q72YY-{zWKD5z8lmJJ z>G(fwhg+bh&{ERMG4f&OQdN(*a;bG*x8=RFTCTPGGd^5z+XWm$j+{Um&uC02&j+6@ zB|sm}AwYGVa>9Q%X*mj*Hz};Gy%Loa?ZMNb(XLSa1lrYm$ag}5*6sAQ1lz^%LS*aJ zZ_a~Z3nqB4xKl-zaRVcgVH|#8X7p*U!S0WjpY8W6$WQhB<71)Kw~C6!h+llZmBnb_ z@ceNTZvcKrO_oxgO)qF{rx~?vBiVru^#!R$jJHOMQmhRjRY_8-x=J33C+o^&LRCjO zBc^WUv*=j{S|YgtxETL3hqOiAB*QHcflr;!;eAEfxX>nq7dy74#hs@b&An+~pQq zX;*C?SM3urE~$-;&HUwobxEK4a*&pf>;3a-wBzdOpZmxG7hYDp{O03Wx`IVjS1{#Z z=XEENGR$WHY$xF8wp<;ZI0?yNMZH=-032+KKlDv#e#^tFA#=&;IY5#!Z|5r{kq9L| zq0xigX7qTjKdDNnlIr#Huo7!Yxvu!e5c&NQ=V-ZY^R`({Xi@(%8iS>%AY_wZVq5{$kG3{-q~*1}*MQbyIG2xH(G$A#Dq|FHTbeY3b1cX7#X} zc+3S(tRPyw@a522oF4;^)O3dQlE@Izz7{odzjPLP=qXfmQ-JpF0@njCYb~?Cw#-oV z7*fRj{qeoE+gp!s1lbNeV5|qIg2-^${?&y_hHQjCrUghGtwO6fAk{)Cw`q+Le+?yw zM5S<^J0{i4jLCredwa&tV-zPs+&`A?6yU8x^Strl<<=f=8;f)3rYr85x&F$FGQX%3 z#8+YUS~(<Zg*O&KD}pMh&t?^qs6<^_k@p=GP1aV@z1jVcs}MoI5(Xb#&p# zd;@mYyXi}vdY_$KRli(S-7UZ-G;uYNRjVjH*Rv}DxZ%wOr8*s#8mP(`1wPCLuq zi+AAim63*^l0!@~yoKi7OPWNSfPZt@?Z6PpQOE#%9d1ef_1Czduy0Ru*FD;zex6zL zyyb0~TY@y4$A{?#rL&JUv$Y2&`ml2~^b*g1}-1C01)+^n=Dp7~Jn$ByTf+vv(XXcma09|NS>I~z=aRR1Pd*Z-QT1|nFAlEU(d*IE)%9*xqdPrt>NcqGo}kP{Px)9d&PI_9a@$Z}Gb0n#M~thI(V?5@>B-Q1%_o>JR&(v1Fh+2yB}IvXqkota zAZU^Fq=28{zrv9EbNP{LZ3R%ni=zJ)R!T|)RzbWGcnwXw8&w3WTdo`_s zA2Kp>-Pd>gW3N>^HbUFszj;SYQv-g(0Sr6zi$>^mgdEo*>l$xdj-%Own*7h+Ioyj4 zXyejj%yij4uZj z^hl(?nQ$=P>g=GY0q8IdH{>6CGPMo$^OjR^iA1(>N2|2rT!R!?M@y8``YmvNdF0;;csI$tym_Y}L(H|`5Ls#w*heC_(|PUSB5 zp0xjUQk|O%^eTVP@{F^3jXpCt zBI}stt#z#N-$W?o==j#Am3phj= zmlN|Ozls^jH%mVd8@knldkq!3ijJ_K)+b&Vjil}DgAzoYmShIyATEM}f7mCa>_z4% zzsWMplXS1OKMOmk>`uB9!Ct@{<9U7`R!{U^E|Qb;P<&WbMarY1bcUu6_&eIiMd-5g z+nNz0pOwE-dyK)*3=rEvq6mV@QMI-cWo+-}?-Qnuy-*LJLbf7$*N&Fp3i~_9*HFn( z7%D|x>J7G|m5QPdOC->?P@3~PH~yhN=F)ES+*;>ASqZH9*VcOE708i<}CesafWZsVX*Ywg!xM&tef$Hh!pEnF%bkIMy@|T2rsnJ zb$;I5kZS)~Iqs!_?<&^!ztvR3S&5oZP#Ahn5~g2nB2q^81?b~H*u9d?;xT25Ov#Ee zv@j~&Um{VhgKALk@@A#IFu*h(Bl!QtTn>94$t0f(FJ<1axAOBqUbgw5EApVo0T z1x6_q#N4s-4a2Sy&+by++h!$SK&gSzXj3HjPslZaXuKy^TZ1E0Y3RzMU?+Gz>W>Wc{PNPp zjp5{0%B$3I-($iXM_2mFM@@ySSQ<i5El2W71F*NV&J6nN_wAz`23nNKGSsuj z)#3$I(40cb#8J9P1lNn&5fDQG)ETs=b6BBF0C-$=9y!F1CEEpD|9%kGDtd z9)rWwNNBUCMTiXcPmLwzE$nDt28Mt!l-?2_h%6qDb#x)Wl^;rVuPl~?kNV8EzSawE zej69%?>Gn|v6Jv3_TQK9gb03WA!^8o=kaU%&8I=;EH-jZ{he2n~h;sk7` zRT&}dPoyigXv9FH-CFu^*`&=bp}a=777*}KO&Xak4wK{2Q?5rkeSs)M(w`KC)5NMw z+qH6;W$u;AYL2G58Z+@cZ(L3Ndeh${Jp<)u*uGJEo;e4G$bFuUuOc7Ly5tB;MW#Dm zK4(YaUg|_sevS101eoc0U6u;BIFt8Kb-k(99tiRFu+~NG;hos#Wk^!mR69;~NzmMhLTj z62j!IORjBXo3tuDfIDi>J%SIX)#F{xw{xAs?F4dFUQ%yCP z)L6UZ#K)n?Q%!K8&<~-6!R6cJoy9o8e=53_y|Fv|Z}?JtKCQ0W!uyeZWNTH%S^aP4GF*Nuoo7aumPFx9aDbjdy|Ji+#2bb&z;V zqd2!HEc>Bw8)^(t41rg5@AgoM9yO^YP=J^78(3+KD!3gv@^pCqT5!UGv^pszM>2NLqb|#}4vTs- zlY97<&P&Uo603@@uS$;?VSNYcvR3o46;iax&^0dQ&w>B@L&*)f$$vK;AbJQDl9oZD zfkv0Fc^c`EFyrK?544m*vZ>(B@{B-ut-W@A8<(wNq4Qr~R6${X20QV{LS)d7*yf`J zex<{&Gn^vsJyO4OuY?@Z<90oxuy7DxZG*tZ)%4qz7N_dv{{JDADZha`+XME13VOkz zBfthKPnO~Z{sAft!z6QIpAuTEl34lci$rBfcJ-M5dw+)nvd_am6^?ArpQ(6=KAMj; z83w=HaT6{YS^2ny*3L&r5h%!&mdTfIK~4INgG_=eRwn4@DFA}%vh%YpUUao;ukn`mItz7~frZyrhlUHI~oZ3Y_h(3@*ft$NFP=5@ayK$wW7`9DSV|>3P8puYlc^dM#7JEpx0?}a%M3qtF zF;&WISCuwK#Q^H|g+VJ z$!F!Ihgol*L=(=j{niyW!Ep{;?NONy$!o}DR3pm3l{&FM+4=o(zv~}y=4+OY@& zSKgZqH(#^Z6!`6ddz-mpv#{E@jqNN`MQ+ZcD+LsO^t{-=&yh7h6r5K1hWq7vpf=sX zfZ>k%70mJ9_fBHw%OA=Ss4_L0msnD%_FO#OLp!iTY6s>e$j0kcYmeDNjz;KCGv?1j zMqPQ1#XcW>zo~6uKTHW(xN! zeyqt5WE@?(z;Qk+r^_Q;ZKxw&V|XTx`@fWLxsm0${X3`&6QYQtLRG0U*mCZ{GvWft z;~`UdZ^;eVc-u`JKr~!sjhY&`CaJ6~WiZ*5*2oEcXd}HonzDD1Wbu%QaVzj?6Z^Nm znp2;87I>037LxlkL}lpHL(vXKP_3_a?m}zi`WWTL!VuG8r?uIhjAKpgk2(71Sci6; zHg{(@`mmLQW3WS&=FRwjWF;ZYe<))GvYq29&(@Jrho%iGf#keyM)IkJG8;b;H73Ja zt4LOGlsL`a692}xp_8d3wS-n=d6U#(>(h^0!KJfyB$QqV9r5j|KNm1dD0L1$NQs`M zA3&;+`zV`Q6hr`lZ~l&~0#km2D2b71d*Fg@Gk=@25(SFhSr$e>rc?H!qzX5^22qBbIDbAee;Ei#Mxgzr`mT ztN6j0u5z{}s+`b~KhVhU3vWZpVsvo4RWA-WExjV9?$Qrq>+E#>P-%JBElQ&{X{^D9 zcd=%$*@;)OkZ&MZN(PbRl=s1)+uNOj+#&sCqXbwxDJh3Rx+GeQhuG(X^?k#;sPvI< zE8rZM&DWtNw78jg)sLEocMJOW&SqWY`SnL#t2@0v%aECjf!vtyX(HY359F<+QyaxC z3POIWTzGtNuwZ@vC}TcqtkuloY6}}GPhF8$2SD%>jhPd2Y4;L4nqD3!g==5gI6xtzMs*IkQvUwP{>-s{P!aETyKx*sPw0>2bT zV{T#q#4Cik8Ez0Je8l*CfXUsab8(u{OXy4=rXfjB*lKHE2T{_)0vi&9pMQdCI8U6z zafI-wYguFM$5R$KYMc;G0;g`FoN5H}=+}RT3j5R9w*f_lqfqvkDKu7t$NY=@;)pED zR-$T0Q*@&Ur8mnI=T?Icpmreo{l?c z%RKN)FIT_wXL`0{_e;mp#lgB5d}($QhwT=y4~moA2ia`XZsMy+P-k!ST5qa!Yjkda z1$4Bbm{{@km8Ih^y?hS9X)B#yJgTitXcqeldV5* zq~P03L-}<*b>bq-PU#=-g2cOUK##)jG)_k-2nQ}f4aueo>OgnE6k;@pn}3CJ3yRwX znnp`qxlJpJt=rP7zHowuA`8~%@V2g5GZA*$AB(f2U1s|v9LgbUyB$4>IiXIbIxlxx z-f?>{%uV;hS{%&}Z_27`@`_$>N`=2kC>tD3U_URrN1gu{pmaq-y|fl^S39SzFYU1{ zhv}VbIp8E|@@GJ(rFMY9JC=^vtPp$rX<9X?@{!fS*RtD>bG1FXmIjG@$?J^=WCU@v z8=+wO?_rB7^!ae?o?XQZNAC4-56_>3pP0l-JriCX-q0P* z%DRrE7q{$rEh*h7T9R!yUocSZrtcV&?${Ul&A4mBMdGi~snlCcq@4qZTi=VbkdNz) zX=1i>^PtI1Ea4O}VS2v&wE8`7iQ+|()L?^238tmX4AWj{Z4pcRKpNaG9=rtJdpb!? zw3MuMd{nAHn$IPhKpZMrm+05N#z}3_iF8ZX(t$n~RTlsA_g%v8*PdWbay@H!3?1VXLY4$=hgd%i*zH)Rp!eyCPtcCN1X#!zaIYO zfl2A3gLZ1@SLe=oVjC@bg$cBi+4o|5M*TgPk)6eDe~38?dF#JOkz!^lQVS|DGzTDI7l=81K3(JwPUR%v19Vpyh% zHDea14Z4Q3Y73VG8th0p(5hr+Bd6^cT+rUO52AEmC>ud)KYpg$sVY`YZ2Jk;u!SGz zZ%NP$lBp2Odyz`ebdv0-QQ@Jn>l0f?tbbLiIlEfZ=VsHHC;SjhA6XClflj0fT>IAloIku7&gMxnG9mXqcqTvpE_Ko&MW<6^Vpi75Wb8Y4KLi2SnpezJ{^&2?CTD zAT~)&dNjbIX-x24%>KTV9UHbnPvNtu?aW(Rhu^iRi7+Y`>6$RzwPggvRL`}8WwE?# zDaE%Zt(=t-R|D$OJ!Cb|sFs;Of5!+)y`Ve7{nhef%!$@I|CFH~gh>?X)$5q34IG!_ zoZ($8{8a58p62-Se467cX(el-w(c|g`+v{i!lH?HKV>r|wEx4BB&(p?47F9J8`1`z z8in|y1tQm7j=>DDh->X@O`XE&{Uk=+w8oTu(K_)in(ya-LCqAsXT-R{*Z;$IbFaE? zE!~WNH(ld==oPGHfXT==jCkYoQkryU*#{hEFN?DkmJ>R3{Y_S%vCL>`$W^Cyjf33r zTh^W>By)CiZPL=iyk7O>zqD&lOc>!GC;RLP0q31Kt7?ib>bVVD%g%$x{gfSgYC0`F zzjYAH;(g8;9KQbgzI`9F{0mcl0K^IYmA}+(yKeuUxq?&b@=}UA=^cVKHVz;D7LYvk zPUE?jbI{4ee9KTao9)XdaLUPf+kI32bYW=E9(!(#mFsil86hQAu4mb$MR%{qB7EF1 z2grjM=p5mLf#i(%Xo|pU)_UMC;`5 zC1O092PdZ9;MDO&&&*0n(l#vE&dzp2*U&vd|5y5zk0QFQEbL4ct`5e;8JMLIADCPb z9Jf30_GG$_nkPsSz7Ag&>Dg5}&sY01yz=ScZkV^zQk5~8+EUMi#YRDku^2@EiW}x> z3=lrRW*+!3<^Z{hrqh-cR6c~(!33E;XB%AdLQ5S9>MQ=u0!Oovj6Dq%7r*p&87hlH z6ffe3Q68KSCP@34&sj&z6=_1W1y!+a0326MOmo1yb~eP)Ixy*NhXzTG!B_ACPTucg z??jmUJpy6Y_TzPhH z_~P&SuX6N>F;2)4&lz8rCw?qJl)jp7c@I9SAw zc4Dt3?*?c&_6J&*d|9{9pfYy^v1bAvF!6%S3WW;wO*6G$@~J6!wsSmiuLNE6wW~-B zwfQC@EXYHyl5l9aJgv=KdKLtEv?{p|9T_wU9$CDa?IGUA&_m~YCj%0~>3E@n;%tF` zP(121*Hq)5$?NqeFq;~9*p8Cx5O zx4)QB7jJDbua9WI5p3Ob-8KqmZT_?Mo!1%-6U$&>xBAa#7q>?l>v{eBtVe|?NM~n{ zi{Zp8M>FAgQI=b*jmagu{T?Q}jurp)|LPYRu~7_0_W!Fr$971%0`>Qxe!#|lezR{0 z8qK(NnRXWVZ_w_#O7G{|wuxKsVVOjNCwECv;~#Y4z_X?-<%Z#FQF1$tj!#Kh@ID5( z0J>cYX_#6}uY9bl3a}L^nBA-ZSvaO|3~*86#7TPkqo%wUS~hJB2Rwvv+qg?oJyG3u z`j14K^C{DX`PJHS^g0uh{_iH%JvU%LXm=-TeSA@D!KS@G)nk^Q%|zg+XW$J%LB^r0 zng`xH*34}ir=7^K|FSvd!a;-X>y^HJCiMImKDWgB$=me(mu{z@57lSQxH$Up&XnB4 zTlwAZa5{Ob@G~M-@9WDh*&m<}KZW>yj9yFFNTDcGmcLIn0r>p?bhy_9io)qjrt#jg zY^9ZInH*4D8yu#zn^ERRq#FNjsy2``FrR_6Am#&t)@nJp*mZ;?2N>o{fOSL$Iz*sdbOKo5buGsu$ zBqPSnhrJ)FsT3(_U~|ot93zf5WmZO1&gSQO&`gU(eJX_#w9WBhUK}P^=DI0B)({dw%!lPFtbY(0?>-$KgrL@x)YWs@1&rev z#+|%3T8FqxEg0F3I^hy;yC@MLmGM$z0p=D$k^GZ1vL0eDb-V5(U4}U#h2+>OX69kI z8+#+};d`R@WN?oHd={Ts&J2y1%U#VQt5RKK-Z2aZzZ<=t;S8$2puNxQKtFd`73DR$}LpBa^D&&7Hb2wD-HWW{Gx7uGWdn3d~U?{SMptTcNr_k$;vK~G=h6bKdG^N+v5>%)p!-qr#uwhA! zqnl)pHmaG}CZjwC&R478(nJ0%mQdvzV!3!<$T;=eP_a*B$Z>|@@|dQTaj@yjX19AA z6G3lEbb_jz@CN+a=-ZVGMOwPS_e%}%dL`Bgo$oKt4pM{aj_l?@#ed|Uf5rH{;f3{4%N<8(HWL09s zjC;bzjpnm_qn`K{PSlmihI!F@Gn-2@A23TX`X$ESQgGDua=%&7&>eUub=Mo>3 zZCg+~*7V?6YUs=7-R9rx1kJCjIk1^+H#WjLn~}3DiB7OL{N&4AC@cPWd|&oI8CRd* zA8<#r{yt1-J72p8^p8(q?~ND?HJU(^0#e7*<6!u2GV#| z{c4@!Kf5dj!Y0$at{h5Fx%yAln-_?Rw!Pprj02&V`K_WQkkHb#lo2jd%$YpA#M&yY3Z9)96@OSOGJck!P$hsn`F4HmRL0eO|8!o| z!YGYZ(7fSd7`i#|kK==qe>VZp`D7(x8`%)#okMoa541^IFJd1N)6Ln3JU}8s)FeXG z()Q4%3ASJ?@86q5ograszB7I-5(egRY8FI77HO;WQ#Cj&Y2#9nxjtR(#f(oO5{ zGK}#OJ9PG8MCM&I1o4wEoV(K+gobn7V8-OrFw;2B0XWZJeCpztwVXvJGr;#FZ&uxf zel@Z=<@~2gmhk)cgQ~46`PZ9j2VYlTk6CHF_veA6zp9sz1y^Ll&x#^v-%&$OB< zJ8r@`7U4#tSJnf+@YFo3&r%>COrXM69r*<5UaDm0P$YlP7ao4uw|xHl6S-ED3%mAt z_IWX?9i#b?n7MWXU!1aN6!p3=FZM~DnwD%)Bd#D@TLM{=!Dwi!6MSD(LHRGe8ahkg zfaP9rb}eJ}j0HTs)XI`q>f$xl`enS-=2bKvnttYU zWaBwJ_rI3`u}Go^Z(di%@xdanOxsP8A7aIoi2ThZe-2$4we|~~6CKV$AH4^I>)gJ7RJea;^-*n8|s^E~jBKV$4Y8Fx*p&J9F62wSVhb z>sjl0^?%m-JzjVxJ$KKFfK{TZdjv(SLZ1eVUM2phQV z`3gpzJW>y8Bdkr-hO;{Dd6Oq;ej727+CE_Q7@Ddd8Y-aGsrWQjg;L>1taA}t3yMQT z7I!LwEg}eGzD-o;qmy2JhGqqY@5h*5oqU;Uo(m@v%pEzie`)N_$~IN^=(M<9;ozGU zdi2Qaqpx2#wa$hz!{7?>E6h zS~qbQ*}u{0;J0PB!N`1pMgQfY(8y4?UlW7FmTIE5vP>q*Xo!J-H!PkL;$|pmBvAM? zB{F&39Ry1uxBIAocm}WQFibo}KnW2YQuA{VvU54Ta2wS4q$IdyAi;W%4}N&yiV)Yu+>J-s6i$8M;E%C;G?Ws;3gxnsW^m)DXJ)%Uun0DM#CCjuvoPmgZOFUd^K6!A z46T+i$w`rwo&PXl0kLN^Tg~Gp}Kn8TrJp40hthdCyiCQ5WFpV2)8|#Sis-KKS>;<`Ib8w8->VZ zHNJToXjy#hhVy+!4vicbkysjA22P_D0m~+2^dN(tw=eNo*yPRYCw%&<4D`RE7Jop% zf^_=*lj?bYQ{Wyz0eQHpmi{K~Go;ihq(x|kr~+49^F;*g=jyiOiBtU@Q(GV+S&v%j zhmmOy9)E0IR5|aRQ$?x_QNu-AMfb*_Q&&L;n`;M|r+o*CDvzG)m&h zLEJ?^EK(it9jYF2e3|du)*=*TMUyJJYks+Tb4#8-SJnc~<1@(V$KI61Ezb$pU(jJO zKv*&nii|Tbs9EUY(*;4(Hw%>uqoHAZ-h{&=1q{Oo&*)xh%|P~8>%L$|VwV;C^=h_^ zmi71V#vGiy#Z*8}XISe>^JPcyz1_%H2th9hlf46Ip&RG*gy>FrrOkM^-dttOi?=2@ zEyQd-S!Lc8lap&<;XiN2&=M+EO&7#=J%su z?h^{qXMwg$7TG%>(bbXbw&F2HrBo=Q~-pjVEmNla@1f5_F{rbi2R`-#KnQy9c|R zPTFmRtI-d&o|zu#l%k!4%RrN`8U69&vy9bPCg+0ZP%ZQ2ms5#=q7r5JGbhd6+HU1L z)DL1TWy-MbuaJJ?jWL|bq{i6_y-~VjwLhHJCEGu`>bOwg<0mvuS(Vz_8%I%g!AwbFnEO!d}nG{e#k zS${w%lCCj1r$wiw$)xETGqk#;9aIUHF>)V3;BLYc=z9lgr={NvaxVyN3~&Jp?~LWu z(C7VpW)f1VDBv#j?R1>Q#HEp%g5})KM^>ZR_iJdOD+2;u@R(BmzG;p5?)O+0|MN5NeCyfawcEixKD;V=f6PWeFhDvM<-T&aIkq7_CY5PDF0Ei)>5B>oa_B3mFv8n9lx72; zq6(uc-IDD<>?nLHF$$CpPaz7?KY?(kO?M%An(nm8_q(|jU9N^xQnG9|J(Y}V=2QIjA)=QxWg zm#Vo*Pz`#DwhGN9$|Jm{OrkPM*y{lIPA{wCIk;r+VDy7EFpz7L?`la*cm_v!eQz*^ zy{694MpvpPPj{@VH@wvF&kw(PS7EjjJ=u$I6iuy(FB0G7aeD>m>7Zo#X<>X9b1T6f zGT^%}cfFHGLyZX~3%To5>A-!7{q#uDRA=qq>p`L|((>e`BB2VH7m}n+#ciTA&AB>4 zOF>PK(ud~`Mv_L1NugAS-B>l;>#k*rTFk>cuu8uys%Sheaov^EfIgk@>+I$k)pFIz z{M7VZ`VjC~{jZm?vDO8FrZv{vWc?J#1^q{~wX*a%Q7gpX7>EUp3RK1IF&uo7A0(G7 zv8uU(@9(&}e|RYp$S1s=Tn&)B@gC>Egvg5qO^z}XXHIzI@ArP;eu(FDdy%T(+e4T* z1yUztFMJoo!@6-mF0XBNspL9L;Ug2)!{Vn&NnCZy-MrA#wnI^r9fZSh=iS(v;Hm#1 zC;lDT0Lh+U!b?9act6hT>sK2$Rhmjee8_hfMhje^Wl_Df$ogn7Ntup=d^$Mn1eqd? zdjnVz`2fD4pCbI4D_U<#vHUGBmn_r6%1#P($LGyYU=}$`DZEKN%pv9r=%Xi9FYSvG z$4<6g8JrqmKJpKuQEqWTd=5VW1cyJu@0cRMM%CfJZ-gHB44YSfMekv3jd7Au19N8_ z*@aD`SoC(AeVdO-jOSl4m0I?Qri5G6AvIS_hBfk<$o){Cv_DEI3s)>6Q%RMphs=oybXpFIz_tZ#Z$a zMdjYDQ)C5tuE(99LLa{7ZpNSOctmP%@V6XON{S2|U5?5Z#C-Qib8bj;YaNRuaax^+ zP2M@o%Z@x+;xG08mJ#`vUXCYzrC#r;(uYrvRHbAXm{e}&3%Nb0@hra5n*vW!wNSBP z24HNtL`8%K(Nf|J<3NeKYk{E+>9vLO>lE;3dEEPi(=*IO;!fC;SEaPeyJ2u-Wx^?3 zKkYL66J5=F?0L&qv?602-WkF`Z%$yfbR+0`VBxjm(#qz1K(^n92VWH)>xxV#t-^53|{kj<# zv*9*&oFMR3^PV>z8D8S!rl>$^c1ow4Ta|5TcF1Umd%D)slBTmx&!3-b+ScbnJ`IsS z7=4>xmMQ#zALX8YYcWn7Kr{*po;^&vt6F8;V~}wF)ZN|HY-Nk1hm5{eJ=jrQ*6%=stIsc%Lkptqmftg}t`zLK|80mOFyUR5{V2?+AI#XzAPzYVb0lXP^4 z83*xUBOU-X@RghpZXqYgj<(krr>(S|w`c5^T%e7H6@Edp%79QH*d(I7^VkGvDG%!w z5OGp~!%RP7m`c+ZC81>!r1T}-z4og`VsqRKE{U<7EmIwzfaN+#du|L;A;yy#O0tGo zl2Gm)Zv?(gH~} zjo4YEo!FpLa8;)&_`x}nb^$Hg{@D3cq%-n&1bo=I8wu+-!vOkLAcZV%!?&Q)3snbc zjW`wAyS%3u9&<0=*6%L;F#k%Hv$W`{MNK9JkxDQ>oXQx0M^Zhvr8S>Zyc zI`}ZMYXH`-`FBH4+VTe3sE)F52yO?~7>+pz72^Atg5qF*1E^}@6@0dCAVmn$UkTs_i{NPhQ17a7URAdXcQP zAZA@J5j3|IF!4W@qoBGw>l1F@W@HL92>qaet0fzc&f_MT#9dqd705<{Eo|PGU&WEk z@=M^_HGV=}3Qc^}=#fWLBU=#ZcWlD*x72)btUTUm=R>{DL2zpu)td4=8pL<5e6WeQ zizy$MXtvc4U3*NSOb#AO*4ZsQ)|f3mr#o*ERuIm&@?G%P zltX3t-VZ*|z>Yj)*xi*+ft;A|VKV4z3w6%IxqKfK2lX4=#ABvucw}JsT*WDZVl(9N zq9r#mEUL9_)8!$F6HZ9(8p}DlbL!w|uG`CIsOMSUgVVp{tZCfL;RhXX#(tPGylwIH zy>Q4uG`=u0+#dgi4}jC_-v@8uF*5~skGc>l+eXdtEc!@GSf6i22<8QZ!!g$#p&47vp z&eQeTwpnfhZ)9@Cz%i2H_-Y+D%H0d53QC}Zdr6Y)suESgO2UP1#!te`lH&hS=-jIX zR-1+T>_{s_yY)fPgGRW>x*nlaV7qp9PVP%Q2p$k%rW;`iZoI&cy%wt>B>B=C!M8j5VgdJnElC~WfSuUKy^F1Lf_R84CnOcr<|vgcWZ5*W@Qn|`N;d0+B|#YG^b7nE7#d_YJRQdov7_@l})*Kcg5s_ z1ybNwj?uV5jUw%}j}DIxEVXU9sO)G!kMYWKmRr1SFMkss_CK5PpF(fnfkO*{aaRBL zPAtX*H_30l{!Q1bRxi@^H@{iWZ7OziX8-F5^*W*hY(a}eMd=@Opb^?Eh6{wX$Yfx9 zZh47QTBt6}O=M^wJ3(@|x8H^CKPA814?WKj^VPbv&AaN#@cc2h$E$B6zj(aJ^wSfB z5xN)&$09~1o0hb6s`$|(c5v!GEq8uf{C!zh;SSf^r6sGm|FF=1wToP_?JIBB8Op`Kk-q_?ClP*{(5rt%7Rbqk-g#C^pjoI?%l+ME2MU%S&Ha9fWxno_YpY=RRW3^ ziu6P8C7ft#g#6=q+@${&mYsF0QBze&lmEt->{?Tw)!@zTE=8nwozHo^A%24Os$g@)xy77F*XaFK%3K0|bK z-m+5ff7LE=8x2~EkHjMOV51|(!S)Fiz*U#WYG+!rdhT%isx=Ny&#~6pxT&cLS>&#~ zk*MoAoHp}l%aq?=4ksRjqu==)qdrJ?cTmj;iMsYfjRi_~sP2oPT(1i|&ty7&!OfuXtpDgqPZc<#6c zxeJvX)0hj*bb4?yDJ7V7PH#X^#Irfkx}y&~afvxX1Mb|u{u6f55#dp_<6i!8HK1?Y zjuIkgdKcXwIngmTGh|@*DW8G7# zVG6H%Zf7(U8G3fQtN~9@ufZcb?=VcWx@BjFR=&y% zgJTlX3oY9kq0b4$U;U5K=vnVVH+6Ezsf#90cIHbQqoWQD`os)Kw7}vyfc`+1y#}H5 z>u!)WV@>*uIi1$Tr5S=e37k6IDPqChRp7lc$r+30?xtS_ly`Z2k*Pr69!PeK6+swU zdmh+G69^}#Q&!F{sMnT`)*2Gf75E(fI+nZ-#g|VGJse*P9yxdy+Vv%&U~KWz7;%?l zoow_aY|sh1e@lbsUt>0_F*jDzefX0kU*TS&4zM%BrAq8dB$wb=-a?un1}7iP-i;~L z0ie`M!QtUK!_Y#52fyvDV1-vU`$XKrURjwEAN~>PsmGAhbWOx;v2`spDpRByj6VC= z_VoA4h;Mzn(o6c51N)&KBah~9NfIq5vDst@Wv8_=aom5~{aA#CL`i7&WSXmn^$tFX zJ~z$W6VkEP-oj!#$ah&D5bkOZqAp%Hh?<#r;zP(`itqM`T$PlC2i6{Zw{^>rFTWl~F3~5P#O=k?2$}FaOwJ zGnoYu)PS!50iPU?1_I1jU~;6NWWAr@4SR{t!Z`~wQ~5m9Mtpwf*6kBEFKCTyiFRqo zn*V{JT1%Ty{v@u0^g*_7$TFPYX6GbKEH4VK?~B^*F&@_`GLq(Al|DsY!zGaF(+i#R z>i`>2!N`z~i`{8|r-ZMR8M++B)bdNTziN@L>VKFWrRiiv{3Fz!b|ZB&h^7fQEke6)oh(w z!qF;tqVvxWy>UDx#Q1!kf7Kz+N5QJ>6O-@wJ_`uQ52p+i)VYh3;Ki&I8n>r~p6}(^ zmO#1u7J}tSLsFh!kUf-;^xq6q8+l>K^hvEzU3tF8(D9lq2iB)YGWabC6S2RoRY~-O zboVe=ly@b5ucePo20m0>lGc{7iE@Z-Ukf)06-)T2d+TOZ!f?7Kl~MQ&k8UqCCHIPe zf$P{qtvhzk&>{bTDbES&-W_ zcd>g~ld_Ru1YJgK!4YElo8vFPglZC-D)3o7xd1ja)~OGVRl2{jY2yW6fraDU66R}q z3v0>kVhGrjD@}u!iI9KE^$9sNG4_9a-hB*;{NgyoRzJ zRIFcWjAGpC-amz))X+1uk00*_5HUZKreAJl9x@;#a=#&8LwJCji-*e?UhR49lS zl*Z7;ssWOYPcUN>Tx6UPh1eEi@sV*1@^79A4U@4uB2YX7?B5cDGPuDs&F^;%rzR<* z`qOa0{m)AAq?f@EPH;xJNz?;32<1>PRR{=&+ee5Rm^H>eXoutkq;#JyS8nxSpzN!T z@@rvqm-g*7Z`P8@ge{A1&qr!v%@fCvivkO6v-a8hPGA0BK= zmlim(UIQ2aJn@#KPY+~MNYU*=buh}e5HVl&i5K%_Hh&@4*TAn{Ue}73{R$)2*N{@A zFEAeu#r^QkJqNJaTf@xNU*JmSIRb!n^abD*1U>w#1i;u@FopR{>;KbFGIRkI>|u_v ztmDhS8>nx7XmG!s>>+&s*LLSb?@}Pqi=Yl%jMZf<<}0=U1zq5erML9POM(Cu>|@UG zTeWPRnrHJbh%A!w=OGD-H}Uq`Ul`f`LE|yQ3m03zQIp>)drwVF&ausNq7hV?ev{CO zo3x6>ZN+M+-s3;1(KfLP>XK=(Ed8Eg*q`chBPwC6gAjE;O8dsp)@hMPp}}C;pVj_APViu9?Wx)k?(*m{D=+tW)@*upHP?}(;g6(Csif(wLKA5fDFCHWI;97@ z0Y^gvAvxjT$FSq{Gkk$TSBp_i34YTIGM>(*^_FAvd>13@R4~D78-JS~qZ>be7k-x- zZ1FB>0U^<<0XJKFZc1MIh@o+1%72{jv(SB@3#SU-=WLCvXA|6c^2kkd44q7A?oaRI z%kfn{uN#`~e4EYnip|NGebs~q9<0;FloyCj;C8+?{j_isJBSALc2RcpN7NAX;74P6 zOJ6$jR^IOMpKD1yKQxKPJ9Z8FvQV!T(J@d-DxSZ3n?co)Y^RI#%fD~gZQPCg& zx!q;)5c#qYok_!z_jqlI?|byFgwDe8E++$rm4`Y#F655`8lW^XF$T>NTA4VI%WX97 zJ&VeeZI|ZVoFphiAO#b-LyZ?!FcWvYz|zphvm`xr^(MPbmE}F0E6h^RjV>H$rOBGzgh$IshLUkf`K}Qei(HMSf~QXPdN$pv^x3Iku0P4d*N1 zx`@c{bRS6R!mFlxu5Q=>p@k;cIpt$I-@Zt49gQ-`F%u0$GTFTvpJ8clcOtE zv0V`7AU38qk#0reS;@OFw$S?vj63rvDZqFnSX68F;8lyzx%v3@ljg%)cQB7hu1Qm6 z>V#k^fU|DYaVTBnS%6uE#rOO<*K%p(YICCy!(M!hcUi@LX$Ij)uxHE)hms<(e&OSL zH?NUu(0A<$t1+uQN^Nt=k#~;Qn}$6bbDO4yzFvczOK*Me=-F%F(rw!kav~!eJ<0cd z9IB@8?E4P;&LZ4eToLm4)a8pK607JaY%y2esit?uu140&RBTwDxX_MlY+KBy$4!&&%l5+;%H|N` zLz+Rko?}y@-X0|9{OsX055R&5=<(nCO`{LjI7ig)pZcZ6ovw^$O7^PBwwFL8*^W&m zc-t@C{`=VsO~NG@A65dPx|MeLc!Qz6ca*fdZS-?e9md?%TaQ^H-x|e={E+23+@P&_ z`LrJ$D9Bp_L zf30gES?7RIhphBq^{j%%=dtg;@J~>`L)&h+a-`jYzD_JUoab|n_c=#?BL5z4gJeqj z2Nlf4rwb7@aPOme=7J~uP~D3{N*8LYkK`AKlxpfD4s}MRF*Ju8&?(V*)$1mrx+trG z6wBL%CWCbXx^|S7wI0iuAU;|8Haa8#`5I9ly?9S}nrA4B=l?gMro%;IPdJ_@%@}2N zF2fEhhKM&h3eYXe#n;0)f_((906>yg7(#Le*S3W@Tr5n=v~=WaiWSUVSR>Af!^@Xi z=(#^wQZO9?dlkx*yGJ&N)&kd@$EE4y1K&)9D)A)#5q2PztGt3pKBCLdz#k6aQB=LqbLMPS5m84YI)(0sDGsq$=q`K6 zx6{zh$oP~qNW#`7U8g@lUAtLZ4kKj>>;|Z9xf3W!q`4WZ-+8DllM-tfiRCZvRdCkc z1>-u_G!Oy>Rc2krjZdbGKO^IgF0Jma_-Jt9_fW>@k&|5=%-@h#Q5=vQ?hZg(CJnLZ zCH9$;SdTDHz~r^^N{HQbqDamlkc25l;qXKs$H#- zbMzV5g1z$%kGmopj-5Cv;5Q6Q(@E3muChM=J-~N&AHaKPkCA(2DzZ_;PVj|YR*L=8 zAGu0&ukyHDXQ6X;36`->NRk*b)vt^@m2>dyE;Qx^%kpUDx#uXO!q3>?#j?azTSKvt zjo!0LoA2I+W_k6VEu$-Ke?GtZ#dxtPk?Ku7t`*J-p*tj~_f+#N2-yeTf6Aq0Q|iB5 zDnB$5YQp?kV!&70IEREBC=*$r-w7{+zw_4?$ER!TKv>%;X))0n+i1Nrb)kz>iPxDY z$xC*_LC;{E=jf^ri1Dn4mfE6>)592uZfpg1t%WCc?u` z{$WXIbU?Yy>0CzxTWPViW=IAutmdJXoeRqtXJ#bFF)I6$@ZO1-OaLS6`}@l~27r4w zDJT>z>WAC;N)HhG0!B^cAmc>K!#wT=U=w#DA4MBkL8{_YudJf?;M2MdOUj;Fl!J0}Z6Ue;9z{TN$nQZ3`{_YN7<|u4U4XinqaEL@I;Kn@7u_q%Pb(OyNPc#D=f* z2MDcFOrc_0WV!rt8f8;(463V6ah~H*QU_<*41W=-*7FB6@bkk?#|8nEF-S1CT=Ra) zFL$$@!A&}=I-5C7y)_Wjc}xi&YN=ZExpsn1pu^ulZjt9oK6K3ac4X1E?~N5+RcBZg z2Hzb*G0NXb?K+B3DFq7kB%8+bCe@>2CvS-T>7&IXHP={6s|5&A?FcR7v}+49^-$~U z)>p&2etO%f*;7`>-LB zA@(=84YbpW3oT1EJnfKv9+lmh;ps#Z8db#BVKweFXfbya;;^;K)!dICF>5&$MGe&I z#9_X?=*p67MTf(!Ff)f--57!=(=K{G>h~8Q3$6OF{m`uh%i4Y3k3Lfa{Y}q&^iH36 z^y*HYlV7c%t`IrhjQp04;GC5)WIO-eFb~5)=zOFC-B|oHzDcG4!762-=TM=epfny` zM_rzhv!zb5yG&^x@eO0UG(kcV+t^ooOAf(l zm4pkhJ%}kIZ68Q9jy$>wkYzOxU_hQx+Qua~3$4o2t7lt-PDt-P)Zsv3cZe-sMz4JQSb;#NJ)uO@W(c^(O(VneehvcJ~GAaZLOoo zpaEE~r{eiX$Jpn9L?IM_zfN@{quOOlW#|-tiI4F){_Uf);CQvO|M@GQ*k94zDEQze zP^sku>aCoRE1&?PY)fzZ*;aW(lC>UCHa&Biu9L^BvK>Q@k+#kuu6mgH>l zd<^rAg1$eXus6gJ4L43)88el-byAdN2z;2rsS#kHoIu6)3WP;QkRojq#zp&%>s_G>&|ne7m)`<@KpJ5~ykxr4+_#LWJ}oaBy^l_Ab}J zfY~-xG=oen)Ue2)W^^wV@R9OgMR$m1sq*83tD{B)gK%ucm9~Tu$I1p~+EV$;`q!zN z1IAoZ7?6~J@jU1d*`1UPmS<*`IrSaHU1`%n2~nY^94QLK_Zk^pple1ayIr8MEm5~} z!AZ8=Nez|n1Thj+^LecSgfX>uwf?A@`CTF2L`o4XCs8%$kuwQVP?bnanztzX&)z1F zO+1)JN}AdUefi@v)bt0@1fSL+^4A_QlKAn@#zpr`9UPJ&yd=m5mZ0H^rw^Ns7JV6u z%J(yNe7)!Ki-I>bozl4`v*d?Y8VVcAC1~nQH1x zIp*L>Yb!}j*CNFf^VIJC`Xf!YV+(%(2I(LdWZSd#7me0r=0fuXlPaXfg7p!|Cb5`9 z>d-#z3Cx}gJkMqIdXJXw=D-#+?L&C1AIsFrz`doO6}f^9b{(oUjXR||MrwE30Dw*~ z#FK4Me=U9Vb`1#)6`HAAmv2ydwy6Opu@&Y@v<$>Y2}m~aXTn90K$zQ0QXuY|o2$dO zo^KguZ1y|P+46Ld+COD#XgMSt|26z}*DT!k z7(1M)af+Js7(@KL8glOKH;1K&wA-}EbeA7BVCh-^yW#O7sohHYMf$LQ`QHudrZcHx zb5RC(?a4srw6x)2QT14FqoQftp5WVl{_}cc9TSetX}>krI;6aPZWvNg z_15oDO;k;DuHz)O-W=~6l5?!|)=*euP*V3fhZ=p0$Reln?)#f(P!s3M`^sjw{KGxY zv6o$!b>@-NpMi?#-wnhedw#wjC@*!#Xv^s*@SQeL*vsb&{`-_0pYYd}43$Apal(q@tFLKdAL`4OWT)574a}x! zaTPB~vd6#)z95eHDC6iT04np*yDcT}qWGw-a8jpCONdUPd&N~@8gr%$$z6y&5*xGP zBs<}qk`Uagyn@;rj!{Vo#}8GCNBKa~Qi}i#H%Eip!JKI21C07Km|$^1l`;$qaCLrj zI;KDwwO4``D%$`PKD9*CeuEz~E$|s$8ZN~>IFcMs;5r3d@W#2!fL7m4c7J}Bb7LFD0~B$kt(h{;#8dE4eI%#h zHL(+TBqvBAsPO@w{sZfclXHOaYlvq$2tB)$Qn(tHu{`v?uMAxlT@z}(G!lh|&q>V1 zNQq57+XQPeym(`zE{q;mI?0_6$6nIZ{OLw+6EMUk_93)EwO7Val)jLti1&ft_%*U_ z1K|pQ(>Fl_e2@BaINom9TKmlK3V8=iase|7)5~x)^{a*Nl=>(&8E<}h#L|Tqe3;G@ zZq3pwc>dSbwg$!uGm!$>Qg+MU0Wz;;0G~3hB$|P{#b*ho@VzMDd(2Bnx^H=Oray)D z;d-8#&pl$Kk&t3!b->EyD94_Y|71D}aMmqQh+|&+x+ao7&)@fp(!}GOeW8LbGa=CrJdTUV=^VxpunIDOaa5cXI8ta+35OwaN2q?6Xi5_r zZE8PD)&z1LW=7E9A)$3uC7g#!lPNd7ZhD>6Q3xUkM-1s_f+KmMv4_~4H1v^%v1-jy zo=S9ZX>E(?C`aT-ET{f;HH6hS`=H3#t>W$Zr~Bt^osLD6aZT-CAHCh9dK+`bq((au z=8Ej4O_uhxq;_}W5yBB^3sG|psc3x~MZGYM7{770?czK~siRHj^G{WictoeggG>`2 z0eU7Z-YBisQK;9)>Gcp2OG-I-D{qBk0P&lzuNspK$J$EUP|vGO^dz>}~42Qx?zYPzg} zy%RIM;O)_%m5ZS5y^Rk7fGI09nir}eU*O|5wCd;CWj?@9n~0T~I}hnfn}NY9WgioV zQzC30lpT|OuvQ(u0V8;E0ZzGIgQOd=4J^xO{=lBH_)tD_--1iO_?%(pUs3WX`QT7o z5ys=8_mz-S*j!AMVHx#m+1HmxSdsJR-+M-<=Vx$2&8W2tIGe!1nYzT@fi2|r7Psno z_l%3CcO4!Cd7Q3rE}ijNOC2c*Qqik9VtX>x;iTPXl0*v_zW>LAD_`8M4}zLX@sV>e zcHMr;C?~~^3H#Im`=(**-Ts^p=M>U!*=RRVKV|uHY@e%gV)L8#e^~R}>V$-kp)v?T zGbH#JS)zOzm?woTmBRrz5&&j%dVg#A7+K@y13=|>CO9qZ+P00o;nA@Hyh(lA0-xyd z8rsOGYBcWkY2srUn7#hT@BMc}Q(cnv27E;Ce|ka50kt;+JLWmpbFDQ%|EHM(UTfyA zvLs9`zL57Hvb`tIh+>w{i`!vn-g&3zN40UjFP-AU+Rvg%kr&pLMSp@d90h_a%OIsU zP4GkERwpG7m9us=xPla(dl!a>x^8?Z|e)!#;7gyxPzkw<7x zB1aCQLoT4;m$&lnB>em!kv`J!kgpJ7OCNb^sgaOa_56yjonhO@m!@^FVf6h~^0=;O zT$;t4E@AvMa`*&_4-{vVUHq3DH;-DnK>RDUp*I^0iq+P*yJV`4l`wK6D%TN;YX9f> z1?J*$B!oJPOQtl~-D6_8918zNuCShu>_98cX()A|cHr%Jbmk7C3T!rG%W5zT8R+)( z-#2O*Cv-#$)zWGj+B_Dnykz0%R}~8ap39$2j=arrPLt*@F&mo}A5K#yu+yq-5TX}| z?voZ9fRh$OcUab_Vo~i}yt>*qZN$HR9ZjOr?{HLs~FI#95{`1dgY**%aB~G@tF?}bpJ=JZf z`oXuqH^myyaz%d&Daj0F;w0kbnnWdWcKN1kFL4mDhwC#OS~3*)iaA2uDLKO&vev<< zt-a_D%X^SV?ymd8^Ujpjh#+c9GwUE2emt%m#5e}v)`3~Y7C9)h4p8me_aTGpwDI-M z6p+F#{NBS)8dg+B(li&;PJ3l)g!9l+Rsgm8F2*bg`owq=1xbryt2`P9 z0rN{fA5dc^f6HF z)c8wGUc+QCuE8a&0@bdDb#93bq6_(xPcRC!$cp5$NiO%JLg-SB;k$HIk*$}}mB>%o zKG^rQxofkU_VX*A-stjYQ)us}r;R)X#Cn^SOHTU7-D7kHl#wy^wni?gsiEEq%b}zv z=!_|AVe|vi^d$Pj_fGKIw7ndX9hTe{k{=Lo;Wyi_G9VE}7$C|JQfuSy*e!34o;n28 zCRN;jpqrsE{rK_I66pXR2Nv=>X%jO52kV=HJDr9YkLGP@dZV;Oq64Ra%z`XJG+b4K z&zv(ENSab@8d3IuKIe*0VnjfovsWW0qio({l0o4wE1$bsb7c}EoY(j(BkBzD0z-w= zZcNZGk+r`fzCr6S4ona*aCmu$#aPiaQw@%OV+=L*Snr~E2tBH1^5Yzau}n>Z9St}$ z0e6pRPm1$sKxR*0oT5;2raG8YRsb$;;2VPR2FboAlXFNvnyWV4np$(H_8+eeqav}ASp?}eT+6i1>yEQ8$CH%&~I zZmYx7+UBhANZ!`RkHW*F*>=>`wWyPco0Q)&uLuTQNhQGOWxl)g*+!}TbaSaI_ ziEt<`Q2kVzRe#6v+jqWB)pG)%M8-d2AYe!S1IJk4zRAf=qH(tbd*>T|rj)ev$m4)2 z#b+n^p!RfYSolipSz(r5fv3-B87>uV541mU`GRO|II_)L23VwG9Y4%^RCdPu)+*vZ zEsYIURGQiM5Kb}vyZ_LL3+sA{N}aK%1NgLw@B8b2U6~v6XqDs#yC4C`1|VZtPYuvy zf(J6W@}v-Q0zxgSZoFPZhJDNubF7hrmWWb##)eH>Rz#V!9Jg4Dk23W9rJl|*5yz%w zXd9QZ3g3f6x?X#e-*8lh*@SaqZ;xd(D6?_-{AhvIG@8@j1IRGQ9?j)6>zzsMG0%jG zPp0dVay*x3a9glyEw#Bd!Dbd2i4D^v1-$=zwPBjz^qYsUn(N50;i=E!E)8-@LtLyk zvsL?yxR2cy%yMmk-Dzx}nrf9>Ket{xT6wIhFw{<4gE98_^7EccS!rKFLT|^1gm5tYjBA}3aA%VvF ziS!9UUX1m_%kh^xn4Gix+HB~NI>D+;ay;$IP+e5SzSgmoaiYdo9i`D4pN>XX^OPw2 zr~+k=AtMtjq#F&hxO}65-cWF6lErcl3p~`EYx>wUd&M)R>TKv-Hvgr&*4(w3^XheD zDj|_G-jt^BM2+D(jg$^bL!Mh%h4W>>;>WztE^z<5ULp&%N#wJ5F*?8W8*w-)m5@vkxQ!ME-Cb=zPI(U{QkIssj6kHcP#xbEhi z>E0s#HFvk&_nfx>mhbp|?I+gHzL;mN(o-Lm~5Rn_?Z zUjJjQ4fQXZ$5!vwOxKT?zz1H8(4_h{gC>T^i-VTPVho3vO`U@}!v>5V7af^+a<=!$ z(Fnb~EJljK~}^f%uDy=CYC@BA$f%Mb_|4&D-b5v@kGa6l?;!@lMO@ zrul#2OxyX!39|iJs22rM{jU$UjUE@6=38lmdh9d(N&`ODJLbFJD{;=3y8m!X%Me7a zSItGm71YN3Rc+{*(Gl>q`*NhNYqq!LP&>b^Wc~G>>t##+t0NaVUwxTI@vAZ!`)yp~#mkNtJ{ zQShg{LnA9Ik47Jws!x$Vx`+=*JmI{t7@QzT>;k?|ONQ{a$5}0dj&P&Ic#ybRa#~0e z4cnQVBwV5m(`{zZ`{0Fh%3WE$@F|kU9An4mxD7Lz4ivKL$4jFH0FmA-h26!!dr3}= z)kej6@;tkl_wqhd_=tG43PJs9nJ*eXzkHE7(%@)ia0yyn!BF9(4q=Fe?UUyhiu?5_ zRvEMDIBG~tq`7yg#y!_lPj6<|9BDPHT^zx1=8V{}we6WD$vs!8+uR~QhVbT}JKu7v zuJA~(AIa6t{@K{f=;XhqWdV%alB;NMr5S)fr{B{`X z`I-Ek%lUIecHCF5pvd{afjSexb$q<h!D~tDpQ1zCQ=q|)$OnD|67%^tcpqYIcKlE*0Y}VJP(HI`>`TP za6WGSIY>!@4>*ByT$AT9w2C=#sUhZTuj4Ql$YNAK3dw9x7^^Gj4E7C08#JsLeOK!? z*kk6Pt#%AS0YzK8*)sJYVN`Xiu$t&kjLSmy41i4_fkzbjAtcPSn6Q{{z_ZkKmXW0F zFhVIu$a&(tow#e=y2LK8qYY!L9+fwGFH^Y>O@fpE)cJZ~un%RT$HP7#jKctTDGo_N zEJt&`^ICt2#gqf;38gx!4v&CI;+0`bi$oR%VO0ZR*`DWEJENrIgYUsle+k6peq(p7 zQwxmB!+prOTgpjrn{Z_h^2!s<)es3@7QB#@3g@Q6*YVmr-Y|ZHweIL&=|jQl)f+PP6*p+Y0p|?^yfa>7BAxHQ?f3 z8#Y+kPHevgqlt~+fD8SjS#AAo$xsYwrk$yo4|yN zSxH&?ia$pT>j$`U#{WcgtZ z4h}KSn1m$9pc$r>fesF2<7Hj(73YTq6)uk{)9aW;d|@p6Pwf!>cw|1`oD2yyQThd;I2H_Mnb;k{dAB zk4EW9N5Vai>efy+6a=DqpcxB}yNiI$iT+!?m;%B$#r~mS z3rjjmp0ra}qzX9<@a~S_&lP#}aTdE9sffBJu9R?Kc|*~MMQjPw={)jP)5a!3asPXU z{os46P@g*$3!6*(`SyXVQahU~%gUZB#RGPY+aVoi^QU*2AE)qr;*W~%U2ZKIyzH#p zW*^E-&(pjYsB0J^0OL!MFHl*b(TLeP1jmlCBIV}(#rT9dU$-Kuuw5=*f1Xxf9baFr zag0%Md%1BauW@M5tS#2Av9ZR-=!N;76&$%cR|iT0BSJ^;a87lXVwEQFnUe|67nluV zL$jq|=WBp!oefvPu4$Nc^4!rxjWHRce;fOLmOIdyK$Af=#}wp)>Fnm|XSddoDfK`D?@L(syG-Lps~b>-ALjdq zv!Bpy_0SSU!7GQ$;mz<;Qw5!!N1SV%)ogpD4LZCJh#x_@7Fg_lIAo2n01#M@|BR6#>}PUK%~4c)C1syGFlCbfE#mmB`R$OGWv=!b*^m~ z(Ye(;jJWwISl5K*;ZgD_(yAg@QrnbarK0#}Go>+CzxJH6-dFqfY9M?U)2hOEgxEYf z^qL(FEYz)W&2OZXEyi_PRY}q!RlMPQjNmkidA=4@Hg0$uc9Nt0$Wb#>l<;wQ6*||D zyEX50`dHx$O2s<02A=m_9yAU%7eM6|;C7w?rBLKU61}%We%hp#YnM|t0Z9>Wwb>yh zJDE8r+ArgcqB@X#4E2rr^yJSOoan9Lr5Y`1#`XJkQ8hu)&AIdvct~F+i~_MtZW54l7dl!& z;hIbAQZL-y@eaCclq0jB98d5CIO4bFeW|MFaHJ$dihA5(T#9hF7f|42)S_jf0&6O9lYCnO!qoax3+G>RmC zDlCD^FZ5vcF7eqkCD9;pm*6s|ctOzTTL@?cE6E1vq`;282-&;k^I2PKxN5z|1eI`) zD+XNVjQp1s-vSiLJLSUgFEI?^DB&+sL+|nyzBi!Z@O1?mgEU{p@;s`g%6Da*t-`^#${f3(Q_LL|C@FXT}+zB_+!ah#$?keXfmZU5>#A+;cYuTnQWb(vgHem~iLG z)qTR}zWpbUm{>JgrfW@uS#g;krxm&ooV(*O^_AgP!@-sM;owJ3^^^S@TBp8fpR|E*C#QD6gtN->lJ%E0!s zEO7gx&H2udb#IT6>XG49TTSK?sV=$b9?^ zV#oAra(ApLjaJ27zj5t7W|(|BV!Sf=G@Db^BHC)dCB8MQ^UaEBvLW2xc)fPomDflW zefmjRMI009ED#$%3-(m=a)m_*f8s-e${@|J8la)EP$>)fDA#bT;tyPTXN`_Xn|pc@ z<)t%omaIS7l{?()#?&;!qW9achcW=MN;;S~yrus<%LT5c_VDjCtReh17n-#h5(2Cz{ z^qbW-9?iv{v$xS+&iZ+gT{_-KEh7izNP5xbQc=u2Td!QY*%y8^LB#n|s)UI?$V(+U;LZ`V3Ol-;_Xoc(Q&Q%#gk)&I7vBiffnf-da z8^cSjx~#*7y2`dA!7UMp?&(93o$fWw!|V~m<4Oy6i8gnh?eOz)va;5CtQL}L_(#cf zshXwCh4{(s%d)?}*|hh4P62m-_dB>-&Ug!GcfPwogDbtv;F4GhOD*J?t5XcZKm&Zw z`&LQc61SEg}sE0 z59fQFk2sWCS6T-haCWM+D|=Gn=#=&lWpV7GtaCo&b82ns)wsT&`-N3zdVk`2MLSg> zKy{Ttg@nC}E%CxDW?mk`GHtS@tU$wi!?gb~`hrnv`I!iB9o{9&d)`5*^cnY`8KJlP zF*-HIUwj3!oxnTn6kl1?MQpU}1(`FGLh3}u&8UL^z6tVR`Np*xhL`JzcTx?ZY-aUN zT&;@n`R0)T1F#S2yc`zBf8a8OQ=C(HF0cfB8ELO3C#^QAYnMB12bL^dBw}neH z^hec4>;FnxUF9X1DO?X$*~8McTf97|cdfR$VWhCFE`F&C)lfEVboa?~^`+&0`7~Rj z@t-GBleZQYWF2{uZt^tI;$X3S{pcy{BbAB{iR{DHfyJx-HdFp5u&zP0r^w0Ayjvhh z1f)P}zU^3GVvLAE+?v^eGnwfY9pG-w=EAyx$`e7|1NRQL3qrHSQ?SA5<5U$tSY1q0 z5vnnCxpc;TqOGy3(#3vBJ4esqkDfKXLb`Ah%LW@m1E|<+9vHgoX)DLE0njc1ib?IX zS%n7#S%@}^>_o`UyO6}c3T)7v$Q28&LJ}Y^0HRE6@giJ>gZUaKPvPo((fV^C_mIBF zyq94U=fg2CcZNUGTr>cZlM9GP#5#B$3$_if2A=mwd>AKo=`*YHZiKfNbUF`b)8PWw zZ1#Ad(z-~-JLU33_S+X4>A{qYoHATmyVrK04s=hd!l649bw}!X!`;7wcI0s=+(lT8 zKf%8$5EmAj1@cefVmrrF73xyXay~~H=qcM@6UlQ%S^rN(qljfC*()u&TrhKHPGk_}pxo8^YI%REN(Oefz>sTvzdE=ru*18n zavyaYEyL80BI$XSH)Vz>Ae7;cfvaYYqHQAvgpdDrtHW%H4saGTN}DH4kWSbgk8j4A#9M7H?( zR;Bu=)f@f9e0sA>bBOI}M+UVsW|M5m8~r4I!@Q&(mY-{ZOJb3@vz~ANR`&Kk%AV&r z=$<73kg7*|y231ca@ODvz&jlv{-YLw5AH-$z61csOM0;NcYxFn7alOxR zsX6z`A%FB=Tx@V54($O&u~i&9ZZkA;bJrE(f+jIPlC=47#>F9EGyo2U#3hWG948iN zK;YF(znFZ!PuP#k>TNe^NHJ(INsdPOFjFMG+y@H07_Fi5dOr7<{Tt`T{M4CfpcTY>NCE=>e8W#Mvwb(0JC1qxtr&FcjE1LpJJ=ZzA5}Aj4a<)I&7g6@9 z`OZV7U@iV(ik4A+{?a&GL9lN(;1mIE2apU#qM?=^`I(#C(0IT%8`zD@pLnI4HQr_N zITF6rKkQTEWJgKkH;QS(Bf40b+F=>L2{DkCN5$jSn8fV@1K|U8PA{+@-kGC!0V>Z& z&F4*cppLuE8?9S*Xfr&v9jHkL?9lqwRCM-*(TxoIz)*E%)rHyD>mc52ISRhs3#t}v zKVF4D29UE`37mv>qQp|GwV5fi1rcfNWjI!CNZPBiC0J;4g=Eu$=k zmRgTrqlW}*w$MgxKfQeM2@&wWpKn|`oXs0cz`w~#XbrwYnXa}I-DGi2mZcb8%F}AK zHQRC6_Dy`SM(juDn~aj@M_C`6D*GekmQvo5pi|Z?^netx9kCKnc*1k)3j!8!)E9p* z&DJn>Y1N?il-SMIRMY2Tn@o3YMT4w1DjPhy4}E#sfa#RddkHX|<^i7aT5MJ94+#~{ ziTAG4#729;9wdvrJb#^|W(Otl-EoMH27qd}!pgoVVFd?ib#^7lE~S;Y8=Mi~dWM7U z%xcSnD+se!O`n-Lf(;%PVfCQ_P?eJ6S(Z@JX^rONlr8Rxk+WXKo>$(nEac@L~k z7P9OR$w4UoMk{8D^t`HyiPvMU>-pi~b1xc(ZqyB7exy8fZD`na&U@c(OAP9s`nb{K zvMmj{b6+nQogQsZn+7l;=xV?Z%p>-Ja}ji=8pR*oBJLs#AhrarJUk|Ft>xRcuEAwm z>!6%Gh~duF<84mWn5$7p{9$0Lzx1V{arlg8w66Eafk@?SBwQ%T({L=2?vK9|2>hl#l=7AsUbyvd}sNB!-a)^ zLUG~%pZB+ijv|-VfEuz>D!+SXhe#GD$y^s)fqI!I^?aav*doAM4=u$Nn743~TCEEF z73iO%@wQ~#af9sezVarDX?Bu+xQ~)Wg${z;EV?2P#ZBd{NvHy^iJXAxVJEZ)nDA+? zaa*1fk+t1zy~3=l`ks|qW_;+g(QY%C{#=mp=IE8a&fRtt>&4yfIbrbLOnSS8>YF1) zPP1&QX}bVBt0s*y3^PT|; zdmbJDNOf{h3NX!KzY0zwG65T9P%!!V7V&GjsN>ko`d!O!f1gz1;G}Hw{w zadt-NhNQGbr0}i5hg}8%l}McYB_;q=-4RVzH~tO?vt_@;(jOycvk^Qr0TjZc>3omz z32+=$3(|od&YBQxoc0vqS%QvhK{oNPVMQ>8CHd{o2vuW|gUTo|$c zvbn6-|DTsLx{q<%WDGrZ_A9;T&E$?oh0wHa3>v$4zv7{n&xQ{Fv1h|Q6{kJ=_sR{y z6Gt`Hj#Yn?bJ@+EglOettZm0`Yi%>XOs87@z=i{^?bx5(thZ5?OOKS zPZy_h%bnlN`Zi_q?^$LU5Sro(Vt7?YFF81ky24!#m!2$XGPk1qSh{PegVM1}?LMOO z__eZjuX;e<

ipx^`PPttw%N1h>%v&jOPx$mqK3&sQ`iZD+kzpzMF1YN6;@$M|m zE~LyChu`q!h2uf}u&&3RI1|y}qHgW~_|6-eO2wKJM#`Wpiq;7pK2+ zo|5C;Lfq-EAb~z{VE2d0`f2r;Q~0RN-C+1}sSI{l(Sy6MZeHt^+>4X(O`mkWAAI}5 zY*8_jcw_EaqB`4>(NVQY*;4#L%+Z*GMoAL((y^rjhkNek7}eb6KL-f2;Pc^rTHT8) z0`jTOktZ%NSWJ&YXNR@(>Fnf#Nxm*lM}|U!osO_j?)pIsR@FECrWQ;hqTGvCS97vw zuJ-!gYz#fdpnUpB-tA7Wco)N(bL_f(Ue(p?h*A^s-CEr0z$16^vh!^AB;h1ZjqNFl z<0^DIS_AW_T1+`&g5OrdEf~%L;17o|E)+Rb!dxx5us|dLDsV%{W&yqAJEz7}B<`v9-@{ z)fg@s5OilofaAa7GrbD?UHE!;5=YIL|CB zejf4AF#!-GXm;=&hx6mb{(k+0xe23aH<7#W_!J&#j3AD4=dYDR4cF7+AS^W zRs7lmb~$k0ZD)7h4J4_1tKMt-bkwP-aJ?}U`tbj@vj@#v(QhU*!bn3s;lV5#=nBl{ zAiv!t)ny7F?IzC8I8{#%{C{i6^doWWHs39iKxl%2-S1TcTfh5YZieHVBKU*E=oP(l z>_n``dX4i=A$ws0MfNW-bxj@@M6f5E0a4yKMVRb5b93gK4rri{HxcLQR!2@tN-$i zT*VYYVhxd}fDj%P|7#3r#U!5)^}_fyF^E8LV+!UY;}8G2@&Emznh(>iV<)pao5yHX ze-GH`LzjbDmF^LtT_Lc|RAWI1q{6Jkx-}~Y7}{Ebm24wu;IunJjfcfHXX^VEyU?Fu z9~%jS%@3tbX$E|W8z!3ucAU7=lJLawPIKL-FvH-2`t-4+6K~;4_`ML4Ly)?q7Ymmf z5}XQVq5C2%yTiVq`(3+pkB?ODH)_ZtB?^41t}b}#05LAKKcNp(0%g06Vlm5iFgxbA zZ95H**DB12bGs+Ew@1t3_~rOXbM;KmRGSE72m{l=0888|d)ZvmBhnrj!QWkv!>mxcsZOcl(I^Zuee+<*t+ML2TD`eStXt?>R$SB9(9 z!#!M6EznnX5QfdAGz(lkpXn$1Dk%1xNSM~Qz8pp_Ly^mDQ3Y1TS+@9g@ufhHM7F)E z+IutL(ba3++Aw0UEXhXptRy2;1lp*Qi4T^c$d*U_4)@SwqwKnlVU-ehT^rR74~s>;h5QY)_IeNuXXE)YS3YgBwmj=$E$~WgLP97X7zktj^XA?$bYkOoAor(PRNWoC8EVZzPkN&I&jZi-xGg% z>8IvyFY9AJEgi|#kJ(5X+w=Aq$?;FF+=y>{IluXKw0ml+*2u?sR$>C;HSSjR9_F`- zLxGl?vajE5?cKBka}{oUw2Sr4{padQuPRp^xTfO1s)vitudW?`ndm9*5WVjrs!ztk zK^O(J_a&K{9c{kOTp%r^Udlcl)}x|*6?(wJ0HGB)lvat7U)%bXVKFLF$MLUljgu+5 z@?-N^3&WOe+|BEl%D%Slwu(U0Q#_^bz_M0#49urwGDD-y%24RQ*$~iR zTAYRtFRd6%Y4^_T!DHA1iK=Gre@s~zBsdJ?PqUE;V6414-t2Vo!ez^SVHu?Sg$DVs z$X=!Cb0?FG_(if!-rTeauw&eS19)ibe_pMR5uia=WHLl}Yac0x#y)wVDz3Az9N zy?0$in!*;C@a5093^#4e1#obWlN%@5tK%j+#)we~XPkmdX7DAHVr5_%>_8 zD;h6AwCk7H`<3$5voxCyVl$Jrq#7%FRv7V4QrIW{OYGWHtk5!_xCOQi@#!WQYsnPG zc^v?GZwe1zJvtBr`}Snu;SBEj)ChZ^x^Pua^JFuVWBMg8yID>n;Mp!v}5&!`0dt*F__Lf+S1wcm$R(M)Kc*;OwU z-VNUXr7*+8Gt$ZR@2Lmz@NqL;Hb(!A<;$j!8Xz-P|NUq{;jY-GoWiwrErZ;oL#?Za zEG&iPU0+Pg9 zMvlHQi}StJDC5yS_mrKAA~(Nmc=3V~&%QLWqEo-0xlpzVe z*_T|ov7nrDEN610VaksFLiD(?f|B*(O1^(q45hlad_LYiTv2aDefVoDA~=6O%O^dz zy0l$og`rdbwQR37rF67vm>iR5O$meI4wpC<#HO6SR~7ce+t|)%&H2Lg-G`?Nw2Irl z!kBM6KJ5qpi9y8FC9GVMlf#k$*@xAQl04<%h~ zcEjMes!6PBI6N=4QA|j8KaqR9Bjx~K&SS#)iB^$f52@!GkcPV$fWF_apPu2|cjriw z$rJ8K-3}w8C**{O=)tZ_MRsYw#DboXw^^4(mfEbPIcwRKn#;Pr&MoW!fd)j;P^6#M3LVw_<4Ay6tW(XzuZ)Xe!ZuicZiugF zfwrUu5?Vk$&;z}TKxx4f`$=O_7cHK)j^}vF65HP*zFkJ#oZ;^gfr5PgdwYxzouOS4+Y*1%jQ`o$p-&iw)f-0Q z241<-9*Q;y4v$LO&sHM4bER6i<6R_0zt5;Tu7PN8)94Q=$tKjaA;c zH6OOnkVdyLMPRgi5_mpP?qeJEv%Hy(b?46WpH!BD!B$6Gja=JtN;2?zcwhjukosZ; zNxUd?c2L7p%_5g~TG{#!6i|{U*69WW4BOQo?6VjmQuOL`RO#W-G1|(oWfcJK9C*y) zd#>h`y}kn@>`c^Dbb9VYGS+dT#Mj!k{LW~0|C0WrrFlL6l4DxivTVKGIELCX=}}bu z0U8h~e&+#zqM(iQS)_)H1$6@(pcp%T@|}r;(@-EV#jFO1h=5!EY2^X9&F-|xSk~v| zaTd@=P)MRG5#H14#Fw3!XmqJgD~xXe;B@Fapg-Z zaPXgIC?6G&?;zu>DIY=;JBG+a+=BS>%pvOvr!@U~I&}E^w-fs&Q4kbNk$VW2@tC z9Wb2!C8_!dy2 ze1{g+EJn@FU}pK1+TBE0t0a3-#S5c$ZVbEsWfRBUFB4tt+3j2Y!7f1tJO%QIzp>t( zAVlOST;Mh?Kj7~pOe1TN9iH|MLqRPwNk|!y91xy-!0GJ>HTlTP$g&q`Dv_7^xC}P0 zarOtTwXN|5MQJT|Y3}7np{M@d!Vx!J_p#lEibr<&yMFRNww5)e(H}DIl+GJ=LX@{p zerpR^`&3PCgWNw4BmQ%>DUj~~yRmP%xoE5t#p_hwi)^z8!Xupjv^w=l)?RU z9VBSs0%lllzzu>gplXyfg^k~+b~OJH&Jq6;||Wbt@*8)rE;4HX!A z9b33Sf7@4DoXs=tBE_k0f=+{-*n^YqAVc2?KbHfzRxxYd~jJl#}>ir;_5tlsmD={a*uxLcS@J&Duf zJ%e_00(p<1+gYt%8zClVpdE}h9Kf=aZEU1>O+v(8FV$gzW$_hUWKRcJZ%yrZ5Fc|! zhmv5b>*!EV8Tqp*l5wdu#-Qns45i(!>Dx329^Wj+DfGcMhbdFT(M_H0SOo?vLrB%E zemwgRq2r%6)9pdWB@}U>j0kNuUqS&ddZ0&NB=YT?`FDHq_UP8pp!(4RgD{5|p8l+k z&(2X7(0#$9nnO!pbumX*5C6U+@~elWN6VtNM|M|S~sI~-OJ2R`LAz3 zsMK0bdHDlE6clfwx3N!%ySfPpk2c4vOD5 zY$5JiBq*TWf{A+wbUHkNb>L_Xv)^C)l-Yq2+yg^Nw1lDLroij!3i|Yp9xKY(Y^y^v zI&x_tw$s+H+hi6DW25MLwn&;T^{yp@_OBketBp&azz;MOW`UX7>SXFMVY*pL+{`9~ z|H?*`9t*LH&`+wjy#EypoZDq~lx2JezVW(-Z6L(q#R23hJ%9;R`Qa+sz-^4HDfV^F zI9dPJlXMcabJB_JS4484*w(>oGyfi~sW)EY@-IPkOucx#YcV8k4Fu8ytsqGjQBqpj zpDs^f*}!(ppj35)f8r&%o9tUiR-8GSzP|!td%~Gi&m-YD%011j*GHqH6H+7dcKY5b zc&%K&YU^1z?BUk(<7I@GLZf!eWS|YwJ{*~Ly7M0%@_)|H=8=&=VXjpIK*vmlPDCU0Mk1xSkIVgZWbFnF&H^=I?l$NkMYp{QgVSOa4ZS-qp z@mMJ1n$!NNCm!J*S>T)+d>K^V9B!N}d8K0S)u)HR^YB-^jrKskK%9RUhv*_oQMUnO zhZGKx9lNwVYoGG{O9NGMoM3>Ok)(H;H*s@;NQMnF5xIL+`XNn@pp7C*S$11;jd`%h zVs4J;@$b}QLxmj6_+n0PqL~ecf1XQvN^G})ZmMlQUjCr3WqfhtBd+N-@-jbEw|}! zT~w}*`6wtPmd!KajY=moGb_xkY%)L$swWgxWTjB91RpBUCZ*CSE1i1DG-V=cX5t*k zG3%md9@JNh0jXBGo=Pr_)f;t-BS-FDk^It(nTyhG>mjjqt$UrvZw|&rXZFa4A!2`w zwsNM)tnh4%aqqh3BVhWea-7;AY#^BR@~;y5VD;kdu*oRaO_SK3Q)Uun{w79od0t4| zZNN=b61~y2?5wWzcaa}!CdX2Exw<>KKicecH7}DKh5<#|0lQ;?o?Zm***daU4^$)D zf8SaZe2=LYJ9gq?hF)k1a$dA|_*M(9DE&Z4OH*Azm-timv&@e7*1Wxa^;E{q*5lpxGEAejuHSBjSsr3$O?)u& zc}i6yXWu~Gxmr3Kxc5fM+2u`_ddVwMHcNHBwQ-x=&+P5aX zc9mB7d@Qd7%tRn@~VOpS+eQ||P%{$Hf`KmcshISzxl7A26zoan4DE8~A^>x>IYT zY>@o1@@?qNjI^(66cNT(dQXXAM-m_~owjPnlWbBAAQ#Aq<4PgcRB*e@}5y zf2TFz8Ej&e3AX^26eIO1=d5A<~Mzl{^VuU4)@smW-@}2KyhrQq(>0%6uE`)w_{F)?{zcC{^(*m z{hDTK8m1h+J^WjXu5f;pQQG})bHxatipjFPFOtS1xS8F^cvbDeRw>RRP(@iKDnR3a zFx}#zqR>fIeqZs4HX7`C_R|CHEtM7eCzI7vS@(=Q80QSTYR>->dwL1vN8W@T9)e@w z0FUJwFSYYSp%Xj@(s@XT9u6eY5fxT3Wmwv;rE7XxIeT zk5z({!hZ*j)T;{{@Ve!M?rCvh6I>Cm0*!M?Oj0sc8b@SPNT4ud;1AV7m4yP6aWiqf zF6E?i&V#_u*#qENHaQ-hbSH1yG5jg!fD#ntN%DHEV|My_^?8k!lwRU&_6)~K4{T`Q zEfh;9_eej0)vYb-&SIn1Lt3hE4l^udPo?oJGUmr=GyowIEMAV{PZ#(RwnM92$=*yE zMtha}$R9-QX8cy3Vtm1c?y+N=l!T0nfwQlqT5;sDQmb>n+w-!fvCi zlMH}m)s@pq&0)sq;;gp%u~%OlJ^lu;?g9Yo8E5SuDxH6<-pWNIND3LaMl0YCj}nYT z!z4MpA!t^<%e=EQx4EZHW2#@&STUtOG>HrBASe&%QUIRQ*)=}s@xd0wTmfQ@Rn>-+ z%Bk5p`H;}SPVOAodt1ySLOUh6dpboD0L<8;kJ#>0j{$c@9ppX(53Yr`?Kq?HA#6Fb zLlJHuO*dJyYpPB3n(Re$+%Xp4rmsj*9@Y-*yQpKJUR82-obq#jT0~laF~M3CvL!ZZ z)SN*xpt-3r{-F^2hdTQO@g@nO)S@jwH8Z|#M@CJ^N0e{_IZR9 z9$@lH^Ino1%_Kv#-ZsBqVtWO)oCQnFPRoWK9`1En;^Q`nF@ZCGZ8nezig@4QYxD=6+(Hgi0Q zs*m&L1yLcJ(hmXyP6(Mz+T+&!xhm!DN!gDpULRjaa+nx%C;U zunaLIG7A|3*_5N zkc|Jj*l(+6H@ZrkY-AC8qnOH5x8_bR1&wmMR23s__$@Z32??m8BeWu-I6%b0*~AXL zXp)kd8n%p%vuCY!rfd>i;&d0cY{{u=LNixjU}~A`m4L0F&#%#oq8rM0W|w)TtNmw(tp|f_6|Cz4UgtClY)V8|bb_W%*w#db z_wo6!nUp$8j7LJ8cf9c(ZgtlTBc)R$CIkd%J_x|Guw^-J;4JVsG&{{&QT6PfRoTQO z#tPg@@xXp8TJzB=B0&ijB{C4WL6(&ZUOQ+*y208jQXWOk0Nzj{qTn2*0!KL*Apfji9Y`i(hg$yKjEDJu($CEd?Rq=-3rp9kI#mX_|}50 z04fM0wx1V-a!xqCwl=<~ zl}x95eS~Q*Yg=Ef7Pu(?jR+Q}OlPFL8w&Y30V4H(mfimh`u;R!k_7T>oaeo6sR#tOF$3J!J|72^E=^$A8oVitE{#kUYqNo?812yL?S^>fP;#pcm*ud22b*% zR-^J88JThWvdpJG#XibXQN>Ok0o+IVIbg7$p@Fl!(w2p)A6^wdyL~*t)z#|<#Z#pA zuOYM~u-^*!C!iG3b}2RjIz$FYMkI`)&`YB$zY3$a&vRBBqSLwbJI^GdlxS)IeJPHFKWEdc?8 z7|lZknuk&YY&G5vp|tA>n+OxI#+hXgzBb-wq*WX^e@$-?798Y^R%C#LTsodA{QH*dDGm+ zl%r;rzJF`ihedR+bgD>ps{U$cP2hRnIx08_90@?%P0#XZ%-F5OSo(YM2fXlWN)(E6aMWb-V_aZgTM>JpJed0d& z!OjxAH5VHL5`$q+A(^1PKvXo8QgG&Uzq#^(k)i_K?d#U;TF-p+yTGu3F`qRM)2hJM zlPW6WCFFZ~bwk+u)MMk!6&b;qelrwF(*8K}_tPgiy)n;?vxyZy?L6b&`tH5v@7u(K z9qI7VAJG5MTsi?K(v`N~>|RaEr{v#`8{k9eU>l6Yq2jvB|u4XJXuP(;q$)lb1_# zeba=p4V-a-i~^GwFT+ad(UXE;Y}ek(?RVrd8cV#;jjG~GBMx3({vnN*CeR_d3gbsz zc$Z}+D3N{&jgIy=y<3|e|L!f+6sA*^@E3&CRu5(hJ%JZFG{+fV63$MAU*t-I?aAra z^{@j`^jQgxuJaIFEDbWNpM$i)-kfswiGhqd9S>nE^sxc(14&Oi2ija=> z0>1%x{3306^VyJ)Z2vPbg#vXjqRqGhy))BDM8jVq%wGUH_J=%JR}7 zcb}@3{^UlrO}}*tvtyyy)vIyGH7G?U%`CqmForpKgY#0ujCLs3H%Zi;;4o+|_nrKS zX)b$d!~cPo_?(_VjxD-K1CW316#oQJN)tli$fsb=0Qu9xbs0bp*H)qd{i21cG(Nxb z@87F|Wf2)QfWedUyq}l^kn0!L#EkvIH4qK|Q^^f9Dmdr$ex4^UNrE2l3R7CRbDid- z{i9WrU!^Q+|3aM4vivYh`|yIulK|)BNSG$)GABypVhKL2lsJ*jf(b7G1mlhHDs&yN zNiIUR;A2qSx?f^0nH|l70thTaTMe9Pj?q>O%NvwS**uSM4B}!# z__DXh=&E)}g+YX*`HPf2UJK3Cf;(6r-;L?NeS21=9y6x+IpysS`&eQFq6@vv%&G!! z4u$gJXG+XfrgLU(5zIva&@#$KnN&RXz%^bS1)SdgfES$mi6oUswoEcHL4t5n{H^@msDd{%)fjIx6vliZC&Gne{`+dJy1x@yb*{-ebie{M~?Ke;{ZSc2T=fOV~adx)1 zH)i}je*$W(3zS}Tui%IC7IXe}78h>?<#*(bwRSK48e|QlTBvF074sI6!!ta+&;E022ax>#W3$7{A?-|1v8r%FFv&q?NGCibGD!ldlO^PGrJsxw6E}oE3$wSp zyS&q&y3FuWYiHb(aG#t#jL=1Ay80++tLNJ01?8NIdmK0dW-QkEZeArLJzH0d#AolB zGX0McE<+LW7W=OzYC+lpKC+#i0lRaf0SZW=u4utSUepY{QuQ`XFKD5is7_u1VUt-q zMIeQG>s7`AKFiBxhmj6wK}-iCJx_S`{Bir_G$?y+qGqefvt;p`wA1s<_jHBo5$R44 z3Vw6k19Q04%5r`yp=+NcrXI4*g1s@RRCUKA&i;#& z|M%^n|IZKe3ExM=t}?A!0A@b|zfp7st?r?iQ~Eg7>J>C}s%#IWCOHXta9VL25KV5S z>x!C1ek<5(1pC=Ug$~)Qvi|zatEmSd+M-$(v*u!x?^@i-YV|KHT(I07dm%b&y&a)p zul?2r;_ulryr4fHABl6}NMrG`8~=6f_FojTm%Edxg2()2_mKeseRxEYgv7yoi8;~% zAB=f{xCuEtMl)Ln;wL+sPds;9ZAGFA$J#N;&W;EZvT6<5eqA*2df3rOxAkw9S4`C7 zu$nypyqUZRo3H$cmH*im1bxO&<;cGWHAGVlsS%hf z_)20sb~r$QYolttdLY{?f6m@a$`vx3^H-$zOh?Yosv-W=bwS4`D?i|Q?u>Dt`pnS{ zknJs}TQyH&;>0uq(mpro>S#M&a%r(V(CuAGc4VeTcBDnK-=its?3wZj%n2MWYO=P` zDK_SRY@xF*iZ~s(W#ml_kY^&(t&~vch|tTsh5TgvYT4#L>&IxUxTn6&247w*8fHdX zSd`77R*!${cuMMM2VGoi%yIx03*u;s(H?6E|C{`^_dG<&*GRK{U8#Q)0YARm7~ZhC{FSM zahD!Ya7Rs%fl?P?_qE0DX)6JBIfIMYh`b26W07+0tn&`p8ssiqEA+l0(k7JpZK;HD z07!i*d7^iJ9hy+B88+{7TESF)Q+KOq<%g(&t;W2biJW1{sYMp{)J}VqOHl5MFjsd%!l~* zNb;guXF%Gfn7C*P?tW15o~<_6 zcc706kZIEt=|PB-G-ab6aPWB}yz+SoZ!|nMp4A)3{RUXI7Zs{e5MQBs6&gE+yn?I* zCeucsl_gNn%LtSW&w*P=gF9jbwE9{p+o8#?9G=49oi%ou>=Hr~-<`2IF(|q>sjJZJ zW0@KGPDJk{U7aW3rhX_TCuZb5SaYE2UY*hD(N6CQS@wx}lE18?)BXFwp=sC<)Fl5F zcXd(BpKcT2uTrb*nuxc(W{GHDY|M~5M_CHHHPF&_+OMH(+5t9zC7O3W_jj^ z(W8L#hr<`D`+TY^YEh*vL!YCo6{Zn!cu^yBu%qX{NZtPn#NHtaA74vM!3qjFO4U@J zrpT@d%hLs_^ElP5nQUj+t|`Jn@P%_;pkfmd=L)^0MTt=AG8d#9;FHV&R6mO!XijO4VO?H#(9pL>Mo%Sq7AVpF(VfbAik%19>J~7hCTl* z31(`MYK0qMV+NrJq|JXoid)iSuP;5G4+Q-N&u4b^+WgeHX`@5^XRSHwfKk9g1&q*cW3YnaLw<@8Z|buEMH{u|EJt@9l~N1*@XX0ah^M6Holu@wP*Wg+ z1@6rBj&?jWpcX?Aeu5ou*PnpqC5Z&sz@*>${TImkFrU}4-8a;3>y+uJX`P8mN%rV? zb!{L}^+5Y1m)`nFoOSabncWh5k~>lD9YRBem{!pt~27u0Y8SK2Kis&~H<$?ccg1+}56UEouZH zu787o0f5m%0E_jeyrH64K+8muS%>U_m%Gag=P`hUceDm5qIjGXdx9AZ#xPBrp20e% zl=xw!TQ}L-_*q{B%MMf$`or->t-LdLFN+GSxHXFr^JTfY{BFu_7^i)XZI%!keCI<( z(r%n>cc|cHPR4l>@BKZwcNt^eoSsZ_-Tu0ABj4fPR{)y&^8TsKw!=-Ik+8w+Tjn;$ zG8B1>wju0w=O`gt~(q=o6OhS4(@Pe$ zrgxd*l9Y6CuCc~QV=S4O+U8Y=m&s&0^5>kh*pZxrut}C;%Y?minh;#~1kPMDV8rdA z42whoU_{VpbC~Y6v-5;WX`qg}|1>6wsyP|kHd6@d&hB$u^I|{io{J6vu^KULNRs9QUXH zd;dL8z7Cd{dG{-eIhj%D5#u3&ZVJo&F@YC(1JdQjw&z0)0?VX|ulBerQP@mxJH0iv z4zjbb?$yFC-@Rdr6+A9t4+-zK|=W|tzbML!OBng_IiEzt(Ul^9>Kd=w*JrSI9gxsoV8KC zTXX*Ispm%x1NUtz&xi`yc#s)@&ggpbJ@^?T;q}--=36mYb#WgO(|j**?ncX*U+L-) zedGCnSEP*P+fhrqy=4fae2VO#yXrja>2C(^~u6jzxw0UAksO zIe5+{G6N+T~vjp$F z9Dtl|2vy~}%(t`=AB9Je2qPyW037b^dd+MOcqsKoCE-}ngWV1XrGi(%3rci zRM5Sw1xA_K7-ZYn^k$G_d%_*fUBQ3@^?_cA?LZA%iIk@*VWNBgBWUKK#M`jq)(V0ispv?{FXq)qXn7LuVc&2u4L8b|~7OIt4nU=0% zl~j6?tSNtJO+&oF$Bs=kKtPq~3N~!^8gJ%aq2WMvLxEb8!WeqP*RrOmzy3>~pzd?5 zHr}Me-=MhOMZnRo;t_a$p?YJ9U;5=hRcTUsmh$?i8j_%x4|$`Ok#{3fh~7538MbSq z59djnykkvhj=z$Dh~pG%vveC`(#*DPKq61{t0EI4%^-hzUcs?z`kSWL}J0 zGf`;Mzg3=(J&jGKs#0`F7iRk0hbY_0elWl$t{e|iL8wLHF+J`jUmq?WU`9Fl_%ee>MR*Nh3ECjyr%`MnM!Pno^yNu#+*95pUzv?^EhWWx4B9luJlL;Z zPHO`9?l9{8n0RP94ip)`$gfsV&p7FP2WPoHCm2W+fkt>WF3}ks)_yq(pXve43icv# zN1@TgQyZTf7m4&E?Aj}l+yi~Ll8Z{+gg-7>XuT>8m(=9qJt#xqy${i^;A=QCO(QY0 zWp;Q}fk_8%H|-&HLOKL7Nt{!GTIJo-n+41atPa^IN>JeAg4~5_W~U6dArTDn=Kku= zuDEuEBmAX6$XWKa-ZcduNL=g2n9#G%ySdHfRBI^}3~QLWwiD#%Icz^R#n zR`9VaIiWwT6Q+zHJINLKed<|Grs2{mNYW_47?9V)F;inDWIZVPK{>MbpCw0-BzPB^ zcghpNiQ-qPsq1_9+ThgxX0N~fkvUIAh5N)TM%;Y%!Q%(uyeO|gFyLsP@&S*=4}2nc zAk9v)eQ->(GeGxKl*z$}od~UmI2?eD>G7hgB>=3zY5$vb{-<}b)8DW*<7xzOeCz}m zI{OL+!_VdNsf#5p>-|{of02VGeL|I@V+dP4mdMrsZ+x5zG+8ysCL+uTQQ}o?Xokd# z0fy{LmRMa=aveGH$mRTFQi{;<2{vVBX3FtM#c-JY^$Io`n*UH)n|7&@(8^hDld)>3 z&EAo>rsq)Rwjjx|^Mx%{Kq#|yP3Z$A#j zSHxFJoq1o;5Ww;wM${`lk^(p!xjiCrL{OGRu)Y`ci56YY&#VT~A!y<@Nba=HKT+Ep zi(y5HovehktIQ~y-g2U2m(JuN55qyf-oEeMk|OX0XGskbcaVt8@~}scX>b?W@)E_y zaK$btng+*7Jmq)Tc07bGp(9>GrtTJabkbQJ$yaV?Y9gn^x4IKBdA&nH2NcW5X3sev zh7Nrkk3E{JSCQW$yC{F=xD`C0mDJ|7pvFUkL}TVj(N}uSbZ#OGC8p^sUaN_GE!alm zhW5y27~4c~f^BI+`LYR;4O5e}uMG~>Dk#0b!l33&}0AVMP4_DV@cn%Fa5~)Hd zCdp74xop{}{ZUG45D~Lvy1GfMU8jK;(TPXyj8XrRaMb9%@n8ETm6B)mG;@Csr?cV> zcE(TlkNXiK1_~>@8orP_U4k2BIWYs#{tGgIPDpqz^8hs#5sg^zW$U^n>Vxxp003cm ztMghYWIC~&cgm&D&p(w#mVb?w>|v*`nDpM1L7>L?XG&;R8!NJJ=~`Hbjgw>N>ocME$c~B0MBOU$L}N#;4;^a|LI{+Vwh8*gBnL`bhm;tR+K4 zOrz-LANY0#_DLt3iG%wusaq5BwukD~ejRF_MJ#<}nu(&qM220evu!tIC5rQJXK$6$ z6%L45J7*2s4#DA@q4T|8`g>8sTG z^ZLDB_fF03?k@DG+sb6s`R7{XdDmBGXGiY%Jg}wDJ;0%6WZrx5qutl|F<^yD7t*;{ zo~9TGoEZtE%Ds-~rS7Ctf{H}g^KGxbemvTLfW4B@nvYUPdJhPyu1kp8KsXjj0pjq{ zl9_KJwJhCc!*%IA-e4Tz$E?f&^hKmbqzuW7{3<7mKJ3t}Fsp)mG87n+29Mhkw7I`< z#=;P#MLN~GsC*qbzfFFB?{YpqLdaQ5FkD{=Eu>jaP0h|kSX&%9RF~q>|0wOOrBkH< z+Yvz!_ZOPNmUkhl2yCHi-idCtNR*hGp|`&jngzs2%dLucIYC3#-HFa7Jyo-Yh*{I^ zot89IN;M}{qRf*? z0ydO*Wf9#vn`F2O>IFhi4U1c& zrJ^i-w|5mY*E34P2XlQp=)Aq%d{4*Pu;C70Mroi%Wo&4E;8y(I7!QBR$FmKWYK7xK z#z&3Xd|u`%)pi9frBwj*tn?CDC!zsBAnuCH;32?txlSvWekGKX*C6Er;u>Fn#O}9# za^S_{5=ODb&Bchu+c~9VS14jE(^-FFh)QgP9Z1W}2w!P)U`Ch9sWg#kj3`!(m6HFc z9H^MVoY*{N6Ix`O1bq`9iz}eKMJp?Mc9OwUYqq*jiRF>K zipbW_pF#VR9yoQszeLpY0|K7abskpxtM0~bU60E?>`8M9ZGWU6uHiY|pJLajRncFY zeqrJCU^>sUi=ozKYDr_=n)e=_xiuFOA@u^E9>WThZbj6D3)AQ)vS<)_@RUC48XM{c zl7lPKC^ezH!Rt?xaZ;QN$Uvn|^m|LhzoweWdX9$lY z=Nh-W13}5f6cI3(HIdwZ2O2A2(vO-t0R;$Zn>cBpVZk9_hI7EoqYBf4)yE@~Z3@F7 z8?yt}7~Kg&J$k#|bo;$}>Q&j!@WPE%Fecsj9pKdATI~!CIT0AHz4L>Lhk2i=r)QDf z4gyriY|7{wB#sJ&Na!!YGG=J8+wQttq$R{EXCGNO7M?-$yr&ty1Y$@|Xb_)4u zQh)@F6bmsK(AUx3eHJnU0Y4+i3apy;{Z-Jm{`u+s`UN1*IW`cfGt`pjlbVl%b?PGG z9{!Pc{NiM^&WrTChRny`UiOvE$9(ymelC1RoN*b0difx)(W|r6QzzVtQsd)q+06`? z$Nmm_*~m7N>C1L9tsD`^P1P5O)ii_4fQl0xp-x+WZ0s>DIr(QxG4?p+RZ{;P!)O-vazY zaLYdeUCmY1cNFR}WBHv+0Vl+?;qLaJP0)NwGj{!Wp#joR4Eh0hAyZw^7o-HRL~#{g zv&nl0%BqQ}hQACbj2Rh@dl!N%8XyYYKtzh}ieg@;t^{dQB2#8|ny&4P*3Soq2Gt!+ zlaISI=GJ|Zou+0wZvhep7BO~GW!l@U%fwj6UwjE8!M=*-FMlO7x%8}{wer8Hf=_(J zbPIpu6gfor9!uy3v#Nl8EciZC)Fptaus~gJgH8FVql%#Yq3#58mg)UTs|lVEUPBW( zY^AOB8^NYjTTw5EZ8ZMv$5}?_ihIxvItIfVqgkcL5I!F7GAWTNmB($3adQYLADE(N zH%PndoK3);@s(MKIGJFDhBS94P^-sennjY}9Lb-s4CEg&R}pUHVh41560fDegyvZm zlmg}0Aaw*^LM9?85oQ&l+~l(e7zQ?&jM(dy;%)Ox!}p}_eLarKPK<;{12&^Z5<9R~ z-;nr{Uhr=$We|K6k;_Sf;cM|AsQxbam%VWE`ZM5kavJ1g2!pDg%)+)O{qQ6`(0SDtl$7 zCv0jm9!`84A+fvLP0RNb^MbzWE)GpHJMbgAo=7vu@TBs3g##(4WwEx2I$Ss_IR%)q zvk9Zy5&0uNc?*&P6OZ;QbV(95FrgShQ0*7T^JbG+n!x&|+yRlJc{89PLN9TZp($OLR z@Y(3Rj{y%L7n%(CPO8#jeZ{H(fMYNPdYi;ysRZ*Ja!qp9MLI=e?bMazy|78!5KN@W zFan$|lH&nidF(XOTO8+i(|hLQGF&+W_fm zfQWll@jDv&3n>!Q+wu#Xu_<~XNseg!AkTVg3i-%W(ol!p?V2Ech>@ZnFl2Ffs~^xtwM|&i zIQ}7^t9Zh8{PCoHY}n$=6A5v;QR>Q5{EdB~JZ&oV{NOt>(MLuhR$_jt{r;@z-r|x5 zQ3UEYR(?yY(uZd^%ysG)(nb#ST3Q-yE*Cal?pUmUBDGNT6mFJ31~Xd?*^ypnUDAk< z)8x?FW6oYZ!O>xDt&lk(Mew2hHV(k`&6Su!`9*4yO5Ol<9&(#gcMyIE+SOx?-hkXK zBfE;?3Q#zx94sSD6mtjou1{slV4vxIV7rLrmFyLW#_Nb99YM~BlDL7oB6RTYpfNmr zJE?@OMe^a+2zD(!FS3Z+*Ee@y%b8)V+VL&ve`l8GctvHO|CD%-Xc!rJ^uy&_vTC-Z zv10ndr7yhK6B7$B>gzfD@o$cO3Kg!v3C9KQf0*G05jHc*Pv!&1fF5)(ZJG*mMJtQ7 z2qIJ}8gvN50tm!(`{V{@m&X2_<6(w|Lu5Namf<#W@{l`Jdu>i#7gt-;H#ZVM;uO*l zW6ZQG-~y#6)G%N%eIH=O3jh(uyb89PUV1`3U8X9}r#L8jboJuC@--;Ca2H0669H%! z1cp~a70cU-t-Tep!`2xdLB7nQK1N-#YzV4c;(MkL;9-H7`CV8Si0ykoCPEIlo=mE~1#oCo*ux#I-KwOJW{!q+ z_ZrzDq$$c`U$gVJfog|9RBKH#-=~*3wY8TxlU!Odp;MoHSJLpSqmZRZJ!B-`^0b{XPsQ<&nz^df2E;YgM?1H3$M zYjQ?ZSy<4P98viUE$m`kkuUNzLVup4P@R_sQ{G^gD=^gNU4U<5C#G@Ht4JJ>LnmaB zH^Iy_nN1m~Z%^eQOBY&DXw7zD)IP|5hf4}9zScqXt7TCf1Jc3X**3#XG|!#~9%sxt zSUTQAwK_9y(mx#1v%_<$oyOd69joT~MAW{$mmNNCUG)c5%OdWv%;{9H#p1;LfXUN7 zg3JS$r47j<;Mrz1K5tT?(jonQBA9JWY2s_jjyxq}vH_W^jlYz9OmxKNZLv@qkY-#@ zMU9V|4$g7($9?S9JOX_r?6s+Bk?*Ckb+UCJV^ac)N!>weNcn)T^+2RV@aqy;6yhRP zRlGV`4|F zLQtDQdK=8)VfV;6)IV{EDP5+C_UrkA6iW}h;XDtak;8=Dm{!HAmsLo$V4CWR1j4m} z@yll1acfEVz0m##A}4n**t(Wh6y^bS3WKLvIb8-7=fR5k+({hnlz!gd)}p5mx4 zW~z)kZ-qM2upv}+vMmxEFN)B^rHyWZ7zvg(`v=Hz@xli&peePci1@hkemIYw?igls zwQrVnB<`Z*qsx4Rn^B&ikGa2$vi)0%wKsG=x{T-UV!GdU#O*+FBqt0GKc7Hd`a)YFzlqseFzm(Lyx8cx`Kc^4}{G16T~R?#N^QdmbwVFnn$QF=wI;hA4;wI7~;*kHyU)@MfU^e z8NHo$jlLdxiaIRqpsTUBlF%KaYx^jwvR!>PfEkHv)3a4z12Qz3#fYsc%9xo2CcEh1!ZXa-b^*alKlaNwg9tOH@GtbOJMr=TaRVl=4;r$%{+9N494z!(Ek=;v(ORLM1f20J5 z=so+)&T@PGFuQVAiJ6V7w5n}zn9Hs$|4eOKq)WcYvz^zHXl)#oGHdLralJL=FtvS= zpFDr}R8qh`UGE^xAxHO+r?x3u>}=I`viph>KL1&LfX8zUJ((`JT>gD10V~MW2s7OD zJ<1}Et7m(Ia8BEjW%GdN7jAb>J2)&NQwO$oCH~#>!BOef`(;{WelfN>Ms{;{SXt2t zjJ@amNG}>oIIBg}jUWdMBS#_LgH^jHzrQ6}SJ`uj5D2w@ZrkVyOCt|G2(<`Caj|9hn-}&iTrs zRNvR{EXh9l;BbYd*4f6)ut#7tAhc_uC4)_E9>i~s)!RqCsLG{QjMnnz^FOJx+7~XY z=0@VxC4{JSJJWS{Jhr%Pl5F3`Z0y`sR=K4~`0WiH(;R#`TEnng{c0yvb&hm!UF7tU z2YU;4o0tY2$v^VsaKzhk|H??B@cfTt2kW-MjxWA%JhA$zxi(~P%j!Q|V$023^NQW< zI@j}dTsvyomvD%2@Sa8jQ2d_V-SXcHn*VqIUrqae)_9i=i%>t7Oj0E({hYb<@Sl^s zgedB(Y|2}_WRnZf$c#T+S2~M!qlChhSh0{ZIcNlcF^3#S{%=$s5HTp39jTwLDMqo9 z3xyj1;lf_RdHz~Dm9KEQgI!#}$Q_O@ZcJ4Cj-VG8uyX8ig)S$8_`^SU4~S0ZdO7pI zAx}Ab$oPMjm`sB$$OF`pB&jQP(gqQ6od4}B*+x5o_WNy&qGjx#CB;oY^!VR)wo~7O zYm45|F;1QK`e(^69Y7s)H~$KM@!6%Pa&=}mhF7F;p#fu#^PHue`9H^xEhW_DTNZRRUv1p6PruXbso4(6 z4m>C*U*-S?3$7f5>CSFn#UHYHaBFWOeXR?4Wyh^jbMnqXqBTAfnDeyiqlPnD?AvpU zKN6a;D4ENn@X=pD0bu@q$|tXvn2fF*OJV;qnP>PURagtpDU|z0zo-@Xzu^E`#@=eUgIcPvxb$fYl8pNN)He{|@uQzM;s zb`E1e^4}dvtf&sia%e)UM65m|Ze4qs?kO_@gQu$?>AC|h_!^~mbn}4oj^iYbaCXX} z=sl$ya6FTpRVn>+E%s{CP;1^oY8Eq%8jj-WdiGO8X49!C$MYg|iu0yHU14quT8*$H zJdbgux&fCmwD`v3;`y?$?g$7i+vp=*t?1@Ir%w$-K9VtD;I$3;E*t~QKyz6<5+k0y z29qP@Olm0ldf_iHJC$t#%_nmn$ZVk6X#Q5LejgZW zefMF!H{iJMVt+Pn;#-9M^Qwn}xhGqS4|Wq}b^gJh+=1T^N9k&3?NAPmwda3`Qvkx~xFy{sZRdI@Qopfd+a@|oQy zwA-#l@2Wlaelf5+oW1qgOGP~P1QD^s0tVB`#|nMW$*`dc zYSGIay};vlgV2dpxTGKz`ThBL^(f|5zBWr;*r|qJ`BjgWQE{oNf3Eli-J|wJZxk_A zq0WL+xbgn8(+g}9;u4pL_;vDMV4!ZQE$Qo^LmsnN0KK7{GujR>ez0fEN|;z}^BAQ!k-2IeIK`+P8iOz^}?;z`Q^D&ypIG6fnw) zn?(is%eGP1$kgiJML63csJ3XzZdQ+ZRN6dEElkg9d{=lZF1=97wX46C^{Fp2TXM{_ zZcl$W+*@o~y7%xj!J3!%`Ml5JKfF9YyvPW9H4VFz8vnDzVy>pvbwbW)!6pGkIqIKO zEwTw>dRYSM9mBO?*=!jhTNl^_Ivjrv7#0*w*s@mGX`$=Wd%w1C9;H6c3t|8x?31G5 zGv3g2#T~3 zGXUVgN~ku@a;~-4ZBfS`eU{Tg*@O&~xyL;CT1jeX)p!L-1t&>dZXpKFjJ=U};1Xth zOtSTzBgg?)On^PKl8lAyre|S&E=P@ZUWgtZ5qn%OCc>C@`~A7tWPW>!h?d3PNY=06 z3}S?eApq!cRDT@itos6r%(YNFAjB7aK=8lSG*tGKD^WmDZe9Y?sQ0A`)t{em0PNNc zu=qG+m*yA}0#}OB>0+pcE<&SRfS#vLaSSRM-2%jEb&EcDU{b5j|GC&2Jvh2q(UsrA zlU)o~tO0{u>&`CX{``aF=SeXR{RvE^x3^}7st*?DozFE3_q6y{TNGQ``{GFBr<;SN ziqYdA3knKqsC3CH>2AeajtVuHc#WcmSV68_H(~dRughkBd?}QS`N6swj;`g$ZHq*B z14P_re=S$9BU2V5Ht;885!-?+hb(7pA}$`Szdi9qz`V7QGKOyjPPripwO|^5_%Dd zLR>SU_XkC9Xg=c1G+Qs+;xZ7C)yve2Ogix-lwXHlgB4r@F|$j*Rk0lT2#p96wtbONf(Qr$EzYKZg~@C>fXKj85Zu`#p?Y1 z*}T!PA$6@gM;{4Irh_q;FYXO@eEl$%arvOdrTUn#kJe>Ojja4H#K0p5CwwMLy~n>a zpl?vMqx?9L-H5%oiC$nA=PB z)jT;pgJOZGPDi#Fz_RU7f1E$h?8ZePoz&O1h`YkVl#w{ct6&#N&V!NSfoVp93?;^; z8Ez{ywcK2DtVf1|W`BcmS&sY8Z7J9=HHSh?#GD@~8@ljYP0fO)i|ie^&%bj{;dwri zpOq3Yy9Ymq6@PP9fvJfr$seZ@K(>_EaDGM*CltK`$hTfXFz}@0EgaKMO=4Rj(uW^pIy8U8 z=g9%nhs zF2v~hz{7E53AP9|%#}zGqBN8;h&N;#`jIt(R{<48X~V0t)yM(RU@bV-1a__11kh$b zt1ys#Cmq?#0TM*+;;dhTIla!W%oOI- z<}$klY;|`!c%P!5nvj`U=yYr-2h_Y(ThQvcfdd9npohPV?A#}FdW0Z|NlWgohn>YF ze=iYLl|-?yAKp}@=gP&a4)0m*6?W2I?grR=+I}&f|BC;95>)4Fv4Rhr5lzWcBnuFR4(dK%7)ZyHn>GeB@yt z*J?c{#9!4?^PcI;y&m5W?#Pj(JANqtez5o8t5+_M3aZ`WUbtC*DmuLohP`YyJTX5W zceZ=7L!^9#WdI*(8r=b|3B|1s$@o81M^{f$*1)u+`O?r?Yf!WjT4STyft!n7QrZ1S zgSCtC{gUsdOQ!#NaqC>!ue7V^Ro1vGpeLhFR)O9YmrgqCBc>2TIM#wgO&V^>?hTCd z?g4*ub%mmHo)9j)_kaKXRVm*$ba27{=IvOhXzKvSXGCe^p@A zax=!|(&GsK5DxPja!P6qL^3(+WcbGuma|&6vGWsQypRO8&hXtEy$fJayIvLmeQ#de zRrIP1Nod|XD_;Ud{j(%(9=8f19D$iBmjHj2#69$Gu7R(K0M++ZaAdx)G4{*b4b>iN zI@*-?1Nz?U-#M2=I_=PBzw^_I><2_iol$%C-m9o z^?W2$_TfFcxp-vzSerWsEXbN}nMR*t)kXDY{8GS3&;{;?4U(I^uXSr+Ml){MhpUU$ zshYiJKexKXVOB}z4Z9{s3gP%{md9lVX|ILhbV+Yi`Tn$PqTgjp%U-DO~__NVf9%pVIxJl&a-fH(Tl z1GbwA*aPltB}E4Z+E2U2qqZ`Wv0$;MTaq=Ivdcr5S%oI016ze_B`%*h6h!ui3zQjFe}<1BhrI_Vjd?| zZ)-{`>BJz&IVxNIt=^MX9Cs7R&xP}xAB4n|WTE!==OJ%z54Hq6duNV?TyV(>2e?+I z4H}#vj47EyC@x=@(hZIu&}7TVTgkakQpKT>BS1&!KKT@Un8avWlyTNq;a4F~J6&#F zQd4m>g3a0%ndwpb`b!@M%8tD3xhw29hNBCLXOB_YaW5E`@p(Ss#ZQeNXm4wO>+#)W zO@z^z-u^m_=&QJ5HY{$$te6>ySwMrnDhPtX)Qw2WPl9NGLe)a*R8}AI(~4uoki3+7 z63~}ob!r&_RrN_jEzhM}WJ+xKUd)cB6Y=}IoBd}pgv3>|fC2Eoy#4%i&awuoH^9Hp$vzFByUaqLHE*PRWW0fLR zciDU%&4R^8@t|0XJNB2~c&u)!O=J0q6Xx8{^!JGpraY4xMnkq?WICFH=o@gZprcia zO&_(F)90MbTF#M1xc)VCnBJm;exZDgqEGgE0sX#iLdkj4SB2rAqm_uGhP*)nt3$1s zn+xTF%ElKjU&?-_^x!TybHa$j0B*yR7yYy3G&<96F_|F+w!L=LFqDSk9li1ptW@2a zA?^mlOEi}=g}vtd3++hZzT$9uA1*Gzo|Y~jX%J6dsX{Owwnb}zNnt2Wql`hY6l?2z z07rK>MXy7;*{;PK0dq(-eZb4FzzR>&mwA+dF}JbfWqCOW!Vd;*57dVYW2;%+$bpN( zH`GXKE1MHb6|fBLUO;v-4LgbW<;z_jabol}3y8tZizsL1@~+p3n3;yg0L#L~sDG{Z zq%U)iqCTF`t4%L|SX*0_?l3h5>R0I|*>d=#3h>KFltdsqA!!?Io@RcY)WF4} z<_xUbc5t^`=+Y11GMgWFM4rm?l_h0vxqg4=_wFUY`?a4k%#rLQuLhh1Q{jYy*r~Yg zjQ(Vn0bkmA97lVDbf<^3{i-0`8M|dBGCNrc>vhAs9ob2a7OU4BOK9saO6WX)xaGvy zos$=iuS8ie?;L$_a7W4v*=&v`2Kovw`AOv81Tw7prysK`g_DQ=*17O(q=Uppp5eS!0dAH$R;Yv!LX00U z3GR$Q* zDldHbpOEh#4Ql(8u5x=@ggg2Nwy%%fafso$?rZiy&B)C$6Vv{C#fKk_{dy1wvv-IM zP2uHf-`UpUvr;P&WxL2R{&=TD_r=VkFiFPU;Q#7vxgXDZ+^bm6_pGrd%3VF7N$?Cg zzD9Y~wPxWtBs^HlE6}7X>3@-%F!~M$@}d3MB|oC*jF|!lLFs~UnsNdJXX@t-_-bW& zB3l>e@ys9uJmcTK^7yTvz_v(Gdqg^4v{Bd^DVj|czKIKbg$n4rY`ZDHN}c*B+x*Xx z2YjE2B#IB%*lRhTHR#VU@_f!U?3}}d{1CGjqR+e}N|hh{v!o;peGz2w7n*^%urcU< zd3~ifySAWy_@Nc5#h@kTlS-X8lGDxRFA7s+#a2Q)`CUePx7;7EI{A$TbU({zU~?S? zcE)C{*u@))?pup>5La>dE|jl$(je!Ml2OF`%JL=)@@33!ekS%?0QNLWYR=;;N^!bFBMM76@2lopbNtA)-qNF4L3o9t0 z*LI}Nkz`gX1P%?a4qQlnBehIOn__tWG^18Si{810#*=-&^;tfAnbM)q8L$Dcwb;)k z6ggOQ0F+Y?sWtyBp`B$}fviuk#_8cNS3Q$HZ9L5h23A@1vEb#y&e%3 zPcLFKBF+}Sn_=X+E3u{Vjcrx<3_27zi);lb-eH2swH5sn)1`r#Lto=mJ?T*x01-lp z3MsPt?K{gVjt0`V2Z+Q*^Ns|GL#mDGAoI`4pzZ=K{N4Rxb+If4fqH0K1CHzYZYY>=J{ zr7E+C5K8h2zQj>gtflk=$gYW)ne+^RJO&rMpM9mTao!~}fjO<%Oc4mzV}3p5n zl+4qY_0^_x+NY(Sa;rF8H#X)K2Rckk;b>BU=&iZk6b3wf0#1!o+D?DLNujJp5+$zk z(kQg@SKe%THTtSTcPI**I;7Zu)QJp~_f_{-lN5#{lYGsxYLv!AebC*wkfMEpd~hs8 z0FK6i+cUE=E$IOW02#iw(;*ynxKiQ-^OKxyNCCp3=_CP$S1Y-OfV?v&sE>Re>J|Vo z&tyX#I3hpPeyEkWdX!H(IU=IAsd2R1f(7W5n!Muf3`TtqNBGO2b-7VX&5_Ad+gi}| z^`7^}U-i>{J>uM@Lkq_v)LfXC-NP;h9(4cM3!f)7X;*~CCJ!!!o;fUf=$@?#L;tj(_H#EYsn$eZ-9w5I8 zIcvZKo;AS!WnalSN6)5oRxLh{+w|%Q(%|>3_k?VtoJlcK@aVNbY8c?JfnCUoFJQmk zBRe8{pzrQItUaY5pVOkOBWuIP<)H52Yk%o;JO)#w%|rE#g`B5^+L#)>j;oYa-22=| z6ZKyhQDF>3+IFasef2RR?|KSDkpu*Rt5T+<4?zs*p+7fY?7zPJf4^Zb>V7q z*yB-fTU!Qs_(W%;3IEy-hmY7MXUqQ20~%bevlXi*yJuRqv;2eazlmN zZT*>e>w<@cY2!sv2jQH4VVAM-kV1? zmG(}?J?$u@A^PN7Y@49`Le;^RE!rt$G-{1KBo<~7D z$Vz>u%_hHWr@dXjx*R^Pyy4dq$H8od{(C>PJXubXf!;3dV^i!(2 zN|Cl{rBuzqOz6`k-Ra#TFgu>~^7(Wy={gvTvFRe)hZ`&=1aU#Tg5;#HzumBek>86p zhCzjRVD#o1If?9LVBTCA z03bYhACnnma{(?X1RDoLSEx5CFls~Mjs5ch7@k>_bkhe!k)P9)I=xvRw+=3Eeafoa z0*Co<3H#-mvy9%qyYKZde|6>E4A+{8L_Rg1qfli!P+t)?SkhyEcnKe}xY#YyLO_bb zB1s)J5IcCAdJ!{-YoeEb18>N{vv9e)B$Ks{@|vOmr;HI7yiLt`%ySx|K*NIDi<=21pXef1tuNP zXd(vZ#?o$p1`p`CR+jTnE2JR%F8?-90pB)6ZzLar0!toDv2-Cm@ZC3=0jA?E(5|#t z15jne*<)@veeAbLREJIygy|_+nJ=jgY5=qwLw_neYr#c=R7@;579-k0>E)FDHM$!P zW{}O``a)E62t7_J8`W}+6ANU)Cda%gK72--`gsh*R1|^U&26zh8r2)JE7o!2pC!dys+yh)NSMTsxKUZucIjkLA0mEhpB%E^5Hez;=uQto z%iX0_*Lk~q$H5rUjdSL$RgXIu`JFc&3`#1%*b5svju|WxxH)84ux}rmA#$9NBBEGZs^BSTT-W>$eoZ%GKd7#*x+X zrWTiC5||q|N4PbaR=PROT&>h!aq~#yuAXCeFADbLNCwHOJ&fj3{;s~d>pCArMILt& zy7-H86O>G_BhG^Jn#a+gj?p<~ZINsOeQ@qMat&IS^L0KatmbE3iNivc%8A{x{_kDI?JLb1hhg)kC93<0B{t!6rEQFhr0z} z8~T^5Mw5x9=oD@^Hg<8yXMWVOTs#8)hD1z(HF*qpv+rytflEzbueY=bANM*ZJX{TB z(X&bIzxGw2e)fcMO*?{lcxJt6qm;MAx7?fRW<;2<2$m#A1$_nUD)L7Wm;yx7=#( z6T=+#YtE)-(f(TcEPV8B#lUko0Q_2K0{f|65~#mK;hy@?XQ?7*X%AXP@`njgUc6U8 zZNY#tByhEL&H|5iw_4~bm^##)?q5;`mQHO5@0zYsRY+nxKyBA|@w3ycs{!0~!?eKr zB3PIHBQMaL-}oF_5Qsq(OIE2hGXzbg-}ZR;8(O@DT-go%*N811*swa>t$ zI0JdoFCc)2QgwJdichEAWNE>2O-otVffN>&5TaE+j#!O%-Sx^F*1xh-7%UG}_zSv=zT2tjyK zl5cqqVrd@YX)+p#%i%O9IH8baGk${4zLD(ttPVbF!j9|rNM$qpRIa6FP5@0(LyT|m2QW^!HXHL@4ahSHG^bn zI}9pqE9C)>evP!BU9R`Mb2#b{ynh@}6jx#s@?kenLdb^|xfo6It{CoIYgJ7}%waf4 zkjBL%O1Fr8f1b)hnZHUTa67etQsw2A$pfW0f0lC3e2MjP>hO9qKvXk!T0;p z0VH$`BST&-xu??)DpYHr&=zWp7FwX1#M&|hd-P+}UyOx6B3lU#IaidSxAqzHwc1-c zbDPGnKZz|8Tm-6d+zJskscs9mI&fhIs|!Aqj@McVzOhK4kUl3erL?guz^pn$j%+S@ zQnym%3Su>+RDm!=OhT)HI*Jw4)Hu4UJV4e1FtBC}&#m($W8v&j-m~Jm(u+CCcD=4|jweJVKdjHmb#h9~ zOinzq-L?f2^3u@l*o#M_FDw#uE_?}jmrU?dzf)diUo`bTd>fE+TayAF(n^5g@av=@ zt7K=-h|Y;E;X`Bep%@_&v>1mm8)iU6FlPw>x~nNVBvoEzOd)}V5a``xGPF3;QdC!9 zz^w5wp~H#e))%-gbyU0ScED^PKSU#h`>h+>V8zB`L8*U?!_OAbgnI#57qL%#PGm?v z4!?&6n%GZKNp(5`IUtW|=^hMuf0a4jV;lp#4cSD1Yr;Vc@c}_LGr;CJBpbs8qX`a% zt+w+Vgg2?`M6aSZ>ox1!qdH3t2B>%>m~oTJd9#$^LP*rC$1}VsG8`hKrELOA)FXuE zDe*5b?o2lMU_t|*K-~$ah&3M%pYkBxymIc^m4e6%bsMyuc8gz#&Wg87Kt;e0G(JJl z7nv(<#b|_dzS$;S+cS2gj7PAPBr@KBHpSL@V z?9+W~@@3QGr6R8bvqKI=%zj}+f7=|D58Ta4T?jGD7nE}mvqipwbHYGaUy#h}Y(grK z{rk~drTrAkG6>a(yMdD4p4B;I z`(vIWV#uN6sKL1d%)StSjYl#&n;AFR8>H*ShCLTcn80|b#(?qKsdZkO3^BxSa z^3q4AjPG8pA@FLR-U(cq>#^#y_(ESL{R=oqMg+|HQ=@38uT8Ayt!7VEAR!uelYMC|T6>}SPyPgz z>ajV5!Py8x^OEPthjgF&!nCTY*B;&;9cHfp5~g!_`53k7TP_ND)8g!;>)^QuD6tX_ zz?|!`T*KQz4@m>AFkNIpwidjK<2txCd+0?Lw-$m2xsH_Fl-|m~=UJM2=Vy~5B_+(T zj%GLM(W;$65}wfh?GiKPZq9Jwp9NYMhM1*`uy>V5VhiEpY+=1&RYe~dL|nck1FMP- zOZza&OwOCCal}n9VU3c$rj|4KIw6qEm>%BsS*>7c#g^>dF-jZS$W(gA`SazQL7=?ilOle0l% z*dcr2E=fimP}t+nU9Srjoa1%!v&RVt1I_S)rplQitCGrO*mj$E>hGs` zDMZ}gMBh14zt8UeJ{R40kG03Q+diwdHat~%H9fIAz}V1bG+k7ZPp~K`$J*zId)fpb zvZ+3R3&DreN{j}pOIt^3!ou4a($_f10d%7{DpI49q|d@Dg1cG!CHKLNcE+#}5H>-K z<PRTf(R+?8MO?7(q=yk7P)PSB?cXn(et=? zb7mR5tqJV$66XW-<|WpXom` zt(=!%K3>&C`#I%AuUKy!72~bb4GJkt&zwJnY3b}6AIVtmj`xXmO$Bhg;0)65lO`I> z0!E@J-QIg8r2x8*DJy7W#9g~NS}s@)6taDxxmzKU1EUW1&C;!~7Y~37fB=@Id*Ms& z(V>F)k98aLVk>qu76)_uLlv5fG#(%`k&p*ssL4hGO3A+8=E55GV8-rKwm_47chNCQ zuLm12UXFXRqptDkNfZrACvJ6d9WD=!YD1gD;{uz;rv$)M66za9$UeYzd0jE$Zs{EE z!?T^6?^yN8rDt=mMdwh$xNPIMLnv0Djm6A{j8Simu7EiF<;5h`228o?^Gx67A!!PC z5MY~HO@!Ncn zbj??pTp%wO!DYk@vH3fAs2EeJY?lFf|6>wBlAuH+{ODSN^-Xlz2K9-=TzI)KyIZ_$ zRWi>f-OF>9rPy6e-aUT&cf9NIla|LBV+ipEgzyQx930q|)@#`nbnfC|9b;;TK8t90h@8pq1+a1g;9q)bC#&0ZnU% z;tvjL;J63OR~P2H-W^|$H%DZxX?R#@aWH7q4WI8cWKms9^c7g;U*$gT9{t!6GD{z& z2A+VofQz8P`N5(Q#8%h%^(|Oi$P>ggvKzl?UA=?PF zXyYB!!w`G+%Q&I_&=EjtJl=*=5r>2;cS&-u^kNZU8{j0e2HfM7u!E~%1g zl6r*40PXHPAeN!FNM&*B#YRwPt3Hz4ZekYO?gc>zarD{vr|%c_tI$S0$|QKBrN~Ob>~paM&m>q zrA-4KHvCDEGpO2*W113exD~WnPVQUrxiZ*GE2gj$6L0BT1c3J?4|B(etDC~O!g}cD z(kPrW6w+Pi5Lz^S$rBDg+FL=;`cpUmMxa7XzSzM9&$`B%znhcu#d(66uknz14R@k` zKk|7!KS0x`$iWk9O8dZ&uGP6%34(Lta88F#)|CoLJO*FulRoX5HQ^IAhD+oryai9r zxaUtLtT9Fr~v zGz%O2t~TGTjQk)z(oeSKdhXIVS`j_ka~LFQKW%6=$(R;8|noub4iu#a^q1MR(IW|rH&R>*?0uCS-P(#TH^w< zvZfD};T+@Kn;Yv8v|CCBChHh$mDn6?y9eOs^`N#IE=eYb0X=b~I_4ZMnk~on!`;-< zg?)LuFee3OF$)|SvKcg*Eg*6e`IgNjJR*4rd&y#bAe&rQBOl5{gkzu~H?ifh$c zB_Eq3FBRBNi=15U7k?qX_~>9-^0C8e$+R}-Tn=8mJ2@f3v`d_hornNtZlzv~RGl%d zTrf+&h~_{ALRVOMoHkq}yywzfL&+Ww$E<_Ln~-HOjU6X)KcB@j0I(R$K1d?+y$u33 z!du3Q?rkWkXm?GXK*7eH(&Lgbn#ho|iM_D-3FADhSo~J{iUZj(lE6apB?8&!xbnkcm(2lkUaALG4OS5yh~;#jL*JRqyW8eD?>RjK~NKENqTZDc7x{d8cl98wpZ z>8(wz&Qs~+fuV{w$%1j|YAU$=bzo9+Gqs@(WT#2Y&e&1edLU|v_NQx!ZSuuC(3`tH zS_Z4$stYY9IX59!GC%-6fa1wQ1#y&Zq{#;J-7_!pJM$g(zwf)Xr;|$9pM!QcSKiTL zn(J{((RRpbvG-kONnj{p{?-Zp6_vpC`ku#jgw+R17Z}PlrDy zQRSttwSK8#L7KwU+&=}f30LY3=T0(T8@E3y-2%+V%?bx9*Si{D4(Bo7WwJ>mb5h=GNePN`8 zkwyeHTIJ~_2Ubb3cPqbOaH?`vP<*KY*^Yy(kVBKgAjyL($YIYW_8OKEdA}enZX5V} zanfPw{-=Oinpd)QvbmT{Ke`b8MMqAzqE zctE-q57GYtfLO5&h=8F;f&EQQ7^OP>F^(*JNEic~1Q}&G!ajnXoVHQq2vYo!Vl40L)C}>~O7 zN(s)kmw%eo6XLMPQSuZ7Tq1jk#qqKl+W>|EF+ewU=wm-`$L42|?pso{%zQAfIAYas zF!Cet6?y<}bXnkt&|Rms{Ei2-i( z{9AxBKH5(X2tB8yPP;)qpD>K@CwyI!AtHeN!IwOuEGPdYxj}Z$27E-hV+=XC zsRgHu@d6Qsi>$6l%+dY6 z1i8s;%TGK9N=?VrKOUysbEiLOY>69k{B&Gx!Z_QHby8`lUDgY<}DG+%pFRD}(FH!i#78H+hOoQxN9qKpp3iNHpPXjLgZ}Tg zNo;s~=WG>kE;>t>1)4V6(`I;hCBJhP=s! zYcWwDKdKu0HwGuaFInq!tEXqEZN$zC6Q-fFAEkre?<`lSJDThWd{~%YGGMhBT=bX= zo(GDC`2rhPNus5qNDo}g5wRSr69g7dzbP)zR*LOZ9{NyYF;A=7UUc3-n~=Bn@}i7z zp8iWfuVArdR4c+-;|&>zXRgMp3o|d8Y;??+mdFW%0UHmSn-l}JkNd`0?ngAzHcY1z zX_iEvpc22Od8uXJtk{7#?pI8)$E&fT;x(@xc%Oc&QSBi8Q4H5dD?n%Ly*X;Ek&74)U@Ta^W0X9{B5eC7<6dY|;P>ny|@Us}k;X*jVfi|m!Su9<4x`D#74xZ($5958+a zQj_ov+*N|*cHnFvvq(!FK`@UMHICErPt+3aNxZ3wsp(txH=3udpw9*}n|(MMb&2k8 zGS9cReJ;i{en|>WYX7C=?B}CRs_`ivtd6#2F{Mv_&rk7j+0mVU$nfQpyT6?D-$>R~ zzweaqa4`7F__0I`W1O0YjTck71v2pdtRNkKiUuk8hRf4q%I#wi$$)ac%sthRe4sgH zi|YwE!Qw>uQU3dGQcTf zPVdM&ecJ}u_GR$qr$;9Jw|T+Nyqh8s&c(L-foI|BV6Ln}vdbMg6qMt7>4!YW>tb=^1d&>-9b6jy#_u zX}Ipdl8TO%zPJB8Qu40fczAoM&c)d5Fzub(t7iG)<&tc&o*=iWQux`+lN8KLjZwhU zH?THN_gJ0bG1iiDUK%I3-DHMMhyq^4?Rs-3K-HtlwwN)iee+h?sLn`!Rl#+9_g!MK z|2~uc?pscjv;?=g>;)Hfr#pcikA`1$jE-I_u|OUK+KO*NejfJp92_1V92{-}n|qyW ze`RHt&C2?Uy47U|h{P3kw-2o|+S#V$__$4dXU_KHGU@T#Wpe)cujTJhQJnG%`66>r zD3)*lw*Y@FAip9dv(8Ro`!^=aC`<7BDn@KD?L0^e@!rQL}EgA7=>3<-*~Y(ia@^+_DX=lFbVXfyYQC$j zSNZ)pkC3y;wYL<~k}m^5V}HPvn(7wE!eb8pOt%lS#r)4b@01*^f_?}8$oxlN=Sfl0 zU(GoZK)@&2n$8Agim^=Gr}OmkG)e3R>J-O?F-ld!$}^(ttU`hfE_n|7c`z3hUp8!w z-gre6r;}=PR(V^JiSJeImB(%OD#QCg{{1oj2=y#+utmo@k=E4ie(>4OqruZ2cV9nS zvAps~p6$=`=Y38e**stpF+PRr&@%NElg%Vy48Q`hUCRexL_l8sVhsQ$&AzwX^Ht_i z2%mZ`MH~dYdKTu>K`Id`+J^;!%(P}W?#=rMp#w1Ee<3B|Y0}Kc5$gdWU@I9R38zcx zrBpdiWtF(Z9;B3m&+xdV2jmmD2P^<(Ocnxn6hIw32oj5hba4=lj{Ex?DaJlg!a4B9 zK(C{X`aW=)=zu%(W~Xm`!4X-UrYr$5SLu zW6eM`dNaJc`>IRSae>}-6A?n;I`&d%Fz*IkmuQhwwJ$JsRD*P&^aCY9&DkO8u z|AVfpY@8GTEXo&~a6?f*N|v&Y?ABvASEtYO=_%zeXlgAm=}PdS7BsTA#N^w&sR+yr zIlc(y0tO`2=B2A?JlbkFvQH~O{}fOl@X(J{A>b>e2s}sC7`MXRkE$-V;v&^a1Ba`t zyV+bzh;lESJ51F4P#LMyO?E4V(eHhkkZ*#?M*D&|OU#2AP(+As%_ka`WaIEPlp;j5 zC1%>#xwZ;rKN5zn-_t(3gSHxN-j!>YIU48S?V!J&_4ROqXvTKH5mv?kPFTEI)K zW{}cas(>amT19fx%6O>HZ?iIC*G~RPn2oPbWDK9VE-<*ftUUpL{0`B5@1s(_tr_ufsR;jeva~N(W2WP3 zt#*1Xzo@5rK2gch`@pOJ$d3GaFXcwm(BV}-6D!$wVjz1sh;QCN{ zk;3&bh$V|^xh0e=YJ=Axb~Yf`f1?HnwLwv#d{^dOH3`43ha3hewG)4en7Z&wgK>`W z%6O}7=hhw(gTiYhs8qe&&Gtu#WXZr1^o>CM)%sLL=?G2wmOMcaU$l`#I*ZF>=h)&6 za#F_~qMv~AsD%#!Qa2W(Z><;{{_ltb=W{V5mpDQMc-zr#^yEtE`zX-nK58b-?ZQOT zZXiEB#eAdNY-aon?3RrP?KVBxg!p3T+ej4c!6wp|m@MUi_sH+fED*Ux24E#+=CO49$z(=l&)z8*d&pwJ`7NF1c(M? z?2=it4b*x+LAQ`clxy>S^uc>imvL;?k!}ZX9D<-4I^-XSzfjZD-9Nfx@>$XvbOn4) zK;Uv(G3(v{I@5u8CS765Efwn6>h5(gWY=J(Ut zPPaxUdHFWkyYpDz8#n9j`mHe2^>D|9-L9zSjVkF))6Sc?9qzt@2o zy64mzROVzOQd2Ao^Tz@-c$h|7oX4h7Bmm%sym?P>IQY^gi8sm{njZ2Dm*}1BW{g$z zF?WU9);#UMqIuzE+sUrE;tG9z&p+y~#w+3e- zYeug3XiYXBs!xj#`8Md^%NPp3lzc21#yF^S*ZxYo{78>d6JOn%ViV@xUVO*S>2peP z#>X##OO>W?E3UG`-PxdC9?}@U{n5{Ef8vy=7dq^)Nuz+YOuCW-2RBkRnR9E00Vyf% z9PcD?9Ky7*BfDm(!~I#S1S78!=A;RzP)q`(rx?9`#@ z{)z3ARn=9Q>O1%!H>Y^-$WGVwdmnG*_s;r!$j&2sK-8lw=jmC|?3PHB^JWtzVQa7^mrL~5$G<=S^A2OPJu}}WLton zwE!E3R`MXV0R}-3Pa12plN;zO0l8bh^e+Lf**+uDaaV!BtqmDBiYzbdF_}|Dr{z!K z5SB=7Fi}R{KnU05B-Vuskh44WTUmR^p>Q~4(Ud%cHtgf3oy>IQG-<8w(T)DxN-^#) z_1ZN*f@so0#tk3uC8FkZBH!36S67s~Q+mRURl4e~SB@~cfFK;Iy@(P}DOzu<55%o! za_pDBRE^N}rR}&+sBh_Zc?!JvRi;pwi#s-OgVF&C4tkLk)sqE@XW|Fiv`_;852@OE z_4w{O^Uzpv7z0O2TC}sa)lE!F)?8rS+tXrPW*dYzE;}NqFHlKYn z?|qf2wUIUgtG9I7=JRrZv-@TM4@;7%v-U8^aSA$DcG-wGHGP%QQT!H0mZ5kIDcw@l z>M~ssz|+Hurh&QH{TQ6DiQI!@LdVErQA*2~1Mm@e}} zDw6d}VL6fwH#B$X)Gx_7Q)Kdrj8?L-oC zI77NtUg^kBZAh{4Xc4;ETiJ4kjX33H&paKL+-Y2Lt1hXzY$AHAWBKCUw4qzoI4&V< zaQM&Zf_o=!Dfz!Xk$1px_#7e+6D!77!doLbc>-=RJTrx;#U&spSk^`M!1HT5kz}Q|Cd=@)+P8ur&QAQL> z0bI;S+TUMdqo~|eGKgGl`oJ~=724*M`mZt*XKCNL)0qlP7DKv@D*m1Ny}MeBb#0s{=bA0jbFGU=Ur+W4PlE1k(eN13h>26}JA9aW( zy}pw=wuBVNiTbQ4Ye8AU0CcKpfMO6tK5_b>r1e(|5^6_-ATAg9zQ`yl&u;Xq%x5E; zD%?xyJo3i}{I^*HE6nUYhQE&w(z0Za*H5 zw^3+Op5JQq<>c)atr4oe=rz?Jv$TT#s$Pz=fe4|qz&?nU_WF@A#nKjzOh`CJLD~bz zSeg2q)&9SK_*9xpfB*w*Y)Ov;!240-R8=mIH{ni06Zw{CMEA#n$^nxSs$+McagoKN zT$g9Lkf=djuL-*hux)apb4X;&7*5VlYeSclE8>?PON;!WDN~1_&C5Hg@5mj#!*epT zx3Jyjb3gqZ>u0hoYzt`|u6<2WBI(&dnWY86LVn(bTLlC9YdShcrn(H;Z6HP!C6 zdGB}Fem>?JA7Ev@)5uLc4r06SFdR$wf|#%q`DF{HiYB>V5KUcJU(cAm%s%vj{mvc4 zgN(~xWqK;GC4XYxVXuQDVFc-?{7UIg^z){VtwRhg(V0|Y#`G7HJ6YP6@z!GKNZmE(p?o^)s z(eJ_a_q5N$fB$S@Gz+;Ka3e>9aZG=Z!W{Xs?%+i01%E#7muB}8b$n0Iorq078(*LW z-gpU0w2J~l)U_A_Hl~Zz$sK*z?;*T9lTWsWeM+C61X|wC<8oK;a9kl;5_z9l%A+Ya zp{j5owQ}Ijur9s0>EghU%c4q0bvP19dBt81WSzF16mXhQa)hN;<@5d8PYIgd+@H@H z*Y+7#UnI^c7&Y651nq8Kv*G57s4otJL&D=A_pxtr1>`}7K<6F|IhABSW0u2JP+E(0 z3yc%!YJ?n6I0RsQTc{7;PM6szv~4mX1WgAGw{61z)EfKXa+C0+;DHQ^5BBBCRVQR* z5PYLY`(!RH3nLqU+PC0)^>Ww+z4TWb@7dtO?D{8eR1+87i-ff{SN80T-)VAby7*3c zrumZ_K13$hX;(x2(5-w8PRh^=N;b7gfzpngw_zg1YNWKUGHF--_DaYXU^Um3kK>lm z40yYlRJ&yEtIQrH>US(Feqo5-=>|&h80irPAF~uimm;#i%KS-xgoB#tPunCL__E-C zy>-6IydOdS{Ur*h1AXUXzi+Sy{HgnK;*DQrzHI(>TpxkH&_4|S5~${`vlSUoyR=?Y zAs-%2>ecnPR|ekCiFbDyKZ5IxFr;089%kg`+8F97XGuS!}w68LE5I=6|+k^Y@ zlzwcye|x(Ao1LW|Ad%xHyb7TiLYCix&J2qRMqGAQ7N^9W^ZOOe$+&D6u(n+Kx_hgLeWx`qVyH= zDW^Fc6AuOm(L1J`5@@T%zrq}@r~M5$0AfEM>Z{GbC=9~juR15UqRo1Xk;h*TO=u$R z7285Ji>oj@fZO^ziDRBr87hdjl;`sG8!7UTlm^I1{jL`J zKpzx2`vsRzX9lJLenL{gSOMP+N^V-r`UQ>>?S^wb`%O^G@Ohb7o=pQNEljQ9auO(O zyUrSvVF3}&NdwD6+!ObYVJhn|D@-e^R0mwU%qHmHfom*!v-h@4-D*J>DS9zd{9`6R zIrOy*K6`OaaQRlM;C_(v-Sdq6aqY}3n#=q}5o*kgzZ{|g5I}G)*$Flp--T*LEA{aX zY#aP5uEFVBcS*+3m};QhI#m?z^fK^C!1- zCbC@(9a-YKZhIfy;{VebF798m&MiOMbN{0a`CnC_{?T*%d&~GoBlT}x@E_anMd@X~mzwr-@q=ZXPs<=_oPVw!=R|tJ>`8IWn(7U}oQOCi!b%@Zg2vYe>LzLR~L={wj1H zetKTw>+bSs(uZJWF|-)4NXb+Xy*g^G`*25t-P)9$mnRcV48!cM200gM=ORPwPW>L~ z|LOAG3`hGf0PPy}2lZVJ&L#Cl21_CN7Qx6BdJ@ROAT}rNxY) zQ5IkF7;w`RaD5v+6KFA4Q-^3R^jNJOfKCMwwG=Ss%B+Jk1sGg8V1df3aaYa>57-$# zRWTgvseC7GRsGXy(kS6hoBgfS0V`)gMIo^*23lddrBVeB;gtonNX zb#JRCToQ+A@bc*}sD)X6mAQCm#c!6nGlyY!-%IDt+}$(eATVwZFkH=97efw62|U2) zW)GeNq<~%`4e|CgtM1YOh8P1L(v z_E$9z%nW;E`w;fEJx5&r&fD8-Kbwer6Qp(<{LRimD|cI0S4j@~=OX9`dYx!a$N4Ky2!7KOR=_`^O-aW)F zE2*1FlKUD%zww7ewE1zKtgqFKgNVsPPHh~oh0qx{C*cOOTwF^P1HZP1az=n6h0L##6N`t*| zOgG84b|HXeHxL_X(y$6qm6cWl)spog=EpVs@2x{|ma3$XdI7tM#6v}U%K_?<6(lzA z=;F0XI-+S#PV5C(r4gGPs1H0i-ZPbsB&@AKezaa^G{N_wzuUEnrzh2z!(Qkmo?uIS zcPMS-gUeQ71|=Tx>7rx%8RNG`6}rjgmy%TQCN+VUFHHJO)2DC$XC5<6w1fKE3yAf^ zd;|73V*kD+nAFBkk{tGw7OrAWDo0VGe(qq5)D%TT_&jv?_V-A6UrH~)3qehKadBXS zhx^GvkHd?qZfR+5h)pW}m9K5I2UHc0+&-jWdB;4beEA#HWZKBjU`S|Q)f1dME`aAK zLbdf85{(Ky&lo`2Ze4>xxa$mc*Oqc!=wr z>Jtv7X%Lf(wykQq8i>@`Y15%2cb-H!n*& z3jEGwrjJAT+P%ot6Jgya!#&a*cG%5VBjz*c}dr5a-; zphj9+<&nx8m!9=h$kpo|H^;>qYqU%Bj;pAb+!Wp;1sG0<46}VgebDV0VY$aj|7E+X zzvOij)J{B1c5rSo?&&k`#us!w?D;VGzhp{JQgnfSZJp`^f0n+07RyAffpG`RlL1oI zmKWgBbl|lyVRg_1$G@hvFo;lq?+*&Z?Vk8FvC(;3@U8#b&DxkzB#@@7YEx41c+x=V;tqNGGh<$1%8w-n$;L0Nb}qQL{^kR+ zRc$}2ae-94D=_eCd@)lE5zXkoXU{xHCv5rs~i%lhLFC# zyx`TEXpvpte0xF(=6LxZKb@^+K1*LCqqC`hZE{k2$ zl@ZuhR1}>FhLw|E6V{9&HrVUMS{hxd^frRbV&_Y+jntpXs+3FW?Ja0IfqW^Qfj4~m z@w(x=@`&Bjeh=+6W*9sE6b>aw*XoOYVZ@6Ok0^b(RaZpV_27H?tEfk$0l?_eb4I0{ zjD_5y`L7r}PUI-s)3u7IMDlI9CZEWEcym?mwzs74X6we!Hg!F@&w5)pYEWlejadsO z#(O24#-01c4-jzDV(QN1+ZZ_qtxSt$7J0eKc|Le17wVsOaA?|cxn{Fd>&xQ2b-uE) zzNKl{n)xZ;>IT=8qx;>gbDhYKFBK~B9v>8!e5(B7s}^3KIMly8At)sm5csK$f0fbJ z6AD1ge;4YMW)DbngNt;Kl@fF9R7T0 zj3|r0_OX70n}x}CV}94G;(+(?t#+yPMviDr(N~%LZSRZ_X=?W@%m5a8G(oL66A>@ba+$XC5(OULS83@JwKV z?+D;>)^2VrZ{s;o?P`vkPns%eDGKBL4o`{>|FRAUuAzoya@Q3m+rcLk`Id=-s^*bD z-%KaIydU)^qZbiLT>p}A&coVRh`{K%f;f@8IYJ*DJ> zFnv7YEWxGMq~2TTmi5`CZ_cUzg>KBz9;Y_Din6k(a}CG}z)dVLYhLn=q-aQcsO6ky zq}mLQAZ>$ak?1`@qQy6EfjBGD$ON>JqCLO9CfiaUb0i54PwRROrR{jsU~zz9;KU$t z#Z%q2@lPi*^Og*8)xj2Q7naTW%aW}{gN2hgJ8!{#Nu31Trk;iO5wf18_ z(Z6?kzaRUv2m%B@Lxd#RH=DC>kljRXln#oT02i}hU$5v5wZ&4*$*3;!p&Wv%Dgqyo z=f7l_&&vp~cjzW*0m23A4B8|J@E5)BgIc`Fa|71!N(n9_fPJt%5 zm>69k?u$Xfdfb8=0Ut{)23k4;Zf+0NDDu9Q{P}2o{e2ak8#q%n=iS`yL}`}Pju=!a z{PcV7lNMmPEJQg7gIms*TTjH>@n3E{f9K<{tjP>L>$ z5m<2NT8Q#fOT|MVxr?Q!1HC7T`Rmz=c7na;!Ex)_zB*K)f2zt*v5IY8!`QI!@Gwi){?x0fUB&}uZf<*aA0h@2*IJf$ z>gWAm?7ewZQ|Y@dNI4cF2AL;`iin61l}Q0fDFOlo5s^UrY zrYNJ#Dw8rKLI@}nGEZR$NmOJA>?njS8SL%3>z>tpPTlU`{dM2ftNZ-^bFo~>&bPns z{oeO^pZ9s3&ZZo=|E%clxORCCwzUw~Vt?e`!$-jxzKYew%2$dN>qqQ#=v5l!4*qKc zlR)i1y65<{Zjxe{mdjW45tW2e)`nyy!B%%rT&Tc zg6fSS#;-d~j&_9%o<4M}``K*L;%C0pH|F&2%4)rXFETs==F9QdvzN0Sj6UUf`?0kD z(Uo1Emj7?N1U_Lej#-2ZkdaVGB+CBh%A@e_ejAZHVDlW^hiKX-dPM!K-zb)%K@-W^ z|AoGP>>lCtzZIMJD4L}0kRmEnY!VE3VK+0I?t-3d7@Zj}-pNySM{hSasVTsxb(jW~ zG-Sn?7&jgqw)}nfU-~S^?Pqjv)h|4u8pj@Kkhwh6IULhkbAt6-HQ$YVxcPHU#JDi$ z?;r=8zkA1`$pR?7%14LU{L?QaSi=C!0VY{Ae<>aeDnf)?W^_mQ0qe0@ox~&k1pS!< zW6Q?hR5flJ^$6rDv2x33?O%r|Ls17mPHevR0|cQq?fvqW?(N?F!u}>J&ep5A0;5%7 z@78R8GhGN(p8pv<`9J?9mj7g(dzsz>N6_DBak_@R>dL`Vo3X>O>Sn{+|wI-KVly7OsF>` z#IlgPCiZQLw&@syOSd?%xoGR>w-x-F(yg=iAK#3FG|N~&u2L$BV^mnom}dr6+t8mJ{MPhpA)U|;^8CDn+{^Z{|z3r`u{Z#x@iq`0`7`kp+-zh8EHI&4Hun*G%dSD zTB3r;(S&5JQ6%tvbpK?3hyd}k~ zW8-7-r}SA!=W3=$JyH5oYp~(IAictT*!yFVoMW~@Jfp$2@N##d*r9y9SzE^*o_zmm zQvj2fxs zVZDbxL?t)p^t`(?Sqh2Vt<0i3J4fAS`#RI^F_qM41@x9*s5>F=1Y}p$Q~i>lxNOE- zpt`nV&(VNwovZFS0}{=~5ocfqOJ9~Uto(Di`aM}dbMxAZ&t3X%_X9Zcy0++v3n|iu z)TZ)xBiXLR_5F^$6M9OVmA^~u{ssvjTP*FMGA#Lc5K%tvk5;Cff-ECYRSVhpk-A+q&k5Gg|Q)NU_s5HWy+xrp~e4v_GKsT{y#J< z{^whIu%+Vf5~stdfHxePO8;ki3NrsMK10|FLeEfOUm*}wVAZ75Fe?ZgPCJP)_bs$u?!6f5LtYCiT zHO@My%GLEM#@1C=4`RH$Dk{X#2+4oyP5rAv{F`p`{{t0ru=pexKMRY~falQ3vcga( z-p1!EHomTZN$#MWbzk9*{Hj4y*<2ASt;n;M#)H#H=E8RawZ zrFF0C^0o4%Dx&^;%*tl?U%H-zpDuWK@Y3!i8C1f!c$iwNQOfowD3yo@$eWB{%2U(O zeKQF6P~Q)BZrA<-sz&nJ;on?*fK`_TBVLI(5%=O?B-&I!Ehc9Vh5U z;JL3D4S$x;U#o-WQ!RZi1(d}m>IddO8Y=%C=WzR{o9p4ybb4!X%X&m84U5I@Y*aEa zku5Uj^MOKJ!>_%XM}jSP1`OICxnHKLYxey7x&Ou|_z%Hn2lYR5AN~goO>e?(!C7J- zEHU+h%fRXZ*9F#Fdn+qb9Nrd7+v69aPx9mZYun*#*e6@%2;R#Pl67t zcP&0V=Pt1%~g$-J_*Xy`l zbZF_Ai-j7&19ahKooRw>0%3_{7)2LNFwU8V~8udx^(Qh1?PfLM9B z-ob+D((qz0Knj`*z?#$QI0uLoY4e1Sf;`yf!8(haIbb6Ll zUyzWzr{!DVCnBVy@(+gDcS1Yx9S{#0m|m$8UV&_onJnv+a_@$`5=etZwk4RAwrM^) z^~v*kn!7t1EW9{arN833Hz*93>2Wwe*HqJxoPW3Tg2C+E@Un?E83J*N+;xJ0?*pg;c zlnrq-R=mgrl=y)bL8$8Cx%zJ>e{bQhyyH#wJ)H65!2Z#5%SELT_X{mRH&z8zMTp`t z?-JGV7VTJj?1}=o-m#&5S7_uLO!_@D^QhM%Wglgzt{GDkRGi8f)Tw^39CYB^6}JZ? zju8-iO#J~LXXXEZkCCi7I_clxW5}m-41$jbj)XTy+3TI**tCog{vAHf%!@K6{~LTv z@(lqhP0pLgKTqMlUttVTr=!H1#l7(++G7Es_2IiNBh0*pEsR@+J-#9#@%MlDod5rY zxKQ}d(NbXQV4%QW{2r-N-$UI=fQz13ND^FmT5%c1tt8e;98;l1l2aAZW7*L1VrbbY zr&YUsgI`7pgqqz8k%9La_ebab6h1xm2|UnaG$pxjNwd+cdBQ`9n6x3R*3N9#KbC1+ z6KTW5@!NUc+PgN|ubbt&ycr7(Xc%1$r- zF?IQKCPV_&!W+DaP^GsP7P&dkp9d>E28)LGH9S1Wd6hYE{hZgp0pIKAqe$Dp$u{!t z7K38^5Njp5M^4w^TOdq{nm)hbFZ=DU1zG8gkouORIc@uIJiRA=dhb z1I~X({5Re?_UJS|4$G>wPD;87Hk(j{v1%#RRf`6>PX)_pAM6$$w^%qtMu3>v{?Kv~Z}o*4I9qK9mcJa}ectBzY^ylko{)Ga*Z>$@)_BhDow z?(_wSAbfheW|C+VZyoCYZ6P=VL@GpoWt+r?Ayq+Lx+tCn&_T~+rpJq60{p*+=If1#|{8Tk$iKbzpFYd@*t}h3Z+liZj6-fDj z*t@aZ8ev>m(*wXdBU4MC!nQ4J#CANKLcarQI!#YOPUMEu)(dX0sRL@iUblDuEoSt{ zg7fdStwT2%y+||2uXE$($zvxx{&d^+seCZ^wfwh>+L9BmI)jsc56-^-$WOzy=(X>j zt9^5*zont4fhXg|cG^o&H|{E$juu(q^^^Ykmg^_zMY}-!-VPCzX~m2{;sa#Bfh;hh zZi5UTxM47+&KAz(z#l66;7(e0_+~IZ;_WtWT81qdi}>J{&1HdTNT%@iO4$ z)NvKG)6G^tlP&Mi3L>+VCeExqaQ#UhzqU39f%PD{i0cNCc18gE6vW6YtJG} z7V%(rEsXJl-&|?uvfP0HObDH=iSgSmyhm_)ik(rWq}vQ0nXVQ(R5%p-n?|WD9l^dM z%~ByAjv+~s)MnP=#qwO7^bbcRk#-YRK$PXno0j2G#))h!F2UrBXNT=0E6kPI36D=7 zFWT(Ro|&<6Yl%G|)P+)OI7jIX3f!mN{!^Jd)1?y#G@d-LQtx(D2slwuY@cmkYP6KYog1mh z9nb#S_Q}1wZvRKeleoqXXSa*f;aWQk9;QA>Zwh&67jn;+0$ucI@GyDlTPfa#MW&O~ zL2V8R656;8S8lPJkq2=COT3^>to$4jq2J>&c|lRu+v79kBxLDqt#i-MU#C|@WJmAbvSM~pv(6FaPj7TJZE~yVf@_1+J2%Mh;EOJ?n)+L zcn0WUs~0Z@gp#b@y$+$(s8D{-UYzCRXMPDZpP|IaD;qv?4|-!%;C#n`lK5w7f#%QR zUNU%^Z$9&|d^!QTU%sj- zQL<=fYG&4%`>3ku7Q=Bq%&kfAIK0*aygA={Cdi#Ysl$eZw5NAE$}dEn>vVmj^Y!FV zO92ycWYX<{=!OvDJH*@(YVm5irX%=nV+1*#8_R~PQj9+j_Sk~8_?`t-&MDg1#vmkQ3I zt4A*uUwBlQ*y&m{g{k&C?G&!_+KZv^DE-2R7A*xOYy+{M*o}s4&|^sTeO)VwAWP0~@udZmbQX`Z*lqADWKlI5iGkZ>Xh2;piy&NzZqX2f>NY6$fPARz93CGVWFA^AJO(05Ku3WK;CFX6Zj=?y zR9?(C=VjeCS9_{AWil%3j*-J>qIRS8OAB+nR8ib%If12;rHD$|0W$fQcvLPHXtn|q zd8jv}oowC#@@{iD%R!CS-hHkQOQ}il6tpL%L#5 z9>2>Egg(2F{){-Fszet#bI5PVNpQI})RuW9jKsQ}7C%-CZs(lNad|nY4Hoj0oFIqa zi;P@`H`!=?Vn@y0txXH2MWgHJ>(7RBs|V|F7Z6s&PwVx*cWCAb_u zv%|2^8E)s$nazj6Bt_odYmSk%xn(_!wjo;`I_=zRqzzCiRQxW%cjh&=Ar*wdKo?U%M!W$TW)+qb<8HWPbkC+Ca3UM^{&+GZqMS`i+>j2wsr|+ zH;Z>5V)}@ zcV@eB_s^?bb@E0{;H)q0aWiwtyPD#4d+R-a0M!YXseRVv3;iKFA!fX6RQWa@7mRfH^C#QNVgDcfS zM?jm7R0q@8D_m#*KouM#XF7T5l&u~U77i&LmHj5vqN<_X^iRi zJ4UTi4nv3*ldIua{8eh8*@k+}rzr8yk`-X#kE}C-I5cJ<;dOYC{R=qFQ?XFTM7GZ4 zVf-oyzogHSXx9PnG=qh#4c|lu=MEW$$D^jwO@>)-c@hW~t-Cg%HnY)4Gj3qsAyo))Fr3YM!~`HluxsHRfm4TZ>Kgi!iTU zpOG8Nl$K{MgofsQV1Jl-VDpAyR8>zFs;}LWE^aX%IJNvDjMt`mHYGG#AbELaX6pD8 z7Xz=E%c$_seQKXxWnU`NJ8-{iV-xs55G2%tys7L-6Y1^wD$wDkaR8vID>a-Mhupr( zlw<}_b^-($#*}_DXHRHX{&8UY-7u}@xQ%vu>HRp_cWm8{!>gg#LR(RD0m+}Z3B&wS zyEZ1eN%<}@`g7#hm2tlgs%kYfm`ZNczgr}<21!64yYc=Sj-htu`G|!c!I-B_uh`>e z3oz-BMQK2j-(7Q~0t{SHEWL2_5pewuroun65WA6A00sB_v!(~0PtizMau0ICvbqv- zao7ou4X@&em(2}x8fpsW=H){%V+#4~XRi-+I_^I2On8y|bouN?rtvXBW6*HJ_IN1#a#n=YaW zU^SD^&(42QFfo;<5gpo-<~~$^Ud{4xJ>)A$67R5rJYchy>UrAyRsh~cPW>u84EkxlQv?>RH9KRSrjC2#S}iAG&$ewoK5$&qIN%tq5!wh03vd(YX2MBTxa zHP}upS!K0GR`}~kq2;dh0^teJwjvX()tr30a-<2t-TLG+bth%d+`CH;??PvfJK_7Q^Zyj4rlfgouYXM$mB#Dy#D;M+;ak1n6Q_yzZ=H89=oJQ+!Nrw|JIJ z%-?hq9}pE#4g`c?T@{DOiuTe~)s>Ahcq=V0(^09YoNFJKj7j0XP+bt>B!n?^rS+v+03w=AJ2ULx$FBR9<&=$lGNdXROEM_9|`bz4I z3fRbgYz_l@=FqNrJa{~A)xiyo(%1RIaH;<~6Z=rIDwo!KwL+Om-{9+%P6d2pwA zXOQ#^yWaScrj#5+r*pz*EWi|=)OjcyjWs`2gP|M+=4cRtBag9wLB)DIQVtqib#3JI z?f9$;OGEnhTwzVh13_foW>(7D^jPBBicCPHEMpu$=@1W@OeBV@9B{bciA^zHj;?!X zTTPQ!%_vsGk>Ve8QgNlaIb@WRvf~0Z-VqI zYBqsh#l5#m7b#m?b2^FeyKdkl&%mRp9$0kmDrH(3wXWX$3J&CZxxSrZFs6tQ>8!2c z=Mu!kaci>aX_nK##i@Ir{+K_yhLA@#{p@Vr^&w%8Q5SGJ(q>EH>fo+K79|+h)M+ z5NFLgy^`SCIAFLUOHCGrvQ7mcq&;jZy%$W}W?=p1E|pl*jM5~do?b;MfdCuDgwEOB z8!mz3gW@UIKYo%f_Y`VfY)D@J{qQ7sm)&xHYvS*>E!A3ruaIu0F8sOkG^8!CZ5Ta1 zKcgvnwOlVXX)onpCEnW7sFWqcXp~ns5|Y+> zopKilsud`M@nWe$Q7MUqU_h@9UcWkKv2W-a5T9ISvCE__Dv)F=B8MM?Cw&+b>hg5y z8b`9u@L=syTd4hpIZ;u6GB1tPO|1P~*K818>>fS+4El`UkbbzG#9>C)QY>#fD~l@) ztgx$NgR4B5eKmN?H>&IL*P_kdWRZ6pUK}K`ya?xcT}cG1Tb$NSdrV=QKF{m{8Y*hA zO>+r#($aMmr>wu=1Ca;)Ag7b3O5$eE?-Gj?cFE!nXtlPV8jo}NO+T#Rr`vmv6Wez; z8Sk?G5bo+c;=WCJud4h<`ER#ec)MWJ9;Ln!)#8Aego`|v&*k|&6Q}zu@?{Gz`e53N z({rJ)& zYYyLH+$dEIc3tMp-EaA6c3LusMS1>^b}pymB|hm}_$hV9=Rs1?XEH;%3CbV|)N1qV-OH~i!(oka9NO>LD(Fh<`r0hm31)Fz#)OKs{Ez)%P&)!9RuQ`L@~Z@v z{;sN}^qRPt17N*R`MiZ1NM&)|eg*vE>KruL%(hl*+5DaTdPae;i-9EPI@AulGn7zf zs{M&uLfV^433gnD<|=D(LIn*yOMXH>1+hR@7jF@s2g&7lZCfx3dgKoTi&pX*r0Pya zRptXE9iGs1zezi(WcbtI>bDxoTNi?|-k8(K!Nw+mk#*6>n;C_X400Bv`{qi?dK64o7}Uo+ZmHn3^OJbe|Hm2B9hHzFqzs-VX`_*7HyC&Wcrdo%OS6 z+rXVEHzDuP%e>MQb2E#Ia(ApN!;u`Vkw!AN_)3B@9C<(~m3pNnfz*(-^p1s)5#AUQ zDm(%67f74!`LG>}MD4CcpPq^~ssP0!2__1CD zc);g1l;}X6ls5Z9{Z7L{Dn{vCk8IKxVo&7pw+pWsm5cHLOA<#kf9be)`PTr4FJ3jo z+>hb%xtDR*>oMFB<@0|X3s+QdEJpHMnw1h-4t2^FyX`9f(v8RtewcN*#aB(c(NhFv za$EEub)h4|5TL6oI)WW>$3h33>mVtSFl5NIo(@-A+6$b6*hcnmDRM&7R9ahP(t?J0 z9dGnczOm}Tyh1$Iw%=T`>a$1C<7ig1T!kl1L4aZQXRNJdKK8v;7GQF7RFz1^21k>k zqs(0!Zo0ZYFkg3zi1@bw#lS5g4(4B)tmMyqIoF+ZP8>m^RtqoZ ztKniJAu%461-R_NT<<#jSx?-qh2vn=CBs1@onNz6nbv>IzaI9BaKB;cWqvrUi1{)( zNX%XSF2S9v5P4IBr!(x(6Msggm-T7eg{zkg)_?Rs<=b&9%Iuu8TK%-^oPg0%Xoom} zYOr?5e7%205ybH3*d|Mpvn#tVgMJ-w4IxzJNO1Kax{f-ZkD7vfs3Zn*tP|Cm@v}J`KeBv1?M%+gDe9UZ9xeQ4}Zf zzk|7;8x0$+oxBEHLMK>g16z)H<4yGtF1X$J$i?#z64Ao&T_$4+y%!!-C4ZMN$(Q2A z-QiF{tt{?5Y$+7@Y7I&WH1K9_yd$d5z`QE_U3Oy^GMRFcGSu>uMcMIJyj>|Q^gV_{ zW85deCHLwH%Dz_S#D|7DcO#SG@CR#1p`5V|3EYR^DpoJ|DAAac`%h&2`8)-7d+nQ@ zwd6jTcLyFST;BA>j1zk7nCin2yZJKbM??1Z^wye|u!Z!<+~Lq2Ylp3KGEN?T98S0x za3=4mYMt)|lY$RJ6{o)x<+-16=yWhqe7rXHzvy5iu>@t&Ytr?;R8nnFg}@C&bzzEy z>HxjE)g)yNr!dh|7(m?84764%G59I$l*7iY52denzh7?uBz5Yz*ZBY-z~ge0dQ)y> zMlwpc@gs={oP7ly(_ct!Htc-ml2km&Kfy26ePqx-ZHfMbxM200W!G2g+c!0TO%b}x zq4FuGgDFAyfz`*0Uai3x51DSml&xv^O&Yur1BNOq`*_=GgDdd}mT?|Sq)XHDi|{>Q zv42>2eq(DxR>=Cz2IF-sF1*sq^E1D(wxiIM$VpErbqpB^DdI$Vm8?dGts2liohXZT z8p+$P?ubt6Xgo~pHe9$eH5iiXYqIpw_xxep2z8qvtdxi3Y){DSV%yv=fT{{kT4Gbli^9*0ommvaw2t+jS&?R&X>QJe z6t2+Lw~&sx61+CorK5oYjj8Qi&6fKRO>oNYog!9e>OT7%NPDYNFEi5$^ui;~ zj?CJJBYRLwwgMu-M0AHbn@)O>m12P)90Nz#HCypCXRwFWj8+sP28i&*Nb3h|XWkG; zu@LhF`vPCcM%^{Snu$;jL*cq2Q*89{uMV&7qMB1`IL*mRy&Dh0p>VSWxQBj8y+g$| z!*4U|6q|ub_O1%nDHBK`L=6|Cn$;nw`Hv5$b4PcM$L{ zPL&@R4s$T6MG^>Bf*W{Ed$8V5fCT*@gb8A;X#|QjFl4!wC?JQmMy6qVBUtb_ql<&R zO+p)JE%l6{MX3x>5 z*W^kfQu@0@z#=Y2ya%i;5h{aK&p?BAZwX+mqb8Oq)1p;LFykG}B30t;*)|=pUADmm zmK+BXjVRlacw_BkbvU~5>C2acOgo-ARhKx0e$mJEQR0U4aC|71#9)M%qodf4DbFw# zo54At#AsiM%u#cW?_069*(t)%LOaiMf z+n0KL&iCZm^V>&AH%Q=q9JhyqP!k^QC#n*z0#rKbo`xFOIioyK9d=5)Q+%u*tQ8HiUpsmWeM_62I(d%AlRP}! zm^OdqH4cuBu#aVAcd{039+WFj3|uHweRRwaQASk0#3X=owkumo?Ve#)HpU4v`n}(i%6iD0@HaT`HzdbwusfUoS~L3 z<^5Op!EJ@CNnM@ll^fLU;vM4d$n9L{qTk{ux^M9lv}Np|YaZ(cHKGdX)J_kbGk}AeXt&^)uU&J#Vz4Kf5s0VrkziB^z z@%ftnZ$6GyriY6aT#MZ53SEanX|Jt14K-f4eM-+JuB_78?wlN2lkj-KJNYL43iTIF zakiQdJHx}mPwuQNs?bs1`v<AD8>jO+Mi8@1XyPC&|j{RYqLZNyGirRi^XXChW%IT*JK{RT$_V* z&Poqy1pW_nZiKnrNEI34E) zpH6G{$#KF0zv^ty779f5)Qaew|!5tX?6QgqyV^kh80 zVGvQJXF2K5*Ej)M3Y-bH>F4^F6}P(IfPiqq2}#z-pkBdI+foqS;rWMFUnZ$J0OI=J z5$JdQ5T9%_C*5G)YcSS{;&|d1>wFfkk0!g@n~g}A*&%vH)Lm5Fnpg4s%9ebsSe#HT zG`K!{k6A}!W#P?1gDRy155b3z3Kta=DWWt>W16|c32P;nvd-2>S+s1Gwc+72eJ!tj z;cjxP5uMb!_4oo&`mCs!qGkh6^5HRWw`f%e_Y7PMvn6mq}XM~eqoH^NIJ6yiNV5T{&+_x{V)-&>as#kvcLq{3kO(;{?MU^G_Gk=Cu zg3$L3Ctq905j!-@d46EaoU zq=$H2p73P$`fhzYSdV@|yhUnrrW_V`i={<1&_0YdIS-zCCd7zd{lmxUc>{z#dws%$~zL3u7;PYL- zo~mmqDXq-O^YvP0Iu1U~gG67oT%FQu9S;9-*Ven+zG9#ZeKBLuGjIHUd;Q@r_qFVT zvl0yv(w>ay*Y)2elou(6Jk%Wu9FIU{j??rC@THxR4x%qJm|kCB z(}iqI^EVqdf}01(@h71$97$w)P}{Ms*B0QrsD9|QjqM$+WeN41Z6P#30{f*V=TiD&VYAO<>rR!^*jVlJmYGn6!Ab`;Ilql zWdP&B;A%y*rN>y&P+gVc9s?TR#5A**FKr>IoqM!YrxRtRz`e>IE$_BNnlw`~C$9{3 zu#bfuGjc9gJ(lTnrdY*FaLzhIwZDLKU@~0mz(*&CLm{>yx}7UOXa^&LP~m=2D(N>; z@@R0SxSW>m-TDn6KAAq8hZ*6Hy2z9psr1%)^>i{peaVEF;hhn@BYE=w>0 z<5{>QsG3(pOMYf5nai;{`Vp0seSP6Y;3A!(G{7~&ZS`+Fj*Z9!0vK;whdhRy2Uekp zGynAmYBilu@vBu_$pp@#K^6wQ*M4RqPOL7I=U0qcz1jxk`S%Iw0gJcPS@S^x=ASvC5&Myn@lW2{ZuR zE6LQC)FlygCoO{{4f zx*jEdnMLb{8o4nRlWovdz2!Fa&osLEF$Z-(JRsAj{-UC206vlYi)c9MO?=uM>A@ZJ z<5%BOFZ=0Ua0*V!IB7Rk1{JNIUSuzH2mrSHit!zO)}3<5wI|qR%H`+oqh;~omv70H zg{ycBCO^uU%vgCU+AV$$6|RUkC`QDrIkgTNiBdj*+!^+2?YjhM9}ZKrg2y8WTO(NZ z%4;Y&cl@AKPGX=TX4T;^Hro3O3s+s&HK<=@$sU| zcQ6yqjcO==Z>OX6*Zk|HN=8Oig{z10(xjW~_+Y+Ya`khd*iparSBiV{p{~p-mrH}^ zMIX-{4y%&13rT*&o|A7KhYG~9Sl@6rYd4$oZx<|ebtPOf?(ECBvqk3I-k4pw-yr-t zriKU|8?`zVl;vC$rH7@(YZ&wsl|M7psD3L+8+i@sw^;K8Pk~JXndfOsg3Ws?)n#=U zLo-EjKNR0ZHop#p^(iiz!uWG3NA$wIu6<^>rWY}LMLUI?fI$#3>(@rzrk!?#qN9CR ztb~WPm+^ERxOSk-peL6QtN7j@@DPGm*_SK zO1T9N2u=|PBen0f_*Pv8LoW;(iZo~6m$Lifvw5$mX=jHB5hdE4f|?yy*JC3Ri>voL$9@%L!+3p$+sxlw3GQl=y6*O&|B<->n6D-zQeWH;toMeOso% zl0XY7FQqOIZcXB{;ri(6{&pB#zEi%BQIq*+xWF{p(eVg z48z?zIg_xQ?QHIJ?(U;$+cU>g9%<_NL{I1URgnwTD>zzd7554~lI|aU6}jNzujAES zqjgb~%PHchcoJ<-$$3WgB~Sxb;sD_l-rO5xtat~Q1zAW!v#U=hAn1~Gj{=(%T6{H^ z^m+D8&v#o@g%AZM0)3~4>u(LR_cLIzYbY74>q}F{%($m_lczCh=_7>A>=ads_XyAM zYTlTk)j%dh=9NnJ1p8*|7#7JaWb%e<)qd{=asefcgOtTbwZ|;FN1L5*7&4M1_`>3g zDbJ}xrK8-MB}8D=cZnln1%f@O&4#$r>)hS5TR{S@Xgxl{182P|_1K3rhJ%f&4sCS;>M@#%bvTJApW-B2tGc17uMP@b)a zbD5Nu_wl>M@27lBt`l6vRJsX>H#yc6RD?iwm&QnZ8t(kn(3xDHC6WU2!rkvr? z$$7phmHy&DLgt?t7CqPfWOBm47F=j&?i@GRQA>Qktmn{sN-xNBV%lWVYv=E`;`EDN z&beMJR`=;V6P%*uKxa+mYOCnUP@O^Nz`AZTgAy<>H-B)AMwLud?gdFa{3 zw%#ajU@VPzl5hfuh%@SoA2p@MR#oB5e>HS4UgE{8u z9UEIyUuH&_^mj%01lIg@y)H(~9~|U=TJ^mA{-NRE7dMg=3C9Ev+)OJYM2gZ|aD|YV z!E^02>pb30o8TKWnn22W48hB~SE ztI*mQTeng>$?$wv?60$(@0}5h!(g_K=3ud*(p7aADkT*Tesm1*uszgJRZ=_9Xffcpei^9}aT7z6}JE5rwM z3OXA6tTp!Cf`(*BKY>*C_S?cU@181-xH3_^%UN#5uIpbfoenXqa$7>@qk)SnNl@B% zmjrd)#+eSo7O()MDk(_W+-CMVs8Z^{lk$Y*gXY?8I?%FDhlm4yK*Le zJ59CpZVSR!VdY-^m+XUC#fut;csF01YRzcr3)C+wTRb>-$t|jJsP37c!|};WE|0D) z4cQg#9dvv7aA7sTZA2?9bM9%vU%D4+4z*t_RrY!Q>e88GAx@>YheP+# zQtsX-S*LOAASj}1^vhTkh7YzNFo=X3h83ut4Dbg?_qbp_UvD%F&h1lm_gH9pH&9olZA*Pj{z33dM7~6H zK$-Wqpe*&j#Xoc8OKYJjx z5LY(;p;8r{Q2D{O{*rH0wAx<~GF81)20d^-w28ybp1ZgseLJsTRd8|RSMb`7bj(Uf z9C-MP#I+&P?3~)98BHKBW!8)>^a~aII^mU!VdpYt&LI6hWe;+?BfjWgy`+iRn|0&a z9;v-MU%wA5PJs5~@3yuFt9%3X8>lF^;jcWG!{xV@=8Rk}(63-R^8$X)GucvhqI0Sr zW0^N$RY8p;42Dow9RzmbQR-$AwD-1vp&e8u!s%xdjY;+P=pDjS06M;|hbfQ$F3~04 z7v7jEE!9rcsP}6gLAY@h6T_o#g$=|DIHw-vx8Lbx`Ejkr@E^=#61L^JCwPTmuL z=lYP*2{+qYVZBy*q#ux8<#zf7QkxHQHut7jC#n`065#m07AmQ;&HlQcHmSXniD1@BMF{myu$ritiGVkRn9eY2x~0;#jOE)&|L?`1RYu>g-PS; zL}N9Oa?_Bnu(v>wCw^2P`QhXAr%Zn+v>*gj^Z6Gce zdF*;|_PnpyeqY176`~≪mw2Je8d6l$tZD)(P9Dhyl{pm^lHC<2O zw6hg_=t=iY`z*V1muq9k&YOkK*iVjIk8ebtuKwrdB1&^?(O&cVfLub_KF!(i`n}KI z3?WUAerwe_+OqO0i$YvhPdQqbx$tYs(dYkP|0%vhL58ol2~0ZkwhgraTkq2-tey={ z52oAki)OaoQpV2DkIgIPn`goj&WdQ@mjaRB+f&p)MWM`p;OY1bZ-R;?j z<9;FU>QAYd9PuMV;*#5baT1>xX9eeioE8_`ZDaZ&oT`em?YnonAE zTygl#xnY{`iIE7grQ=vf3=iG!Z8NT$U$@w>ecZ~l@^a3uWrbu;=P9lA@LH@bo%+F< zdX(x9pK4_%t_FjE_Eh|=O9t;Tno!Ips{L2UE7}ZJc<)STWTTb50+fJ$bzOeu-wZjU z=yM%dn~C!jY}7WNMT=(|xsHXv_d4s3Ij%qElCcvZ&}rm5V!3i*&g1OOCFk(6#hU@n z)!vG(mR=>tMw)J~)*Q}D?Hs%Qs)nAMOeZOY(KVVV;tl-Wp~bG%Tkb8egdT7_bEN^QNKB{+4XyAo zyv``?8-LegflSNEKEVt$qX(*=1og5u08Gyck;HBUBA6sic-@6)Hzz9C=u0Nh4N=J- zkkU-iA>X5|JnE_}@28Y6&%bZ@mwOpm_3aZY;j(yBj;6BR>ytA97E7pV2!+gX6a&Hb z!zQvC=paZx0WTxGq0+Ts8Vb5hyi?5nXHBlWS&Sf4!>`?!FjhJH;vX-uAFo%XX&R%Q@fR zTHxZ_U8h~ruGikXWcA+c3H4rwhA>j3KS{7$1Uv#*FJd>|R!r`)!o|`eXN`IOwBfWL z#MoeCt7!Hzd39fFmppI}2ctMH^2{g^Iv9(Ry9?=@RP3JPj63DK7U9NED@rN<1E%{t zSH%a+2E@Sn4#_ZC3!s^IN1I7W$~cVZ5PzHFs6@>Ne{$S}q#Vc(kmqPMw-2Yb5vU~? zUD$-plt0IGLIf{cYEfF8v`FuI8eMvQ(ftRBTQ1UYmni#zd-Ra-TAoK=Mev%shCrVE zJNW8h;!1Iw&FN`k&;I=Di06x^0ZK>Ea4=j~#U*xU7nt*h<01^JjkeKp z$h?29mnYTWO`|FxuZd-s4@S)id%TAkcunx@>+t)a??MZpHYq1)ihMH z#qgte-fZcgQt1)QWOJ&V=$A0QVy{F6s4oukKXoa8$m`dhe>i{f5pma2*I_fnv<74M zh^?ygVw6UPmz}CQ-{g`d&JFSTmcSM)cGwhA@J0Lrgruw=@&)biI4Z{H0*V(yO4?s= zjZA~>GTJ=CH>>1!<3ri8y(GBz!hIj)VTll$Oz14XsfGsd^JTG=up zH6n{S0Gpz65vwSc;zMea^?VQWfxEjp;iTk41Ef`kEBNYj`tl3DVv=Iwh7ReG#BT{A zo*NIJaa=%O$7BTkx`OFaCiUK*>0HYRUS;kfhO8&ROfW zBfM*K%w$!sr=R?2+Bq{c!%RJ7V;#$;#2s(?_fPoy!ir_4^6`y--{;jF*Jzb542pG> zgdLMf^!QviOMgU_0_F8*{IF^NtmmGt#*z!Rf z{fA9RcpswiyhlJwwyzo;PvmuNIoTZJ>o!&gWREOLOy~#d`J`mZJg)hy(Ha2jT>amu zZ-rf(O|=WNv#WLgJp;*DTj|ltHn>r7trClIQA2HWjXrv0mxNJ8Iir8HJn*k_$7g<~ ziWqYP2S=A5NHLijM2Ukn-NL(!iVQ+2IQM?)HaseVt(ZnuARJhnAn5XQXz77d>*K`M zqQ>E2oU-zQB=B$jye~$xu*MsUGjP4<8Nc6xs659b- z8d|z*+tO7GT9sf1{2xehWf1YXkHhDuXvf69Sp<*t$O?VyW@cYlWuUJd&p#<#D1Yj* zX?yiQ8cadUcu?<`C9^xjY%xzc)adKUN6it(N-$n($_m5bBE*LCoR#1DzA zVq!fup0E#^7Nqb|DIG3cssjErR4BB@>G#1@B)dfCC&1$u(#Q&%I@XpP0;vQhhtcqr zMXFEt-d}A^YTjqG8cZ3T9ihPAT}3%YsAbxjRR!Ii;p9DR2>JAO@*CZu`2If~!3e)y z|4Y|D3i9H^$KHk&YFHhQd7|&~cp^5oA|uneL-qL=9m&7ANda37vaLYs$3#Ik^q?HC zC$8#13K29QDE2}>Y95kdzt3k``+Gz8I7Wy1E#T=L?mUVQl*u9P!rAuMi>#Y?l14;o z=J|<;yiSFPhL-jZn`Z4nGJH66sQeKCFX8cs7nFX*B=U|9BtP*t=p+l3T5x7UVt^YI zJZj`crq-RKUrdl#gY%V87pyX)8pep#o=X~=Xv{ET4;mFu=y_fVg zcwsZ$#dZ7=IW){axN2!~;(s$fa$#Zc9x~55qV5vZkuSCJwaXWysy|IHs&CN; ztVZX{J{OfNymwssRO{_9yBHj!BTBRGj7WJsM$tZ5dz_->6;f-8`uN@~=g>2MgY)^{ zV~Rqb)D(<%pr)v5#BNuKG^D}Kc|foR6tk#R88%302RN7NF~!gYuBbqXXG~!^d?Wlw z`M9JpyO+o8?F{0q#g_ZpmZBX3CHo=2(iz7a!sBk~6;_eK(}W2k&uRT3vjNsaP9W~X zyRz0(h^jWbMIJf!Ej&suIo6DKLS1O6R-I+t5ClAwF#3LopTvm{g0u7?|4`VPTwW-_Tx@!^6@5I?QvsE z#p#E(nG*RQ;&?CHdzv^Q`}U3e)IE=Jw%qGZ`;Xhn?hSa{->IbL^qZ@c?nGokwY_~& zr1$Kw!?MPxAacz49sihSo#(l>d%=5yj}`PC88+xwu{U-oh^w19i#%U$l;>3~>4K&u za%kaU+%_6owj+$AxI;23$NR>aeas#@Kft+2?vZ%~MYDs!>>bk0N zDc<}1yQtJ~#e`P!9{msN!4dMtn8RaD!okZ#5BzD4k4!(n@_tICSs7MMEq7`1eG^Nd z99r%y8J>TmX2D`LzN@l$spYhk)wJ~mR9kC%OU|};3AgdPC%=OJv$Sq2AR5NPURXAr zJl$=+GGL-`%lX3~{52CcaJ>GR-{Hr4S%)JI-sJ9rcOAj<<(dO(s7OMNMEgk8$XYiT&@S(FBZ3VTC`^zU}11UCeJ*c8nCX3 zNAQ4ei6&Is?JJu@BN&X?=%E)Ux>Vc8{CU(~Fs#-_x~a2a%&~o6d9Ss3&;8B0^cxj+ zB@1m?8z((b-`|9?$k0)l)my?dk`b5`l~v*8PXqgj(!*4lW-((W+i~2U99`*&(Rr?V zIp4U4+w;Swxg3(?*gn#GMc+iLE=u-{%uZj^K5niQ*eZLYVxVHDe>2p|*64>rM-@dX zD6Y&37!{>QnAVjWwK*#p1T4sWu>R3<9k%x$L!$$+9Y@plWqpcpzT4VG!IiZTq84wsy$kFQ zFI}m=O1qYJXf^Q0`Z3p1p9?uEc7qJ3QVU{FrCLqt`{}fMuvOG5IKZTRAVuLZ!bpLN zI9GBS953t7aisolI7jqu+!FnF2wjOc9m895fC&+j9gWg7tCs02NatQdBU9?7aHxb{aEOD*)SBH zBT^ZBH~mGNCx~HEb3C1Kr`bCSo336JArP_{Z*Q!Qq3WsFQ4A~*Wgc(l3O|LTDQ9t%V--NeC zM<7|g9FPO$+fVjU_ejo)ZsmsZxGXBo$nw%SK8ms^NX2cniN^}`_%d-FhuCBVFXQ}* zrd(z!-u3xM2i>RDgBYWCUS@Ry3ua^MsvoadkkbY|eGVOkNVHi5WW;%cSK?#YiH|m_ zgH;B6rTEIo82Jv1&N6bvvX@?A*%GQO5HV!|NWRUhQNN8#r!zk`pUss8O%79-T8Ag! z!2wA1Qjc>p`@0l*GRelpy!8iNI=Ds-JhqvdvCF;#?&F-fj+z=oF1h=p9zM+eU2%6tv6`vNhr9l?mE`IYBZOLpZ6Ef;eA`_XC;oCeb%n21h^3!3HR^|`iv%y`{>yo5p? zw_CeZyy5lNi&!^4kz}P+39UsNCbMFYU78ECX0eWqx%)0H1+3_$e3AUe)oc|-ic@WD zB>m`LtfKff2!>@Zv2m-HqKoM=25ALZfE(@7Voop4o_R51W>&^bOfrJUn`g>uBhE<^LOUY#orF0n(EgU2S7^5jyR8sZ%8{vp&8nHE;^U*Pij6SRQKmQH`x#uj^N<*L-%# z)m=kBF<<-T*=ynXmM_OW;$eP#QEZuhcp|8@*B%t&FHNqA4F8;9%7t|8MdMO!A%AYIgDpQ?vLCjm<3b^Xb$G>> zCM9kdGPlnzn$7N~m++4cAr=l1q!*9qL%z1_P z2Gy8SGHM`XK<}W7;Gd;u9|8*;9^{rEyP9w8*>)<+*v{@QEzi5(-f;xktV$4I$j!_Z zA+KF<5^@)!fD!N{m_b3+w;YOLy!Ty zZ(Sa`;5cft|KoDbsi_6G7CO#+pulX*%Aj`HJXFUUt{^&3w}RElP1Q6BYi$56%b8vq z@>41o1s&^TrmhnIfI)`}l$Yk&J(A%cF|*aT3vi=#c(gwVZ|eg4z%LsAYM=^HlnX^z zEMjF8`rqsJ?Q_)X{4mz!O1TgnDSdN7O}&tQ$uN`$aO)gQ%*&>`2CL!jWJV zU!aUP`qMOMpNi{acx_XVSdZCj=2f)tJ9r+wweAez|wXp~Ne(hMsU4lKs zFoJo3UYOd;IrctX|I)J;8+X6Ixt7`-Fj?+cde6ru;L?c@lUFLfqo2Nh8HquoQ>Z*N zj&Kw-$F_gtn=lnZBF>lup^^)v@;FPFyF;ou2)Q|np|pvuhqJ$D1fPJuetYc_cv3A} zC0MR=?B?j%&nQNf#8V_A3<21Fv!Y1t8)smJE%X^EogobWP+RYZ{SDi0qll|%5Lu#= zXSMi%+|mRfo5tVs?tn2RI?(V%5Ez2ks~37JqxBOvi7J>j^EkgoYLM1^gx8nPA@7Sn zql$t9f~&C@OU3#CDjPW^HJ%rx|y+`mtj! zfNXtaNBU2xo$GOb;*rrMOKnYVTax`*h*HI0BxA%zcppjy{!pR`>e^okhoHW~j#p?| zNRcxS@*xfAY3;_wDfIg`#l((t6-x;^s&3J5E>=3+cBY?_LK64;o=oJ-i}tukc5O3m zW?TAzy1fzI7ws@OWO)A7nAa>U2_c7+`158qHy)#cuz>q!QyIO>V_E)}+yMWEt^!M> zM*ok5Tqd2gqE;e;#j=Y;ksmsOdCUE1Rm7iEr6q)>RSXv{hwjOq)#Lk&r%kT%)_YDY z-C_1R3L-(Eb;O}@!tXOcn-@0T()RmYuL&acm|Ho(@ zz&r5PyJ7OkijRaTi85b})Dxqv`S<7_K<}?Lm)8^3=zRjnh|OtChRh|ys)H)VIXuq3 z^BzD{?0^ZcG&tAIoVoiYJGs;4OVE7zr${be?L}kHSQ>Sz-1{R(xm0V&;&Ddt8S|>n zi<)PqD$Vz~P6=bdn7eNGE(OBW|M2KM7P%Y6IJhEG;dQnqcc5ib{_NU;bK^vfIa0)r zpD&CA68DXL3eqluh~5EfrrXMJ*ejCvw%l$e{JLWV;PX;?-8v)eH8ujw}84zkm<85KGdh zBNbXg&kD2EcjYUy$w%}p^%sgqf3&piReBftzn=G{t{T4CNB*dy_^COXE^%%-s%aRu z$6LtT^*LmRJv*E0g-kDqKPR|yTFQC zA@POX)W-RKbOL#AQ|&Fz_5y92aL6~#8kw@FZJW4B zYVm~9BdBtVZMAXtBij4vT)LH9 ziqg@18GS|fwA>}reP>>dmnMA$0N?guZ^%2S>~J@B}L} z`0+!j*GBzb{W?P)La=72N&&v$={&^agZSta(-_2WyvYP229aE_pRdqf^{LOVbb+Bq z;$c}Wg;h&!>7Q5r$N0D?F#9$$^T~d2?}>|tA9Y{L6(1FM5mfOfn#I{arGiOm>P!WZ z3YL#{7L#l&Aor_KmSX&dM@Li)yTV2bO7RHSp{kW;EJtZyx$EtH>Z8N`38=Xl4G(Vm z6)_hrxYNe>DHrLC2t)1Ihaz)MN$3+3ZJ)B`EoKMU4o%|h;12sk>a2DKs?@={u7M&X zt!QmtdU;U#e`9annqDgz%Pn=PHuNzx$!(wPOyjFi65mz2QI18FI~c!PqD|4mpH%i- z%{pUm?1Fzg@gx&d>)bg!PGNwZb5CdihO5-0R-YE(%|I`lk^8g>3p9>kA_O+7Vm3g#uJB%13Gp^KB z4fa@zGU0oXnS<`N9z!OPHG}$o!Y3NIP;&GoMPvm`_bbq63>ABUK`*>-$eab#*5;tYN~^>9_jVcb zwrEs_&VU5;+H=^36lq3KC@%UEbe4Dbr3$ zs*xA2X3xIIeu7QYBv+yYARTXX78fBluR=Z4WW?VErs8UX3ZYwDu^%Bd1-bVT8-{6M zz5xCrKbAdPSc`j61!A+|3D9PA8^t&Sal@6MsI;s3!M^?kL|b6 zh%14(hq_B~#NX>PJv$otrdx?t=c+M{-YB-UM(-WJb6-jSf6?H7gXRTE&^@6mNViYh zSl}a}TQLI4Zrr@U37W@EoVHV0BIr!40w<)pNi?sleedrz+LMV99(Cb^U#@rm^@qc< zma)llyX8}=GL=PY;aw+o16!VAK5Dm%treo__tOK%0~bwf1vft+A`ZSd!2Qnr{D=Eo zh|R6sC!b4q_S;pSccvbT3Ux<%*21u3vn_@y+EC!NP}TWeA9p+IGul)?ms*^dkK+`d zDb=by8#3y1J=1o>33RhSdGRYE5dDD4`vCX>9bVPE*+Jo5xT-I%>S)K=7?$@>sokD^ zl{gZT|IFN7*%P=}7(cacslIm?z82mxYP@P1`*NvL*_ij^bqt5k1TMZu&T&Mus--@0(9s={g1ON~-I)iTD&+c@ehDR9-XC z-D&YnkYa zJJdPjORONor7_Gg@Fd2NI*DdbCA-{trv;ZGZ@`L3Ot#YJyKq{&sj0);byc7oo&yRZ zawu{}VBfr{UC6%wL~;lsv8ES}@OL~8u0$^?Pkm&1{+sJqij9i!kKP^G_!sCCBUsgB z8NyD&d)RhjAm#r-U~fYg`5c|IRz{UT=F}Y=K zxo{h;f5vrZ;WqciJY2SSI zP16?{$5YStzVb>dt`0Gm8+4}2GP7tA%R(Bu@v}B(pZ#D86tc=bElLiNZ26Xq1r0+6GukFbCm1FPvkV4PP!HYPtA^|Re0(Sc$jwJwuSrX}kCA!j zt%-e9-7W>ZDL*Q$X}XY2{0Jg}#0pa6Fm&BO6sFmK#oZ>U3Qpa5A4N|l8lwfW@w zDal7zW9in)2Ws;MTEzrVlKh#_rQ7^#1y%*}=Z`*bNW2=X9&F}wpcc_GkbH~3%RGU| z)z-`sAal@(cu!(^S0#j3`?BF@IldUmCZI*xb)eI%yXD*&^5Snc>> zepmmg;+0UB#?L9GZdtCoOz5urypp?T8^bx=qpi9gCNbdiq<;fX`hjS#m_lUbp_M{9 zRCrwOaGZ&C^gzKcBzE~D!ZFd&Hs1ULo5Q{kmb0TE_barmKR2Y5W3+?o)IhH_eWQ?9 z#SFMVK9iltQMz>_+yohEsn`V=0Dws((U1(oc$bM&-#F|22mnRygksom8C=V}*JShd zwwE8b(CQsQ{&D5wQ)03aHGmA6H?j!H z=Dn1iBBMO^iUq090k#LwReeY_U)eEZ(OJGFC-Q%Qw6rh8XXImDYcR($FJ@a}mz zrWPc7p{I2?`gx1^dSPTTELCL0eiJcs#-_N~Hs+RUtL!>+&rd1r`UB=Jf{F+Z74l_l zp+>f~I~Sc?#zXgz9)f0CP%RY2T?fFnM$%#4=le$zX|O#lSrv;7K0e}vwP|O*^aQEM zCch4McMj>S2V8a3)>(=J>%WKy;#U9@d4}ep0J8=3#b)>giZDnQwe2Dowg;MKZzWmD z@FvfFu=<5Je+>#N#kC-kW{=j>t>AS~SpUTR8k-ZNA)?bUwbzGByv>$Tt8)3L-UBru zSXX2_F7+Xi=M0DRq7s^!+L-%X0VS}E-HI4H*#*3;1j;l zZhY^N@sne&7{?rA^sWdWX~$zNMzrbKwY|#JRV$%iO5cnjTn-?U&5=rjucN&Ub1Q-j8Z4prqMH4QdCrO=o7(Uzln`iv&#U@VvU;B)-mN1it`Jbh2x zRV%ss@_3lg5S>nJ8WxqxvE9Z0PZHiX@5SZv2$m_)Pi& zIuHurWk#}44=4e{IkgsrXBVu-()j z4b-6fLYN>6TM3;cr&I;1F9qUCAST$4IFSd%)(p(6Edzp4jW56bu)>Pc8sS3CQ4vpOZD_M&GSWRedu!6` z`j%U(IxjHN!0*UfzR8DGxK=b80m*X&Pby~>D;a_=H_r3XfjzZg5Yw}A01F#2p5?dkBRT!zz^ zXGTYYk5v0;+x*%IKKQOki9g&+*^T!S7mD2Zw*+-SAUi9K7AUA(M!nclIf@65K^?iAm!t}8ALG;7Iylot^c6NC+g zMPD@4te!H8`nK6Q{k^_FUeln0{?HY>YWynSq0W8vi1}I6%vQ{IJMZ5!K8QZR0Dt9g z?VQ*yE&LJw{G+b3co*mwURZBpA6fz7KMaG~d!op@oAx;$s+2b~6n4tI_SJ>5xbphH zr@D{TYy@iz08|E9>WRr|wK&hP$+Buet-+ehSKz|41O`zt9Fd6D7Qswfe3i02MY0^2 z>?|cgPZcy!-t9kEXb5Pi_cQqimGK zIUukWP-PH)Wgq-UI>Pu&j(9%S=yp960Uw~IwB>+&f*tc@;xr~|kI0Od-kJS?h2HV$ zI>udP5ndf2(TKf{JjnTWIo2i7fl6&``-TSmo^E}_9juaMwBUfavID^(k;t-ygSZLr zMG+B5Nul;C(P`e+quUdATFJfC=|<>&KTOMu#(A}R^A&$zd4lN1^Qk3w_U5Hll51oa zqFmMKjogv9HTjd7>z(W0RC;b)KY;vD#&)#-E7f^RFY8IMF(LeW%(T#UP1y3q-tb#2 zOGBhwEx07k1?3Y{m!P%)2K1Os#f~kO5 zZC8iiqb)ZelSuEbWM_Xk2q~()^8+n^c7UPNPDh?>TTtV{qiY{I9v6Fy~% z?u*NAL0=1wweU@}^&YVOpFM<+(^Mek{rI7c|S2o#7_rqX>aih)SuDN-y zJl|{|zae*tEP|KgCZ-Ip4j=}G8G(uL&18Y=6!Rj=6O1~n&0jxd z+33V}hw8_Ps*~r(*Gz^_ef{S7a&nIBnYn44V*Lz}g}**SfX_j#-xMp@rs7`H+RemT z6!)jp@H${Rs0f)4zEgxi>v9a?M`BPo+4=l|URw^jhngnPM`8d~Ns<4=W(VK5);L|h zvo8H!6*9UAm+Bbf?NHy6>ex70voURSFMMHjrF85|>kGQ`;Mh%ZT$kD+wQ;hSG&N7t zHHA9asqnP>!E2d(e+~!NZVx=PP)RKobvb&mwHG<0B!nvdG#7C0%h`)-;X8*~NEq>h zvg_{)ly!AqX09Guy>IaK{OZ^0_L*9Ej39_Wv7K`tKk+s~ufoNps-((JU&AJxtM~nO z))#fI!Np5=k>zH2sIS8;kQPU?VJn+FL*R$M4ALg%s%aVCRQx=Axz@5jyOlBDjkI;6 zU#cymfe5AS3!L0?SYBP3L=I`!1@J8E>aE{dq+BdL(>`)@)aqQYUoB&c*kJx2;GDA( z3W^C%YY%!G#0BY?Vmwi+)>ICwEkZz)YP`*aiNV~2i4(gv@JG*j4??~}iK7#gev6ip zt5>+a?aTYTBhNzLsKuF&)Uw1%MH%JNcZzb3VX#HfrPU)$rwh{YWs7fB4`^IcDYyS5 z`Es^perLI}Lq>@6Tq@FXsm0-&t>6lX)6RrN;7(9m}oS!D|Wl3!s6!=%t^yu_$toN!&EDrA^rA@hu=yuLU8!Vo5#HqV#1~`#q{d zMN!qsG}odGpU0)gOe!u+jg;@Y+KzwW<{sV2ocC+63m~|A61(+))em8EQ1};#L~U9f z7^vq|b~x$PhEnjV!M_HqSt0ae@9@TQRnfl|CgV>@c4mq88r^uA>XiCB4%SULId(gHyhb61b+IksJr~*f=&j*k@CENCNp~ZEZ>x6vm*!1s zY=@!1!9li$Qa0B0cTv^gMZdCH@6|18$CPtP)|skM`^>qr+LFUhiih?wvdRz7Oyfs` zTc&N!i-Cma24#vV;P4_zY#7N#tz*oO*Ec@19%tplp0R6!YeAQ|PU@9a=_(!`Ak0m!S8NeTE+J{9;4HvN=RK;s;M zI~5p1qA7CV!)R~?=5%Rie-4j+WvnR(kn}BNl)RsU%n7fnE71uEUqw=y$fr@6YYcX1z>RRBQ6qM z5MG58xx`fi3f__5DmpUAmyxIt_uvnVhb!vRxcPUd`*e@A^3@VZJa5|8i?Tva^gcxI zuV}@k`)cD|g*QT4wv?uHQtc!VMCawuV7kDG?>uyx>)9Mll*M1+)zt#)(xK-w(;t@& zN`l;mFfro1dr^KB(~OxWg+bM?Q`c*5uPJ3qkx)hPntf+VGNRfR)~W7PCpzszspGZE zQ7}ki=)K(2e#fG~Vmv*wW9@shdpp)`il?7UWaWe0Mijv&|0+5SMwcMClfCE%Xhl=- z?S3e}MuT-kYa5^m--ryz(4`uY3x(LDBGb3770_m|N1RCm^z*{%g^-q(dLPWzo}T(q z$wwv+iJgiT%(nC1ck+WS@QojEnKGWXrJ!u~ry(O1cj35>BhO#H>}nnKECs2K{z#>|923J4*GktnaD%smXO+%L@h*g4_jqpx{*L@r93e$nzDsZiRfE{@h=2 zLC)ern*-3?Fo!gtv+v;8R7n9@MY0c zn|tC6U>Ql>Nxd0!i(~u>jOv?QH7!8{psob4Lg50*E1pa5RoOys!3Vs{4YjS;FAnm} z3s~0X5EX-n*KYbcf?<@|wk0;$`RyH}kk+&XSW9G!bJDZm$DgVT{$lrGsoNz82 zTJZ{VB-><$1{OtSi5CIQz9v75dYX8X1RWuF6F;6FusO~j=q9Bins&UHwHyuJAZvly zD|%EO4b5XFT_Rh5nRrZ$3a zT1n?EQH)oMyzOlV@ynZ|M_Wb@x!<5m1v_*#b>XT z)G*4=;+^B?7;}MN-9xeN>cekAro_rb$f^iBNdgX$1mNQhV4d&MXUK`1(3`Z`N&I6_ z84Ab2B}zFTZbFKx+h|A@*y1}qTT=oyFM^15Pq!xbU+ov*tlRz%bQzQwpNXy8nN2dQ z2*kigk*i`dFrqmOE+j9_jioKJ#(0APN_d%+Pq-k-o;dW5ynruAlu+fAyho6X(2IJl zqIin--Mw#Uk|PP;`ld1V7x^0p*8ih_69K*(0$A*$)10zSXM4BaCUZ(#)m8>d+Gz^c?KV#JMis5N*dtTmg7mj%nS>wMN%A$ z8|FYv-%f@cstJ?x0xp?5ziKKoP{OxvOq<%42wEW1EI zrM?LvYOiFAMF^6B=BPbs>Xx7yDlG?rQVDd#DO~Ghx)%}LtZgDc#Rnbk)eglMutCMX zftkJw1qbP5x6zg0B6`i{vbEWuuaAXgv_Zt#;1!dxhD}ZC4S4*0?8+vXuW*kykWPT( zZVqyF^P)>cp3utz zm&YY0)*TjJ7W%$Y)P=aeLBVYKr~zGrf}r4Sw3gK9N+ec;Y;k^T594{tL4IOX0q5{l z9}T1IKYlUIK3JzCdqF9+cdRs##$Fijy&0-@yK=PneEL~H6swtbinj=_d>^4F(Y%CY z`Zw?G%EzBCPwYAN^4V?fd6NbDlj@ttI7hTkmhU|8p5--KsqK()tmMZi0(km+62-`gTF!xbyS-LkektR9Z$*U`eziy5J zm3^PCYNABR=VtUj5vr;7p;1MBCAf!-LGR`@V3CW3^hn+Ck6e(o^WcUL>Ndem> zIqemwK5GgQ`DiyNrY68;s3xG8<~6<<&x|AJ;12bgO==G$s+l0q%SH_5n>n1}W*F|o z7JGLK{Ar|Aw1UJ4gP2AucDR<|Qbyd&UZyZErAYq$_f&OE`Slv}mRG_5HrPoexZ7<% z^;Riq!)sVN<@FuSK+TnQmH2@_mb-`LQ@c3^H(!inTIk``AF92caNhw#thfB!C;Gd> z(ki+W*_`FfImj?2qQ6PO%uclohAE5i$E4=tNr@b9(R$E(XK!T}Gz z-WNEcGvG1b;$T%MNumqo3yepF{5#w}!m!qp(Tdy-h|NVviyZ5ztz#lXE-IxpN>^jV)Uevq$Yfo0p-YW7 zm&vB=_JL?U{C{CR1bzM;5+}y{+^YngYRdQG;jbq&ud5r!*L|G95DtFmFulb={2Itqz{4;cB*1~L)=x&jcjIkF zJM1b3B8`ts5@yO5HN2u0<<&k_b_c6A1tCqMYk`Qw%?e`u^;fH&_7fvnx}!l~RRnx^ z``G%RVSDvr)ZkzgaS-tk#Sexkb9QSGnU^Zr`Av9A zbO@3MY~MSn{nWiyF+6W>+K)T69O6zenFXy_ZlRwV?zvH&Z`dzUo)8`44RrqyWN|DF z!i{up&IaBud+=B8pR)G~uNQxkrMe#l=K?AsAlG}LU^#~DbX-CH?vcdu!0>r{IcA_q z6`J8H3?LXvM%om+k*7VICz`L-Ll4;bVST#CEF#1O)g=Kx%Bs24UvNFNbG-NI)fh%( zbve$TEg-RKm96cEuy+~?qVu-e#qc7Mzv36~yP|gNi{lCk1{Lhei}snth6aYCA!~Vo zn3t_1tz|h*Td0BI-6B_@G-r_pLG!3Lcb+5xl^2buDRQaBD!W z0z4O$)|9WozK`ir>*D7muSIz0C2y>&&j#EPI**we<{vumeHe=Y)!%)Q5){GO5SJ1I zD2a^yqu~_BH|}~W!2k+z(Ig2kuv_x)-7CeN_G)n2djpT)ru$;GMDDzzqLNU^n`4$P zm16cFyK54P=K)WszV;YBFbj{Sm`GL$Hqn_I+b~2Y`R?K|(~UNK?BW6$37IlU8$dvX4S1sX#M->#O(tb`$Q*b*xhadzO8hJQpt&PyMt6u(3Im+ju!n^xi_?^>Tp2e zf1)I*qgNZzipL4tv+LU$$6M#I=mRdaGp?zG!H-wldR)@a8jLuXR02q}+~#&hj#15M zj^XG@8&g=lZG00k8KEgL#-D|T`CDvr62UU4#bT@1$e~;J7|&1eINCVpKNtD?e$9!o zHv2`^{K<#m0{X@(T&&@fN=xT_&b-=niLc|8suKj$;{$j4NkI(XeT7y*c;kgim7D|> zJDNk$Mu4#-UWu`yi>zttK9L{4q^<5L#g3zx>}$-Q|Jy1!s=~Lc6YzBFEw`MhEEb8wDx&$CEQ&v%*a##kZyTN3NBzw!QteKgO~yzp)82b^YdQ7sjv|S$4_E z8pfD)(!c4yCi3iuAI$y=UF5~GN?ZHT47e9B>NP6GVz`eP=WgC*$s;FyA=m5AVP>CA}vBl#sZNhBM6KjM5IKdL`pzvq>msX zT|x;7A{`Qiuq4Iv_`mi(?`~(G|9h^puj~CVAGq=*VI^xl>sj~xYj^md)_O%N?DG#p zKpeWSkTfSV2&sc2ZvNY*6cy1E#^TTM-!|EA&{P`{&J)+Maq$^R&33`oW@fxk<+bZ^^(dd`c;qD59fU}q=K&i< zUK3;J8~BF=-9WmI<+@9L~n#HXl$T37-h zbkFfFc=VM)6GT)F(_efLgD>D_-!7hIE+^WqF)x8yx-is(Bm66Jy%;kbJ zcaPZ8Yu&u}O`E&!942>SzB|CzL`@+N-5&Dml%?lt?uvyo0kY3YJ@ZILz4D-04)x9ELI|RLU&s?0?AQ9MQFd) zBKdnOdiQa~Yw=6AE>!~r^?bZLe`w)ihL-ixB6fAbSl6nR!nMMUNVOsq%#;WbaqClL zGT4Sps{34{P%VrD_14&80`SR4KAwqhL;NTVA_ACRb)?NEZO-I_H?ufBKSqu z3x995zUje`3nfcStE?ZHb791zclw6fyXoPT{UIN8Oyy?y+aX`|*SPzv&DX`Mgzzd- zf*A}nZMFc~0y&&~*pEO90>l(>zvpx5W->!U63?f_3=uQhk}~HG@`BIi&blin`R1Cx z2&h&*6mK=4?`6+lS+@CC9CtKwc4N&e{EOE7pMeo6e3^j-&12=(R5SlqZ@SXZ!8ST= zZ!AxtD`)()#O?$@zTI&UszCb$CPJL(PqRb9NKpYmr=k(C-|FMr!;rNzO|~NM76I-; zC3iSJk2_QZnh30sg96^Zy{CENu;Cv8Hnn#3H)rE>119HVszKkbQr(QW%>sg7%19D> zcxj8^A{sbEO+8UrRQDQ>>x6}etNwZ;K6&7$hO9Ha(ODL6Y^a8nRM^Dat}nKa zL+d_QPcxcGpssID`W*$1-!lSA&JGq6X9I@RP2-U5uEP<-O*-c8FU4cMKXI$$Gh2+n zZ`~Y@zKQQ?PD}Gv9QLibdofMYtP>1g>d6V<*x6qY?*w+&M&Q>*ZEie6e8<=$HWl{+ zODBrC!@jn^9*v1ACCChfV)X);*Uex&)s^VbGsJC0bB1CKZXSXgw;(;%@;uMoIle(o z=V+_kko!1o-d85_v=;bTeXR24GRSnWv z0Wp|Il(h*!hiIoCs|@SM@Q3)H8|v@QpSE|uxY3~NZg?rhPN1)T%TlN3v87$r1Zm`NQq_q{@$<6l&zXC)?Pm0RJ^erT*r7=P%! zmC|u@1&jU1jy#7X_}$>!`-$I?s$p@BFle2VHPcGjgwL{$uqQ7UU0Zxnc;n;JA>|U>|Ni+MDtFp^z6K*h6Skg!su%w*Q@D`s0VpglILWj zDm5+jZ^@d4zNJ(=cozq!uq)n z6Oe4!Ho~O=q2i?1O+lNy6AE9YSjioH@Oa9%SLhb0|5p4HcZre6=8ERWdah+$RS1sP z{{U%%^_neuLh_S}L&!Ahf+mjK0gDC&bS4!d+uvzv&OZHql1plV>d&ft4&;*V(CQVV z*jcVRyGlzHo0+Iqks(Ky%>J6K*nqHkz;CgusSp`GORtC#mNkslx(+U{AO7`5fQ_xB z4G1iCu{jZxT&Oodl`DASt1|xedg%%V0O0TuE%7d6ZaiOo$n3}h<%jQpn7hULccR>u zowOgwbQUPvOe{8K$TF}y+6?xg2ojs&(s-vY&W!_lXOE@W>BY56Y|FGhE`g#jlr;x?qKublB$5C_5*n-wvhQ#%uq&q zylzRM=iIQX(Ng7p+gnBfX|<^pQP~VfjU4AY^k?ZhOO5Fh%(eL+U$YM{)@YnFYwGT; z$SSO*wBGO@%qUSRz{Dq(B;PC!J?U&GDD)DUHn)O%@iTA(nLHHyI(z4Pr%{{A8auUNcgT(tMf6BK zH+t^0}~In+xTb?{9-UJfor3ivQz&5X+z}L;(Y@?H4W*$%`K8o z3_diffRklR{EK)P>IEXX0VL?O&wvq|(~Zd$^B|?Ljbn@T0ppKy$9hb>H6ij;{MtaH z`uv1WT?~756o^!3#m4XTB@&Mqd+GZ=7;1l`^KH|m+1!z)@z3o`1*`Mli--NMC#N-S zE(bV0n%>oEuhZ#F4YaQAA^ML*GOb4^Zh}%Zn^ol}ls4wy6uqi<<_bC}GAl$?yln?V z5}RVrT~8#+LZvM&%7>$lNIKH%bdGI+vPP^{>905b8>Yebm-_OGvLS8VmC}z@15;mq zMja46V@xK1VjQ^yzrr0p2Sko5%u^OzS~FG)RSp7T2`8rD`~ znI7ae1y_Qqf$V}_r)Y<1n>frv#OtGJ^aJ~CPlrHmlceF@8X=P_cxpH1>nLXaPJzTO zmx0jZt9c7IYfB2UtR!xJ`TWk14n%cYU)(&VRvy60ar)>W*7P6bYB8@I%{=^H*y=UhGWa=EO3De29B_*)F(q^rxxo>O_p&*!}{dH3*q($N=b zDOiaiKVU zD!E9$>sV-x{moR(cJIYp{ezm3CF(gP7Ba|;&}}k~KmW6Tus<1GWYX7@B4=?|_#$xR z7Ok_Fh*wBWXwvjS6n%&2ZX=KZ&o4t56!o8dr)Mey)Q*82;Lj{rUugC$>1Gp<2&#ep!Eu(-WYsM7)eR=zoe#hWa^hT9@`dd$rfzwMv$6PLiFL%+UL7YX(O z5BtJA6(OMn0&o}PxGgE=q*``A!u#t}-#>)XZ=iJ5* zOICU4<$krVlqh0a0vpy zrFrnzX3CZH{(9M5p+`LwROE?={ZdW2Sd;$Cd;O$s1bwb@#jPvhI?(pfHbe1IJA;Y# z_O7$yzseq9FiF(sCD&<@0jado@oLcZAEoYabO{oD{B1J7+w=%C(7^d(1=z~g-dkNuGHNp`Sk-iaBSC-b6f!R$n@Rhc}{UkpH0_9{cpo<-jbqwVi|vivL1^W z&Vk379TmSvDVrFx>u@_EI7g}-v$AchdlDu!O7ef`H_4I!(P;3Nq|I}NHydd$>~yOp zA$Qh#!rihtQBljGqjlh+jmh|wqq@{Hl3Po@P|>f~1skx@xDoFf zWaqtvDOz5uBbGb_U`EKeX79biPAdHo%CN!FeNb&+fU56{w!azBIaFQ+iwTJr?Ks<4 z8Rgu%F_Wk-Y1XS#DsoZ!KmPrc;C+YUnVpK7s>>%Cno3CE%Zk8QzEWn z_Z|qz+5bAPx}n#;%gz8zQ>(i6?#JY~N0(|;54L1_yIJTTR2wC{G4u;ti>O-0hu;G` z8Y#w10tCe4_zCYT3pr30kJiGFsy{J>wLxP)($noB9J=tbe^Nnn)x&W&FnPp##C?IL zy7bolp7$Z{cKKICuV@5A4m_R-k6lIFSKe1?2Rz-x`0($30T#^3OPB9Sf$OIn%j6E-3g>%^0(P;^ulyiW3N>`qB&=K~DND#slQOSNU*F zE%-rU%L`zr6}IsR!JL=$fbIY`u1fV-lP%BiRvxvPP2wMUa%KETFZsVJ4RP#osk8m7 zraxkmFogkfUp&uMJAT__Mkxd7KDK;1l*3iDfa<>c$<0Bw7;JJSkw&oK=t#5M$v`P* z;!eJ|ghVz}-0A1YcC>goH#f>8wX=ThFHz&2^Y9NVM2ESLq+O~;;uZrMZw&xm)L{hS zK2&OAT`Yc!sAwK()6*+Quz!3nW@KD=W)fPGyVZ zF&leR8E0#Cjmpu6ug_LK6RS^`9mF2%e>QcXJxd|<@PTvr5zMDo{R2-KJDA783<7#9p(Lyl(y%jJ3((L!Jz>q5LZboUEOeAt?bI(c=Sb}WI8>|m<_z3ksN)x-Q||cwCMHO*QotL# z_nZ8j#AZmID#(v2o=r~N-lo! zjs5lFAERy~qgJvF^m>iW7JE+Q1wG##w`p@}|0PcHC(mxR(&h2Yh=4WMuwf*YK+pd@ z=YX?I?poZVxq)Xd27{l7QmySeQ*LH{B5M!8j2`9X${S4k!8*p(#XHG@19Pm+LUZm8 z$3Zh=d{Og8+p2pNxfm+qWX?x)w3PVbo;KDEn<>sZ`WsLr3GmVY@y{=Z_5T$95L$>) z_+{A0J15jMwe@AydMk)28D13dHrBXIyE`m48*fzc!ur!;>II*LuA8{ z$wj^))0I*%{>On%Y0)nAyGhVNvTp4Z})Wx}_ zzpARes;dODYkOp$5KF9qD7o@c>Rhm9q`~0%v0KSu*u}K2N5YDK#4u!o7L&}s2AvhC z0wp9+SVjR(qaar~mu+n86+D@r#vp`@>n@mcOVaPtpEqtxAGmHJ`)uC5{W1H4PRZh- zr8V?N-qzZs!I;>f{HPkiY2ofZ(ly%MYzgQno23A)bP(mCr~-?p5b_d~h`3iOw538f zyZ0YbZ&`_)WbMSO+tt79EqU)g_ho!eI-s3(J-jw`CVgMhm_|X4%yP2*8f*BHCN8Bf{6u)ChF#-@V$wDHc$ia^!`6ww?OAX};k8-~l-2rvjrXjKUehs}u(PtSmPaa_-yqVI4rkgA>IrfV zy(LZtkhCV2-ZCLM7XwY{KV^p2$nxAz|pQ^}T;Roj7NjS0=9Wg4iXCetYu zrO^WOSrlD8zTS)1Gu6>b+hzVML(&i8@$45k-tpW|hjeCMz0YjM+{$OpuanPbd-@T8aSEDxgs6U8MF{FtG3vg6jysJWRTs(#ZQZV_p zCHdA3SgX`drW#=6z|JqZHd_y$DmrvTt0-O&b;M(|!O){m-_)$jsg~m0FYVe3|I~*o zs7<%c?#8Xj)B}CiNCh7Oc?nLq&sRWxZcr)!-~y8_5Z8?M9yM_nPiE zF$7}-X|6f%eU^{_{5pTPr{baDI7^v`D;`|3jsba}fmiduL#{KaTU zaqo-T`9ytpl&poKjh&X-+Vo9Z3-uj(rH_(9F37`b7}tU}0=I>_3We7pXAbDh`S{_` znibU{6H@^~c%>F7hvH{Ljq@5nl1O%kc`Yw6@mWB)M!8)MKD;f$EXlt}$;c&*g2Q2C zl&Jy`LwDS4vlu*kBpQ`~X+%Du@i@T{(3)My8lG9=BqBP4b^Cg8XwpHij$(5}V!ay; zzrr3)68duBa-Lu|;g1bsS|kT`%B^uY6zk`a5 z*OU~ZFKC;p?ylzlVPkWu{!s?nPUT7RO~;E)=wMAZ)vrFh@b54hWs1DG!u8>MKz9W{ zi~E^L&9K|%_JFxCw5}W?ZvPOF4uW%R283tWn)PRkTe$K|5%G|Db1iVaM6aG+N{p^U zI~6lq6{^Y<0-oXGQCu7NJ&GdY8sjdaoSf8XBJKyh%}()lPv#h8In?70Y5LGmS&A1L zxHI<-gM?uD*Hst+X)A9A zvcHKWD)i&nNwdl1RnjgyjfF8YROb)QGxGuxtPef> z8Yul1?{2$0sTEFU?F)UZ?`hzf)p2q5ys)|{gxrW4 zKsdi}Ea?S!a+|@%X3vg@3FSVO-c^{S|Au8saHJ%C;#meUaoeXN{RdA%ZMz2gLsqLq zyPiF`pq7ykSZU9sEqoHb9g>nz_@hY4R2EqEsruWxWlMX))`+V$d8JFLVX~9iW+2Q` zmS6*oa?Sa-d9F#OtX9$pAEs>1&gX8SPhpa?>c2Q z5J-bxk<$A}Yv{GhTSpEeG6tjGPFzgrbB;@PN=Qzwb-MQ`2Y&gDrh0E)&)xw9DCv!; zOVaEf=7V*cY*f6r9OMF$)%*0-(#9*H`lF>#a#!n@3qv&jVZ^8$VBO>*utLN4`Hhdk zjXv@Qi{RgRvqN2Q5fD0n*unJd@mD?swXg+W=~CMsk>05SOIrt0ucZybM88_$e?{h|glGxIu=^K)EFWVg+mM#%M+JDYhlm(aWJ<%&3KS%0)`G ziGneRyyORNr(8&1;7jm-55{0!0`|uXXk`s&Rff|8h>B_G~3P zKHgt2XP=;I?_yf~>}$h(+)$xGLH?N{Et}vJS66y!V>t9iVA8?s;*)sti|bSHc$S<_ z2<0FYhT8=VbEL=)%_4Qs1q)3{GDEe?kDuER4|qa~4)Lw7(r>5%E9{=lv%#X+2^}LZ z)9DOr6A8WzKpO*v`$S1%If5f}hzz+&4P{!Xh_S> z*``-Nz6i5~H1C6JHx`Q?BChn@CY8Zr=(OF0J+L3KA*6j?1^LBpAS%=_uHGSkY`$xc zTtCKj;y@iEzKLmB9f?KzVxBK3sFU@^tu9u)d|6RnlQGmebfB>bsAbeL;_S54N5b5f zrmG*yaPjU=33KoBcQxxCfArheoP!2qzl~9}J#gc$adkGD{&?2X7Os!IC`T-X|1w!x zGlfvXOSe$=>q)Ha#^<_2;X!PSgN!FozTI9VoXzp@#xt6;PxWE13(w{|gysbVcRb6o zxURdrJuB);$eWv#J;Lv2$(|q!$qP&fH$~|Ld0rC#ZY`+LH<)uK9(ni-!DLKBRBnJm zR$6NALd43wl~Urir~PXsK0o({EbYf6+OyoM`wv&$MRJ!)<2jKYrV*oS#~wa*aeI8{ z#htfre|EWe(k=G&rcKr=lY8y)S*{-0kadCNb762?jiK2n0bn~7;(p;KNQ+m;CxXVsf;gDr&;n0lp5H(`2#M^G7Rn7? zrIVfdN6mSXGtmW&khC@tHXw9n$EpD7yuO6LGH1m~pfyS_D9QI{7Gy>KPnf(!DN_#2 zyOS$-6i&kl7HVZtrhpSqQ-)PU&V}x%ar!Ol!qE}I5 zlr-TW&)KSRs9v_H%&MYBC&I+A+e-MT$u z|FoxGww<=;OTB=}{0GgBLD_q=kAXDX=SVxciAj(y{aC1vAw1&ArAIX>^bBa2fBO4< zOpq7Ffv=~TVQcwLbfgx&PxWa%D8%+;9HUTZ8-ty58Iv+;M@mPIRD-4 zJw|FJ1@rFNIXM$^yr)qbo=KI;2=O5+foThuxN-RVj@ zMQN*o^IwW@)?{0nty>uD=(%I3L$PL-V}I1EzESMAa}M+7Ji2r>_iBxEf{$FX%^RKl zfTn~mvbik2AM^`RQmAy-kxPqCapSc!&|xL{UJC|Bj0ZxCA}IOu^1;9ma^JFjtXYxr zWA<#Qj+%I)|9tN%_M}+HM(d2i)D+@>!Y1_{7tenq-?dB&XMbKDcz5}_Nke?dQ*r5v zImaf2PTO{{O&x0x;dPx!N@d9d0V|_CsZHtusDOdunNCY&Z4zrPL?ljBaO@)aw;&*; z+dWrrQ5=BNT5vDr z!Q&)8@|DxOAq^r0?7Uh`P4D&o$dzfDmu#Don!?_^+;fk-ynUVCyHXGx_^PQDQQbKc zP<3S563qe6u+?3ZwrnQR#of|M+K)hkXc6J@iost8g|cjhobcDXj(v>>Jf(hAzpoF2 zkhLI)T$`YkYgd98)oZYFX?U;i6>$jW#*-{~ScJX_k=<$!}9 z>QRqC^U_9}%+#UL$r45OVujFoXbmR;DRPuc^E~bU%4bYf~I45_7#| z{qe;vv{SHC{p>*g;taXaz46GSj3-wMhK@L-wppbe+ihfA9jZt;BhV$jhHWSLVn$7I zXL-nWR6J3uw-Q~i56U88B=j(f2Xp1h$LKm~teBW`#Vw0QhS5TKD#~i>aTJ4GZ+hjq zH}ia&ykXx5sdCDz*=Lp37H%r(SHt`B?Gt1)&mF(RJkd}aFnSX^n>v=am|SbE;%Fh$ zz4^c8H2;gCbqr7gpebK%W7d12&r`g0j$dn{Jcl8a!LgAECWb!6 z?z32dw9uh%i8_Q`T=K2Y7@Z1l!Mstcx-%sHD*o$zfLy!g%;)oOuU<}ldvb9l+bYYY zv&c`TJmFn|Gx^EU`)}E|+iZWZv};pE$`wZLL`*pMISXzCb~@r(yUU;hHffvjZ?%I4 z1FkGMHQDa7JYD+;4EJ+A5U%eY(jRXhbT_fzbJz~{Zii3_5J;-hd^^C?!8tTB#YEl$ z#toJdK5`f_6t^3kn=}A*z#OIONtVbBMQ?~o z5`a?o4Z!B_kVqrnHVF>^8c&_mMhF6{^~JYMsSgp8nF!!Kmjq+^QUxBvAHWt!6Q;r- z_4V&_Vw#df=ZMx8!c#ixrR-o1e9_m z263MvN7Mp#Rzwbzwv(Q;11KUb3ek=7147fMC@YzbUm6X0#Z1?nk^u4H#&Uch{m`*q z5DlZo7Qf3>!4zqP<&v{7ujmKb-*h7B6IIp$WI83Z68fz3$Nw1a@#r_7l3xs za}io#hh)dCqBc%bnEfD##g_$GSYS7d(ds45a!42lp=mlvm|L3k+*V8kS78(7Yx-h9 zSdw@v<1ha1fAZ70^FZB1c?l+EsEe=s{pr8Y+JC#&{$4Hr$J`}fJNVVyPgD*7v!^&# zF?v1+2G;8Awm4LteDkqcC9^GKr;o`iGnZEhXI`aXuHQ4hejRHXEEx2ig+gj|->>~x zpUCOVi&f_{NR_u7t>Gzw*L*Wz-St=6TF643sj;CK*{Xf)Cmj;zlH_Oy?EUsnH*z0M zYV?<9+FxGmlF($HB(^7pq<$<}8{3xtTUkB2MaJJIQM3Quar_&X7XyWT`RVpfJKWx=)~ENK>%W)GvgtQdLHRQqU$`5)_G!ZdpY=;E%(O3n=+@n4 zvJef(?MyI*c&pdML%lPZt?r+wQ&Q3vyWPer!-AOSs}#um?40NG;m%D3Xm{_7boVOS z^9kmhyI0BJ+M+?9Sc2FOt5^_^k}7G7hP>~|8T*BjFN8XxtokcO^9Wg!9h{Pr>rV8c zRf+NNCT}qIYjqsv*te{gIhrItC*GU+dg3t9?a{MByxdPIdZV2emmLPe#U;VE=(wwTdp{Dp9oRI&aspw#~ z6^YoG_?nM~Q!AqAtj!I&Me%Rby11W!;Lw9~pCJcg3k=yies45_44Y2k zr&{Ez)>(>rSgDqv1-bOvWr}QR(PW)5OV&$h8Krxm^tXX7fU>x|RAxBYMfuStjPidM zleNBng|=v;D&!ZFxPJU7D4uu9a;Q-SAYB-AdPekZk=wVAm8+_V%xcWphr4}J6bjD4QHsMdTRSn-jsoy7@dNyC~Ir?zgIyjs< z+8rjdscyHIURk-UQ$qfY6R0|odr97hv~|6lp>}I;P|hAp zX+=cgwRh2%FQ$;SH7^xf>W&FP*st{&YgJ~ig{+7`LPY-;g0O@CzZtxOkrQUg!rrNC zq>dKSaIuy6$6cu<_}64jFyr%g_QfVR*jl27XTh@mk4IB9JewGmdh zdS-_(mWx89?~ZB<5Vfge|JTMm!in%q(DfvgYO2DM~X{ zmE~m>(aXBa%cZKlP8vETPajxQGh8ooVh@hrDLnd>24<>*ajJZlOemgoO)8dD2|`y& zbZ@ODZD(SL20Zz=dcP@om+eu+g&513YDu2sK$PF7mQC2W zy~k^?-bcDMoZI~_xBdR+WQOydw-whN^fd+_NJu#m29KJL_VOz(K60@c&@iX$bI(#2 zuzJ^L7ur}Ch}!MkPv%)%+K%a4X2{s?S9cRnvrZJHaoerPx#t>w(b;3?sc-aZ%+>Hj z(YbtW1L-X``8FZw--4B{a%+fk`2mc6`8HHS;~vN%zsI2F#as!^EU6Ha;TNOmc;}G& z`h`B#it0i7SFDUDY83J@e>G6~fxZ-BteA;4vNLz6*cJ35AmSP&Qm961x zMfuC(ABa726zQeznb@Pf_1g*|bvK?mxp>&#FQ(nC5|pD*u%o=AcYu3hegF7q0EQOO zx$c8==3OWpLRDK-_Zbn`2LFU6^NqjcK6Jxsv)`HUT?iSx$LaKKA4jP2SISafa853!oPPUR zSEa;OyQk<^UJVgtVyeMYKBe=f>{fx|zEsB&+tDb@6lrS@n{pdC_i&5h!Ae9mQ64Do z0gUrF6CQ$^3cbN#<4V!95E)=8GSfuM8Nr)zJ1_?65Aj_9SZ+B(^Br6(V(e4xf7+*y zi$xBJ<-sh|v9!C+?^kKNXFD#{fSdxX^SM3*vWQdhWCj-3|H&)yw>RSd)f*u| zTHLZ~N5!_Vk#~_E6}?m*B2K?M*n*3@ZK8@X2^;p-;xS@N+aBC~))oX`UgF#`Sr`?> zjA2*1v_^VWll5Z1E=ltppjb}V0euFj!Yj#bvnF1VGQs0uX?EP#ZUpY*7TPiGq&)PE zQUCSYcdyd+c3;mJDkJLRdbfUYYhK*E?$}~qe$(!p=4b{o921f}$edrE(;7uq544}m z3^>LgpLm19qL3pI**pu-d;DkU0>*8NfQytImH=@?E?@b!=}{s z>l<4EgEH!4G4#Je2GebN$4Q%hme`y8D%laKKss3Si zCHGaRcb>*zhtGm$m(=e~?eq)BUS)JH5yy*yF~BJt%<~R2Y%{6 z-RbXfs~nAdj@y&=M2X>fzYd|RbPItvyLQb=Z zPEw22!N)VFl@v2thhtE-^67(x?$v5XWi9UoI0e0)9t?O_p5-1AWMX#o>ImufgvpHe z0b64)EkIw))_pe2Mpl8_g_!bznFAr7uq)l zWs9sdUz=D;{>E(tgz47k_azbvUlhHKu0jzC_C{;_lCZu#gEyX3%{0~P;sX4RDK{O+52YDnk=-5hY zJS=v?q#BG2`7qk7&`_Ea9uz6LOo4ZJ`8m9jKaN*#`NOM80&%;<_rYmlWN9&4vFQ`u z*utlHY(XI@niNnU%pu+PJ^?+XihEdF;UO%Eg(|Uf1*qX2Vhx$t(uynGo~CuAGkU`! zE$Qhmu3fIY^!tAFCuk$Az{=ORz~jwx2bn)Es8?`#ug2bGqa0#p{`^g=KN?r;bK`Sx zel+zy6}it%gYD%$z=B3khOs4)ui(gZA_oei#tDeA{%J_Sy7K9B_m|zcna|A87GGFV`{* znDtHt1mKu)8>eai;F!Gt(yQUWagQ;xh~ORsxS_BJB&zUMnDc=5XnCSm-jDO&tj9CujJC7X>6XyIcZtzM}4zjL| zkv_-#E1>hkpOFfbP6k(BZ1vL78}Xly0SeGWz@5o{=gt;h0q#r^b}?NjB8#_x=K}&n z3u1&xzdoxc{pTb6=X3rqxbAkqxRZ_n?tS>Ha?`&Qx&QGP|LNlDMg7aQZTb7s{(Wix zzudL@Jo3^Ychp<-gt)thBNdGT0Wm$Y1Q(uKA&e|k%wHfW>B8*$x;z|$3ZWL~z9YRk z4MUX_sjU@N*`3VGK2Z*Cy;Jqs#%_7mgeM>#FqPYG`rak(+A95cRf4*JVx23=thsIPC|{ zDU&y^>s2V6-uQFyb}QaW_~V@pexWGO>=YqTlmo<_bu49nR{2tP(iVTFdLlg$)2e*v z)o{sjs1)3$m&Dd#3{ERN3i4_(c{ysQ!npP_+bclvGPT#9y0HsV7Jw8l<@EJq}bag7?Jp)939=|RJxt%)w-&GB?!PGo)b@5mW4!gBBdabFdieB~OXj_5t$#>;S z75Z<=4rHuq&Fsxr>e0qW{Hbu-T^93eKd6>*6DkKLNdhEO96JYf2K*0jh zoMVP-8S_be6HK6LrpfP0p1he*t=aAq?0!ZZ8c!vs`5Jv#>_Ox_y84K>7DWH@i*58C z1Fw5s+|$}-$aJ=U?X3_sM%I5dn+q}d3HDWkQjQbFl4MD%Ungs|CVM)|C@cC=NO6kv zTG9W`0Imm!|DEAptHHRh3O2~MYG53B-;*R0d{+D61@bh0{QJml(e44%MY4TklyYC^x=UQ=^lIH9%Z(^4 z`?`VsI}8lcnOUWo#h!Cj&SUp;Y8tEh^sN^s!(Wl@8k60gHVgCO(N9|DRPl-U*_ zDtyS^Et01>ZxaG>HOlwfro7i1^=4mI$umd4&Ox`pJ~G!|35SmD0D8{JbW~N+;sTp7 z;=?)VJ*TV;1@ka1l*-dU;918}PY@!X;Un9!f8es~2f*H=)X06zTaOl6Q4wTaeSATq z^0B)9dc_z18)5k`+<#$}QmWwXs038S@Zz^kyrWZoO|-j23232+pv4W3CmzNpfja}l z2reGwhgZ`G^cVBBJ4I^EhCD0HSC`yHr- zQFN40dJEf6D0_;-63W5YJK}sRxan=I9Rxp7D!~i5RD3-Aut;`#Yl!H+SZb$PZ%_dB zVhz_SPTYmg!!MZs&KT8$7U@Mhom(fAxSw*UToTv52efHzh&V=t;S$ghZ|wpY^X-;f zlSz=q@FH1YM!<60%d<6Kos9x{!7Q?w0_MgC{Q55gUiFmy_-fnUnLZl9>iLLmsMG_Y z&huH6Y^803YnL1EsJ~fdVYAICsx1Bmk(o8>nsJx^a5yvdi@plw)#A#vuvb(+*=o36 z=mOTYx~eK-cImzNchZ%WB!)Z#G-nlT0{hkDpjM;t&^k@?&82wrL z#<<=7Mh5X5xb(t_IbZ!l_N?;?I$1-RW(64~4_1~|Xr!j|K&6TsL9{ZeNG(asN5nor z3{XN2!8!6BtdRu{xoPFDw={Hy2V;xVwQYHpsC%^?ElFD;k146<2HTbhm+AF*6H~(k zqJ|K4AE>Mqys4OydHMZ4Q7L)TI%j^(ce+uK85VewPFr#FY74d=eY`<8sg9EUIDm_~ zzBKc(^#7vlUBjVV-|*pDDIun^D`F}|C9FcqZhqA&VKE__h*>2urZKCK!I)`>BxaRl zMa+_2vYBLt>}FyxtE^JC88N$6vKjMCF`mq3y?6hQ@0a)gesDM(#~jaip69;r>$=YC zyv}pEfFfw%LqT@gzz%}&6Hs^VLY-%?A-|)oRa+>J53n|Tm}%lOdTXF&ZVIMbHAfOk?aPTMJgAs%&<`w@F0RU6$c_>Tk%a*)rTXW zL=fX6hA$>ep)L;D*(ChMyG^zOOy9RC2KojH$gOUdD)pD-g>XFXNsj=;4jPm85}oC& zm-ocfuMT^Om@Ox&O6l{ihpPXqR3PMUGs&Z&=H=&F4p4XpoYq)pG>`9IV#B+Vlm^z~ zDW!o$58n_a$fSY8;Jk_gjFBqHuB3y_1Y=^OTma`oZJjB;1yJFP_=F5`D-n^-!*mk# zNtXx6`*6<9>05^_PP7|}b&rUDX&Q9JBsfeQQFJbUYRYflZKdrqNaY3w<=uZQ*1mlG zGQ4ofKc9Dp=`|aUKNjkAd(sQvRQq^D~lkd>IiLGjRnDGq@OTxq=tfajO~ z-wGp=MHvjH8MZZNqy|tPZn1)Zk*ycJ(GlGh`5>W6)8DXot5- zcRR;6vuuZ~l`oqZcfHXWv39UtvVb>#%NxN*_e>s=T`1ksUrw}s8Dac`3=n`GIAzFay?y$fNxNCi(X}NAzczvPhx+YdeV4be8^+}!-S$6M? zYoIXgSil_b9}`+9I@8=>n1K)KiK~?^_+UKSj~aGE2+2>6%1lm{e>_#?Z%cUCcHDK~ z_HDmd$=9awINpxdrh(hJ1($<>tugT)Mr+n2y0ibzf!6+80mlXsch@eYh4yo<1n=)OK!IM-3Xgf7 zRTnB#y#=n3>4+n;|2kTw^nj~{$PCt6*?fn_(jM2Ne~~QMaOM-DuI-#F{}~ua&J$6y z>u5o_6OPs@XR-Gnp}u}^f{nS3-2;@Mn*H?-2E-*G;w?!99^eX@cIjoWavRfQuM02X zJ3dC*#1$9e>*DI$Vi}i%f_&=i$ggJEsYo4lFA54^k1xspRpqy@RGcMk0Z}fNWZD8` zf_K3m-~diA8#?)LFL`(DJ{ZwkVgc#Z&X(e64I}A-?0{4eS{oPCdSr&Sfx&m*g`cO7 z`!HHOx$oJ)0o9Nyh?Y3XI6buIPE-;Q)s=gzn65fewiO}-u$#&pSO_=m;Z5cdVwUd! zbNACCx*pDX={joWG3xgt#&dv1b_Nwg zQ{j-g!{6)vTcPP4LcNYMWDD-onl-yVWKb2yN&8e!BudRvoXL^wG(t~=p_8*y?q1h@ zH9p$q%T1Y8v(7;^m7Nu5`sH@dNx#T^?mY8e(?P5vd&SfJZ(pZ&_&a~WSo@fbZ1+~2Bl)WyQv{Xl4HUs6$_i4P zyACl-eiqDJVeX&+7S&s4#~~ZYqI*oGMkKFrlUZSN7b4v`nz}$RQaVGQO2HV?f^-Y9 zj~ihF1?SXTp&MZ|7_QH=KSRuFzdkA-{n1iC2k@;uc7T8TL(yL1C;otnpLvt~Wb`lFH={Rg(>m!>2PZgEE3?}?p=EPYp zUt8%Tq-VKnK;Kf>3p&kh;A0}fDqc+Ui2NK6N6>^RT?Kxd*S(9d zupJc9n(0adZ@Cl(U0IC-stmwYNvD}8?W#zTFclaxA8m)0_|Mi$%M1idl5`YD+-UT*3-UC`Ce>DiW*z_Nvn7p1?B`<`4*eKCp zN>jw~7`ykZex88ay8EI;X%TLgYAL7C{9#hE zu#&ZP<&uNNI z`)V2egsKpvcnG5ybCUpxZn}@4XKhi6yn1SAWnF6Q`aj!6;>Zi{|EJ_ z*P5Qr>t6PFKsuzNvtu#iDD_%!BCO7UXcC}w9SYGUj*8V7@;FLE86ayD=w@cHwoEEd zBOCb~v#{RKpwi^CP8o|i_FaXP9Hyy_cbB<+6J%D9&dh4xeQ_uBxkZ?U2`pUJmT&ZZ zd-DB&f3y9iz~tneae;}_9Ro?|M7+n}3VLH|_?Xv7YBFsI#p%DrS_3xcvsQ06ulE(thJO2{V{qQKwyOC> z0fP=YHb5CkD9t8;UmBgNTvB+gG6Aj@|KNx<%PG+PLd`CzF&3$Dv0V%5*glk)yd(2v zYvxt89??n^S8Dirf9D~%KxQo3(Pt2Y|J?lND+k)|lui#yj0z+FMRkLXL-D*hI4xOp z7y4R>o6PRI&(OY)V_Y#8)Q(CmM-`)d=UbX)_M({){>aBi{pa(#Tax_zc7>Kz8B-{C z-Ri%$1WKpaK9W+K)VJ2*7g-83$7YFryi8w&>E@@?8k=u8&~7gmbYkOxT}>FbWUH)w zQUmrB+TEs%1*O!}pm0}AIq|5-vIY3h9B2xy%&Q?z0m-WyiDnkU-s~siR#}UGk}!#t zu>DD8zAB$Bc*bu635Ei#{y+hqRKRzI_7aa2A?7^9mQZj{M_4Z(I*HhXjW8Nvu90DG zlXgePLu%;>)eWW!9l{){gfeQX$C}s`#M1>n_x|SVSdUG0Wp8f3Rn9i z!NQ<=hw}WruBkIfGxC0tMHuY(%C>%M14fIsabn!P=nAj6?zZ=DqisjlQ$oD<_3bt8-dXT?T z8L6;Re=uh0w1Of_%n7Z^Cb`JGg?7Et6`|v^#&kN#0o&jM_?dL+uXC1vD1>GXQ)wHRmS_kI<``OR+vd$ z@J4ThjYUpM1N)bgt7Wvz#MO3@^>AS!2E8 zf5NrH{s5ulwJ4#%)eQbVvOWXVCVH*P+*=YDedXRoS@hB$q6x77t z61&k1=fO@tZb5&8f%gJj3DZS`PS7(!U}JuQcP4`j^C(=hQ>2F&K6*92QAw^ zls`N-+>VjCRqQE2w(e8n!MXb;Q*C0e)@oA_D4oX={p4?SL`xDg$Qr21-@j0>|5&tQ zr@9}hPx3CayTiDWM%ATDR?1$_ih_G~&1-Bhi+;JDwDUz$iM38pu=e_p^v$V5k4pP$ z&d6%&+=AM|U-J|>*h^1rQ{5sH366G|P=^XJ^R1K5L~wVbqR=bUgKT}u1-y%QUUw)e zowbcPCK>-`f3~p9ewtQK4xc+pDG#HY(5i4M9$ca5!5~f@&piV^&t>*0|BqD zMpmO<+b^^8hfop>cz!8h&!#vGP`X&EJT~qic{fRW;QTMQ2RkW%95E#$bQe)GBDOT; zbYF?s0C{9xjFT?7>_}hT3U>+pgKu$H(_derYJDQbcYQqC^{}mbu&=RJ_l(i@dhPaS zaJy@C(8ki+T|vGJ4I!?9>ufFW6#ZUQwI{G_PhidQh>~NE&cPI?ms7+|?;lv(baOZO z786r{b&GNI&pG3klGI)uTozQfd91p^@6?SPuT%fBLSzVuaiwzCb6`I}9bZ5^KT~-( zK|Z-@WHJ26$OV1p@$l0z&c0>U<1bAnhRaJd7`c}WHlApFqk;YgkWIm3zLruaQj|sz zT6IZpgUp+30as!e>`Zqf@>{+>-jQMgUpx1}&6#3A>{7few93Rw9zVDkFPL}m> zS)nJsyh-8nsi~^!^u*5Yxq;UWX@ywxu{(jHNfYqw39ydu2=80Bx$OwEW4r0q5nk-} zFwaYb8~LpdeYf5>d$G@^i&pgep_P=kbHVQ|?!fT}D{gQ`>UzJsk-YBpx*x3Tggt-T zEwb>>(06-XdhbF@S&oy==mWop=ORzHc&}PCd20wljhCN{cJtdhGcWj#kFP1$ul&)o z0O9o1i_D0GkI@0vT>sv)zXu+o$KsGjHTT5!ZJh1&+OPZ!9}@NhaGYVAC7|fQ1ut0! z!>k>#EFt6HBca$0wc`W7|yC0{;VGV{Oi*AL4tnae}+*GuYq z*;VZZP#8jZ+|;@ETqAbs%(3qL(t!PS$Iiu^{~l-bz{)V2^J%lWj(%46NX2!|NY5+R z4g7k)i1a z{R5x+JuXA*9$AP{gnqj}Gg`MhVAtc@s(+OKfL|;A-#>rLG?iBq*$RzV%$m^6Vvtph=_uLy|84PB$dWA-e5RV(tKCi29p_LBpH{tL3;L1Yx1pM; zsz+5BltsM``rKu9E(*5is!yOLAaRS{Eu^WA(s7C{bk-~Ozo&2fw}O5dd?^!M`Zwxd z^fmBjJot05UzjQfRcwBB0#f0mYYqG^5Vva55$#Mv}S_lWYBL_W$yO zB75Zb&7gdrv!yJbLEHtHg~gU=RW52NaviyVJRG|dqHQ*sGEGMR8?~rLN>>F zr<}QyHtyb^C`U7Q-n}6~MYVi&`p%9nihjV~s2)as-Ae+DjK&;T7l`U>dePa7ZuUMS zjESSiM-mifVJC)b;}+fodksmpvIi604yd(k2n^Mc-kC#q!bJi;>H%+l;PdXSFRpuT zD|E^kLpNMkYqKT?f)|$TU}SjaB>9>0p>hf4sI{fwG+hfPDzs(*hC~C~2G|j%t5?C+ z6{LgER8JDby+sj}I$@xuI9Ef_Y=;q_Q}9ZtYbCcCc9_+Xc=5YUBFYo+gzl7}y0g(P zJx@=6m^s<^8k(tyJgdnX0Doyb3su~%Cz*hv* z>`SZSjfZ>)Q6eE#aen@z4Pl|Dd10hqfac{$Q`5UWM+ta>_Ku$qYwz&;C3{QklfU$& z>b0t>P#hBSKz@k!WJB}CB86Fi+!@jGd*YPvNq*IxvGQ__Y_>Ktk z7sfvvH!NiQ<7_As@ZLX^YJdGW<2=kBMSQ^`B#rmm}vJ>YzShL2B6B52Uw zCP@qx0hCTJ)CCm0b19b{N6+OOBv8ht$MsBYAs%gr)TD5a1Y2KWGSjqwx&)TzDhjt*MO@hNj#ULn}3J5y7@MxlcU}WbZO}y`qf%! z*(?WTK;ENte5c;89>+Ai>ELrv>tajR6V`Yldw~2_nsFODntE5h8~)}+?Jt3rk<8`=je9ZVnT-%pM@bhOYZ|IxXB2i{v2hI&TWMer(g z81=Q|!J#wxe5GcCD#j0!LusN*qGgeCOU#EhAex&j3yy$4DT2upe59RPpL7s@qc{c= zgm>MBKkalAT0-++AZQ_^GBoC>a0h0aUS156i%?jC74>-nHG zZjRxt1!UtS=jM4l&pJkpOL6Eo^g!MGqSjp-o2en`TmoGAa7TN4Q`2cnG#AC|Q)}C= zE&%NhwEQ595&<_ucLW9O?nd7-ML5Y5^oIlF5pb2_wDK2--6`?zaZb9bHo_uxfg7SD z9433f^^gH^WBva6*utuAcrmK0mvV(BU)fDs|)B|D?F zw(%)0*xf|to+UtOIez+Ra)Gubl(9U>Ba-CypDtekDyBt^mpT~JX~;I2Ly@<)rZrPZ zkFKPR`t;KQjLV1o7Qm~!ieoovW=6sKBGfhK{<&+4UuM~jnYS4jL*n>hS#CoO%E1_9qxLpyZ$>@Q!#P_OnB+85B22N z?vHifE_1KLiNgsYCI77eTGT!koL3e68RzCn41@-)>u8BYro&-F{?rdk;?XWPs9S)x zS_K*KP4Tj)wIo|8yr2i6#U~vcwV--JR@trrGJ+^5yDs}!K@h=6=R+Z8`dq{P zEvLc<{V#jgNY~^|*f{z)mid0ksEaQ1RV2HhV6*O*j-b$bk(*!O*9LH zQ7??VS-^}fi3m4b&pH-QHVFF&#-3Hg=e>QNBX*0XD7^?G_t=0$>HSnWEGFDq&Xj;d zL(^1=R24fW*INP+OAT^YguCiVLfXo=65A#D4dB~F(N$WQTMpGZIo7E_6Hi}xgwYJ$ zT>9Nne(g!D`R=yComXS?8vDS7j!|A4pkn(Ko7m9=RSYm@tliyns&un0Ax5~{lE;Yk zd{S|t_zc}gF(YWmA#u9KM zDN$exr;uO25rD4AjwK03=FaZ~PzL|yt#-%yFEkU>*)}%>ccc~BN!8|-<5G_2BPEBQ zlF*C+5?YtZDm{DFiE~ zf0OLDl$bvF#u+7~QX)mxrs>OQwYISWPX?{$h6kmP5hWe-91UIrs*8ejcT-xa3KQ^0 z{Z&sAX*b+=L;XF7JE4^o#MvQ$`StQSeF~V^GoQKUs zm<~I5M{bfu-$_8Pb5v)qq1r0wx7Y~VL`=r0PW9wr0nQpRm}U>gXQ52BcD4fW9y{I0 zZc10M{$#8I@N}G2nE)H(2wj_N;_FCz;3Ay()y|;>?2TLV1d@HNtt9%T+9D>Bdgk4Q zUs`heFoTIIQ?HRz)!)7iQXSo~M}EgQ`(6rQ@?y_C|Kxv@K52I<;7&5#kT}s(MM0&3`NKKiZ+@Fzyn_L8{<>0sDp;5yak}Han*5r*Mkw3D*Gs@``Oqmd3Lzbmd58+K zNq8|eo3hc>LU;HJl-7TXWvH_GG7@cFP}60mfw$ z1zk7cUm7w z@|wPk4wTn^45~goeua(?ymqYZ%RG}=QbYGG8fPkyPbknS#5e&E$n^W=7}=5}QotXR zg7b}`$&ucan;2EdjE{`C11<11N*pd)gxC}lq04*$J)q$sQ&Dfa=-?dP_2}6ru{Q!` z$aKq0pA6O9*`c_nT1u)(a<9A$_;~ZU)GORrED=5Mm)~X}f%QS-A0f`J`?!=?Q%pUeF48x3nb70qhE1IJqsdA4Eve$TIJ$R=+ zxW8tMBho6LT~CPIxz;PL@y6#oYTMt%%xZ(;@nOq>>B;*9gSn#1xM2KV+Be%;RlN#e zQG&nN+9*L8=J_KEV`(lij%9Uv(FW5MPh2G$sfvdN10Y&=_!_tU(m#ji3@oN?|N8gGV<~S4-#;llqa7aEx*@jC%6FrjbHWjb)SE;&sNK6PJI&OuVK713 zjfQtccOsD_11J~7y44R=#f&b#3HvuRyw^iQuTuW%joH6S#J~EZKdwAx$SM-)6cInS z=5dyzZ!gw)bQadtwc}_yq$pXMWTp*5mtL03`+XYQh7wKNaJueYywvg8*)b8b@p&UNYpSKK zt)=*$X)2Crkz7?0eWEnW+;g*^C$isIsh_9TX%rLS-Bpq_SrJh0wNRLTaW{hfpmPV6 z;ZHz6jjHg(wH{~|(QlfJX76QCe}c!m05=^#v}0AL9$+>QrCllMRk;fotyuPYhYmfr zeLWRi>d>30)Z_)amS+%C>KAmK( z*^T5ibCcF+TAUblb%d_QVzmAWZf$OT?`})-gyJ5|HNWge0aQegt2TZ2!wP)03T%Zj zBJk|+$-O3|Y6EglbcXU!DfSPWrlQARoVTdW2OZL^&`rPY?8q$!SiuXOJh}zf zb^CULcy89~QoC4lS0(S$r$@hyd)72t4zRNG+xG3MKYf7dHM2DE#!{`QOGS>_1OSF_ zHky^^$!zQxj%;jA+u!HMqInO_FKS%ly&h2nGz~Jdi=E$!{`D{%US#DxMPpZWbcZI+ z-K}FZ@2Zcq2h`mkxOIl=xq9FKi2+z7Y$?44Zl;9)Q-IiMu1AQ5ecSOO;AF;U*}Z3( z%uE`#<_W*WG?$nST|}KVOX~hMWAcXD&0mK(RuusUy~3}L{e3dxm`(Xl-)5vpG7kft zL|uj=xVKRP|�suxLRYmqgGbP>0cJzj%>X_uf}u~_p?0(%uPzK699vdd;i zqHZi_Fw328F^io}@-n*P zWHr_`^q0Z}lXMAfKAJ-q_mG#v72w%u!+yZ=$q$4>8}e(4<|1k=Pt5OmC3XLqm@P2w z-T*?M=2i6$_1jiddMPBQLqLAd*a3lf1x@z#@J1?r_PJt^98>Q;1i*MpO1hz|xZ8)J+lDgUi#qik5DXBnAm3@y^Hmu!Tr zx~W^T!&2$GL>!m4F_D@+sW%##Uv^hL45BJvt-_s8fPUAV!H_ouLHIeHmOJ}=Z1UUd zQjJAukT-8oZU8$VoznR~c8qi^Q#_I*2X+B{YOO?Q&+HMI;r#4$Io#N`ofXCMfKd@{nv)@FlDXs=A&q5$%6@iC}*BEP!csc%MJx{;XrX zi5JM9BuI^jrpMne-k<7v|mKX^Qcvr|(GzOdJB1)R$3Xx+)UFLg-mAEpZ1?LKdv8Gf;;~Fm^5nwbK znHjOOMRvTFz)>M4K zK)#6dl*#5NA7Xg6IU))#FK~DOkH1uH8{Y+1z=dUnp7C>%Mo`RY#Z;a8mCYo=G>3Dbazmo_;xi z-+Y(1%WrsVN;5Td7Pl-5qfkI z!SdRZ5c;gNSN$3YohwacnVoHQ_(d$qiTsLAMEkclDb6?}bCgz3 zdZI+s)nH(IU0)eYJ24=wxSIBKa&ukiA^!LKFxbo*v{n_H9Zls?#gsYeQzyX8*oBD@5n?0*xLto^D}(GZa2`>4V)@7{o{)yuP{e7@wy?TD`rigqlvhY=FTeG(20of(YzBH>2Fp~hz;z-I&C!q_FU z&Y%!Bltmtuo#Rc+PE+cSK7}(oM-MnhJ^G6MW47SpgG3RsnX5D~m2*_r*pZ%K^ylLYC{-~g5cgE`IN)n#|t>2BTU8C&}ncc;p-eFlw*U5 zgE+i!R^eSFK9EyIyy4Ak84@cC2iSrdK*aK6!{GRmR0$||oc)y!suuQzer;m>OC-~- zb0Vm--KD1LRs!KR0O8KVIgg2PlC5pe-nAdD!7g2i*!C``^$by`_gO6PoKJU{cD1jb zWi$Cu$W_Fgrfk#twWv+($lnfc)eqk)f5=;{9iXs-t>k|K?twfkJW4MM zy!jP;OxPq`B=qRcIl$uqKE+q7q5Qz}# zr4+Qa%nvxvRlY|g50*l-rA#XmjQ+f>}KkUq) z7*Lv-Hs(hKso|HGIFAPdaz&_H@nJkZ8V?^+L@LeUJV1(s$RGVFo!+SP(zP>c@KCQ` zBoC5z54hSZekZ>r8{DK^Q0|cFHP?8nXkv=y^tQ)?jBr~YOkl?_%j9DVW5Cr-l;#l* zdt7=#OzoOPK@}a%5^k3?o2|=o2iwq3y-tY*n6t!+3o0id?6r$@S|nXfZ2slXH&wHt zMs$Fs?tjr#ys9?IsNm#1dEtYHX}|@kd?siqC87$8eRBHa_%B|M3}=q*9w$q2sfECx$Je|-i+Sv@Vav%9&5TLMuHDi$NrDZ6nwJ9W%|T zx7mGzEQj+83&8+(gec}{sI0hE=LL$fgZe9b&|*+J)dBeYdPv!1i^7o%ELph{iNH7y z4brHM>JLaw);4Tte&Oo25w#gn9UdS|>Y=AE=e)lEC(%O`aZQBNMRhmr^F)q^c@O;Q z8Xwx}fJq`+$j`(1B6g-6l-^V^X7wymw>=!EE6sKYvkp({X(sO*TzsM*-qJd>h}yha zz*1ab)ljO5lrs%7<8-B`QOnb#B5ND8_w}h`GQ?NzfJ+8E>e>VU4k(n0l{A^|MXCO} z@!X0GVXbY_%XXLi!CchXzpjDZ4l;XTJ2~mmxo0yE>{njnigx>TT`=5zGAEWFk~vdi zcJBEd+u-iu_78!Q}Fu@$s%vNH{l*uwss$Br~m^WvU&yfJWo9v)c-Rhk(ms zo|xKW9Rh0l!>}Wg^EV6GCwej%@EjWH5WCPeVfgR}=hxP6b&q8>5250^rWwoEgd zd1hFVOHpdewnEe(iV7h5&mfoINeYFIv9sz(s|AFg$nQ*Aan7>bsc_l$=>ejSu$5aG zb0z6AzLnj2`=y6vbxo{aMfD$({}k3X=;%;~X=74(BJ8cOBRRfT8!FA99(E?lsd7(W za)LViZHEXSF8-w*=m0XQdv-4tUsgVTSYHt$Xu7s_aG@_uchA7zy~9XxD0ZQ|H+FM5;T{kEq>7L=3-Fe&4 zKGZnrC~j}egrJ_!4$uCa+}in@{UJg1g^zB32zNz&?|%Vw(w0{$Ee-sRt@!`*Lp}#i zjtJDheT!&?4Zjhkrt>qI-_t>K1f{|KDDn_euJ@yYs%aC}kwG)%|Ms(&g8)0O4XJwZ z{hv|z1683>_5B=zPl2}^D<22QSO36e0XmtPhIwlE9b8`H2>3QH@=LZnP7P3zchZ{0 zBW&+3N~@4PXP&8kSf2J)N7VCGx{3Vy9Hlh$R(I3Eg=ahz?!hxt9K?68=q`f79-Tv5X0QxL zrw-DV<~&sCfY&fMjJTeum}Adf0>najo*?P&4Ay<@-~g2?SDthz?p z_cV6pJDVn^NTT0K+29zy=1tjt7fc9j)g52+__)K=X_FJ`?Jx&O;WWzs2iCzG0BFoG zWNGAU!5uf%8g3xPuArSgDlC8t)c2?_f%yenX`BrhX8j;>2c6>Yio?pa4AmcC81iPx zE?MTy_I<)2cr;Q*>!KJj=^2G@n;(--yjNO@4}xjCpVLW)h0_VA0l(NI+k0rdyTqh< zgqiBnmh7N=q=!5MLexX@7z#isw6Vq>fc62vkucWQ$!Ab+ur~9i3qTIhZdXl89cJkg zKfmRK$?Xo$;KtoE7UD|H6oH&8P*Tb2rUT+{PWQ?JZdwP#q4WFlCViD3^FF`lPuy!vsgWP-VTc&*r^@OJnmXpB ze2-GqNs{FV*_wFcWr`79R`b$)9eh@{ae2(vv0sCzBZ{b&#dCP;?)7ZRjxx?=5;FK& z^Ny)~SCY4O{7iJ_^;q62vX&f~7!WKUBYUYw*iG#2Faj7B_|YvmukH_JQ3ZtC7yz{Z zQO9d=kE|&4U9B=W70MGJu2+mKC^P^n;}b~Z6!N>-S?+tGw@e}4?vyO2q)td!^&#{O z@%n!&x>)-PQ}O%9r|ftfd5@|Jz#)Zvz08)dS@H1Z~)sd$RGJgPZ-Y&x+2-3aF&0r`+F{k=MXvLF(}^{zhVBs8InL z7pZd?8y$s9tHlK`2c-kovXpU>QJUueAl;?T2`6q!kNTmI zy76O-^SV$L8_|`8QYAcqwxF)>jw@~VyqD+u*J(O6g`r$ zXd>vp6(VkrO+eQH^;RgTxDsn_Uew(bn)H&Axg1xaTc2rm$B0)pt&>=rR^~tIn6vv` zTlORI>*?0^210SyzM?Bj7mLNO=jZ7O#hIHTMq@`VKTl!grPU2gMno@2OUa`s$d9mz zXbw66fO2s?JajrjkM%4(-UAG(JIT3F=2c3pna3}M2U;4vX*VH~sm!B`tqVV%f8&2t zh&EEcch;RRPQFB_#08r~&BSh)R=-yOW=7{>)(SG;Twm5y1r-tuiD;2smz@DxU`Iu+ zbl?32Cq(36R=K8PFQs8G38}Y$n|HV}=*`Z_f~R5Ag`cTUX4GcGl>>|3E;fFPZpe^l zgDf#!Om;(u!aG49mPE|!LTTDvTrkD(`sKbIlb$&;AMMa^n*XVR>65EFsuh{yS{oub99|q+BQoqht^V!B=2UPM`{HCf02NHek zIc3aLUd)26N=auStGR}=GNWY_>=CELKOXx4>kA3pD6=!ut-b{!%XfGn3D!XnMdzxm zXGVHz+iis5C@;Ti;seYXaPegD@J_MZiE&eE?XwCeN~iAysJ==jR%_vessIr^mt{lR z2YqVJTkIY%4m))#e-!vSLQo zeB@gl!4(eNtT5Ev)zfT^F37Lxv7{fYicU;SykND4{uCcNO^C+16(lav-d7kZ=|qKG zt!i`Mq2@X}f%zL}rH88Sv8tjSEg)zqqafs*kB|@V;7csT(lJao%S^Vn5S_KaziEs# zB~Z?10o;ebp-&LDI9She1Z|NVpyx|4go`alT`kJKlx!SYsIFCMsUfZT>QR{*` zw?~2fH;xnPJ-no_Ci%b(GA9Ay8re~~3T8uz{Zv=08lrl*vr>z@&R=pUt6rY<8BJH2|z)cE^{Z`q^)U>#466rf~l26 zYm~7^J<;@VhFzAqLA&aHdoWIzYTZXiPV4g~;G~`sC(QE?vxv)01LI>HiiqEgk=TP< zH2?8()dQGk>|P9jo7fygHoq}i5(d~luBv?XE_Gk4@k=Cv==2L5fb4N>Q*&i@99p*j zOzUV5ZSzdYra^|}!o1?7ETX!6nrtzWP-Ok2-BdVXjmhKiGy%^CN-(c}@`Zv#32r%$ zP=#*-fPmOt1)(1pt8I8U&u}rr^@QfqfIFdlzK>JrCI$L)t2kFuDqG^$g69v>lnR*zV`Jr`%-0j89B$L3LkJMF}TsknEgY z%t&OL04jp<%l@(K9wc;UCiHV@RC#|5Og6hzyQtG=<{EkiSXt&pQI;nJNHh4TsNfo~ zNOcmbJ0eB2m4FpR%Ou4M==#_ki|NVjU_!$o+bxyT`ss%E7XKo*uoLV?cGLbI1JL#vR%Ea5jhMdm}ngL^UM(zdGAmgnp(!8_LR0tWy6CXUYpy zMHE3H2J(=uBfrC}Qyv9?Yok0~^^g(;oQ%<(s?_7dwUv}?xlt?fCTp{D2iCDICiLxV z$}RIgYS0^Gfhj&T_0%vx+g*P8h>Qi%1Ps|?|%)K_0b)MF8s^;^tp}N zJ`l!TDFAo`omB*rlUT!C9LrN>Emn@3g~M>el#9J?yLeF zvl4dWQYHYG0d2mEI*ya)LIs^+)a+%Db?DES+dvcTyrt{4q{#E`MjKl$T{l$u)&77OC&L zS%Biz2#tGanG}uoed7MLFp66s=b06NfyHaAL$EcZ5G-6N;Mv^@8uTQrRvsH9rLJj+ z7iAl$`2%&5N_OLSnp%e|Lkoc!s|Wb9pVSVste@YJP*9Dps?~vzl~P4@oH)Wi0gc=h z9NZ3LL^ztkOl!rrPJC62yl^evfBx&tiO0RkavjiHV^ zzK4cZ7RD;9HqFp46c(fng?+`=xsm#3j*pgJp>6EGhg~wzWQJBpfHjgbrbz(lCdLq`puYh%-q^^>+szO(95h_PM zZ`o$sb1u|c1=-VaN`T`AwGP9;J(#+9R<1z_xr)|$!Shq9Y~p-dhJqe zxcVIp`h7_Wj+lb(7R)JGf2U7CH?on-Nf^p1QiE1YW{D}>kqk}tEA%D0 zFM7Js{`f{A%nTm1tD9L9rR}|RFR!&i4|_P=J$k31mJ9ZRN*%_;S*0>|(UUup19{=& z&wZAqaU*=fSU;dKPXWHiUgDZ1G4;ogug2T~{z_N-7_hqd2I_y14EbHfG35!^TX=9K z-n-Y`0S*8JHOk8IQ;;5+HGxCFbRuxihs+z_fO7mi3XXHAYO}K`B~+ zhVokZ7L?_P+!G8cm&*KZm|O=kMT-^Qh25tHMQ3ZGZ-u5lsbS6a2A<&8Hx6)`4NtRQ zj5yjy0UVOo@a#p7on6(LX7s6nxru}mLy2Kg?R}^F=Tr5Y(WeLI&0>g-S|8sb`}gdY zXD#fP-aqo{nmC1_sZ}^RDLUON=jRJidf!0AmT{sfxE4t{4o)csO5P~3=Ub{s!{b; za+tQpW82gvQqc*7P=R|MW4V$&EPXSNK_R0Zh0XNrCTq!r%PDYG)XR9lTB~(2tK2fL z%K65lkZIu@kMNiH51I&}@YYR2nTOi8U4~Un{!$l+ufR z;nE6}RWq2)?rznw!lZmA4THl92c_S|yXIt%Hyt52AG-{DtkLxf`#P}Z_e<*weNpF- z8A#CKoG?Bea;VM*ygP`%>PBrM+Yj>bSY*3N7h_}S4A$s-qZv$#YXbOdlFlcX-vBx7 zU|0B##-;&|cG;@W6%GAtz9GOzq>C;`otbst3x~E|aa`X~Z*PjZP;6pN#jQKro2W_{ zaQ;g3?)f_&wd(^>VdtVVd8{rjn)jNw2v38tGOmF2ow*SSgV2n|rv9LDL8y-=5L%~b z_h@vCqr&Pqd-ayFU%hKtXb0vU{k7=hgNo1dVbP~}F9txoG2y2g7$E+{ngrtI6d3gj z!UE?^09Y|j)HaW(LJ3f_Z~^NHQ?J*xcI&#`n&1Sr17aACT!oGt9y!1ELu|My7gk=I zZ;lS`Pudwh$ePC+tQKANO(|=IC;iO7)W5Pl?n~@h+2D4WUEL&9RCk7rmlsiPIRCo@ z-LaUW0v9?xNB0K%d|@_FZZKZbr`p@E35343jx2gzcRvF4$buCc(jcllGdtbgS0-LZ z0(<@NWTqqbcZNxmEt8b>ak3*>_?asy_UTT(!p$PPR~eUse(I|9*l>dZgp~pA&M#tQ zrva#8jQL0>6}P1{??#{0nP`r)iQMO6}sx?#zQ450owfL_fSOq7* z|BrGZ=vt(=cVI?dwN&ZbdFjhh@35g;+|+5s8+Kj}&aQz|Re#UFx*)hgeM7yHA_hu% zjHC=x=TxBBuXcDhU3e=y;mM51aEOeOBZ2KDJD}(eqQThNa1qI@JK~s!J$rz-aXk19 z$7X2G);%y{M`q2Gu)bJZw!@hD71d6K!g~FG0-ulL3HD8fmPw7|>uWZ`QB{>^kc4)z zxn_&{SS+}@OJ)9>Nu|=ROxO)jzzB1>R8=Z+=xIo=+}@_cm6*M>_sYVR=vzc~@A5>& z@t%{kkP|@Z;OqwE$bHbHZq-C3c*s2{{e{sAYZ)3vCcr5jbh>WabEu0YISAs$)G$bS zEs@60&Tehl&V2#z$Ydf^L5anilI~xI3EOC$DK3QtHxlECrC@d=uJ{{xz2$T*&D~pl z3=49D?=Xi9S{jowhya&X<2aHaz&xk6*?Ya%-%$;D(>jTGkwN`nA^V@$yfQ@Kfdv9>1J<5V!3r{&98%|H!puM zNxfLa!`0tOb8UH|8^(Wo0|jeLiaN0td`Tt630y@Sz}>wt$@175O7i*z)^hlYEwG{L z&k+w31l{3`S}AEBpxEGDy)xMbV=Ng*t12qh9DCDr5}Fi@UOC!aHHPE`_8(gr^^4Q!bq%)GyOLDeQP8*jj(CRGQkuG(zWsA& zK>4Ps)MKKPnc=6#y^j|+l+6wg5IphGRYNCp*bVn8kN3QGZXE0_qaFvs8s&Hh>g^E~ zxennl$u4B<;u<-MN;hSs5I>? zC5i)M(&rKYz$oO0)7KB>jvxE_*Ir^c;DIb(j+j6!)qT)&{U84Ce6uE01?j64+ld+j zRC6~eb_)r6`sg0)6t~c1mIuEJd3Q8m=ER6&(c7A&w56egAGpJg_Ll_yBZ*a2VMn`0 zWVY9Eor9;h_LBU{=5b%x?>1c=@IC4f!lYa%e0C@2!^dFWT}Emmw>V!HJsp(~12`tNZinkLr3~|M90fIE|JlBr zfcJw6Za5GK6<Pve#1*UAIcDK7kaESeUg3>wX+L)d$UV zh7_%2D>inY9GUkCv+25$7&RbhLv7%g8Eolhjv)rJn}wf?)#KpKje==QrDhXNHxRD| zrRnLHh}qATPh;cyD~%uR3M7x*jdU&fRUX@(zr>b-km3=wepG>0Z;3_y5RIHxZwE6nr5YRf zt6U5+0OrvlC$up8EV5t5Sflv`>3kX+;-9H(IwHNNE%Mm$sZ{#TBze#Hbc^jXxMqYK zeQ3_J$$TW<=L)|UbAvA_M42S95E~bb;h=7HFSr7eCLZ6_P8pQHF)JJy8)k$qS=EI& zAbiTyS@7ZIE#=bYz@O{)(v3;oKSKiSr)M!=lstq6eVIIs5PstozqefR;+fWE5F;ta zuB1wy@Sg)2@u*xZ5D=tFvneVR$a(`_H!zhk!x3=5~n^9to_>}72v@9L)up~vx{wcA9PO~}zRP!^@ z9OSUfZtRBX1=sLQh|$Tb=SuprDL?jKBq#0WIoyX>G9zV{{hWab>Gc<7D@akVYW!0n zRiDfmOLw<|&XWDh$PNQddJplI`-U{RM*?`GyPbSvM?K$+edIL^&lffJ+%O9rak^ZZ z>ZTv69=}0kDYp;JpHIc%Q*fy%s(Hla(f2sW3!vKY9mbUjk#gnl)J=}5wkLd?y7FWP zG}cdJEjPeB3eB!_SgVoR9#7aIGqKD&x_ZjlC4OJe>OC6ptfsq?y|qk!9QG|%;-ZMG zB6~s2d|DGu0+dlmQHRHmc^$f~FhGr=ueuaaS1FT1wcBu6SDELK(NKueAnWz|WuTmU zN2Kl+wlrGaPe^L?Z`LI)dS7_pSq!aMSFoIP`ELBe7XKTU9`J4M5%S(|VIMfM_RmxP zZnAgQe%lLFl(V^Qk(a5%&qxbvPJEKXn^*Hsb?x51hlBBidN!mBqQ>&c-8oNxt2Z>r zlRaO5H{sm)uJWR&J$G#*$0*y*I=TM&&*{y5MmIW#+IKz)%CZ>D4>?&A5}*mwHdCV% z_>bwFU%DXinT9lnFZ^9SNRDPfb zj3#LVQfpE=c(skPn$S>qUFi(tS)ay`E9suaNXHx8ota%?Bk~c0e3>|h_pz8rs5&>k zzdHLTs5AXla*GkfJ6`EnNZ=@jKQb@fK8JU`_t4()FGt0JAl9uj=6%^O-pyQ3II+XT zR{G%9t;oV^(Y<{&C#=u;IkZ@1;-U_ma9&q`zVL8~-m;f;kDgDFq;s!HPftYz)f=v0 z7Ew<^)a%%vb*77MByPGaeBGl?(ay|jKfeG)p{ zjwDn}x%%ZZL%VUzDPB}@Y_URzpepSrIFiX%5w9ufw@8fS)KbsUH$Y03-_cW4%1pn{4d3abE+1ZS548JC+E zoP4+NW5YwhYrt%ZSyVK}QV1nipPU^Ng>_C-AwL$fQ?Ct;*!s`%B@AJNB!6;@kaXA8 zhOSwfmPSmK5i?O69mnxFcOPJI>$o>`OI+e8FMZ+Yu&y(bC?l@LeiB)~;BYy^5!~z^ zfOTDhdTSut^W%ETUP@PM`o5Kzdd943<&V%$=IgJFN(>Zq2=ECtW^L+tZ*aKp*5)-? zRUtIps^@B7gn0~rZraobk$?@_A}FuJRaApkQFAq@vMicke`14ETF^oeg1_`}@EEjk zW8NY~YSHybs?-PbX6^S>q7SaWRr-2bo4yWwpOVbjMN6ERxXal8K6VtJ|M~u>CzVIX zy}5C(NBY0blny>uH>RrNDQ}HI+3%2>Y8_3r2XRSwJk3~^L=+w$mrJzIn@+Ez8MG;^ zWQSM+yKBvmo$fE2Ai9jMQk9qfMK-|Kh9ij~vGMP`;C1-a&?!Yr7y|D1a3?hE(s=?Ca`g5%>$Y3IF8K0+0xav&1NHkO4^-#ZiNTlk?<#Sy)4iL)^zn(P!TB2fV4!8)axT>G>WCZ7=ZSZ!ug{5=EP(7uvP*472eR2iyFU{iYofuD zN+*5o;I9D(`wSMbQRb4Zplm<^rml|xau+2Sb!iu87Vy_O{=38+xf_nda!=>>#Eepn z$yBJj^*Yb9=AXrp$A#RRb3OOjy{=y@jK_mV-Ve^DE?%>;{lbgq30^11`7GQHz%}-U zo{lQ=NxQmz;z2MpKHdDD!MIoT1rywq04`Yui~-uq!x{rb4}52;y*=+g7U8QcYdQ+_ zpiV$p;EW7E0>Mk&>l(Q;Z{Q+UpxX=o+_D^Tk<+ukcwMf=1u0Wm)?|P*_Yl~aVNmL)Gqo&#)x8kYJwQ0mxRq$J` z^mgPSj^;k*#rHX>2p|%!#>5bBq1JksQUqANI83zicky~+iy59#zf#NAoJ@j>WCl6x zBM}BUO@|=-ak}aNEgZ>akuWD;4mqL?U;QwkF)GI|8dsI|%JVJ>&kXTtV#J23*@TjL z%>wn&vM<`()F>9M#0ZQ4fzz?=0L4`|48)~PXfR_wF4hmxXBpv>3*ade7tAuUMdhcI z7o=>dOSWkNtXok?URkoNz?a(qedx}N?iYnQFC66$MyKog0y!lgoPOr%`tenj z4ka=lmY-uJW}<>=d$i<5d^*j$euz_~3(^kbyz!e3c6v4&6~(%ZZC8t_HucI!(@x)p z)sfUTZ;GMr6?TKiCf(~rbb3>o`lt?s_m;YTn>PMfw4}q-zK`DYQ`uRXoGPbp#)oeo zPQ&2maV7^qmAaC+29|F1dqS~K4Pf_sFMR#lZobqsR8l2p^gEpWNhO<058 z{}lDH;Qw4T{9@%Gukv8!rNL~Bou7N}tHZk#Ha#hqooe#)i+CsL9m3?CW=`<1^&q~P zu*j%mY~~aD08iiyv{u=T5>j=m-V$2=LyGDLLlZ5w+CMQkg_Vki+=dF;fBTQH;3gRd zF+0R7bW7_irQ&t1tS3mtq?CWtof`k>)AMuqedGSjF>Ehx(H=s}neA91R%&yV;;PYy zyJ&!C?$uGe5xt(|(&f4)&*@erv#Bnm&%A!&WGHs3u;MX8*f{NN`jA0SBk^V;KgN zS0T)9-$0p5wugmgY10dsNlOf+O|50zAa9U~uZ8b`4pjgKaFl&TJwyfat)vZq^e&g} zQ+V{q`GCRE2|^gn`V?um;sf-x8>7!b3HKk@W>7Zk1~40%eu9IgcZI6k@~|kS)F8X* zyqs2SCEy!01w!2d5zt1Kr(CX_=AC%{X_0&FA~9g?9>j5~j{4aDIj-s@U+iih#=0D` z_@!|N+po7`{_qD1*2F{gG>2u)gS z*KN8+v>8rjJSNq7*N-@=u^VCneG|0BPmd6iOwV;LpP0{k`E^dMR&OD&Z03cU3T<&y zu==cS3?)qS*eba&? zgs7y8_5(VBJ1&U2fiEdVZKiA|+h{X2{*Op#pT?%a&xz=C%VmuaaW+kIY}Uh?ls;n9 z@1$R;O7WEB;;_20zpWjepQOE&9-AXF7(cdU0++AB(L*6Up3p3hX!cikY zZn!~%fxBr=bW4rz@b08g!ja>=mN!j-=Tl5t(aS^5yry4{-odEG{zTj#GGjusEB7f0 zx^pM}=EsS`rt>B{9t%058;1zV^UsdDT$!+Qo|~{vGYc98^&l>gNQt47*v-gBG|dC3 ze$qobE0BEv77_+v$?>ooSoc(cQkWO`W-_<-;)q68gyW`??_^%l-m;L#Db!sWB8&OOmS zTq!ZoST3zaFw!dLVk!MQ9>)cY#?09t_!jc>VoN4;BoIz3X}HY%*@{gME(k}^((#3% zn*rh8DYD{A$&7zS=dJc%i#UDy_^yrDtOFbze4We^j-Klk!6^a#6S0NKzSh?Ey9X`> z@qNPFTE-7K&0|=K(4av_3%kHkr{=?b-u-NX$3oEXeArQ$oiFCZpyLbu42 z{QKsS#8>34wXbZhB~{Iy#k~kc(<$cD;cMZ!E2(Gm-!Nf|NuVWSJJIL{@20tA;MOqN zd6LxAjsdTGzIlJgUQ%k9wy|YvAJo$w!c>)A@$AmVL}HiWUdgFKL5KGf??W%f0A`BQ z%x_z}1CIJiAV?RYd_rLL`w{1Q|^Z<&-`7$ zi@uI|F>jXQ#aY*I*dgJj@s~R}Um7{O5dLl3u=0uW1;%frXjLhk-R+B(p|4{uFtWl8 zkOql(d2Av7n!5|^CUfa52DsIBG0g)5y{-c|1%Ve5EyDnSDl*8L5p9IxYM8Ml7NZ{9 zU9Q<{&`(J9e-ljFE|GYDcZZFzu{}VB?4>bAt6qbPcAtzv%3-u5T4Puzp=W^zlqyS8 zw)-1T@i$_K%8j3Nwmdkl=+RW+ye8aYa1Q&Hi0wz$*w-Ybu3Y=NMi#c>phH>W^`?{V z0@VUwow!NqW0Mm9HvUyh{kkkcJ44T7Gs#R^kwI7!-&N`h1!v8P4aWO+-Z*`M*J!-r zv+8|@{lxpP6DO3818&qU7>K}7SiyCY=-F}grBDr8n?;QQ`HG*W~NVyndfEhwx$5gSnP z{wceeg*uw%BYJWCGL-fD7!{U!A7 zFnnh+&!iLegKl84vmn>qe7mOG7YrMd5mJHK>$Ka^QZmL3|j-b4`GC$htKmN57wa!tQX^esVfdbH{ zQ?~`CwlUJf&9bG*_uNCP5q_7cN%lpZ9f@&f(8Cg0x2T$Ib3Ssczgpv2i9hC`!GOwz z^X)}YUS9q`+I!@+>WjKT251AbU?VI4HNZ`z02GZaH<=cV*Su@l0-LA|X1SFJx+^*8 z^I@M#Fe~Ir$&~f(bMXije;S$E8FPQQ>dsxeC$HBE#wz_>?$O5@eC`DMo%BD`_{y=+ zx8`h2W!cCBE^%RzIzbBhJPF+tBdG6|yG8X|gx2n{57Onz`1+)b6Mvq= z@s!!_p1^*0(Xx*DZpkv0Ysvk~_*C4UB|b#=xD}>+J-(zGw3ow*U39C;;jfb5*)a)b ztb{0}$x=GU39G3a#_2bYNLOS+uZhkgJJrth*ll_B!y9QCR!qk2!falW>0UU8!A_+a+t4=;5th`o)#8 zdAQ4;gjX)xF+?`ME6JCiAU>YWNU@alU0jvWwx5&@u=#NPbJ(o?O4#2M@2s0#U2@4k zNenW1UCnFi?eOu3I+eeZT|vV+KGK5ok26r)>z01sIn$+@SV%nk6AVNPHc+X`7@~GE z#)wiqg&C|b5z%Tylu?tG6>v9DZ)KGjAVuXGi#Iw9;?q8@XJ<8ChD6d$DXZ ziI_`8Ir$K?JoG?tby>g5W`ukQb3IOjTee0TZm{Ouo$dM`Fg;>U_H3V2`F2vb@;7s4~N*dGz3`AN9LaC1x`-sbWL0@!YN$5A~K|myqs1%6Z$9P|-YO!gFS+?Chw9 zq}YF#9EI)kYnp4TESfyF{GTq*@mKOMQZvLG0o>VOJjWlv!e)!57T69fvulmC+${>slz1_q?i zZrvtsz4j$=W=WB9yT!;ZOdBH?qo)ZgL%srQvp&SyfaVTN7mYvV0e=GgVX*VXqOyXG z2OlQgk_}963?B>_L_ew@VizA5DSd_%_D=e4MKMoLNmv24es`x{|8l8WdfG!zI+JWs%AHk|O>E2=L(ARY9$a|5- zvX~Yth_48hfV$?SjG8gWUq|8w0UD0Ru9h_3%Ij;zWVy1g%^G&ROLIv8Ex-rH&)}LlKiR>7OIk1o}i0?ZP{eJavq&gSrd=MnR7qEp!7`Ec2yl zjqN2M0!|MjujajJ@{C;D0J{iS=AbacvH7kgSjq43Um^0lxtr`QZEU<@xrh2@KG$LL z6eQxT{d~11w%!D5PHi*g6R5f3=Qaw1r{G}H2eOkZeR*)8^npI=`@?$ro(!{&oP@X< zE1GO2#V%)u9yE`C2lnvWRSgY=AMFVH&jE#qgemCb zKgG6DUC)-4-1ur|_cq?w#z?vSK|$go%kQZ!dd6!Ot-eed!q$7u=BB96PGg_j0PJ|k zv}-nvKtYks-+-IG6Y!I~y86j3NM4o34c0cjz>ELWVocH(xGIHBcCrzK%8aEr6?wzS zmFG*(AlV(^OZuI<;h=)K|7ZfV_dWJ#Y=Hna*U&#t=9e$8Qq%QgLc$-rdt0(_@(EMl zBLbdS&>Y>)tG*>(>mDjCPHA6cW`hZ%(L#^>CrIi!=rdFAhCRS-jod^)YQ15U4x9qo zs;O@jY9M}@Ii&uvXWbaZLY@gK!sr}-{kSI47Q;aoJ4$UwAJsnwH+E48jOEZ$8y^9yeNS z?rdCh&e0|;p!3uz>A_2#q->2bQ)1D{%*90Y2CB&L!mUKH@Qb*ecOL?*=VjB+fs7o4 zc4X(G(8j@e<;`&G7bC%e1MVB7m1HluZ)&?3H}Y_(=?I+BA+mn86 zLGbC4yS&}O!k}!}=cnh;tlhFYad=2skV);Y17-mYZ^#D|s5Rcw86SdINBG`R_uwy+ zv-w~n&IN6vXjyHk`oH9`B46Y-yu&vrEVNCu+1LmDyUyYryVtsjqnV$A9R{X&Nw3Rs=UJU#wJbgF@yFQ}M_SKe=)Lfm(>Vuh(P+tzd(rv7PC-cr z{T0gU$I&aKn)7{4 zvC=}zZKrg9zv4eXaQI-9jm331ffgG9-@j#f3%%!`r@vI*f&AC~`eL@q;{M5AyuiK> z{*(7s_hZk23_~x6zhZSX-D}{b0aOQgDLND2Ma`hv`%>yEQrzDYQFynQypym4SLK5z z_;0ka`0+!&Of0w-aS(`_l^+BrsUBc{J-tVO`7l46GLzORJ0HS3q9mCzkGah9C z;(^Om)BL$V88uiag!S#Lp>9`4^wsrmljd~8r%X>UHvZ) z(=1(dN6!{$Z{jGVrE|ck=l||tsy{@0I54MH_lmrt<=RA?pv{s=coVY zyQ9_2Ecl?PU_h|`L-U?urEB9`urz_-g8+?j`k%#EiJl?>lDF|qDLZt1JY}1~b+HlX zo}h)nQu^;6R#Vuum?-{g%|3YIy~iGy->x4ys4DE5VULCY%e%YN6rZm}3P$ly>{zXO>R2v^KK%xG)< z4y8y}RHo@4c5qSwGIt#2hu$ZSyySB2^IA|tT1Ef@PZN$VLu=X}o?f$aDEF8{u#pkI zVMnQvaq$4}PDl-}CZG4M@XnoqS9Zk2J0ag3IhY63afWV{08UlSgI{b->Bj2mjP?QY z$wO_4ZVTlvv4w|W3}w$x($aX?OG<4^#jKg8`~Vxht~1vYm}Y*wEu(G-3AKo_OSMC7 z#tFAuZY8++-O?@gyannRA*hTi8o<`@72tsPkxW#FwPhN8b>w%{H3qmtC_DSE zG7KqZ8gt|j3$VC`fC8E+uSrnRLF=*`Y)~74F>+LX6<)U$;wS}F3(`zixZ^E*eLqmO zy*4@Oz|r@$S_sKGRMmfg*%NSjYt^~hfB@XMjJ}UnSH~0eKK%H_Z{Sk6Z{v7xcwfc% zG}TZHg`?jpWqJZEVvW@ps&%U>)wbl7+NT&02$)lPsYVndlAwdv*Nz#Z$r*I_gDDR| zsmY|1YCx*$3r=V*33d8*q z!el$y^`JmUpt}owjj@uP23tuMZjraj!}K~R|3UI($`w_Ld~zu<%j+<_4Se))Q^Gr|Ae-8;TJs^5hbG&=k}V6t!cQdRg=*e8+dGb4I|@tUhK z0;A%AYH!^@w8}-x!9)@*kU+4-rqd~)@M;8R&;U?1s zAK7{03#GB&NBH&V9^f>H{_C@RXT@hZsp3K0xOWY@wEk*uVxG59m4d69A-Dvqc_=rs z9?b6+-`GfEc43Su8>T=(&wT^Mv51-pg0Y`!apAVexJ5NexN#%PYiou^s}bo46O#i`v_~Q6SE` z{WwHeJ~V)(*ZoyG7nPH_y`uc@=8?i20y?Q8e?GN5rExSaXOT%LOrBGVk9@lfZp}8z zkSl%mdrtRUTZD<~yG4CI39hUsG6}JCCO=zRK`ZWm-%PYMEZrMGWD(1#9hh~mDo6v= z%plA9xns3gY5IIp7G1pbNRW)vtg)vlrOO0BK0@DQzA& zaDZN>;7kKxRDDo8k&nQ7nH({lVMei=h`r+QoB*51Ei)ux5gnjSoFwfzZ&=O--I3X1xABa5dH~XU?L*KJgeX&}`~yMd?$1BqssCqKl0577H-XbTS6D zqRFaK^D|)G0YF_wV&4j!6(l7IzRRX`3Kkx04JV<>-ZULpq&h3gxJ9_0hKGhL`nBJ zpnHQdHf&EewnRbSWr|l!GIh2wpu@H$gyePvcBqjPO-Z4Gpq6d$W=nI3!l<3P6x(9; zLMwb+P=CeEA<2KtHcaCjKOPskLRPFA8TF>!R}FjJ(*@N}D9Y7|4VpsDo(3%sBgx^{ zPbxT%aM)*k3OzsvYGc%M6$Ub*uvE&VQ7-^}>`!vmb&!%#tjOL?p>$Iuw4+4tto-(F z!ho-xAZno_grCiBNMx;yv9ua=>Z$y`pSZfrF3>b?z@SXXc-F1RmJ+Ws^uZn+P2B*> z1a;gl|7-kZ9xF+WVQ)BnW~7bX-PqOcSc$^0gKFC3E2ZW73qqoE-LWjvJu^jcTZ!3_ z=13``CvXkQj!vkJ1)k=xC#PU4a?^3v{+kWkDjJARy~<0M#$Iy&E_zSjA39z>(QP+2 z78V+o#Hp!x^@&B$(B*;B?&}x=YZKTDSL(KcuX`_IB!!H6y-Yq1m4&K9bpWvPKG+0g zt5dp72>R*NqElBx67C7m=cG?B(yzBTks7-SFYqWdiBezV4ocN!uVg(_6{GHF%SdQH zqEUDNczW)Vw`r@K=!i>!%1wJK4y<{iFjlT&AgLy^Oi5aV6fFSGm`a5nndjBYU)@$V z({i90Oj?2-9&FbpGVkXcm}B4RIR*#uA0bcm$IAylJMQH;ROnvstTEnNX}c?apsFSg zH{)D*h5tgyVUKx~P7ZsVE^FYXj$?0Y{-L&YC6FdnH?`OJ(X4fpKK>GZEaRrmZU|7j z4AS`~^lR=vsCVl6d3-?5UqUh+mu;eK8`hkE$Q9_P`LvGS<6e3z3`J7Ak{3;GpkJ4t zzZ_9^ZYt000~R@uj5X7cwGRLrMT?dMxB3f9<(AJM>crSOU~MI{HSXgzrJjWiP;ve}iQJm4aUeBNLm(*@Q zVIpl`h5LsmMU@o0h6WF9s_9|2kB2l9zhDZ{$WBZ;<|2A*T(|nr_qq08`4RqOVi_7p zSO-J2g&^&WL<>t*B&Z;p)2FYA$E>Y@>&wCPglI1``MV~#{4MEqT4Yj{cB71Hz|R~F z7-UgUxWcIC5lo!Uu4R<%KZqJ*s@UHRUh#Cpm~clL(D*#0MgSFjTYX_|9bg9ud)iyJ zKz-eOTH#bbdX~*!f*48)vXc5NNEz)S!+s@eD;#~DOa)a^REOz=i2exK-&V3|sB}c* zTTM!tdy(V!`0lR{wtd+AVD|GG!?>%aniLT* zsxvt=9u~X*rzbcUSM;^XQ~+qmb;|V+3l`Tg$zO#(eWhy*GeFHAb)E5IQe$oh*##iW z)kieeuvk$H(^N{hZ&q#6?LVfLquas^yD1wHEs(MUx?yT1PGo}s?>2o>JQ3}b8}f?C z)-Mh)SU*)?XExL8V79Z?CVTPAtoarJC#6HMb|Q#A^|1Tu@PA7^y1}o!B=Qpu*kjPh%v^7 z#27OS2ua$b=uL>+A0TA+ld@(yO^Mm(q1t;YxsPje@4TYg>$4Ks$(F%?*~ z6Y0vaZE$cWn6A~iL)>)H=NRx)w_ejdN+qX<;v$#Z&yB)qSviIIZGt1ej&3Zqd-_a$ zIOla>TpG5f1EKlQ$yo4AF`|42DSZQItee~Q zA3iYtvoiK$xT+vP92y!PHvtL@->kLe9>Aau0|gr(c3h32Tt0f08ik>w>TG6<)JsVP zM?tL1r=!^P-yJ9`kxALThQ}IDXrbdzcQc89mZil*7Q8+veVMP>TMY%1P;31=J)W>}s9kcXyA~y1y}O zOecROvQ%n$AL7cug5>axV=ebWdV<_kfh*_Z{Ys7Zo45-QH;dz&(CHh2t1;^Low%7? zfZ+@UB-Ku-Y4;wSi!icLHPpVV9w)qNzB9J{-b~Tyirc={DI@A;(c~9UDqjww@pL>n zMO(!OI8mE!1z){7R^$vIJ+xCm7AE~u=c4)VfM$OUDY#3tN?SzPOG@c<=JJgIG>LB% zv*^c>&M>k{jA3*K-(ber(z;*aejH6slBR8)osFA25Fi3QWx3RWh1P5ov}2y7%-&Wz zQ~FUx?w)0&49SiHXaRWS@S_cK|6Q_4;|A&{jMW2+e~WxWyF_}Baf2jx`;;j)o2|7@ zJanWZp7d1ad&I9L%+%LI<};@&-N`ilLMu%ydr~i}9iAuHI@S)w4SXTubA4;Wn~Ep} z6zuIU^w~r5C*;fD6F9J6UBk#2zjLq#fcPITH84bu$W7Y)Z!lK4<3Vz~38A)9OzTJ4 zMf}y|K!|lMspq9*&qAKR<&OiqIVk<&XTqHYZiYEo6)~FRv53E{S@*#O&p%y|I?-f$=GN5-i!H2aDsj zLy?Q$_IwMi(6SIY#6ODwst6<78OVyfp+13V2hWt+VG-K4p*vb$Lu^ek4t|5DYycxN zB@!R#>q*%B&e)k)50?S9_DIK~f3{g@Mq7c$COG$VhMd1zg)wMBk*uVct4-EWpOl%I zh*_$^K&b3F{%XXfO=mhN8DOK;PE5Up@OeLb!2mRN8I-Er+ z_U(tX@1?J8|Dd#~1-jy=Y3sllv5jn?O=aj9wN!m^Jy%oX^AN=BFLBt#hvd`PhbWNZv2t)BcL5nt>QL-5D6n z9)Z+SCOF$N#GJ`C|F_dKQcJT7KUn#=tbJV!HpzzE^aDu z01V`#Uls(TJY624OcY{4nC+E~a}}LNqNgE+%)g7Drux*+C!eg#t-(7i3{NM0P5n;7 z9sL$EGE3cL8kVi-hX-{leblEY-QUA$fTjV4WFG;iQKlsV0Vf}>YahAtGsc`8SVsP} z9u@+otj=zfGsDlpTDnJ6B6_~hpaM|tTus8XF0!)cIQ{lyn{raN4>Uo~3O%#XhwP*Z zFhYh3#GC10wrMI-*S{KJmf;WXpkUbN=z6e8uQ|wtC3^&%MMQTe1Cmk_sD8!h#SJ}TNqELODqTH(MQ3r_#+gN>sC*xw}C3yvxQb_I-gAx9HhpeEi{%* z$iGYAr7D`Xgkq~%+K-7wWrE$uiiDK~(^%k;k&R|H?Jd)sM)Eysq!w-i6F6eUd&(`{QhUdiLWoo`qDG;rPqPx+w@U1Tg&}@ZOD`gFWd5mOtU16qc zwv@&K_8S)@@*BzLpT?NBFgb;eoV(-%U3oWb$|Lj=oU5+ z{}z5s&KtI=!#N%d+djYxYT?#5J0~~RVeTst8*|h~zN7@p3Dy`fy|v{aZ^a64^&Qn# zVu*YimCbAgAX=36fyW-o0Lnm<0?E5YJK*p->3l&4zfO6LV%;g>uP?GV;!il@jb!Lu z8}%)O2`mR$iD=%wfo`M&;xBG}z66EX>w$4k3HF*3I5ENhVHB?(ey0e{HbYn3lxunc z--X%AN*7|THLWk;ak}!J#8OQAbspgUQXfrIY@S0c`86{sz~IWh;M&D)qeY!xr|S!e z^BX)dao-*^L|oo!J5gW0`BI2GD-s=TNeEL!M#g`gNwsUlKE;LlBqs;I)9s*MY_?NX zegBjafRnJCj3HrEOTgw5k%ItOjU^h}@*6Pdr1w#Y6ofc)oauklv(Cp(b9xYFj`T7J6&1a4{YSHQ)$Z2_%@&?1e-Bzyf zl1|cFHcgPKPMC5vcFLR}*LC^8BrSQcZ_jfs90iaG8%R~h`{zR&H`Nc7_w%_9N%q^^ z-m(?R<36JVj>WE?H#^H+AHXNypJX$)50!~q2611BPy44L%d2Z`?5aXW2%kmj&s2{V zcLY~Sd5wYMQ`)FP?CGpyEhZEZ3Hy2{;4s=R3o$_%wR_BF*~ z8pTB81|*@^90sN%`S6O;a?*>@6KBD}8<`ma_St*80kjsQTSbS7z$M`z@Y%~mVg)2h>zD3lH&NsUS zHoa_k7HVg9{d2%^Z7uQva%n}e(IE5%G5CuX{Pqz?lrbrykO(2W1Q7u-B?y!tq+}3~AuYxJlEMBww^!eD`ghjtyKdim*Xq^v4~4Z@$sWJ&eV_Lk$X|R!oXJx% z7*!KMlO7-n!~*oEz}ds0X0Jp?;4wLXKmzVPssSH4#u(^uuPvAO++u?yg$D0U&&&GE zu>gQSi4@5#7n)N-NaS#`aA{f9DW|(nEvL2?bERh$)wFFCDOZ|<_KG_CHKj~NI&>Km zIgim`uT!>jmQWAMexed86`4vyf1Y-f5Z06*K~r);SWCz)(4gZ{vqr)IZ869;o6v`x z$SeI$pdOwtxfXHyJ{~>h^i{N1t|vxR-zYLlL&zD<+{8r3>)C0w#Zz4a?&?M1gM;(o zQ(`a@dGJ*O+~9o@I0k?_!|9l^TyVqha#Y#2@>KvK411MTk6A5Om2`vcmme9ja3fbE zrsn9bEALnuF$U_$P>g6FbfSARt2cSNZsBQF6^Z@Ck5p`$ly7r7_ak2n0P)4wBrUIC zbSHN*QFcSwv_PvqFSC=A6{WCGJ6<|~>JN`j!2%4zzgJF=>C>Hxk-WvOI%9l>bK62a z^z+n2lUdxQ7l=;mT)Ydu?4^N@q~orM+(r@Lda~fCh0&huB7xI*JMl+HD%GgFG}= z{^5{6Mha6K^bHlK9-V$li9Gb8d#-((h&1#l&(q6cW82r8vL&?k3lLhK&R^m<%q%5LR5h1`$x`gsJkHFyGQBm$%B;$x*M!y=Q!5UtJeTItJJB}{ zW;O7qaoSICUENwVr3pfu_&g6_UE%>uXgZpqICwqPcsPBLVC0r^sj`E=fo7VP+s!tn z=y>!Dob0pHqv$kd=pgYGiGPjqo<4mojK64~Fh2B}RQ@*^Yg$-TYzs(ybFyFunK@i% z#ktfGnXyX|2LdQl=UWJo--W~c=pmp}&jg`x9RK*83-EeiZ)O>~@AD%+$s({(sZ))R z2xFV-Hg}?z_q~sG&5^-UDX8uXxSf;y+nC!%~GQOJk` zaxI$!0E6YPJ@eWsQmHwXVcZKSa}yqX?P>D?Xc;B&0Eb&n&_FNLliXXIzbNM&I@!^n zYWgjHk@P7FD)}YL6{(Ofrk^9tK=A?wPgPo~cqw;hW59zZ+YR_Y*z+QK|BN(B@;T=j zSXUPSD}OW9uCvC7Bve0mP~uid)3g3S+lwema?bjz9*E_&@?8_>D`Kw~ejYJemt(aw zm2^GK=6ZJFG|>dlPWiBLpql93m(Tl9jaP0|g8gCm78v0;Gr>2nKvKj@fE&jmH%Um0 z9-yK?&6xfIsbgn%gLNOrE>33V$6rHc8&iJ;CI$ob8T#E8WZ-QGhVyWvTD%^iv@=Ky zoswuN7@$sG1x$^v#}yF4%ZM80OgDBc2Q5?(G@J)-v<;J)DtVYy?3J`WE1q*aVkXR` zLl+?)T+d!EvFAop37LM$GeqmRB}ir$R%I;p(_IFkq>uI!GpamHe{)|5msuY7M8A;1 z4PG}GVsxr?M^(x38v#3a>}flcRQ$1cdT{>5*MZL@uh)21MJSak>}Z1k^Y4hQUp`!i zvjr!;SSlF-1$mnwC6ZgO%ebplqxFT(TFuC}p`twWVmg`0%?f<|(>yYcmyTixkZ+I zL$xG5gPv#AK+y6Ut4eB#*4`l|kuC+_<@d*M&O-i7;EowrY-{tB;v)SSjF1BOF^uxu zM*X850s`27V)(1UgO|Z!VHz}ULyL-)TZz_|w_ph*47o=$2k)?6KSi@%UITVmP5B5nIy5H`d4CIvEWh%dP!aitKpYQS`cLkT`&^Tx&xSPo~blgN92v$WaD> z1L}@}vRn3pK;0R02lTbC3SI&crLmk;`toHbi?EW?SC1#TQa2S$EL>^#ZmTyi;dIS* z*tnM5=&{`-w9!{-`yRiI^Ki1j$WPOwW;8${OSLf57>d$2+B7@KPaBqVuczjsh2uU*t<_tPa@oV<#T0RQjJ%z zWLlW>w#EW~Ac7a%Q*4^exikLGDQ7&{;}61E+1atm8{KU)xng4w!?Rft5BV~qpo)}9 zu52&mmkB_$aEW8AQZD<>*c`d7)EZ^~gb-qY!IV#9pHdBBi1!LGKb#)5j<3AU-bU3$ zZwd8P-6T?`VBh90jGi3ZE58iIUJQ3}Wc8)Y--+E<_;O_5!w-)$(_H;Mx-z0Nh{M0O{KqD?8xlEe$S_ z2D)f4A`;z5lNbqo#OEaquYhO>$|M@-Dl*vG%&6HI_A1n-r7uG-0=rE9tKu&?QBCYhvEBwL|K{-rPF?c=gcKL5O)Mj3FjQgf|$<}{rGcX zU`JjAnAZ#y_#zw`Sr6o$YkYyvDQz8!!NvN@HGvM#n^nYb)33Q9y)wnEgI3D-GAZ57 zLz}Bk#o`V~2aa&%xPqhGdMuU~f0p6eX#i)cGhmxc&~|i!;Z4U%N(b;Gu}5sYBv#^n z8A1LVf~#!d02&FWF5RXHR#h6yPo*^j-*xvi5F+mV!@HVo!6q)wT=$`wy3&bGw9Oe% z1DBfyE$$ZFUm);yZg-o%l6pPxT7a|jL{eo%Wp%*(`EcFAYGOz?fiVPcg%JV+-9o=_ zw~4|AgR06&v)qX)hi1X5xZw_-OhEqq;>-IZz&w5Twfxa!UT&#}1`5V}~r#fkRv@V94-8Pgxb5>&XyRA+!#i3s8BXIeV$n?1=G=Y!xuuKl% zOnGnLo@Vw5piA2ePUXmQ4BIF1*Q)(kIhi~hOV6*5clD32{Oa(V-i)Y3nwM?4t$P?0 z1+hY2DSY-z03VpMM!p5*2yny(gPn+KGjzwm79|!C?R5LjzwEMz*K2}S(YB-HI_2JX z@o%;T#aob6ooaaJddmAPEpi0PYtV24j5smwgC>)V?rae+L4-JR^AMznnIrTQTR^V& z1ChyfVy)XTBseu?2|5PE`Br6=35%>8 zmHEE;@n;TL{KQ@n`xcA0d790sDI$&q@n{DUk2;bJi8Pm9+IA%j4GZauiHvgcWDcSx zU5zYzS&L@L0%-4{CG7PS7}wo{3(27v*HwBl0HhF$SsQHpTj`OEHK?DUV?|vNh2eU{ z$IIHLXkW$#+fQyX@zXJB>f!YBIj0Z(RMNexC`h*5VY8mo_AM0^cf%Z39*KPK(Lr0S zNGsR35;0;pepn1K1+$j@GvCje0}SO0$5XwYK6)Zwj~;{z?VkbD(>uD0-3a;y`LoKbrLGgL3 zOs`^P06)Pod}L&#PUGuNh}EKatj|^cWw4ByJ@L6C0`ui0Or*>9*f15p+G-LUxzVxT zQ~)=@@S95ZtC@^x`F9978{~SN`_G`_|JMIg{fw1siIm4UCxO$}pbS90;|es4@F!p_ z622bx5CbUp+;kKGOsm)j5rsB`*hA%3l@;tY z2pk9I$&OFfZoOdX&9V!DMTrOKtv~f_*WOI=UG#q-T_3eNAU~gUu3hl)2ap2W$Dd;k~S4 zI!&i7cfk@|8=`wQ72tm<8@D z&e{&qc|Q^Vj-3HeOAoZLw<7T!jU8*oQ-)SQuZ?cheaKV2T{N2(e&F?7BH#-QGM2JB zP~&+zm>mFVyBg{zNdvakRfQ}#A1h_5cU5PD(jFpHEM!4MdZ zlrNX)H5(X8K6S(e`^vq9UdHtk!-Pe??j=Jm!f=m9=0!n1M9a%Au=soOWqwoh+UhEo zR4(4i;O6yOlW@1vRX$Hso6H%>wbrZMWRL8j ztKGTVv(xv4?ZYQL44=r~mYm?0;O9s5Y(Kj<^_m|(318b+&9te``Z9kAJWG!T9ftqR z0rkwJ9=NsO?P$>>**X+bSH1)M56oT8S`$ViQvW(68}*S!BHj#G_6yN^-9p4oPbhW{ zj$^Mct?HgK3D)=QK5!oP$(4f#Wbpv_++aC{A)Qn_{r;p7DuLFM4BE=_5V3l0(=Z`| zQ^tNE#B1nbCk4of7927XT|NPlrLYx6jDH`f=Pvu$FW$w#8vJ_nETuDP}%)B4Y& z_UfOrNSQ<(ZOduHH`d-ywmSKlKTAGwZ}XG4FE4$4%rNP`h8svW$q65D+$Iit+=nwH zj?R&thFWE&GLG3-OK>iZfZk*E3Lao8)n4nwv9*=&40a%W)!=$*OLCfW7$Ur?xeUH0z~fYpc3Fh6dZ*^pu}9`$)No|dN=Bx&@XLW0q7E>u z*k>&D=wV}&gP8IRrtH8#Zaw`zM!UvQq!ASl=buARMkxM%=fR0 zjACjG&Jop}j^lQIJUifr(8l^${=Dl!y!kB0y}59Nd3r*r6F=hfko;L6Zm`vpjrY%b z7Siu9)k_?OP56j2eIO9<^<)_NQWPwSi)D(ZoZ&=91>EEk&oARgPtsOUccFEN)e*!r zA8-~O5p3jU+#*2{s2P-d(3U3lI>e=++Qh70x?ma=T#yE6uCK2>9lJJB$#VwT!C``d z4V>X&%wlB^Uz0QZ7^6bd8Iu2mGC|_{!#>D=IttdQAqz15M1Uer-?LkXV4}+PIxqlI zit!U-z<{~>4oOuoc!&8Hs^`R&d#0+!DmQfyF|i&E$tL*(Kc2&!j=f98(agO3CJWOW ziAgyoVf-X-+?N(6-gn~=P;TRFYQ34W9$@Xb3=MO1XqM>6Fwkm3@*U9c1Qxl818Nd- z$M5{|k@yQJEHk8aWo%(1F7+6dCZWQr(Yoc{Y&8@WpN~2b?ua}peQni&R^O~eNBbGK z1P$tcy`+(215m9H##*+GvXg^276lGFOxDW$J9S6v8*OYn8Rf~b}oeQ7e4#5*$v6~2q{k-Ke zh)8Vpl%4FZO*u4MS+K+gk9%3gVP%cNHhknHa4KnmSj~9c++!~eyRU~^dV&w!8hZj}{T%F6~Jl+LyYic_&jyTuN68(xt z7IRbax!I|;HACRVj<7(5^GbDTDfrcy@=c|s>Swee5Xp7xp~t!7(+}3ubkX);N#?xt z53pifV4u#eNvi|u`kpF)NShE{%B^&UHh%XLQpbjW4E0mq8O<)()4V&v)iv+g&=juS z*uJ8x)G71me8|KOp#Bp$oA(ME1{U7LY#e zy37@Sd^e@5zjgpGe=aeSuTGK1D-z%fSk$lsfEia_loReTD#&t8qNtf8J4SUtNx2}1 z!!A)$48<920^>frU#NQ!Z4P(|uHbQDf(c-ruZj5FCb(N4q!b(iQ6BkMtH^g*URH+~ zGK0)0Z&y$8KATAs1H`WM8t{YGBC=YXq|9bM-%jWsL`Yd$ra3lv{U&}N$?C` z?g{ui=8Ra*+T7lnEx8nYJ>awL&Zo>f^_dn=TaetEJdGkVT~>Qj+rSbD7(?GWbcs2Ul2vGleU z7G17>@vw+!qBUDRSYQR?=$|5`huEkHtUWh?)z*2L9)&#Y0n*!nOu|JnwTxz^>@a}P zK$#h^Nz=wc7qHsYU#qKoR=t;h7VXv_69J0Yr=pcp_R`9boD-jDNaLd1oxw^g-ALC zA$J2rh_Z|bK*XMD-|Jf@$1NsqA+vC zaY-+)7qo6n3cit7KA-A%z2nf=ENP0F$Mx!wKB;GclgIQ;cE_Qqs)BirAmtu3{E|`T z(cJMP1#sMvVXv-B5VOYL^UA|rqE454tnh6u z4Wh;cfVow=4P;V42Z_*8b_`i6@{2eDoPH-^ZWHFcm9&9=lHJE*#K1pZ zKtAUTLzwZ@(j;yGz;`xE+|vUEux4(|3s}3~uQEfgTYmAH$xx244M+B@PCQ8f`XjrI ziYGNzPn|AHCWQQYdYo0~nKSsYg_Z{R7Hu199BBAv>8TnHmt?`ALmSFvUzrzVHh-2A zVRasUQ0>vCO6~6bKK`F}`mZn__>_3>+2Ihk02_x#rXEXfbjPO;ek@M9BTULF^LuSG zIFJLL17xI0Ua z0j%{HiE5)q##XxE>1{^2M}0L$>jQN!2()nGt5eO8Lu`Yz6Nq0|%}!>{xQS&;YMQcjwD)`jhk5THqOlNVM;(t}KRw_uJU@9&rXxQUDX^j1_K?jV!mIr} zpp{Cke)$fG255pn@AAQ4;+L_$Zz{erp0<+WODQr>cg8;o;FLYvak%E;r)`h347XPW zT>tnfft66*wev$otbbeCi>FgVV^3cOh<&<0b>nv3hk>l5yXLZ>R;S6+zVhy_+4eA( z$P*nqV&<5h)`O`dP#O+Hv& z;5)~D`7+hY6a5X=7Y3p7BwXeo*9?;$L-z_h-UAuP877#<>;j-yrqJ8iCy7AB_+Ezlp~levzSVXUgk zi(j9B3l#>S3u}&)0Ig%sLhem7m_7+5(ViyhsK-LeNHF-+NDfa+@V=O_dz%j@T#uH zXaaDL6@811k%yo$h^8xQBVycxOGd!XKOtZ?VKu0BibufckSW~aD1;51cCkp?yjG~6aH@`Tf^^{}<&!;?2jmwm#v+xyGr zLJlT6$tK5l3XVsKgh`~G@2y}Iy6Mnaq;$6=E0=&W2p0$@!)vL_ z!BdGz9uZ(Bxx?7?6t_mHb;wB#-u81ZVm^(){?ojr zBZtBDDp0WalXjf)4PtD$0JlH6S%J>=mbkDLyEI@7OKn(th@Zmzl6P)?wm6 z&}Ei;w=LeMV4R>nS773B8gW7s4|i&RQ?ak@nNuU#3BQcD%rA|l``v5}*fC_fb0=FCEpU}i9V>S((>c*0{*cqa( zlFj-SJ`!v_AqJ3t)}U~+=P{k7eK(6vn*u%Eh`^drJ)=C7Au1TDdNxo0KU2f;L8V>y_bUD9H7+64Ru4mAcvEX z`Ffg>B<21rW`l32VVW=7l;LzbH>nGw#UBgh`TsTmMiR5fpS^Yq%6NYLUQw8NcgUyE zey3+siL5%uFE47{{1)()OhS$P`*;`Vq8clKc`?1eTL3o_BKZb1RqAPU->|Dd<1TFx zI)-c!PS;f{Y}}%OXkXjR*E6~ifMw)Ek_<+^-viI_wvb_ko0pd*|9E)?SLbL?dZF7F zva}t1G=QmwL2NHk>ZquOFDM|d=L!a>uKJ;}&qXqld=(ND0wHvi-0bPlIH4YL2ps{` znWRE~z3!dbz>r|!qoz`jJl_=BoLkiY0kA7a8nu=B%F6A}PUYw(K0a|Y_m_vJA3W2i zSlq+YI>e{70XtaRxV+lF3A_?Fc*E{EFcjPE3zf}J;z4}I6^Y?!X*Zz${D|f<8xKA^ zOPFv)A@>TTFFIFsXXymM$PC{m{pDNHuO3q1@9v%#@`m*#M80}j>MnsNeZWIrrg)%S zDnHb18br8Tk+G}~;DR%U*fGG+09YW!DSwe*nv5B0<9*_dBvK*eaCcLm_qyJqzsXyU zcLrRTxzaxZ0sK-2n4hbPTIe*)1V)S1M%R?@M9&xa+F2vqS2H=h{lkh7;e51(Q;|~x zXN{eqSDVEk|NawX>sXdQ*zmQs>N-T#Scz*f@jwSGDm*>328f)_nwJ;+BscsL4npeT^@_B>FB*muijb$&Sdw z&@2%q22%##mD>#SeR!lqI@){l%Wjjjres(nh_rd4u?sEf z(zoO`KR=D$m?ylOn*ak}H_~#Z+(z_uj{Kf+7sDF5B{7p~03VG+7gR@f&eys!j;%xC z)>m*A0md0d*#!_cOd;*333wIAk0&hkq8gW=mti6PPY>6OeM{Jz{BBeNmNUlv@*POO z{OVR_XXs9S@5@WJKOF-i&HOz^!OMN8+sZFX44*#hbM`noK4oJw&OM~C-nRf)XT73wJQuVma z-h_6%^3=GUDX~l|(5^xzeC>9|wPYA@YwUQ6sHp54mZ@o|s(kGTa0h4s&r6OX-_8i+ zaMj!dW^sI&g_Ub=aYk?5RGl1=;l`j6y4b#Og0M7*usVlbd&Q{Bq22~)a=iid{?sgN zFsjt+^xmh@9D1Wx*b(hX1Bs25y3RR@oKAPi#btlTC@~XlskhtJ#k6N3_mu;V64P6S`zyxP)K7FB5z6nQOc{m>!0?D` zaGfU=ybMcliZta~N{95eA_XR577bF2q&=u1a=+dL7Fji5G{C>6i0Ll%R@2D5 zK%)Tzh`?)u!fonO`?AG<`LZohi3l4*TZcGA*lEgbkd9cad6NX!a?e5mc|5vwzX8`3 zkW*!b{|bG_!x#HXQYIt8f1?!ugt;UpG%o1G!9>P@2bkitdrerX2T<~z^M*f(3%;UAfv`=#iU zvf8gmeiLoa<+q3F^nYl|!lMCQL!)P3tG}p7RkQIj+;e!jE6HfPv%lZW%l$(;$FA=& zdd5l%Z992~=QJ4Z<~JSu%%eYBhJT|xPTNR{l=37l^3`-9My{#d@ZEFb90f0ZfS#A_ zmOG+mBJ3IvV&cpYxI`=zGqTPSQ`}Rd>s{8H0^{j8kq5{&dFqrq%3g(V8akGa^NwZg zVyBVQYM*M^7V-7E%31W zl66pea}}mOT}PlyD&*x(j&s&cmg5$Azf{ZHK=laz>%(aNswbqv{a2+iCM~nDy^*(= zHnk&+K$1AJ!%d5&aTR@?zKr96pvcn!t3JmF#k6-s!u~MNMG6p|2q39cM~tNQibD2J zw0`du0GbD8vHG0j}r>%F@K#E5qR%j-qW7{H7VYJ=`5H6=@*XvbUKwfLr@HR@=Yh>4|=*11v` zP1f;6+*ye|);~2NXg1y9g!|(Z%c49^h#-!XU36n=?r7ViGsRw?-pqvpCP`_vtRq2b zL<8me^Oj~n5%mW+v!${L%$^C&m>ADl>r0^W0cmj3-%v=z*CobejU|VgK1>CIOuRsy zNZ>>RJ)=zD_*m@xiI>{Cm%)>M^)-pKlK6EuSR}~c3Z4YgL&Fw8Xg&)-G8^g|WTu{9 z4iF1+Q|Qu4Bx~3?h5UqSj)qCxhUcSM?M97cN(%P$J2^32NVaRQF#^7ql!W;@>pn*Qm}g_gmhRb!=2Q)kG5nbSoYMkn(fACO zk6b%Z%0rgsgUSTkZ1SVa4@v+8mR+7s6f!Z?_2`-`UmE6JVemBOJ-UCcnnD9Mn{~?7 zUAypgcDjkDddv%I$<3Yq%k5Y~V-og=EJe8jT*Spx#eL53)7=z8Bcy8%X;CyqJ`L}m z&;=Z^4nhVRI>vhHCjBHyZ~1_RI6p|?c1)rj_jF?Gc60fsFGs1WXGp{Ar!U89=6zwhcj13_j9{bXr;QlEo zf%@9{wJ~qLpvkB$%fHKc8W>#?&&;ve;onrul`0e~0mv^XkbY7yX**~aSL8Bqh-8d+Xwv@oQFtbSHbL+y#fPAZbeUo@meDVcUGSkrZRcGfyl0Jp#O^id|ak({L80mDM36EsjA|uun7LIV9dK zf`9?{1t3&~gp4}!Jwn-Z^BK8G)@kYm&`YO*>)*({t=n;jw$<}YQDy8FM8RkT z1$0L_WFSH+3Tyb{UxCZ?*9|Vtx42WSos|Q^tH5b4p}kgGDV}qNlVG540`>3$X{u6U z_3AeiFQVTn*SFkZYN1b)2`o8&Lc;3NRIkD{Z6V>s{g_kT(mecSJ?9xCf?vjd zB!f_Ti#BIvjl;Dpxt1iOpb&lAGJKV1pACFT$P&%Qg->reqJ2JsfHLyFoRKIwMifqP z?Xz_hLeV`vgqW!izimtcIJtUXZDn54#akucmQoaPc{LF^}uT!D8GRN%{WUZNCqgAwQ$gq-4NDc=I|F6WX=sF^?y6^}G9^(l)7$NB0c3;gtI3en+?OH9>J`2sz5@#R@a4Lx}1{@!4x zcF&gjZz`d*UVgdyTW~j)cqncK%<}u^rS+2N21cav2Sq7$3y{;y--0g^c+p^SUrALj zmY;j4T#cvhjetOUq~LJTQ%Xe#m<#JoxO?6&8BURGi_bJs#)ajNYxBaJKK0V4)Yn;a z69QIr=1yj<@p*1s=DU#_T9nnAG!)xTt`23iwurmIS{07wOMs`Q1%d}+ z9>Z-f1q^VO?@)Xms;XS^YpP)Z-p_L+lBO|8F>dnkUM9Kj6cE8ZrpTG417ZW}7wAXa z=$VESbgUoermP6oYf1!X7MVQT5zAk1>i|9<^d*>A*#2lK3l--BB0tOEfh0v3#jRuX zJAnq3Zo5gqTZ*WL&S`&h-$=K$1>EbP`+ar7d$yh zwP%i{Cg~PO7RX6BJ$1OW$(36XZsW z21z}!H7cIJHo21eR2}G zUGEvE6v;J#h050kzi_uz_nLR8o~D$EnDt^9R3OWOi4_O^vM2-ldw$Z5SLBqGgFws6 z+a=1h^ThlKyPe%ZjPc~yE$+wTk6rdtUrW!y66+PNU4gFt9ENZR@-!a$M2y~i#kF2iz45g5oN{SK;13dPdz39At{j} z^@9Qnc6#oplK{RO>h%#`ZE-PI4$aIC`Y*S2Ss(G%X$pjr7=zQwM>^!|I;8M(gVj9( zlqB<}uBc9H;g@on2aMCw^-duz@mCEidUfh`*_@|Wdnao)2?rF|V>2v09n!&igDoZR zYW#BSgYDu1v%!SE#N1Wv`Os*|-vYq1{zSu12bKG=49lGB-gM)9yPU$r%5e9VPd@TB zW}s-(VS`-M-Ov^6B6&D6au+y{6g0e(pJSnnzbF-LGN{qg+bm5q(qz*ffIGb8)Ggk^ z)TcSUumP+YuBUKHJg09O7H>V?chl~G;!ov9e@Gu19wrY=OJ8+wdhrtVko`<+$EZQ) zae9qW2YrV&qbwwu5k=bZGHSElvZA*xB)7XVSIUEKAHUusLNSe&urQCowoOrqBS z3^)nkkz7UD(h`DMDYq|dAYUYjs?`@Am6(G&nZHhvgGAtzYyQ~cX7O)n|Z>DYL^njEMP$5r*c%-)6wGbbkKha~FlW}8{4Cmka z`Y6-M(Ej6XN0W4tb6m(7gZ2Y=jjAgGJ2# zqjI9+UpzB!o&te$VEo}Q}csP6>AK}HuN7i)1bq{EW(#kFi-MPwv+2P z3#A-lsA8SsFWM@w>KTA+k!nHuRBqs3-bBm3kRe@({oO6DNXoq$D9>|ZXtm^U#%Sl8 z6kW{SAhF%ozU4)qUb5~B-<&;EX)5Y$FMlJUv%(aencq|nj!L-7&i0L`K5e|7;KBN^ z>2_Wg?eS1YvcfduH6#D~LvCqGxF6dJrC!U(EOIB zKO5-x{rw`ixCpR>L7;Qob+r z1LWVK?zumQnCcQYT~xDo7`k`3-98&$1Lme>Zn`&a0RyL|ZCmo7J0??`Bs$U9kL=Fe z?rqV|va6gdE(3`}BU<->*7dnXt@XLY#uP?BFX`c5@ty6u?IXVq=eHjD>*L0PbA9ix zdS<6OI?3klJqtAjjKkIyaDfOr;DbRlU^$DvJw}k^Z^VGf% zo&L@Ql*s81RNhw=V3ayE%r}*fi zKRQfX1B8en;3ZKy7X#)5h}oOnfog(9();_dctV1e=;d!J2N<^?#QytV*=PN`?@;YC z;xYKx8Eee{#Ix{SkV7M2O7cpi+&>p1&7E(Pzl{b84k_qIvu>ddpl0TOKYwfbzkcZm z=UcM2lN%UC8s&a9qKRNRGzrJ0-8dw4*|EJp63)7L3?56 z2RwyXXF1bYCrnk>I}qzT=zL0)U~@U>@=vQCq|f_yDQ0ch+(1rB81LNS1S9p}x#arwfBxZel4kBl81ZdOrhkXHYo1 zMG2E2WE)0lT2eK|o>N}&b zrq#?^_)$P?{(m?QJN`vLNvdF=ni`;lvgZcRGiTI_C#DIX@tM#5{nybe-+xp2uO|~c z7(OAdpQw^?zb219V~|Tg2{7OyUk>)jxUBzp&|V>g&~GX~|GzxY_TCy-LHZ(>7riY0 z8edP-lq+}ewNCE6vkEI0*1g~a;g>H$J<95ko0m@MxBs7KoLrP$UBP*iFN*hU05P?3 zbM5Xz8yxm9!ntcfIy&W6VT@O|ajdSXdGUaAXAen`;IJ>rk=X!zcABDw(qvk+cIo`{ z8cj!}+mUnVkMpVBr!uBbKL1l=>g@X={D;GE?A}w=+)Nr5Vil*$E}RDA4jTnset=34 zaI{O14l!2dNIB2$NB9D61CeQ@T+95Cf@vbI;~Cwj8KTpNlf8aiZ@z)G&*)I+obg0K zTgjdm40~yZhOf0lRFJwxy%JLree508Dr+otRm{_Yh>8W;gRK7jxnmVS@7P*snS2AO zTfOy<2YIiHJ-bdMlyP?YPY^3hXDf#&dwB1dcj=`GkGFR}-YA^ja58^StLwewgyZq% zIom6Nu;uB8%j6CwYSyAW;vxo3w$Ba$LvfHvlx<{+=*pE6AeqvXfiZM{E5+xV%JB)F zvc-6IWt!6dPiSEttFSq_K#f!$LNN>TkOD+H>TpQpubwIU0g(M$7iaEQ^cm+@yPt3d zNis69LYR|DwHJ;-I!bdiz=Z+^e!hM@4Oa5_Us$G* zfYdC=fyZx2PSFx*T?ExR=pIL%dlbp2VXsEhhMAfuQAn@j>)Kv8h0vP_KlrN^q7EL9 zR$487kNd4`l=liB7PBhu+4$z|+fpu(BpVVdF5U|-IfWVA!+H005+9NZB1~njl+YnP%trYJH2Rq*NVarwi=(ck#0lJA7P^pgMY!ULb`Cn-+lZ+Q~1sK^}wUKP}zA#{&L&BmQ|K{&^$*c_aQS zcqyU+Ie`4f&Z`FeS3{}{89&vNB0M2?)TVINjX-ul*go*QqJJ~U(y++aK_`tS1WSO4CZ&*j}og0xiQm! zXkWrxm%DsTDoF^%4DF*kr-*d}(wUSDUv2e8xW~U=DnX`VE@qE*mX`nVdSCh*=@(nG zEtN06w~zn6<@yhO+sug7-lNz3T5*+?-n_|jr`KEbzISaS|LwM|XYhYYFZTbLrt$yR z2g0O_A2yv};*o zgmq%M&9edgcqpNPp)zKDCwpjdH9MlkwU1SM)a#VUQb@9L`dX<}bNZ&Ty;YG1P3j{Y zAUX_b@D$}6?x?6J^_V73jD_UpbUbP~6@OFstZs-=a`K63;z@yP>$O^@>Ry&AIdu1R z?}<2=4NmnxaX9(HQ7(%DAOBEs3!WU5Zvp@m-(x(AmUjD_N^k@GexHWI?1*eqN0zLi ztoxb2KtH5hVYeD?{TqDTDXeU5&B1Rf`IyPCglEbhQcaANE8-Acp`xkAZinI_4MNEW zZtn3q=-FjULaG~6qwcAqPb)evBrMpAS0poA>9;XoN`Z*NN!iF;4Q7rGGCPlP9kD*b zZ((nW7PK^VM5O2jGg_wmN1BNejKD_*0}R_aX=8`IXBYUmw`bO1x^L+OBaND zNuFxGT~k|Y%PS&orp(-PtFT%tyHEgCv78t`w7-sFo_OosX4ZJ>8vaFtj_Yf~zO$u? zvEyA{t_xX`=~qKLB!p<{Za{+JUnIw1R@4BRo)cpKG;2?J_gt;R0U53$S3o*233U66 zm~#DtC%cXnUW~PQIr%x2NXaDLtzxPw2l!>5U!HqFX=zS|mQgSJY;;9aUSZbpTvS1h zFfXIYO4~U&@9Nb#-!-o-dC5up+KWEu88)unU?TilDJ3d6kk2&lmv;00LcB{9LAU&g z7X3Z|G=MpPM<07Fnl=1QW%aJ#(gv25dtELctMSMxEIYhFjl^&siY9g8ZC7IVPWDOZ6Gyd% z*U8VJ9Hz39ZG?QzAMQ}kjZ6_R>pE6YB2>#H#uR7V)rpLsBp|sff3;nF9))?F)vbB7 z`(t1*G4ssM4?FO8r%aW9{x=78!42*O1y*~{SY1j8-CFMG#dn^*G4M-(xT2d>G1xaf zqS%UOrqtLM65%ptb30i?wO(2xB;V$&vxNlARq}I@4L7VjTkh8A6r;9$JOwY@Wo?*V zGoM-+y}6V5xs&Qh%-upbb8w^XbEonOC(8x=e;gD3`(b0c?%R7kBA+arRaP9UkB&=p z`9Iiu52&Wnwp|#oM52Ublnz2h5Ks`J z^ddxofYeAIW#~zwbL^N#QOzyJTwch+BItt?zF*zCRU=YF2M zT=#Y5`93bVl8H84&Y{d!xg%aZsDS}F`0`@L$#5)hjYT;7wzIMe5CSJubF_JKF{j?5 z8TmE0qVw~hC-pTkm&{GIesrb(y;sp$&#i6iUW~uCLNJ@|?k;)LIA(%89UBy6kV8E@ zIMJALWVyGUf~0<$OwvE$>b`@+{_b`Empg>tumgcyyD8m zwi*4D(b~Mp&3Rkp+%1Xy$tQnAzT0v0$KXSMhs^eE<1yP4QH+%i->u*mm3>YLrct*W z{hmMolcdlt;J}>*LlH^~Q@a{+KDAL~!BlX`<-j1evK93trUc4D-Z@sii1AYPFW1)y ze;~iMu%4fhdG*Y_4&23kL8bXY`P>5s4s9!rdn)JZB2oHS0(@yP|0nEGh282T`XEP6 z&cDNK+?P{ZFcIFMk!$Oz_}aVYRrSPYTa8LP0jVv0k(a?I=jbpYZPafZ-K?IhHUPHv zOQ@jz+L&m_GwE89ZOJ{{e!UV!kzU090DG7>G)vgD&1+efp~IgM0p5!%f!bCxfD}7( zHhFnsi3mi&r*#s6HX@4GTc@Y?tY2s}Z6P0?eZ@X;u_$IZ!1S&uG5)gpKuul=<8%vo z*5)N*D*X_bx6*HFDhZCBKdlc=}xG-SeBjWIAVPCp3t$@aYe*FNY|VSW!Glj`aDU<&esrvnS-I(J98BmXP#45j3V5{sk>Lp#8TkDbqk2gjZ;xO!u zB>}gxlipY4ujN?ORe$^yp||(*1bQZ;C;bHd5s4?c0_NA&&-ifwFgz+Omj0!fTJYyl zLfg)y_R#{nT}94i_m#^Xbt{ko%es>og*?k{#k3E>T^GxoT3gEjMUvYiant z$`qT)zNINN3+Jl>06LEAF8&cp{VI`hpVU@MKfkhWg4#ryM#RIv-1H3t^-ZD>RMYcg zhy#G6!v+r~TQd{jY*{9#QGfouj{yha>xwMtlamd9Gq8`P7C=2NsDG8Xa0UMP@Dl)S z--{5R_>2Io97|xh?DXO^6;LK?ewEM%3nCUvY-4`D)(q$sUs86FK7MRm`PReiA+^Kk zU;<|TCrUL*ymJcN^@m*Y()KAT1g)zbN}`jHh+KCwApXj z|C{e}59!w{q)%*$SSAl-T_A^Czdyc~8th@;qCylkBpxtbQ*w+c9ic_Lk`rITz4uCicwrNhS9UxsyQ7RZ4Ip}Rc->eJpQy}s2I6nHpr z(&1*-1}cE zheGFcpO#uqR%M!~OTm9pT~DXLxVESt!0%{?RS4ct*fi>{sxH)m=3zVQ zHIwuK4;K_-!+qaL4$Ad+c+zzdy_p;yF88NFdnfDt!@*yEzv`;fLww;rez{uv{Ig@X zBhua(mo8*J6Ej)?Ym{b zC@MxL%TGKg1w!+F2n(`I+h3K%d-|40s?#!E&^%8C-1*og$m49~XN|4Sr%xVuG3-rq z!uFj0Dv@+IO~Z?stb`kV%WD&yX+Y;Q5>gzD@UQLW%jUq4%imLvZ=_jmAjlsRZ%YMC zsv&%k9kjjYfs7*C2gWTYO_#tw+rTd}g_fvY^e-a=OYem#AYZatpo)Lf1|hyW=}+Fu z-(&M{rB_X{VmYc%PE#B=y9DyM*lxD_R|(U4I135ZmosTDnb<{#2+bO#Pi1-I-Cs0- z^i1psN+bc&^42q7B_7T&|2cg9W)2cAE7UBxL*i%AR%zKsmk~Wz*HM#OX=$pWgri~2PT)h=>L%@oezWD=v)`dx*FyAiWsIJ zj}k7-frJzDm;Js<VcbEC6!q9h@`R`lYcbECMWbgYa^Zk_h=1Kqk zl=)Z1;qNZ<-DSSJ%>O@f8K1C7`g!d5#+S&aO0zK}KT6CE1}*=2pI=0YzZP5TX~j>0 z+_!JYJG;v?(uNVS2_Hs&$}!pY>Of_Q3gsK=04}X;oYZor8HQ}GWv`5@v9KM|ndozcWfWE9 zMy}Dtz?;n_w0c1)7ICO*ev_-~w#VAV+e#mAa@{B)aZEx+V&8_B)NTr-t1ryc;Xei) z^f1t^75OlwG0TWMUnNwCKuia)reRv#P0I@lSR6-`EHu$#sWYf2G?)Qd*OX?BBqv9a zM~^m=bFi`+)%DT1TNeMc;!gwHBHyVNWjjmIVgAO<&^;q$+O`T^!+Dn~6f5Q9cp*Rr z{zU_iYW_bruk$48f0)-1$!-$Aj|30#4-;|T0Q}Ck=4_UmhBkAI|xV6j)LyLFmuoEm6DpGB6VY`$2`XY1JO=;)lu2`_7CD49B@TORm0*|tLO z;JpJe>NnMK)`#IHouno_t?r=I;tPk>7hw-T<#Y-v6WWNS>w zIfDo9yk2Y1Q5mAH23jhpHzKc9_g7GhPA`R-dQ44})hOQHzjONEGD^~aYe60n(q7LI zba)d$c0gM9E#Qr$E{&HGJh&?5?|UDfG-f)GY12+9Tj`Q=t_*ic zR?a#4>0Ao==+fBH2=&pT@9F%%*HpgYzoFmL`M&@L-z#9>D`4L%VBd?z-;2iIi^kvk zEZ_Sq|Fzlmz0dM55uo2&liypD-&>R4Ta(|r`Tq~-=Kp)`1yuOm|L=73?O~tv35P|Q z;thmd>GsrU*cQU3>A=(enNxv&zi?OY-V*XT*`q^H41RX>L(Osc+(%}k`6JDPP4&$- zyx5Mv$ACf$#(cw=vt2= zaa5#GKBI$gG_)OhP7{au9##L^ zBBRgnRDI}ptk_O-DGSUYpgK?@OT1pfH%3y-5BmvvPyk1f-~Ls?(e6+#-2?8o?NIJ- z|GPi%=#>dCv0FhuY3Z#YRdm);n_3UYnDF$I0>#qAKc}{jQVAwh@d!o!GbyURFYfch zUyx%lVtKFmThv6N{$i$1^|{9*YFy*UVzhkpjMVN$Uwq~y+ja&*@kc>Lu_P2GFp<^) zvW~I|WuFx~VIPT1t^-wY2vv<1+ANLF+=aE#h?ob94`vhyuGSwz5=alfbRN+5SR$y7 z7}yS_RTF4B=UM6$`%Y(f@u!-4>Sz`Q9?7gEzbt!l@q8gr&r{=SvX;xAPPXKu-PhbK z6@bp==kO+s*YN4AJVt=&C>19ioy{*H1|D9u2=^XV{lLl_Hu4%X5Z+*N-4Sng61G_$ zs_kGZQ`-zmZJv#auV(kVSG7xh zWcs!lAR28pBiZ6fdu47Q+d+*X9)qwZak)cdZW&gwoz2R_gyY6#^66`tw*q!~SQj_F z_G}VmzBRmXurBJzJZfZzimEm8!e(pIPs1z0^YxDeWSzMb_ zncJN_rDFuc!aVJ3j;?ANdxyxnb^&{49D5kLQ?HF}Dd+_l*B$jn)|TrS&C_v(nq~q` zSbYiCt}THIw#W+54;3#7ce8bxEcpTlI<3FaHHhnLA?03IIUL$#*SaiiDQ0_NCQa8z zFWGW50YJ;TItjQ^b+*3u;5Ey!f<;qYdU@gG&2uOD*>Id2jF;oT0p4~cZvUGN{;b= z>xZ95#WHZ@PI|o3Si8TaJUa6OjTNEk@w}ePY|TEXzgcL7Rajh;Y*E$pkZCBz8?*)c z`mbG|fKu3B!Jal1pY0kC1Hznp^wD$cx$44WG z0G4*zW;xGlq!krm2eZT4IQnAhI zcr{lD%xY|#=%UCh8HTScQFb|mTs{v5pyZM5Q+tNpyeK={YmryoruOc{*E-gX7@~bW z2zk?8Lb(jEZK%R8B_a4}P=vq|FYCg43WVn@xLzkjIWwYM+7;ePe4{^p8CCBfYODb#fmwJqRQgnM-~C&K$?T51?MltaxOUH2HD z;g$x)R$p^|vKkjWRH9*OjL3O(|E6EP_w#FnzKql?E53p6*k}E1bD8w`MnpNm%-vai zKL|CUAYQHG7>=YmZ}|h1SSCC*(rC>lZ(#bH)cS58Ha|(4M#bidcQoyS@5G~ACO60K z%Q?VPdfTDhQ2-Hp1F`GpAQdXXou}{{N5A<9;TX3Fw0ZW}Tu-m@g>gyk4b+5spV3J9 ztiq<`=O%F#!VO%um%X6jA(k^c;C`wux!3HNo`PG0Nx5{1Vu_f@F z8Ilt3ENKkqk@zW4G=@9FlD$o5?SIB4l1U{ract4Tt=U9#zY=7?S-fAHsYO@ZXERJ< zmV+{C6swQ(((%zh)eO6u*}Zv!UkC{#JIHcC0#|dXDc2vhGnDdM&S8i(ExMwg?7TqP zB32`{>a)q4pbMz2bC#xLmK-(y^TeCwMt3!gJ94Eh!gB2w|4l=;j*_Kc>Dkzla|BWhoK~zxY{Tcx&{jKRXX=Fh<`*qG%2yIU zuiU|gP4kBiH>w1WrW8drI;oaLLBT5clA*K=bsb+ zD5Y6z4n{+?IUqa+Ldn1R!gzkMjo#wJBLuUj-cWcuznQZ79C=RtnCJ{oQ^Q+!TAei~ykke>do>u8HZi+1m)Y`g1C+TEE2Is&_Z z&rcV0%+5LVqAr`2i5UZ^`|xLAn^Ch9RW_>wy-~@hgxw=zNn$^E^&EmtSc%Y-r1;6{5yxqg0a0()IjAdR)rJiV37 zr6G9hc1ChpMc>TI9!&c0UgPuf44;a~5_}|Tu)0JRBW2BG2kNS?dX-mDQr%(vn4O(> z)7)l!Lket5aVR5X*lgio;d`Zf?}_hn^QYpC2Lw;7i5kRSI@i;q*&Lv?S6Z}y;59Yj zmL%sKcj(-xp^w_MLafk}wW0t|G(yQNDp;!WEGqJNFqa{MEDI)KH7!Ciu?eeK1*5`P z4EnIzsS$JQr{11(8NO`-u<%=)2o?fAf-6MlZtOsTlZGohQ`Na`Y$V`VZN?$0D`(nfDgR}~m zoyBx?4~Fjs9Rbv8fPzwVUymk*h*R*@hyM5~A!Y8#t}1x5xb}$&g6izQw!*j@j7~B! zM{)E#SDZHk+X-#Qm>4S@v5}@{q%Vpk!SkJI)E77ryC}O!0s7;Mtar`(c_yu1y0;K} z3Cd7Bdj&e7A7}*MHrJ2OLk{idCa~QY@BCx z2;=0^x^eZb;^omy#^knS6KTzErBa91@-s+-fc!(XN|pIv0?&=VplL`(7&>^;2N_=) z__sw=(v`k(U+fPpf)hd{cco3MS$$#5)Fe`E^zo=(mV5nkjNI_G%Sy%%OMrRj7#9O> z1urxo7^PP=!V74txeH3!&tJru%#dz_@S$nU*BpquYYK1nlP*wUi$J)WEH{rt-8DA= z_ul2prAHHBWeiVa$_!aq_)7*WImO&;b5R*0A|GvPoXvvodfOvPEDgvkm`W-6gEv92Y`PSV8RUH;jl!Lmga`p;$2k~~lY&PV6 zNaRx7tA-no;4Dj6Kx@)9vfaImt^2cH(Phb4%OM;lY*)Fe+l)rI=86`whq3%xFkj@; zYxku><@~I>H1bi=-gCv-NPM)zxoqs^9z8qn(GSk#hA_pCCCE8($w=CW&)%cYwm;23 zTKVjc%WC`YY}tS3q~wA0W_5@nuoeb$ze;en<`et=n*O)~AnQVhJIv{+L>{ISwxcnS z*E$2pCnL=_7A`jH@DNc1_+wwMd6HzaRzPObREcKfYIxJJ`E+XrqwVac!;d2o;SVjf zMG5rj3ZjmviXK4TN$e#mh#rP213_0M*j^?yF++|Ln=kkpbA58+Sr3;DlhnMqAF|5` z+ql4u;trepvE2u+np?=N_Qm29Lz7O=qQZBp-n)4ruve?NW|^ry04NnWLuKP(rtg#5 zTJsORc6QZv4MkHgyOsydE8myp23wF7gY5Jnm~uJ z7d&z&Y^(US0y4U&U5ct8yhMu5In47MT*3*ytTZb^jzcU`yIJ5x8#-e9;?EEX$dbkh zHydSTNj=DmVRm=lnq4Xr;PE8>2T>;BJTH}x=03s3H7iA+T0au>q3kkQ880+yXJB;^T!sVOafCy?dN}%6?WJW#p#Q4VjT=K= z7KZzdJP1fR8|{d;lhbrK)V=mRPg5y5wKB=kR#QHACEU|+m|VXfXznfJI5w>{jAe2V zJ4wxckk;eXc0J&R^CH7HvEuE*C{p!#@f@$05uKj#bjsh9QuegPEEcYY!c&}o`oBnFn>36)Xts8*&N4HZw^5A7lT3{icc z)SJ_amdf~RV@^=^`TQbK@7;288m`#J7Qt->l0@nESe}h1r{64<8|s>lI$T=_bYrjg zy(PeSk<&P)P{Dzyf*<1R44Uf*oL*Tr<$DuC`~Nyuc0u&k5^4GpuLM7If|pHg(_uzi z?i|wKX~tRZwJLzzgto=VibTDVup8y23+XV13Ac3W=M>uzM8=degX2F~Yf$U6yS_Pg zy8#?yBZb_VbclVVOeW2Lw7TR|uTF}~%pn2#P$AHPR}8U!&ATd2IYjO4gOOTJRObmi ztpvNoeRSDCq7mWd5Wy^*O^-9cG=oWe5YjcU(vvP1Ap;J|?uG{2T46hdCvntn)q=cJ zp=4QTx4XSpd6-=dolbkSaAf{Du@}ZYo20-`8*R2ED1wdcjk%n_i`W;|tTsq+=W2E} zANPXEVicxSM=xTx4oJ@!nS?&VT+hI?j6 zC%?HCQ&Fx{ozqj%72LskE6OZ>R5xpjJR0DNw#g|tg1T9g$ZjNUB&!2HVUnZHt4$C} zPfJU-)G+jgYUz9iE-4pk=up)g#>H_67Ob)dp*_ftLrZ{tLQ=RV3!F&8J?C0-%tnM3 z(-9Y08A;;33rH185<#|f(Rk`0pf~3I16kdyJQ|AFHG!_*KeA-BxgS?%AN69uux!%u zljFUEGz=z7xd8~eq}tkh`i?pHPB7+mD#q&eglyNi>71n9IQX>G)OM(B6Z{gRICs|Z zNN_au(5DcM@~6a<0c$zGU!AP?Nc=DUY=GW_fJqh#DECd2q16G7^lF_1x5>_+37a z+6K;DH0iw}iY6(FY*qkF$;%K1RYd2ylfN7lon3GbZ=aF)TVuXT4Byh^I&mX`wk)_> z3F%E58zusE1HacIWoVw|N%AKrGpn#m7^BHa>Tk{30hanu9&b@#{K3n9^4#=F%S52& zS?tE{pv@Cmfd*{&-BpS#p5#(jl{5N2v@G!Sk1t$b`0K%sf0K^7jItg6UW>A|iGC50 zlpnsEXV3CbzqK#uGqpF!4tjek+}!7q$bx1yy~l;;ihkuQXMp5pu57)qP-p$c)+%i+ zF=e4|l(Y*bh%=4axIQKhjq8#&a&`2grv- z*@cgGJ)O6ybJ{CQADAJ^|KZ&UuqR!4&ePBAh zFx9+wK3JOe_V+$P{dzDx5s(=oiB*DAq!yCl%5LHxl_v3r%{9T6exqTA9H%OA4=1|~ zU2k(ocw{6?cB+;s7ZSBx@cK8)WbsyCMs zs4)pJt^st>70Tw-^6&5}vZwVU>Dxke?YgZgz9XwDDBJ!q7alBbh22oYrN-#4`h)-~ z$q_Hd5v*ThlgqO~tnSikc)4M;eMz0?A^VJnzJqh+&9#HC*PAvTSx;Q#^-wSajfU?raD+SCBmZM^8&Xd-MeLn%Gk=$N}p|0;dsTt=B(^OKvu zVwb8H=cjoH8B&Xbp4iA*;D|w0*0zvkh3C1Hr6;&zB;iCokJM=cd9wTDvh})9^c(?; z62-Ap5L(rY8is%7_EyQpuEO?0z>sm919dS(n7lEZsmUAU7kQNT-S4$_aHQt zpAnqqap+Lb@az)>?EyYIq{vPu6J5UBA;br@8r+DQp>j5I8~A}-hrV{~U8Rf!L;W8; z6dlXia9N=d^pL02cA_>0e$>YX;B3nYF!mRrLEdBSSg$eGP$=1Arbd8es3#x%62`v( zR&$!T4^~ZIYM7xj3VUZgpl}dRM*Y5+$Nmt{o;x3LvBf<{b)bB8@dZ0V&%W5)+OK53 z)@=wMuxPh=(JM1_urnK7VS~$`5Mv0Jz~xf#On#^64*V)f89K)|g))H0jMp!+#@Z>d z^|GDaWL3P5$4KEQ%G&te0!Xl1A2_$QE0JQJt{6$1_WAN8qgS!}Jp2_*&e`9Z{l@^o zrev2GkNYW}U!-BrkC2&7 z^_b9JS5%a2C@T{%z(AW?ua9L`hG_mJLGXww`rTi3v5%&_|nM}JBGV(i(Wz)W`Q`fKq- zZ&gEW4q}!h-Wtd@V8NgRu^cnDS6{R$44Pp1?CvDNj1X~jX>;qDNqZKpHr`V44MCra znnL8EWtvWMqtPlGt-4O?BGc1f-Dz4pgQwcj$=b?w;V4(wmM}`#BY$%Fv*1|9= zW0(4h2r!Z?!4D*S817mpNyX@^#NLxLOl2=fmNiwUTE|8toD&+Px^g+??_+8Upldqe zupcn}XyeT7CeJ4(`$(bjKNr87FT&oEaX`A)i+x5KCf{~RlS_~ud;7*U<6B(XhojDW zF^-=c4_DD}Xi*=|)B5zuFgJTZ5cZ!Lt}Fl^D$QA;oD|E3Gs*2@Q<5`U*JE3HFw+U1 zOd$e@(BmesJLYd`#;iKPy+#a~d*;R3wL^v{OXgNEEb0zjP{uU0(EH@@Vz6*Gqlwm$ zG%~6EEG;E{jW*<7!>_k@ES-F%X4=5^2B*tK$@uNaM6~I*Yek#K74R; z59pd70qG)(0_hO*co(Nw)AZ!*rqzc0XX2f1(L1#9bERSzsSHolp5$rdiO{QI* zyv;?R913vEvgUq8lb-&dw0D^;1qoees9OVM^Cl;hF^aelTpl9=E07eAk1=Z%+8k`y ztFbMCVJvLhyk7ZIqhiG((UL2h_>PzMo+fT8gbIfQ%B{$PQ94VRzHM^e_=gW;ok+LD zpwbBpBSM^{@;;$E#KG|>TJy#b#-`M&}T}qru=x_eL*4dIArKWAJ~2Py!G@f<2CTS82ZATzY!qSW$EO7U>TqC7_CI zb2)#`ROS&?c>(!>R*cvblYK=!LErdXP^h2EO6^MKWCyu+oP0FqN|~27o1sG$8A?ndL>$ zh5Yh7=SMR$?y*Tq9(XIZB^^h#moLh*65RtE+YbTr7+l4?+t?bvD*LpBi4Ff8*aClp z^!XGXxwG#Mj2q`IJR&L-ODyCZ7$DpP-6YEcwZowk4~6ij4y<|H%2sIc%plk|qY_(e zeB_Z4{Kg#l>fz4=CFrqWC>Y~oZa!oRDa{fQvV_(kK9IyR#oPbq7-Q=?mm>RsY7GB`)jKPb*meE zB%OJi=>=%#K=FS{ozA(mWnyZwx;mx7tGxE(+s@3}$Rz>V{2M(75{iJCMF)+%+0078Jn>G7_u9 zp++6{8G<7F7@Vay*m%6))x~0bSVux`&X_^!pw8+ZE{rv}0X9XkA1F*w@LN3^WKYRk;&fU+Q} zb7;x2q0+adLv#xhImtu4#>Ayu%EdL)1YwORi9TIKR23Cj8VH?44=mLP`^VtGVK6N3 z`W1dgoBmFGg9EQHseA&u_E81yDSpXoPRpAcU5{SQvgo0`?|5pe))M~sZ5LZs7KLG4 zQi`g9AH&;-rH-?kY(Q_KR-g`*@sE$e*s*OoSeOSIiE4!1KiE8-1mUCVY2ym8< zi3?=Qoig*$bQ{bB2TX>l#@MQ^R;bTLyM~+<+pM!k(8=LvJk910va7?(N5>Ga9^Z`F zdHTMH+JMZTYbnKf@t@Hxw5(6QmXo8@9V;=(b*QE+cgR6O(^yCk(IT?pyF#&oGeQ^0 zXoeZhVBR4fhD9nNT0Lm?q}_`|6W*T;6aM?=hffwW>h+_3s!p|gmhh3Je6TE!Q5%(4 zd*=Wy(8_I~T4c4TFGHNVl@UuvBckDzT^t?mjj`$2c2sinMyxVHYgh>99fieXN zJ99x{MONrg<^yT6z;+TNjJQ;PxgB+ZHVBSLX&`qKScLXHM6YjpZvVt3l9r#%VRDSq zJ`ZvXshG6w#9c*d(U*or*_6YBzKoD)4P$oNahFG$druQ89L>~+B7jf-)dKx0Q3*>T z`FqeW>&mjP(O?`&y`l>L85SjuxRVgTz40_dypQ{#fX#6}z6w(in)O#p`MbB7IxN?o z4-YEqxcn5&9*t0C3j>t!^#UaQcT#+gAtIK<4UVZ|vse^4{Obrk2W=hMt&p0nc$|lb zTa7^RWrnH)VK#$@b1?SD3WtpXPxnsExQ^<7rpwOVNbz3GQ6Du-l0*#C%n)y2f+tV>17;(WD*<~iP4EK%J9$0#tzSzQO8l6@8aF zE}FD^o?I}A&pJsTD#!kS8R+3}S=|!t^7ekFgUfxjORF_wlRoQ<{qI{pI6Yc)ZhyVw zQP9Y<``+l`yMC{(S!;!JlWZ3+&8lL;pDz#_22dNt{iNC(k=#6fB6Ly!6B=>N2q;kj z@hEa~5s8YrX6}-+iR%|>u5tc+`9iJlK`3$tFSMA^wK@}B-6y@sNJ`Jh#hbNfHua>Z z2AS8~$o*vPlc5*0$Uj^64l zDTc~e^!6ST)=JzJo;)R@V9uSB*H(s?YzxyZ?#X|Hziob8=J{pf`tc59YH9s=lk*2m z;Na-Q#Il*j-L_8W$dZ+ly+`eirpy-EPdH|xgI_;Ihc7?!qUfI%4Xpxq-UR`slue~P z5QmLNvMC&8Q0e6F;Pu64On+!#>q8z&miNFG{E8(jg_mu!+|%E9t{C6t2Hj@sZ-jzB z3S9hg4mQ|3<_gm!MU1?uZzQf_vDVZgT6|yMk|#=~Pv^onlLD~g@D2*?k$fh#jlNZQ zG!aNNVXelpjl7r}r@*tMH|z0sb=)j$jchk13DrzvC{NMuqc@Dll%h;9{_D8HBI%yH zS}sxL=xdMChTQFMRHRRa@3ewI9wF=<~Vmd3cLGx}}R@in$d2VDm5KBLfa`vO1~B|^A(ERMx-_T zAOp6OI0@TpDTxmeZyOOD#H-Dam)!F~%AZrfM%+%PY-u*}r&p8WLnjUa{bV3CUY`-u zsK`br3U$aVhl-5iMUt!#i)A|m47H^|pp>?@RFl5ut5H8x@oVkS2iIM;9p&EcjRhFn z@vHTV*D|I$J?;{IvXeX1tGeoa)VUn7qof|6@aXw{@BI_FT`2|nk(qCWwScC^c8~i@ zcy%pjZF=Q4eV=%T`1O<|7dO_a&p^$%_Eh09ED<%X-25jNeY?K_?F)8~(3+Re@QbT3 zH_I=tNPW8C1eytQywi1-S$WLKirKR4-wZiBG?Ma(w+D3^wUMsOVW~pt&t;0YmoFp?PePQjFF>^T<=d`kNpf$_> zVRh0KY11z=;V7?=^7%iwm}D%4w6lC7rgGdUSGR^ysied z+llmK*q!YMU{_RCJ+35Yv6$GJqc2LK;@~^IP(EJsIa&~kcL%GoK-9P+VC92~aie@D z?9M7Sint{%AV;k_!)Cz914mczua}H6lA5%48fir6$A;P`U72|y$|g;hmGEr%ufbk^ zzAqV8MFLCY3Nz^jy7(`~&hw>&liacPL>1pf1or`t3z{_`2b|9&KFmWcSW4=$Ee`tV zIP)BnBDQ1wjpCNyS0;aqc&d+C?CCDWOmS|M6B!Ga_qA=+Dh()e#!V>=7P)D;lZExc zX_nTSe5^(fA6vJ+X1&KP*8hKTvGSjbpBwgYPvU+Ytv%Zj5*V`QcWR ztY=TcK=+>KRqFTZD@_e_w4AMpJMmU8Z;-(8m&D8CpX0Vhq)x9S5w;eHhrdc}B$z13 z!)33U+pv4ATE0ri6E@{Nu}&TXE5IzdtDUGiaCKBiccTMuFD*%>2#r{gBN~Z^;2-b9 zqkTclUDSR(;w8kJKDuF5;?wF=Ou--7G-q45SCQzZ>J{{sdc#sLW|8lsi+YVT?vFMxL1Q^$h8J}I?gHtoCog|AlmcH&e<-=&MQ;n)`99=Wn z(G#v7a^`8N_3K>;w0GrYm1FPvLX=Mzs%`a+*7LnJ{NCaCtIi`IXzv(zCulWx%Yt&m zJ<@c%6b>%j)y2OV>rfZNbn+J;vqA@H-yA?PZFv*yC$x0@?M;JP>M?$a`x^-|dzFM5 zASF_t3+~|=@ashxmIsLgs6F&6pbm6LMYa1JH!LP&8dvdThGW$l)@;V5wv%KQhlBx; z70;6I5Z)H+O%oiw%^mUL4lKYi!hdCExa(!2mh_(7CfJ&`Rt4T}-_Op~Q2 zHQre6s)yQmS)I+>X<29MYnqye4e6T!0~=d1fuNhWu$CeJ+^Cn9KQE809(XvNvii8b zJ^^b6%>a|W6@gJ}w5)P%JLN8=sw>LeeWoOHd`0MhF2;R{*2P^8TFZs$f}^UQ87lam#|#~XofH9kJrwG-L~8-^c@lZMA*z7 zOA>~#VTr1TASxA)Xcwz+`T6FutZ-RXf5~=L6|P?tLAL}?a_kSy+BEiAfH!(_zWgmF z^YoJ&nUBKi9Vnk{xRkR1r~_*Kn|4AMdEuQXX(Q4$@oP%8DLxUr?Zb_J$^k(0N554T zkl`gKg1mN@?}`40=5QXI)f-)m9`jWn;QK;l{{ZC%Bhi8mQA6BGs={{A zBd1^@O5=I%`)Npv6;+*R3tgPXDhf|PuB;84q6U0CkqfsFtt)bPQ|Vj1gW|n3dyGYB zfZH?77NarOuFq5j!;>AnKJrtn2c9+8w`plM8r^de!SF3Vu5~KJ<+#P0u1x3q6OZaFMqn*{ z(DA~Bv2!3iTe2#huiX)#ba{yh(|fwAtaY*ec>}Gf64DI0926L~(V~&-Ga(i;Jo5c# zW7nP=6!oBnsBKIbUcZwOnm)CIXPLzIBpkoY>JGdi`<_iQX^S5$qV@^%SW?i{Ur2+sCv2N#+;l*#nd&m=c~)R%gz}A)JHG zOR<(IJ??WvcuPOVJrgG`z_9`mCq@m+sIrR&Z_&m-5v&UA(Nr(5{86|1`|@EKHEX+s z8US4{Mc6Ug=){RWd=YTZKxTUID4^F`t(HGjw!vTyWMOvoRiH8 zI7Df2D>!CNmcASwqi<^d0hFvq>x|Ky7yN1_1n`aPfy)v&o3&UoAtRiob_=)ENQHu+uom!>M@nstB~lPSk~iq zELP9mZ_{2ot+I|fouU=9yf2I3=6#M*AiW~t9>bU)mC20HKWy25PvC@y?Z0&|@^=f_ zCwbzn_icpmFaW0fH`}hv{tJ}rNxNH7Ca6ue=`bqxJ``;4B9`88$Bbkx-^2^5IyZdg>MMx7r2tm8&&P8zt zb1P8`MF@n#xD%Z@d!ee*KDJc0U)Lz58peTzuq|%QbOiSVv27bqh8GLH8cl_nlkZlF z7XarjuWm$M#rYo##{4xWI0%(?>+|&>HJ&3Pi7wR)1gBf+aut`xE)O?i6}rd_1(#8y zTAqQuW}IwXZynG59F2EqRIk7^dVP`4Na-u>!j>H~%`PZ^d!WFmn=7DQc3R+v4TLu= z?5TX~qAI@}U3|sk(N&JKBbVCaZ?k64B%3vH6avBf0GIGdFM>n&FZj*vDepg?F&O$BzmOn3#i;8OLL%3DsQJxE zcESO(%G|2#{7t$Y#6{uH;Aj1ao$$@ghY`FX7PSp74`JBvIipQCOCcW7g4oZfsHJ*C zg|K=#G*$=>yt{O0|&!znB0JaD#YG^z0L2l`MD{&P$f_8oOAC1;Lg!_;XCDTAsLoFKT9I6~y zTD9k9)b=BA`+^^rI0O?cIi4=JnhL@-mY1p2$IjhBXFa^?VK4rCerz0>pFKY|`#y>k zNzG2Hs&oajBajWSN+7jR)5co29Q+ea<8FbI00NV?z3R-xHwBaZ!op!gV4wugFn z{59X_JW&O?1X}DQ9;qHPw_uUBd%nVi;!v`9&9>3Z^!Q?EL08|37aEqC&!F|fggffb zjPU{d@lHbCnS?fbo_e%>@%g>b5%&H|`yWL~-gy@&c?Yvr)I#Dq$i9JtWVc}@acZ~d zq2)fb*n&Q+k2_c5GqJG#A+;#LG&YN$MU{oDd`EC_N&E!_vzt$XYz(D!uXd+idA9EY zY!k4y0qH!jyQ(~tHN66&#f^uByYqw>2H*8t9_VACRD@^y3AR*x0Q-xP)XLVD84H8X z(WxW-=CIPfJwc1ao!qI0d_2bcS(0HB%D!o6XP`uhBQLPyMM)8+?2{%&$5SIB0UWUNik#Ie0@a9!*UqQa2!qEg?|B3ICkk}l$Hce19UM=5(P zjiAUj|WMPw{+N$o0c|<-#vvvtrPE;38zl zck}CbZW*8up~*_F;%WNye;&&Bzw(Ja+nFtC-M^&n|E`W~&8>|RAMm$S7`!s99y8_O z_6ae-p&FqB%P$Qt>rIVL=3Tp=E$g@Z$)6gA(ef3b-d`xlxS;A-DH!0tI;}_e$iI;k z=AJ3;%jBTDyPIpDb2Eer8~G!(qKvN+m+Od%keYK$_Y8ugRWJ#`92+Y`$!;LRYg0a> zXdc8qowG~tS9CcpmRl6-lcRWm&I9{1`kYkuwmDZAqycY%OMtg${@1I%J*5bhZR*St z*oY?>;&OJNjeYWKEnK#;9XG5EAxE$q`Ye4xOQWW#vY!aU@3wBJCe zkJ*~rK+9xu;0Y+M{mL8i%oY0diX6cWa$=+Q;VCawcX3y4H>$B!u%OSsRgabj`L$*! zgk^3@Sh!Abx$}{~(D`wFnMX@HjcE$_xExkbAHG&`WdCRQRw6hot5A`|qfiJx@)@^} zyb8d@%{zv0Ny=@$77&@Y20A0PW&R8|dI8g)QA4>b*#S?mJTl}wxacrUEsxT@@CwV1 z%^W^?%+~dz!*B?pUSn?=9T!nlQ+cbPwxo6>$hphhdt~#IZK2l(aOxx596Hzm#{?RE z$Xw^UfrmN{W~HaYH;^ua)tK>;R4wWf?lJIy)X)H|AOkW4P3{VkH7oyMA{+(0?@td-RX}qDZl-(7qXJ zV!_8#S)G)b3DK!FSJSk~W@-i6^hRFsO**#sOOQ@QAekJ%Kk1>72 zz^yCJQW^cBp4A-157p0ZJl8~23QUu-h@@LYL>eL>ARwd_ks(4SX zf^|OrvU~08W589(+Efty;gk^q)|flvY*tUZ5IoZEp|#Qu0;O%&CklJAZJ5f`YNA)N zB9_pJO0MNO<<$5~A>9Om7Dr;%}kZCbn|=oD7fg;5QY)u*@7d*8FR?TL0%_#IWj zGEW;>GJ?be(^ag`<>`q74t95ON^kAZH_OXe{>w|ffyQ2siY{I|AK)jyT4-XN?PYRr zlA@fNRXSa7dx38Udv=s&rjcVoq}P+iXR^scC2Rf!FrsUDRn4<)WRsXGF5Or9YTJ)A zUgWTnSeu1m?8Zy&ADXg>l#Dew!NwK5SwXZ56N=ug{zNC%3*-XR7Bz)ZY?1=BJkd~BaHFb!T zysJldH;ta`z6;xK6GW$P=t~%;8bFA-N=Su`VNex?+OYSSAxvVUchU;3b5+Qa`1Cfl zojiJ6u#U@R8*}#o;{Iu~T~%;aILEig@>kAdp~vX%C^vsv@fL{<#fI<7%ardI)z{A} z6=p0f_vYjue)mVv#V$!}L<%(}k-FDDAooGeDXZ2Z<_*8#qxnzdtByWN`LZIGDfdFj zqXKpwJXR*&@?tFV6{(h{Eo7)#s^4o@|8p?XhG9j(tSuH?A4^mrMR{FMR8&)ki{G>d zSlS*{s}|&e31d=^1xn%%A>sr5W*dZu0i9XZylr$_A3=qaK$Rxk66MV0_gBfB(cGQ3 z!-(MY(7#$m&%|ZvIPtAN=mW@l)cdI+`<6Pm{0ye(34JV!{6carl_N55M^&~ri#*x% z*y<#mD*r2Ps05O7@Vtg=ueHHFw^_3t08Ic`l1aVGyG66i>qS#5Mkekp_domgx^`+O zzuQVddJ!=<90EFLJHy;;(`QbB8lgKBcPw{cQ=zWoPCuJ(+BV-q74oZ749D~P=PxhU z39iKNFOMAg?@$P%4w1&F$gf_j6j9pZ5SW4CyvShwZ_nu!72q$5`@RoCJS`^QSqq)TU7rUOOW^JU97V`nzgg2!kqz zcW%uHa@Z>8-b%sVEDpQcGc>Brue6pH>OoYo)G@&Zf-J8vi>Jhv1n~M+;Ze|#Z`2lA zvGtge{Choy3jX?SE3VdhrRGg(&O&p0+a&K2O*%br1Q! zHLCOLbnT*$zh`Ho);lO+)*C7x&Wp*ia*VR7nmLufbEmqZE%xMaLq@-rOTk_tkJO|~ z_ob?knhw&hVq#g(Q|3?n4z1ia#d2R7982<>`fg%**G#>0eHl5Dwrgm{NJ`p>0o)W_ zrKf6E(g(3Sot#3zv$DR!*QqXmmadiq684IEo)SY3jB%KA$|32J*fOu^9Ue!Pa4OWVkA@YH7<@WbCtH2a0sf} z_8cE#1;#>#H)^+%DU&gmH6O!H_m3|hmBP!l-6g|cv4lbwP%4eeeSbs9N%F) z=zIEs`wxvmg1w1=5t?y}c!55IFNi9M;P;Ivusr3=d{`mC5+em+A0L- z&g(eqF|w6Kz@*Y^UV+~N8woCgK2wcG>tZ6gN@gXh!Q2yC4#vn6HuP1YRA6Nmdoe+~ z+IqH2hOiA*Vy_Kl`ztX_1}FsDf#gY>=P}OyKQU)rUN+SA1P!w~7XRDor>>=+1;*l5 zv+xy%eXE0drL)=($_Lh z_dDCTJ#@Q+oh1jpf*VMQFa|l-`DxOajVIV75_I6$H&w48~ds67xf(RtdmPd z+0t7^lx!~gP<>qs_icF?dh}hHqLr=Tc!5iOx!=8n+-Lin8v~94(na{y58?W)A+twb zajZwsr|O=39>Is}Hn^8yzfIGW>&PAVCmFYXOEcLz?|)@)@00UGv8Ek>mLG8*d>hn~ z3SDJn8d+%>SxRSL{DiH;#a+!CF!ftCafMTD8+r_Hg2uOA+<3RCOsH9E5oS7#36NB9 zY0jAVM=g(JKMwZB1sdSw^7a>$y76xmF5m30$>{BJq!f3VYqfk#(z3l<_`CPzz~W^eM}E2d=#3HAFKy0&l!zY`omGMD3GC z!Wy_aOrB7mM`}jO5H>=K5e-tj4Q*X#coXRi5v~n`2xck!@b$F+m?9qF)kYAkp5j?8 z%IlI$n^#Ws6D)%B{j9Xwi(K4KwH{b$LViN>-9(u}Fj4b0YAr#5i-;D>!#M%Fcxm^Z z5^ylIzAyLFkgD^^#zy#YND+)`v3I#znb{(iZTM(vVOY?ZRm#k3^77rOsC*-_m>yi@ z@IZ~~G}7zpUr~cx6PxFzSFCZ$u4$~~VnN4I%F%PF54;o)+_G@dbZ0sB__~NCNKMu6 zDICjrMzmxwy?NV(Eg^dRl~(d2&!X`g#^bl*0kL}Qj+qJ+llTSf>O(xLz4rQf7|}|W z6&kYXJ4WAz%RW}5+%;^2-xk2h^UFKF;V7-RJonKvgS~5Ia;apEH_`N`f68`$>-t3J z2hw>?#gGv@J%PA=M(qexlQ+iN>W+8g4$dRI_LV*Lv$NXz{4HRfmiQM9h4ysi%<`@M zsql^E;j@sKQiwrT4U$2M`)75o#Wx4)=;3yHC1!4rBjeo41cd2W*CTmQ{b_*9(8OKb zy{ULNXCE^)KUck2@h(tXG?H5sPOsWNB$-8#p82epJVoS;&N+Y~HO} z;^iglbob@OkIl*MaXFhMtN8NCQrrSZ&A#-i zIvygH5X^P!lv*f&=F?+M&kWcdNqUM7$N7c^Q+!>oJT7!SzQn)jU*sppVigpWIv;96 zelv$O#8Rn(pM;m;-~t#-gq<(Q6Tc^K4em&~DFFFLIlUNdTAbG$qblKN+gV%a-tNaZ zSE9TGdiXrfkw^|g%5U`NkEK%ql_WBVVw&=-HOH7Pw#_-n3jn7jM>(O16uSHG@JDI{H_CjcC2UJ_K|kmw1)g(oLQ z!08P78rpW@v2LM1mbbg3S-CO2xhs~wv0<+_``Adp0;A>HNITocUY~NoPpsV6;*|c? z=T_^(fy`qTghF)h4+%MZAVaM8F*rpU4~&CMt=Z%)(}TB8HL~Z!Y^(a0+_7iqQS*yI zBhJ?J=rApT%F8|Y0RMoTG^yO1R&o#SVPpv)m_@jWUt=W2yIle95-!k(wNCRsWEIJ{ zS@nZiYD#b(y4&z%>WA9CD#T`(+LAFDRiz@>5z$#VGPSCcxO(C3PSD)zu~gA5mv`Jb zsAV6Y_<-6y)t@S;I>D_Lg(sWN)k`98Cho#}x>mS7wAx!U z&>(ZIuJhUc4Jt?QkIxQUDO(A4;5}Q!2B0@{2ZcrK#OrDmn<_wyE^voZi;aDsl_-5j zV`VMmm>m@s^wcfu#J4tH!mzuErG6(sOh$`CHrr;{ya%Y@DUu@Hi+l&E+~C3n!k^=i zKD`CEe)#hc?7;DkY;feFqmc?#s%BrciUO({k8vy+2ss$lzQtEb4ufge{lk!3SchRn z`dQ2mW|l#z{6JU>Qg1e4MVM{DOWQ!QaU5);xjoOYEh?a<)GMB=@L=$zO!Pbu9vRmIAc^o0Ae5>7Y995b_kr34|R|jzvA~iUmceZ z8{fH7x(I8e6&L<_A7IeKPQWed?28w#29|M5vUPXaLixoAe`kjj4n3wi&Ol_M>F)7$ z*&s{fT;C}3v(@2GzJr5a>$X*z5EuSbAV=Vct(Y~m10KX~k2P=z2fdNt5Iu*-v7FhqVo&5)wuifv+ zvFiy;!^O3n>P?M^^7&w8d*R;o%KMS&S}FQA$3xLUuvUTlpHu52{}AuT53=i3&6`P^ zmr<&I9wz-&g~?&kD&b0Fn~@t0UzUGa*W8AS%V;c2I4twEZ2bP)Ev&1(7O1(v2_xZ7 zL&4d!Pptv8W8Y8C@AcZ^b6S8fz7~D%DxrzcY8s#@+5tpa0NWT6?Dxr7h!mI+_J5?Q zxZ-ntCV8r2a%lOCc<-Rc8sjP(w>GuC{ngZMn5N$Y%E>m>uLkmKdcErFGn!RG0T^$3 ztQiSer+i9(jZutg2OFW!Ym4LEomEbw2l95a_z9wd3Es_u5-eJOHsscaieO#@XYzu0 z#K=gqys6Z1xu#mC^SyGu(KWOIj`+^{;S{sv!2SMVckQbALGxm-N3kN03w>V3UxEWS z&1PSZa8!sdu&VQa+oofrxnEC&IT#kJVO@Xj8}}Kn%2;vlCFoIY3JdxUr@KqLT#?s)p&mXGE;vGbhpf^|Ac2nB zuo4F`-_2ujsId)3oW+=o!MU$EfDa9ozgd)2r`Cc#z;akI8%E%hMr?H8lpMC{$vzkg!V2L2c#sXac&h zsdzg5l9BxIhYH(*d0Z*w+^gPY6{Ffd1E7biuXt7aD*mYRq~6eqSZZFNOzbc_K;w12 zPPt7}1keL@exXdIRo+icgXi(e(>Ls1m~YOEhv8s@WiY$Ly2Vpwq~^}P$Ll+5bSTNz z!RMn)aLh9vvx%?VY~Jy%&)oMtUPbHTjk*xB4>yL{l~^h)huSK7xtw|U+woW48+#vI zbPRiD=Wu;ID{C$zeav*6)N%p1{aj7L-SnP^9<{;zeF# z`C`B@4&Dj2^JQ6ZPWiP;B*z-Yd|QMuE#F{5*JLI?wa<`rmkwRKO4I;iN9$J7ng%VN zlu!f9CMOcrYN0(!8AeeJju23Lwk44AKNB1vTnpkS<`fHhGQHvfiEpEdyEN| zVXAEpgG!2-;>Uy&qCZGi7Vq?mSt#WOec&eK1k(j)2p*znl1~RY_LgPBGUyC23m*oK z38))By{8@Q4?+RU#0IHCyvJ96>tDLH8;%7BhZGEBuaUQ?>f82TuIVqoigG~tkcSY! z#;r6c2@t_^06o)_<4C)^B9p`NvvlYonCF&pq8egO^PHoXWmH;Ly~^0<^Xn@s`p2s4 zGNgScqpjxs#}ne9#kuu@-7sn_C4uCP*+4*6L(a`8q<_Q`8n?5zR4_TU^-qM&k17h>q!~Y6UvQI zg(e8`rXy0@r3B__>k`+aDwSfsADz1vYI-eP#s{1TMMJ%)3}>M{i~pq_W@C#zgg~%; zm&;M5w#>RKA4zvkeZ89AL|QjJs%~=0(m+9H)i>Sfu&edD?$73?zJ`^dyf!WV;%n5O z^$vsW%^=6IHj33iGXao;RR~_aV?DIJu({=*{D5fYg@guU(ald4@wH~ z-|cV4#Xw87W-0>{eFoAcSx4MaQJ}-2_lY$|xnrgw)PsGYJo`VJfYd~$X@*h@ecfk* zGu*||+YFP@8FK#mpwB|)s8xFx&N#TP@jOzS+gdnlKRtHds>txs7W*Vksy8I~*5-Oet1-9L`qgr2#psN{CwWH)1FxlPcoYl;Hf4y6S<{ zOa4iSKT%&}7MCMGUiw6qC_PjWv}T zQ?+hK$^jy_2V$^va~WmUY$vSq3{U&U-k0*sx6AZ}?;zi7Q}j&psc$-aqb-~~)Hy7n z0*ZeSOv=4TJ1TU9FLKC=g!67?2fi}el6Zssi0*3D4gQx|yFd9yq@heCEU3Qa&;P(4U$RDaxd~rDj^NY;SA2nM9v|aLP~k$IB$SsZkWZ)&9hRBhE_7Kt=3!Y__B1m|ICl+=4&*;SPwzzXGd5NCzGcIHQ>;PcRR``b}hyZXX#+ z;G-Rby&WCt?uHYBRaQXykV>1Xk-(+=74Jh{*$WQZc9Hvfty*ABas9GSmoJoO?rZ#;}fnn%r02wcS^Yu33 z)%xq%r`4JC8;hi|NA!v-dAtg6rwCZl9tU5-U&Q+&+mKLYCKsH}?X>+i#D4XN9*#;C z{G#GlFW-xsO^z2Ptdj%YJev=~M3!yJ$@pXIGePzhaYfvwJlL)Y$h!WZ{7ay~i`O}^ z4VPF7o`1#yCY-EuYt(qY%&Y2tFQ;l-8Zf}j+H9f|ea8!ndwL$ewbAsbcj2!&=G&em z452YGuI~yLy?WU!=x}I&<~P#X(N69o>L~h8txh_rNxUyXkg=GvlC{;&U9w@TorT1I z=Z_*hDERG%#JjUJMbc2b(DH{wKX1%L^nlt%XL^h6qo5M;duI+xOifucUX|3Ccm&jyT7>4CgtTtpws#PHA3eHKMm z=sTS!2mfIZ`9FQ%mscsm#0nB$2_@2%oHV45dnSUVXcy02O!@q7f1#E3wc^mrr~mj} z>0bXK-~dnmkoYD7Jb^n0 zNxUG^-@aUPBc3A4qx_KQS|*Fvfjq0V`)^-tI>nV$?Ad#C{ttRVSALdo)PU;d)B z|6hN{zgwRFvGvfr_jYH|p7@jZRE+UM?y|D7JFFxnBO?bunz1bK6Y9X%5lO3jZSS4>Pp)4GyV$-YfvL2D!0T>+>FJGox+_#=?(42S5jnFV+AJ2pCnzpO%Ac>ENbiIC)#Mz%ZQ5xa{hk-(>$O0-@S1aI2-|T;_$cEZ1C5A%H{Grp+jwB> z0sF*}oyV`93or8xJMv6q8T(m^e67&tp6A!kgE}s0V3yv;Y`6Fl#;Yg#S5u@c*KFs> zx;g*MAuP(JPi!v@(37*&p+0F}oA|-!_lcy}zx$re^13U^&yCmOpdbHNk_G-n68H~E z0`Uhly`$I5n0BAYA2WEd#}9qsr}3YzNxlA3WA-HNUoraMUt#|}cb1J}XTx}`?$3|P zr8gENpLn{Y;dw-V*kFxepjd;4i5Bk2?<&WaxIX__RdP+Ruq-vOH{^6a`Ato1K=#*p zPPB9UhaSy^NxA3IJGNhHpl~JU6(Gh~|7Z%kOCHWUnbI!SdNF__sE&mOZy)PlF7cPP zt?R3{@t8aCn*nOGZyNtTxT*2!PHyg+IOSf8%It3~CWR^ae#q8Ep(3>%`1v7*Q#sWs zcTiWzG1*#h3Md0HmCrwF8|o-ATt};YpL%G8){VSBa_bbiblPOA{Dsd28J`zk@isiX zSonEp^z-rYRel+D6;GhbIjq%YHXLAojmJ|Wiyw8!B$PxaA29IPeLh%QS3ya`(SE$) zoJ!*C*}G3v&(|&y^cjz`_kK{ne=#-9(?Lr-*gWw^>*Y_{UHVO?;o5bEH$L%XLl}=X zRc+fHJc-%iaFD)o^-^*%8Sz%sBF8R&`4Pcz!aq}?N}b7=c=?2%_O`=`+=!(H+~Zxs-{N=b;zLXO~jRj_}LbIBqKvICMdyYPv6D<>a?(7yiV2rwi3#)-^zTU zTSu`)K)N&L1rT-AHYb^nh0IMr-fY`y(-04JDCD+*3Ub@Xi8qit+_Uh>kEUN=<1^G< z7SP~GD*5atw}(FU_0;AUPu`63ebu(5OHVjH-*MA3owv2ZAng#RaufV)`}?iXql{{7 zyOf(`w74tl!0dVpyBo!Jb-j~kgJyau-(HQ{Wn4DJwz&Kj_@weU`|~yp#i)+!pOm+s z?;5^%efur>$hO}rUVh%@ea43S;5w{a`TCL>Wgc=YVIMlb(*j2Xft-8yzcc^;-+lJK z>Vg1eXK@iy(fOa|`~L-X!5}7yC4U*VfqGj!^ecNwHOuuEDbr@#KGp(Y(;u#FaeY`A z#V*r{KWrQ^dA?7_?9-d6>C5SdwMuqfjANrd#k^LYkD|xYeI*;=N!~I0#8IrX_~(J( z7Kjv69xzmx`75Mxw{%(hxVq)XAnvZ}nlv{n+`@bGt*@@4vN_HK$}bt%e3h!j@DpL# zjdPYlZB?y^V<~=V>T;IMo8=vPkEZyO$(~w=W>Cm~)bFZE-zsx*1cQtiH$L(fi_|1yt2cA2Q z9^Laj)E@2l%)dTlx=fgoMq)>yF5^~zHrxC|f)EYK16cfD6#@gc?z@MY!5!LP2LxUFeGimA5rz_;!-38Lvbj>U24l^B-jW5ExT;B0gQw-;rbpaoM zYuC^M`GmXizQH{PH(_V=y|3QFgOoWCmQQLlkyk#H{E+A{LH+$c4h{r=KL)ny3BXRG zXTkQ_U_P~xlEhNo4LZ*Vr+KVa^ctZwj0@*3&hLXm*+@ma$gtx(A&~p2Dh`}G`exXi z;jx{!t@=8J>G8>oLc&s}yTbit`tD1vKl<)qLV&%m&Mo?M1Z6|d*(lbEr`WO|{FY|& z3nS4M@&*#i1n^0jM$@d>oW#mfR?(jFAF{fER~-J~G9kP*?`@h7PJNO&{p&3FBOJZQ+jU)0%Ill}Dz zV!ql3^OJMSe8{ zpwoZ+&AyRikX2oAbU!rHoI?c|(`{x!bP9yv+rW2_40JMm8xWw(8ID+T(%UjlpVpJ_a%oiqujz=sH4j^Na1p3T~E=u?kYSK|FbBEH1-rF z4ZxRLdIe$*0dGIoeaFe@ClaO3k#@8_9}`<>o0*%g$dR_%{1iq803b)0T9{lK^FO6$~F zcCNgsX%~5OJi_-VR`G@NRJ|WjK>7<0Jn)GT8RQ0xcfdz?iUEBd2 zO*+}+NTf1thtMBJL!{S$R&i7^c8R2b&**KVZ#LUa?4lvy<70wMMj>qO(i>nvRumrV z*U?$t5fmM-mEeAA_1iB3bFuVsE{WOB#?9$-yO`?y1EM@r92Ce!M&gJv)5MKXS^6EK z1YG-i6xC;TfZz*C4TrE7i(Ec+Jacs)PJqkT z+xqubfpXTk&H3T0RI6gGz*C)Ny-p#G`S z6BB28)ZkV?o-bu;)hQF1DAeN~1IFtW)i111htgp^>O7AJY?2O(9;TlXB`?pSWyF1I z20cPw-r?ca6%)xQp`%iB*KP8eQiHCogx}$m;HUVG6FhXZ9@3#dI32giD ze7x2v_&7vm_rIR2^3j;1Cwf~B}&^CA3BIKGx84~Fk_7YQ%Ij=UYq{(5ZZ z2-%o1%r^KZl`Iw8OuN3{x<9-4^;(v=>RM@@OUMw7d@l%fv)VshFfu+@HP9LJB8oW? zQ(r&fG&(|kwYvVV1Qg*A*l0P@SS|Sbs7Yc&A65lgsOWIv6kNeuIoTsVAZ{a9Z73OI z{UkL0NIMLpTdV`UJ;$W3($uDi%Fv-cLzPCQbD%w_;P9e1x3Q0a_T*5b#k=Zo+lTOg zEI@Y$RzJ?98brhb=`vj&2y1H4EtsLN$OQ0@Ikv!12)O%?p<2Ed9LL7N&SWKm^)nw2 z4PJUoCa1j#u>+C}gKu_b_X94D~90s01EPaQIf;A~~k0ECuAaxOa#HU=^V++MxUz`F&@)|Q4}8e1Fz3rX)X%b zop}Ma;TgD~vIlOiK5;(mfwyegr;-%%T6scL&|gzPvAka32gp8BvUfQC8fY*z%h1s} zL)KQeb5!S;?&Tg|vy^kA>9mNTi|-2FZlL<_t|I!4+&Z3;{|_nRpd3$yj}|4;lwo~$ zUN#j%G_TaG9xn zdX+Ga9y&6}|40Yl>z?>CX%jdrBdqOa8saWe<;^(3VJuk+5JZqt^vik(KeOGL+}cP} zPYy!SMQFuKxlQ**O2Mgi6;MqHTe2E@4Ha}n>IG;Uo^GoQEBfraXqL0YV$higJBU6n zS43<@R_|dN#S=A$@d^&`guqy6kJ~(5=SZHHfhvEkdkI!I;epS#ZdHYoqNl_yR>kM1 zSz8TIn=B|kL=9*#!2l7bw&`QTp&WckG_?y&hUT z?-qJ&_J-avG4j)|`F^XS_GnX8s@|U#aVcwk)3>-Ztsku~$7Xnynpl*kr^#Kw%+}?0 z6wfMWr0azhnXiFJLrUy1I1s`%jUEPL-Y$!`L#*WLBpgRAg)MER-~f343&F>J4ZIqp zy5a2v*)dbRg@u@|O}~{#YN54TKF@Td)z|dQE$HToK~;}^D_&1<6Wu2o!DMb1BP+xK zHZPra)lC@#V;^LCaw9p&6roL6v2V9fH-T}4xhK$lmM$l>?WPd9(89f^bFQs;<7Y5WEqG)FSouu-3jOg7~A5Hz6LX6L%exCTW?V=Or}oGXKD zVIfMzyRfx`$`<0rP>#F4)yDvb*Ss0#n&;gOF6*-f$n0=u#tS&%d+k&$_lpC&o?EWb zi7%wa5<-*SUk}}&l2kl?uL;Wkx;UTl@&eb8eYJ8&Mu%~+_1eVO=mSB4iVv;LTG^e<5ntGyNY=r-ImfS{H7P{q>ji)hL33@J^UV{X;#uL5) zx&|q0VP6BXUwoK1r+QV>;&yqi=eOp`~=zIsv{|!q7Nhk&+eM2D}$=3a*DrQ++ zQY><7_3QIAKx?|peiUc-~si@6?*_u|fsZ+C^y z7;ox7TuM>Rbf#ur`DB_oyY$sBly-IcOX$Pjs0#7zb_KCd*z)oFsk`8_x$*lY{)>Lr zep3=}0{O~5%rnwZfCwbx%V72c^}QafqbJPM;sYO*=OKWZCUR%VV3$_@`g`x!@=)`@ zOJctL!xeg43+dBdRL2&g0Nepv=o9kf=X#<{`d!r0u^G`>>boBj@NOkxuEU=r-zr0( z7;#KE=9%KEhk|HO=7+@ZOQ@wz^@VGgh4U!EItFRxxB|Q%{`H51minJp3VrSj42wPDsKqByrXRSVrMWKnNgkN|XcZ7$YP7~Kv zS2V)+cv@|)2nHtR80TYgFK_BZV_nY2!*~B>}C`h<8RCMHmFd)xgC zrw0XE9c8H#R>yl>YtrUt!*!PXF760Cx@WG(+1c8<@kl+5-}vy}H|Kw>Apfy||JDJ3 z*NH!OV7-cnN*~L?K<841%sH0AtsUGCcfhU1SA+8;BWc@aMx|g$!P+61(o$zj;R4P^ zo#I{c7QNuEAtm?TVCfPYL=!V81gwb`LprS3-DC6@Z5?64ei|@No?jGP__z5o=aEss zoB+Y3wqc-nXjWv9G^( zbg-gls_Dq9-$ppo~4_(mADl+!2C@7 zK-$`{cZ?-1^b}Q#Hvqwk9N}oOP`BIEjpHWL;+UEp<>^w&eATpgx;e#DtG}&14QT`R zVewdT@YYnvbjNDmW_gB81E&6g@!Jy$U+Tz*odTwt@)vU#%CM0I>r%&~YXw?`DU`>5 z6_JVrGqLOjg~v*#>MY}f>FfM%Ws|RadvX?Km~g-f2AXeqV}IQ3bQLf84u>jz!Ef8$ z?`hq$VwHiVnm-WL)3tU1ak-22f=^-p2D2!NEjuxP503m3#OVPB`URJas`75L)`K+z zQUq3nP~PPHdJZN61=;B(kk>XG0$`o0RsOIwj;$BtY8#nz-pxm>41IW zQt1{d%0NcRQu(21q{Zn_RWvr#Azb8pb6@uYI|Qp&y_n;NE-)<|vc>Eln-o`9JB+WEU0>916D zU)$Tm&H4THq6m@?7#HdUk8p59=G?*r$cW?I4gkYLz=wc!M{LwXM5IU(jtmMfvh|{o z^3{5Y7-{1ElsL1!0{5&^^iHbF+E2Wwhzp8DrFBNE1o0&1qSz6*PRMlCr?8k)wp+0i1BNyg6 zepB!Er1xK}hO=_R%gXoGRHfT6)7M(q+RA4ZD{k-nb=GLnfLVLMw?yMvuZ>>NF#l{N z;rWZ3uoEAbPpC!t|3jqh{>P=En*h!&60*l6)2l%`gczpqZA8z6`eQ0MCJHdEdAX>& zqxx_Xl!@+N!o{GJ46Nx}geRcF6wdrHM#hJ{G7mh~m3$WVDrL>wL^16?jSOK;eq$0- z^Ezu#04sGy!2w{G-K`6suT3^OX_LZLmU-M+WHjH;t? z!U-FicMpFN(_T9d(8%?tMYMl2@bjaC;gObnoG51*Jp{!6;0so|h%$2rgg=AMFEXjY zg$JhA{-_A3Vvl+F!`*J~CYSqSmrHO8H>w&AW=J0X%mzx2dk_I`k(`{)u>_G|U~E zRQ&#X`S-14i|(~6YkG&=>}$^59r6DCiDwa363*WM&t<)uD7(R$o05crV#OV!Qaud{ zV64!LD=AvIKYLB(fKG!27o$3o87AX2vmjm=AGHqjGCGnp^-s_B?u&$ku$?(LW% z`=A;~)s)sQHF3LmQx6e^Z+B||?m<&Epql=t!b6QabA=0ed2fm0+gy36HXeZUUqwF8iTgECIo#%YgyIY@K6_ zI&X&iTEUJYvG=u5O6jLI$@m;`NY{%H##q{VVvy z`(*i#uP?n&+u7P*_?#Md$arUf%8q4Wfa0aR+pjMc|6ZZU;wS5#Ki$*jNJ)52+DKXh zIxCo7qK=Tnt8ML4^#$fygST7B47xJBc!BpJrquWB$v6s7@6{L)wR@MdT;I)D88|{jq1SY{RMD9|{Fjr99GQbVM(HmMhuTwcZuqHRe4R zgTs}>nfy@LO0_mVxD&^9PSl$dT2sJOkuoa&%)$(V;jt06xz3T<>4!w*@@@^mB@VC^ zyZ-3%vGB7q;!V9mtECX33Quqh@7#i_UW^Ab8R(3<+GLz~2jS3Q1NteR(Ng2A+@Bo7 zQ*r0%MOP&pKjVG|8`4usKwp14qozYX<@~k!*zLMPicimlnUV|Hcd-3k*DU&p?t#2aaS)73eH# z@n3f@t17kD-2dz@+zcS#R&AxxwaGJ*wYb=9)82v`AMwnu1EFJ3SG3+GdDyJS*D&kS z&z#vzY0O4(U$=#}(}6hBG*LgEjJ(2^PY z(j#&M266{?jh=ApA|DT~-E?s$pvF?>WO!~2v(2?|zrYwi)^-*S1eVca1$^|tN_z$s zRB_3!`$|f{um=v}?(*#Xk+-IY#&mx6csDxlzjeCD%G+~aPT2le-%(HE?R7!C^RXukGZNKZ zuO~Y1HS)=ydpLa4eQ4SE^C_J#M-x4h+8m-x9ioiwTWq%^hCkflddS(z!)AJf97l)f z^s$-6RXwr<2}%$F1mb(I9dHW~h-J$`hxR)p352fnA<%xzG9(T;zX8`x4dzZTGLE&y z3_A_|xpccg5!RxRs&%Py0| z3{=;f`-b4cJpDs_gK-E1KN%&CJcNSHDBquapdC0u^L(ULug9;e4H$fJMyFA632k%J zY!fg>0(3=5XHqu-UJJk*ptJuW@gXAx#;~B^5Rp>XZ4_)F?j#APOXe3N$+am>q$INH zGHED}v8vIE-V8>CQ11{dxr?f>76-`?nK#uE*Y^Nm0k8f`e1}UzUw|yRWE4)usrs&_ zMB_{qgE@3y{ct3!ct#&PaNQ&WB($AkxC zI1NCz9q~#@F?irfaCg7V zw;s7gORxBr{-4=RN9YL&vcn9$!wlO1$ZoujmoC(8c@b*%nQ(L?q=J=$4TQg>HqV04 z=0^f*y8w;5535hzt5W=!7$4L+FznoVhx9j(rhLn#ynw46NHyR6ZE#)u*RZ{XbMWLp zjn^nPiPY53TEdutTZXM9Fa%`LLhA4V&A&OSk3IhH*v)^P`@ajz|B4X*ilzT6>-m4- z)(OQ4RsMuUWJFY8F#e|xGw5A!B@VK-R4vA;X~FKiFou~d)cGN645%vi8kFg-G~b^w z$x5sdul*RLB4d$hAN!^@(U&uX*D++w zUwotVqaPA)I+Vb0qd*|i8zThczHjHE=0G}Azuj|*&MyMwGEcU%=n3vMPITVHkr+KV znLZB^;N9)bRDm(;Hf!~H&@<7RKFrw!G%eMtrn0f$v$8JyaF-p8Y~8!CyzUP315@Bo z(_0;uW%42H-cD!dfmPe$D=TmXh`nweWxH zP>%{O5kHb4H}VjJI~k?Z;0e`pfOp>z>I84YR_%=pR(rv$J~2H zHI=_@qdImFqf`YUV?k6(EEGjZEQ1ILQBV+&WE7PqWKfVOAsIypJ%2@Dgn^9GLFmM(<_$PGLK0*Tcvdk~nd2F_mSF+V9O_z@KQ$@rc}G8tn66t_5?XP%-PYOtE{m*@%Q? zgXi8rUgBMifa+zi@Fo*FfleaU(_|G299rBI);0_G?;U425?1!vY@G6`(hy*QQzG$yy62yJzS zB)Htl&@M~CNs`1_kC#`UJKa?rj0v$Xad*#(vQcNH|9oFsst7B`a7^SEu9a{79@rd^ zIW}6%A!PVhRlKLVPzM0vhM6&kN~A$i4Kqf6khvhfUqeMB6No8n^aq8kpxv4;2yUlZ z4smKy_3^blbZg7KeU9{WC3W%Cj@pdAt_T{5@o#!0b8YTA-tW=!_c|44YKYq)jrBzcu6T)`hvKY ziUo3d><(HgT5rB@EP$BQ2=d7o9DR2n

z1d+3`&oq4(bRAWerQ=l>^PVz0?qL$Bc zvUmv7kN)Q@x%K_&A4@#T&DXzX*QcqXv)$a4oYIozk8(%DD3%lGJ&kd`UT1q6uaHQb z*b7b9qo`-VIyp=1!0F?F4BLUo+;|}!<6d@Z1@%57pkvj zeY%mGtoC39kzY^xa6w0VhEyOU@ow~T>5~$isF?Yb27KpFP~QRp1#||tR!LXw3`2v= z_Ge65gU|pd_~$E_lCs3+K*ctlc*J1PG52ASYVwk=L;L?GubLUGLV-$69oL+qTwMCZnQ`$oas|Az2P)AIhWW^Iiv_~ zyFtI@gom=a2u2EFyAXC|p?E#OZ705~bvv`|#`4P819JL^A+ip6tH{X)3`v*XzpF$x?93{_MjTy;FLw^$2AyCRd*2H z9(s@b0Vl&>Xz+I6Z*c&x*p?#B@7>b5Bf3oOqbaoYyy}za**_;ZFj~T8zF6lxJ`NfL z^7A9{9kk_~@o@o1-lxYm(^uz-CSiI99JBAV}_^;ChiYf zbrcs8NL)tfGcq8Wq7T6R1chsN&cc#upM$Ziy&gjZ(xDCE;BEw$#I01ya)5gWGtw5gGw~{CH6TVkAsV$ujU5aV|9 zY2YXZN)_DI%~zo;G5t1nL%7+G!<79+f_LEopgAa&_;%rPL;2Ex`+qWUuHe=GIRQ=t zP>0YEo^cD{Mcf&IG-0oZo(#x%1{5vD)Hk9?o{WaX{Up$`8j$`GFwcu}q0R!F()Wr>&N5GCusBf({0CEy34qWIo zcQn7*+>^r? z(rs_FU<NeON9;nnsmxll^b;kS3_?{r3h=7GHVeg zvk_0Hg~)FUbtyzB%j-F0S}|3ECQo_NkL;?^`yqZ~~d7Y+OLmi<0pedn5Kc*Z9(l~zN zDz?Id0A-eodRoe5e>0sZ~+x9INK+|t)uvisA<=)OJrEo zdU9!kNtXjIRIxmxitq{9!6&gO9%7vKbR#}m<*EJw_6N)d5O>_-)7s1cpWHmGa624^ zk`+}UgTj@#u<;GZR*I`4o8A?K=9%fiK=8{sxceSZg)FpaRT!n%X3)C75jN4*hWq6j zvt}zEee!E)aM=4ciS`T(Wij`uP}ZP2iwtrhPjn`CWejsP#046e*{di3B(}bzz%_v< zd7WED{(5TYBYf4qNz}I*y&CzeaL3bu8QW*g7Oc@Jto2aTh3CTrz$+fKQJX*1j%PXg z(b^8-&T$37%14?;pvofTpxRj$%76j2j%pd)*WK!qQ=%Yo%m4{L{tn;||Q~0*>@>7|05l2hj zb4bk5^7ux%nMZw5$@9xvM4k7ot9C?;Onkt5-Uw{)KpP|Pn5*YxN02qnU>J$;d44BT z-)a9N^(9RKH$0!2Y}jfDmq>fp1Wv`E@1WPWdx!-BPclJ`YvuCS#aOZ;#2>ov#HxaJ z1RHN-6$^PJ8tN|0Q@X)dRrcUcd%^qxC?BIo`9SNHx*U0pt{<84lVa8*e3~}0Q5^^; zmL5_Z=)mc4xGBc0!ZKW1ENX3d;eNo_l&neQH zf;foiUuTS;X_4S`KBbA-u*VE3M&=kv^mddhgN9mq5xVA~{xAlfROwL15GODwJ=9(_ zsQxbxW7ddabD+1jM!|m3#9R*NOG%vx7LwewIr&CZ3NBo6+x~{%Oh1t0gsI;=vbq|N z|A%}|?a4YsX>?ZaA?N%#{flRr)}uNjr^tE#&eEZlNB4i%JIlIiS!O5?iLm?GyFZ+p z1kUJiSyjF)s=J%^1JwZ5Qv#edF};#@kv&So4;I6~2{(5v|%; zQgl))29_VLf$y9vBj*PU6M6_(Ge~!8Ac(gsEU<&RC3~>7%#ty+1k@$%a7p0UrtYHr z)E_w@KLZ)HgG|YGAo4<5U*@C_a9WvXa7k{#yuU8 zl}L!NHKQHuZNrRGYR_i~BWBO=%8^$CWSkE?q6ip~bTx=I9-YE<%|BB&Z+j4?vM0hv zjt`$}QuMMOLq25Q$?AIo572PbesN<%bqiZI%qj^Ro6Ri`&Fjo=Eh&yLPWYJpBy1!) zC6Y8JJ3Qgx@!9gX{15jgrpLm5&fClD-m*h9Ylx{{VXSdi1){A6|^ zFvvE=_Ir?gwql_|J{f_&&4ez0EHpt@j;i*M2Nq-;l7ae^B1E=DVbWc==~;GFEJhdR z`DjY0R-YOghiely?G=93of=!79g-0DOW)p?7uFTr4v+XsM6~)!D4t+?MDK<0R%$Lp z0{ka9PO%4Y`V}=~pDwC1w7g8=(e7yaE<~A;Hyau8BEW&sYBWvVd~5P}=0cZtJl>3a zU52q?%^MZYS~~FAxHDW)I!>Fze9L%&-T-89sLq9AJI3f$NnZzMBW3ZB-J-jURACLe zOKu!t<^sX(-d z3O;yj)lOR5G3IUJ5Dbmk16OUjwkA>G#@Sc8wBX)8W(%qQB zxHVf91K44om*r#yM@m=52wK(#d6rLChwFre)m=15W?5{1{L0|@#ymIGF{;eJEM_(< z(9%|Ro`3dhoNxHG*1iX!|EN7Kdjh%2IFJqTS0*P{tzRk^3Dx=@$f_wwPl`$8^ywp@ z-(m^H8Fc6z#^_Lf8lEi%3B+w=CoE{2_;CcW1%g}rIbJz|9OwlNS1Yav4;Fsz@OT)0 z!@B;Cq)&JHB=sGw%?u{O4 z3#d^Gr9ZQ~b%TQ0>46&k)(Hafe_+-0LTo*h{Vp#lTl^WVt9Ap<V$Og#b1(4&&yrrN{FpOF z4UZ;1FOOVcGR(l) zmf=8a>p+a*47A#ZTb-*mJbeiWAj>9OFd6@q}6)VLvEE^^N){z4Gvd+O*Lu zWxRsbl`mnMh02qwNeY!Rk%oo=J+n5ErFMiEZK}?~`1trs=wiTK1p2pq$zY#VqMv@G zLq{Yt#5IT(g$Btxa7!uf5F^#eK#J0(cmcxkIL&3m3pDEYH=nDZ*#9Fcwsw?E7iY30 z@#|SO%{t;wTNEx`H?LmHUHj<-k$VIz$(5Q^>K-+KL2i-A46qbcYsr2h%pJ9zgqY}v ze&LDnhx-+Ky9<~1BOBlqyj9|az3z_oX(HS$ra_dN^!O~9bhMR!!8oZzH1w1k?@aJ0 zApO-?{X_ZkPcQ49SZKv=ohW?$yJV0(^lFETJQlUfn;;zvIEpWMG*qA-%6;_JDHvJh zuV?~&Yk2iqv@%5V&?&(oTO_>2t*+vLyU@L+c<=aBtM)(6G&cDcUa!ZNRG66aZJT1< z%o_)zWRZWQ`j;FN41p1%3zveDEgPiWXdv7Kwod1Qrq+olWeO6dy>oY+j2wh(X zL@xnr*>t^UJRm5Pi`oh~(5q%_E`(iX$K*eDKX~XCCUuShn|I5^5;?q8I3oj5RV`dm zf%FhIKz+9mbMtg&4FdE1;PFF+Z*~$5TMGmw&@Wsb&6PdDc}MwN(!rh%XH3dGPbt8yw$Q%w>vH=GS?jIFP>PIS92 zmJxamDM}wi#Fvw!bg-&P(&!|j#RR5h-QsUEWhz7J=vYD@j|uNU%_M?Vc0iLxF;x_` zO>9jScnr1fAjk7)`d)OS&VtRaw;AT%$b07&9HLleYyZ4|>mbL}s4H)2xtO^z%+N7p zZVMRxVt9$l7XgWT^$0L#O;bMMz&ek~X`(WEetRKW@$CgSg}Bx!?3^@XrzedtWIa1C zTmyd-aoc+sE5eO^3uvWbm-DFAQj>%z$%Nc{{Na|)Qbr+-goeSZ-^7J61o6tPU=Q3@ z?{YM{g$u=vvZa0P)-*}o$nKXRW$IAaT8!3vJ3j6oUgIzD3Y4I4S`}SyHesDT${lOe z8YesWKaYqPrwM21Y#c*MTQ=^(nqmY?h&l;PL69&>*hw{cp7imn=XDNk z)hDjNF(#c7*A61nq z6Pn8UnSgD0oG}(Gnv~I11Vl)yqrJcz2mtvnt`mI+2jR!#c?-YCZ*Hy0MGPjp@`D0- ztv^2Xzm$OP#I7IuqvZ|7$9k)LWc^x$JaEEZANrJNlZmXF}v$=fiV~dgGxa z0q<|%asv`FaF}!v1L3Hh(z1h^q}KJHPUyIT zNJ5X)xlkqpX*B%!;MXvF?naKa!$DllWLw5w?@{ynt9~U^L+Cg(0%X{~o zI`PZz7gxRBqcMf>Gu0sv&xVquXw&J!1AGd0M0cvyC;{aT$?sgAP1^;GXZTiTAE!Gz zvz44jpVzvSn_$+G<4+J#!m$i9&nrD0=qxG#zySkxtES=-5NF|L82OEL-vP3&eW}Aa zI8@A|h0_hmRh6wa@W3P~)$XymQQxCW;=6OKO);OFes|@bNg6k=Yeeo?er-@5{Cm;8 z1E4~1@rcv`Vc`u&-e1Qjni>6)y9V@mCBc^4UR}9t7M(M>yG{HsC9Ct&^`MRIImFbz zs!uD(=}{3@Tla1ZFY?&kb$(YrIfZlE?OC6^Mrho8O8j~U(-_=8#vi{M84SPLTba2* z=>i97w6;b+ zdi1O9<43JxbNwRyqV*-~yLKGio$6-39xybw{@;Iy_lGKcTLQ8xhROpt;n|kHtbh`C z@U05(*zjRiK;<;}KPw=6=YRdf2_G!?E^*J^d8&L*>h{uuCtdUx{SW zMwj^^83)7&M9IW2DfqWWNUyLRy|eeCyv_OFg!8q6x>Ge`-a-as#0y8CS(=9?8- zbL$*;-Yi9}PxTMGjydt&D^xLbuImq|OzlC5U}Mt&j4(0UDKH65+zYOe~nG-&BAEQTzz`jtsY7)7kM($EhA8Z+QM@$-XkoL(^#Gept zY9}ZS@`3~MPIx1na674dLI=fco!oL-<0 zz^v*)GhlBgbC_6-QS0##t!~wNM1X@%-~~VQ0BLtTCS@IUd;*)>^$m?;~5kyP&UQEJ1t}yl|vBHC#cyg|yR52= ztHo0%-hPVqjH?P?U@)m8T*w2EH_@&dE`}>A0s97gIi?3zOFJf$tBasPc?T4;QHh|4 zmy{^cM2JXT;}-4L9KG&}3*vlRAeT}$XwdfM&*9d<#ACarj4$3%Z{g*{jk%`GF=cq& zx1YHQ)Yr^qxXUpUcXh3r$=KHN#EDO)NU zd_E}29%nYPw|y_nE%RIIsEJiWX>=cDz3lS4CBKprVM$ZYIY z*sRl^9WKLNbk!2-Ag-Ygcr1%i9Wp%;Z6=Ay;dYa50*W1WAB;h?j<|YP& zcl&|n0}6X#*Dk!6{T3j8FBX=SmfTo4Oj!caJAGjqSUFE%6JB0o;HPWGoCFYiFMSIZ z!3F97@dqaW3oP3N^81iafVP@OUHea5iQPwugWDuztTrVWE|&y_{M3&N)d67?uMlX5 z$Cto8$^Ww|jV6mXb}cjdWBGQkJf`#h9Ul2PCr&*y(w8*&q($j^y2S)HHs?jR zNceHJXUpx%E^$2lzccsH+O9R5sE&owKx7VY{YSrLj3z-P#WKgP9s{G0MeRb?z)VG; z`0_X7*YpZ_nkUYcEqlaFrkZ7SnV}$|z|Ql~`5J}wjAQq$%eH;+u}dxW#!iji-5Z~w zp#Uaq!3GCl9(u6^yjnH24eJDtLKVs~O#uR=)`tOj+0;ohrDD{6Y07jcm*I5vks3Q> zgH@&Z#kwu|5FY->0M^g}F4;cEv>Of4%lfBfSwF_wmCZ4$|sQdVmkY}uoH&~n2s zrW}MqI*92s6fFcmG+^sGkQqt_jUF(>O=-4-KA%=#I&imyi!UR(6uV}HXJ!@~9^fDE zr*4ITC9`Yo8(We-d~#d!aL9=#kILhxj@s>Pe9F21aKFW?harCzSKTLtHW7v!^(%{> z)ZG6)XT)=2csj4&a;hwXxt{ulSx4_A3iJ>y7nL*8S!e)cWojj)_COMBVcj6>l?s2Q zZ3GTd2jNvBR4R54T%bVlkRQN%CL6Wtz9!@NWy*El{25y%ZPkni<>wWd($E-%8Hn(xe zbE9{dGW>J4f9JMJev3Sg9T`*C;QJk&&xA;XlE|VUT3gtBJb!!b09V9faxRrzJC`!~ zTi}_OA+h^YSMuC!A8lv$r0ycPwr{sO$o?I7fdMEhCOFYk;?u$k)hWtJO%^w}FSBsJ zrhJ|`w&q(K&ID_Pi~(UGJw>qcmxfw6TMAei)FlwJ4ZrFW?kRwY;)%c36Ia>_*L>=q zGHhK0Gp)NB9PJIrW*GP_omI3}M;cAh)wO}30u6cM$7sqUEt+H?RCH_vy9JIHjmU0_NNKH`6zCaMyd;;Q zBJjk&Ggp&hMxF4aU=k@fBxFsCl!p2=_Ip4qYqsbQ9{X&>jfvW*UnZYO_w8_SYme;7 ze%!t@+B+jJ&TX{9Bj@!(&^W<2n%l^at54ZhRUh~Lz!?pw%H8@IEZACZNZ(tjnA^q> zbW0j;wmPV>n`*abE}SW}bz5E3J+(SZN(E$vHctvTYnO(?o(?mKpilWP^a@*RS0em1Qq+f{cxkfKfl~Q; znC4GRBkZjow9{s%MpLL}Mok_CXU~S`g$Gf782ovF^}e3Zz8tF{*$qw3`KXRWFxSx{ zF>f8Z)*+q4z*G)&qUfB4w7aqEpvmM`JIWD78BfQktsbr4to;NIR0aAyh2q*VX^k>@ zih4sr1tE~k6e}N|6US?Cgnp9R;YozCZ;{RK zs^Pt^E8i~t@jdjlLur}pkMH6BQ|n*tiFC6o3X7W(GR}L)h6SL z5V5=+G^;3Q!KB8i&%%3YkQC8!{wMR8rftN9zgVb83@}&er+-{7(M_Ci!YuV9A=P|UWTi1d+tm#EuV^><` zJJ{&OwE$(&V=_IE^DMues$I+Q9~pkFeXm=sg{iN>T>%P>dR#8ykVOcH}L_<=xuc_Z#NC-R!OIa=bp z8BeGC57;ViSsS`2)O!L6FK>L{q(k>`o4 z;c&;kU2CmxX~wxp)Yarde%B7^^122`Y6cW9wo9swyO#2}+wvkZ$^`|M z+uVyJ*eXPAlA&^fw?$OQmWXk;o!rFdSkp`E*^-uUQ^K^oli)D&AJBmsS+2Ei7Xw)X?0#^n5!qY4pUqu2a*Ewi*xYVG?0o}0I7*!B{J*0Lr z8*n`9y-J&YstL7YthfgNsqm!oCM*n-5>xYhA4*t9PL5zWX{;4W_u)Aly)m;){Box9 zvTelyCOY6#UY(IqR)n#vvOK~pwZ!4{E9*mE1#YirU!E>^Klwi3K#9e+Mna_UP`T>^ zK(DxGLyIqh zgHMb!JBSm!zMfPtQq(6OhhoY>PHqVbS;W-I1K1udYZT2kfs$En~ds?;0#h>J%GV zjGp^yb$a`oj@PAz-W7e01{PjSi`u)<+phAwyGCB_=ALB#_Vb6_^Wl|Py>=O;6d5g+}`UbIpEfOjxU|gGHTpRQD zLOtthNk=SZUC^iiW=pIbPL56o)QxQYoLAhyKT9B|1sL~sG~DZbJZ(6}`m3grOHwa~ z7eh>C7<{#uVg>IPAHb}F@owV>x>rSH-~)y(9UJ&YP#9L&f9?$lGM zxh&=ID`#e8P!zBffvKPYHP7~OjHi-G02 zsiVJ#{k(_LnB$l8^RAf9MM*u-nd8fh&)guNU5?gvX)nNq8G|ZdnDV39wrJ!cpaAPc z{~lN{B(iEq-s`oU2R-UKRg!gDa(t{ZTw42%YRITX274{cdyW`A80_{~;&nPWx4+GC z=(F$kN2&Ap+y9gH0ROGmQb%Y?X&<%|wYW2#)GYvdgCSGuD?b1t?EQPuj>9m($}U+d zz{%^^{p)9Akop2@cG36R*8tlb_tV@y$DMo6`0)24WX(B%Jg2BY1L50Tg&)BE3bh{r zParztZZQMSjq}7#q6SO0N&SGe*noWX8=_6^#sz@n_}@B1{{;LKKXqja*sFkET^k6I zkV^L01eZ|J-wWn21RTL^`QM9<6(?-liyZYY(P>w_%=~_TT@6Al{2dWfl@TKA5d=;Eh=sW4gF>WY>WD zfLSd`(z|&DUm(HWeK^Jown>bTG#$)S=1cG^0;ha$IBS$ekEU{sV_#=QheS?pw3GJj z3^#BtIP>*Ye%QWOuXdgclgID}7KmOG6{_u&a7`vH3Y~ThSC4vuH$0111hxs6&&zKS z96@{kxk#XL=>e6ctBD{2Fah*y;gJV6QB_`vmGB+}wK^hiIlCMVpuxwS@R#y3AI6c4 z!zOe@0EOx6!i~|pnDv(6Rz$a(5bw~cXA_-zC5%*Ke45(-vHF_gxIMD_d7>ATMRx5X z=+r={B)d(bgs{s_o-~s5b9Wu{$lwQPjyA@hS|CmeXE3U5Q#37*tuQy*3P+93j(~IV zq8xzDC?{8O-$rty;8a$JvEiLcSlBk^(t*=rIR&>y;B`M^jTCC7F(ezZ zt#d^p_a3m3&m#N5O+yfZ#iW$<0>5N^-PYOYJ=CV@n7;bu@W2vC$j5pu^pgj0p}fIC0~YJcZx7VPg%hWbuy0U|3v($gg4lJpGI1G*CG7}=sh%48QsjL|LPGSEDsY^~K~ zQg)P+kBFGb&F4fdc#mPsJl8bBY$ME&T&J2-HuT$JjL0jz!I#9f5qS+svwBBsAz4-V zc`IAz^D186omskJQQ7_BSkwb2)Z@1pwTY+nLWJNnnH0-%yx_2Ic`7og2llYx?$bZCW&XOiUZx<6{E%M3t3I|84gO6?2al} zeOgn56J>v@Yg!gvt8V{@Z!B3ur5)4(FLQJ@$y+7@wzK(_fU}{r(wo zXPYNqve7x7uGXV<73#y2;OI&MxosHK)6pM6tXtj$xhm`WtJD_7cel3cGdm{C6+Ps$ zg)Q33ZH;m7Qc-EclPFzK^@mV2hedP!iu*RoYFM@sAk_n(Hl5nlN;ee)Jf zgaz3&PDMKNcsS@3e#}TBpXmb>t#`Op*p2FKeCi_g62;)e`ZtI%`AbiiAI$)}J=9ni z_EvS8ZO!d&M5oSBc08GBIh}43UmzAp$)>bv~g?6M}Skb6|1Tl%_8qT z0~NNE^cIER4R??O_p8so!^5~sgSgE&(S5Vcn&e>EP9ar>Yns#~De4Y&%RqBN<+Kr1 zOHn4%QGijW|4MNad%7O(w+*7T-{9V)Y*ARYQI`#>Pdls~j8{@XdpOoqZHlWaWF$Z%?X5P?6z7JcVuAvX z!BUAfW39gcRs{^JL(DQH`7KA~ony9+^PQ~^4KrA0USe~LSNVATy3Duou~%80pVGRn z___4-DPi`JcNt8KWt7r0=-vF^ix^Dklu%GBV$8e*e1nm1+(opI6s(Rqh)>-(g=~4+ zs4&W$R46MXYy)`n=PskM!nBg9y!&G)9(P&93fB=jx=*64aPQ`^1J3>>7$y)@MF(y6>+6gIlkWqb7lo!0A9Z1@mo-WSc=vp8d?hcjpLc4 z^#jX1k{^^d@)2z^Vxl7ddY%xmBeH69W5%KQ>py0jYyJ6x@{iiYvaPU-lnL4u+CW$! z%E6(VH$_|rsE`=b=4shZuy5kmUN^eKZaybw)4j=8{hy~U&6>%HL*8auroHOwRfA`39p9cMEqpyDOxmOdgRZ!cACrrvl(rA%=V?5;z&Evlp zU7+10)~(>-lK^4(n~WH7uC$&F1-7?3zp=yIp4itoba}2N@TZm*#R`yfm#GiE0pl68 zZ?~%W_=4O!+ z{6@X^#jZRubT@M$ybrKxW{cN>fkMcc3 zISipX927=(>f}FAi0}@n=7^ZvNnH79A!C^Ju(T-Bd?1W!4#_^uw{W00P6x-REK)?s zL11Sug`@vu(fEJ&5^D;LY&k16HF_7micu zz^-TD5ee=s&0g6-ZhtQq-yE_U1x{ z+vp@B<;MCVSL&m zfHc;)qqL~$<41R2hQ*!++Y&0l_uDmM81*G6RGgG2ZMeX0*Qg;_4r-fHU7=iPgWU~ntiv2$% z4JEFNw)(|C=&zUSuB{8iZ!+0@ZLt>1E#6$VHNW#z@w7odINx`UdIe0hJBK)P!bCilT=;xX3G?W%wg<~ zA-fjOs`G<(9@aiT4gWmoJ#uEiSLq+qyT^l!9sNE|+w1}gPx~ut8eKPD2?X4oT|Z^$ zDVH3Li#u;{@|5AF^FRFR_M;XXJi#ZxR7SKrRZ-1xIRob7a+zmj=u4;L4GwE57og^> zLrW~z*;bnu@Lz7c_&|5;+Rsb9{JL;qs5_1;WCq|Z7ORiK4&^Yv!)#4|>((%D@=nkV zv})Ms_=s)O1JFzz5x>gXnyvraWTw=o0cjtJaZ64M;LqCO)=XT^M1KC^;{I3J+k{R< z86Psnyxeu~rAEEH<|+6KNu-4UUl-1|Zf-gj2Wr=BGco$zqU)vbw^;F8io+k_VddoW-IDiX*2+5)na}ZDKR~)=$Lv4;AL^x8sZ{yEKq((!;X|LdSo>Z;6BjLckB{RFF!Re>t+@ZF=HMR2!w z9c)n*bhbemRF)F^W1fWU=k>gCyB|oM^yey6cw>0kCBvH1=jHFh7?In`Bs~#`fd&oJSVj2U74*b$s6d6=6V!6fCr;q>#vx9=Ah zD^A=w_OeuJ*^_4&-EY00kU91>sCEWrL~A=vyZlerP#Zpq`!n2%rYwdZ{_&3Zs@1sV z%^YeYWxs;JB2}r)p27Lzz@Wg6fN!0o4RHJ+#*voi{28YZb|4%Wl=Jncf%yVIUaP(% zYc^jDH_=7`3ZWIIaslF2uKr-XQjuYUjwscrQsr64s56q4=Tp2$r5V%I+^G5VsR{Cstl1n+ntp>ltz5UxnU?lw{lySRv zva{ZD3IsDi7}{k2ni=!IYTgMtH?IDPy;ZTfJWPKrVfay`%FVXAwL80Rq&B{6c3e=@ zS{O2I0IHd;5?nV^Ae5|_*{|9YN?W7eZ`8&{!A++?i(i(cD@8&w_3x>G51e=tqZ51> zV)KNn;2v=cuaqg_u6+w%n>;<_MB0$|)1X{5&|w@~8PI2{S^~!21qVzjc*2*S30tYz;*I8kznuzPq-%40aceK%k(>C6R=ZwZ8CAM?Yc8I~@!BN7EB zwJxH^*(AWDWskG3EMNkcVuw!=E<#YWwqWdT$}vp=)|`Tdqa_E>+Hft8SSMr4PAVMv zqRDoYA>4De5&HVsk=TwlY=bBW39J!&GOC+ zay-7z_pS9b=*V5*)Zc$9m@@RWzviWwn3!1B9vk9qSvckUq^xKAW7R%(w7J4QukK!k z;A0c#feJ7JPsIIyk%vS7{;$#!-Dvd&>EDaCS|~w^Q+pp++dLfMXfpjiJ$F#2?V`=k z5*MzOD>tlAr$D&*S=s{ixf(+q6Mo^?B!b{cTtb_^sW?r&1#=NRs6cT5#Gy3N)=kZq~oJGzX< zIk`%VyYqFwwm!NJ|01O{(%&Pu)L#F~uD^YOd$$Mw!UZEP5vbH4KDs!hSMMV}3?1>8PXd-$R~;+AK3_qSiD zMw%xS5+wIZY-iTFw&PNKVCFGAsBgD-!+s)x5KUgvy*6+!k5tu>x8z;;gwf#q_9Bn8 z-W9(5b6=v2!>kJ}<1&X)e#^Ygdiht}-91O^s`3I#$Gm!Nhvc=K`h~{5=^O7CJg?-( zhvwD47L3PX6^~j$XqqJb2lzaU14Ul-ZrWPh_tUOw2Xs4oVI6`49k*%lO<>dgWTp`T z1QZd7)Zh1NShz`s6mcDm@#|tjz{rC8$rTXgz)N8yEB;=z6C^YL?-z9$`Re-jq95qs zU@#kIe(eTpW6MmVr4$cG;qDOUwy44U@r)E;zh<8T_Co5Ne=mB72FnOp&rL?FpFH~f zz3A^nH;(|qDJla0T?*dst}@6#QT8v-iW5IW3}|l{;4OgQ2C)N$oIV+=?gx^$7%GxeWd^APsnwlgn`qd z8(5X!rZE6d{5O&&DAza3@4`8HW z*GKsMFUVWJ(|R^nLewG(1tmGvYqHo=M)t;a#<=8}EeucKSQnsBJ4?oGfH9G0$#mll z>!)*C|L5aaK2EcZ5ePGx-}n93Dw+VYFaNEHDF5F>dH#Q9Fls)rGyZM8{L93qu|G19 zWhmsh^Z#Ggzt5sqrzTggJm8=G^yr!m$1Y;ii+4HQLfN%b9DtZ3<~G47v5mXNX?GbM z$Wsue9qW|Cmn0v*`HgTYp4;ifcOYAbIjiJ$Tz%mBc;k2e^u#s=DaFpzucL75&}(yyQhE33_n`F^i(a|kD_+tWN#-f@;G2p-_vet`i~?$qc-^V|Ty5(3O)}Dn zTR8!v+Rc`LR@3Nspbg^s4qCbCz?OnhOJuMOh2^t?Sje{#FBc zNjno2vpmNW)86=#@nt`sUu|Nwe{bn3c`|;-eA!2KIXot|M?%H)BkT; z{(p;B9}LptE8O~Lng`7~qLuG`B$zZb2Q@PyVrco#nq8b=krenNiUR574@htDA7uO) zZu5ha_l76NCx0t;{|mR5E2`suNoS8GB5gu#AkG>`v8!?ycF@F(Y0AfNd7xg@Z|-5= zv1gCeeyzp#7kv{VD?xc_`+dz#+RV6m6F8uSe$#iTpaS-PvG*QOO{VSEurn4$5mBjv zl2IuFLPiBq!DJKx0Wks!5`>HeAw-NQB|u1&UIY{b6oIIeNC{D@A|yg6qo_!iP*Si^ zC82~zQs%$?&ik(Oe`h@FpZ9#{ocBBbaxGadS+3-H?s{GO+SlISu;%C#P(DAwi{-$P zTZn~SiTGp5J6Kai#KKM>5CMeyBNO;IMacE!Nb}0rr)GY`)m7cC7cCY zb7WF=Z=Gu%h~yL*TkNU39H6yKFI{J|Q9yZH);K{P8;@M;To(+3!Q}ZegehwPHuwV{^CpD0GL)CDp+%_o1ymyngg? z1XY`CU$1vOYJ2qDkoEKh=RGIy>K&3ko>yG+t|4E7hctz;jR@27ggvI_^koFi<4_V; z!S{pm_N>@wi7hX-I~(nB<6_#8p8GZ30q;K8yWcS`Ncu#^$vLe_jos;#nBIl;Fbn8m zGmlSx&Khv1xLf`bd-W9xG>aNgcu@;>kT%QtS>v%jBr1OL-+y#L9E{2T^&oe+^w z7>zK25!l3BsDqOM1$Sxl@Sa#}V1{JfS9eNqH*8wBKb6M*Lm@*;dXy@cS zUDX~-ZRfjBp~H7K+T(xy%XeK)*RUg5Bnd}mlfDLP)8v0B+$sB8z_R1-A;EuaZ{D-jd9OCfwG-(HSDV zw{BCRX`AFxbp@ZIT6iaK6pg?A>yvK1u*e^C+HgA>Ro~aX+-+aVdjP`AxdUJ0yL}+I zYSo$XM2)n`)~u-o^@YI+d7Xxm9fA zTwnjx=+bVF!LWM5yE`p`)NRjNhZ4eJ2DVl2ho**b-S02X++t=qNbYtCFmaAnDpw;C zw%z;jpYg*4@g#ZC<<_U*vLfrc@Lg&d|l9jX^D9IFv3N>t43szHlpCXYI43S0!9 z-aoA!>iJj*8wre|^6IrF3yF#19JK~^2K~LAll9}=mwZ#g$L&x0Amr4j>;kx>7JSO+ z>tf5Nv9F(nYI7bum9&r>ULj{3k>@z#^)irAjLkL#I0D@4mp|!=pQ1L6AcuVy=~7+2 z30H|~iOl5?09G!5A}tL9SfPvS6dprA@dezI%ZtA@ks-Nvks=&P0r)?qi{1MsdP4Uy zCD(<_c_4p1rks-BLjJuCWLJ;+!2NfL=c!4Ty@FM^Cd_dZM~Amba}g_`w8MgoyQd#O zu6BL0^^n}jF5<4du}fCCLM#IIxSNQ)g3ALf+ofvC zRMUp^#?vHSzYurvF!B(9DFmEIfWHy!S+#_l7ffKQM;C)4AG|9qHz}v(W)`@!PY$fq z2~It*`X?=G?+X>{31lsa9JER3?9??+Nr0^&DN7FbnnWj=(A-;@Yb2XT6zq>2_Ow0i zfDkwBL*xaXdu(gc31kwR)_u6yz2av4yze%7-O^56g)P0dSE9HRiWE3=`GNn`FOtKxal8cF5Y06}L#MYx|7qP=P#UXtUr2amN+DyPDmeNBr1#R5+#*|ouhVc9}WJfgTqGCyF1dJqGjoVJYFV?6Qki-LNHZuc7Q^-Oa;h*<-}a|JDff# zF)JLaUdX)WcI(ujjdp`uCTXSI+lTjj_0ca3{m!M+>r2z=y}cHRO|XiE=Xh19v5&8o zz*tG#A~Z`WJzw*rTnIE4H3?>DVRW{rWEHy|$M>+edfH3-SnWo(J0r{6@9rrP(9$fv9SnHpNa>+i^70#}v_kMcg%6ZJ0Oe%uN2L-tW;_b3G z=-B8w_?1#mP>ONoEJJMYfXJ*G(&p4o57NZJ4yvFjE9mBxGJEPf+fo#e&}u7s;pMYH zGj~7e+zHH0Qg|rVwA?A8xj>Bti)H)-k%IyOU%)KNLXF2%AnY5f3hXbgYK;l-+ude$ zdmKA6zQSJBtYl{JOJ2g9!M63+U5a-c?XA7;T!VW^#T(V7=tCg_7&o}l{I&ft$Wh=8 zK3`MnwK}wZfz|$XIz2v?Y`AsA+3BfZIHmA4F~-ZyZpfnK<27BgB_KX*LPj!wk{Pq# z^%GA*Zh{ehc*Y&=)Uj&HT8Sc$qQM@UdD&r6esTaxv+w3trrJcTxXC@-&sM(^zvI4E ziq+B&Nwk+V_DfD64cf48&9(akcN@@a!%I@L6?}@FOFUQJ8L(f@p`z0y-1yD*FF{#B zcc?79mzST`(cg{kC1}53N1DEn9R9kPMzoT!y!J%SYT$5tTreL~p!N9dJ;`AKCPw0) z=jJOI80=>iv#}&Sz^m5`w==XKXgfK5|C({?@aK=qZ&_Is%c8B8bo##5t*)>w2{q~C zhQL@TfD&_&4Pi+~r@7M&$SCs4&s7i4)btvYG>B#b{54}!90B$^Lg6T0-iLiWQ^3-j z-jXm@aG3MSO5wGq5YvE+M3@T^H%IUriCdwwBk&CnNx9}N(Vfry0r+$v^g4|AcgnU# z9`36O*PT7lbvl_h9FXwlv$AbY!HG4^DOAT_FSqo5NP2d;B)qqV8&cKRK#5#n?5TzB zhJo!<#c8@bw4>6U6X)u=Tu|SdY&_@vINWINE(lr*81?>zyYFacT{&q_64B0XWveGW ztMqV2x;T^>(evy;Bcr1CxdvXWns2n(w<{sB!fB>`4<)EQ*`+J8+^0UrD=SD|lT*7S>MtKexj$36UTj@7(N zt=Vg$Lx1+5l1}($s~=t{Jl%{N9=yq6vY5Ft?>AY@$#Ag_PPUC9Y^fI;lip-;l1%rX zN3HYqg!jn6Lv`0VJ|slA(`)R@IE08<)-K<^&{s9gu~gqHLoY%XFk=dPGe12%BTZtS z+eWm3T|X^NC!ZfBdZ<=2bP}5#i3-qJSlozE1Eb(*3e7hOo^*G)I~-B43OrG()<#_Z zF8h??PR7j#SKr6ZScXZ~{q4D4SCCqm<;z>+@`ZpgQBIj0Kql(;Vw$cH9^kK0R_h3r z8JojKiWv2})iO0d+xg7y_-@#7TpF>jY3R~9@$iCse-xy^+X`JDbMJjE1= zn_!^@qcHV3@K9Hs?MS#Y*CtXz<_}CKgw|irvV_3>-kO0%9}@B9_e`CQIk|Nn*)Jzs z$>f37ff~wH5@;3H(t;)g98s{KhK1iWhFtFj@wD=IiOe{ewosQ6nCF84y^RqTCkhB* z{OkLS(*unuBb%-`3j1PBrZ&y*m$N}B4tflhPT-}>9SU#=8nZ-|ndNGdtvb$DMdN(E=bhpSgpID3+_HjQ)3pu>l zIGHAJqFprfE%AEjqqVK$J;9W-0(y3^KJp|WZJHR~E~v>;aAnm_met5&9Vbw)KWg^x}e9tV!l6}1Olfl=AskAX6>!Vce2s71c@E4 zmPqNduEZ9mdAbTRWvkE#1#A|X1&6k|q!bf%0iWf@s0dlPI|VcpdxG0=r{xl#?L;fN zIE%ehUzA|FIV&4jJT_^q7oNz-^zsgU9ZdCPc|SyBgUH9|5@jIvA$eJf!~tu>tG}#X zN<7fvcy;U6%NFC~;U{kWVzE|H<-?54#*R8?>(piglgn0H3|qJCF&G_vUXrK3IsMw| zmCM%RlJS5Gy5?O5{s)55?d%M2uAWN`O?61NT`+x9;$Y}A)dTQO^ZnrtyyWJtgjiF9 zB18QXNhqvWjY-jK{mROZ{?~J7p7}BFF?k=p_9uc6*+TIu8M-uwlpW5KX<^mdhQMFj z0|dr5wuxq^V3NBMme@v3avNXE1YmzScv3|;2&DnO-@zp=1t@D~c_eTc5g>j5g$veO;E}T>1qvZclt39FWhoaX+qhp5P`3>~wL>nmToc}FK_`Y$zZ=CNN=XuA+w`Bnp2raUG}&;QI3```+dWgl_q$yj6QhLpWC2-ad-5yBnAv586wvxYDBxwtD`a_y+-Loo~alt*`>aJlQ zPn}LZ+o>uAHK$tF>MQ~+Lgx5wfyBUxS)#Xi*ihDp1BlRg?57#K6aMj&!goV=9mtj4 z*uv5-2i+`y%%Jm9Ag`}v4PfI_;;y)iG_a^uAl4_IDL+{`A+^VJ%nzZsn3U@4Lm%82>LZdnE4&LE@_z>)+ z81qE;hi9Ku=kDs=d+&~?{xPoCCO4G-sZ5C>KGueQ^Dck66$NrK;hMT(F7d*)65o~(XHdA_4Oe}m_~qdec? z(SHjf|0zcJ4v+pB*Z#h7zHglG8|Qmn;d?>uzam=ye&c+(vN z&OYwieqnB6+kv(H@7yi1xV74;TWe|#^RKVB-F)m^D$4Y7#jOwKp1tlm4=^94B$vI# zy|lPnUT8;R=KH8EBS#fV_M)9`73{7b#v+5v9IuDub7suGNiMgMY-9n`4<(TUXkvo% z=`xCinw9PBllFxy2B2Wr#xDYX#J6eXmF3OLoKO}wS*v%{RX7-y`dcacdfIB={QdAq z(huH2n@=1#|MQQy89kZ&i9ePYZXq}9CQtTCK$BI8p{RylhV_SC(D-&LZ}vvcX}Uh8>hOkZX=7l!GB#zFy!^TDH{I4+qqU&+~`nPG=;n{ERTQvFOG!D_v2L-sXV$o6vnHG2*AieYOG(za@9b?%w#2O6~X4$U^7Ga~JsgNiSv6lLZMF|ciA1gj;rr7yFxxchhxx)QrDdH92=!?tGK$@+q$A@ER? zYa8H5tzIX-LhiICZU)R#2Od%uUP)#xBMoxqZc>1Uvl0RI#}bMkDUIk0<%J0Fyw6p_ zH3PiZrj$gN;Y|^rj`dpX5){LidJw7y4n69ct}?QBuj#4_8KEqB=i4h<+aH227-g5` z@!ba!OHNki$OFdDVI0X_vQ&B*2H)o6Ls;38p zW{b&7*#<5pQyWiphcn>ZqippIe_}1P!nnGpV+BBMgcf2Ar)=12&Rh zY^Gj=9kM|XE8A=q1z1F)zKJR2FpY4EY&m=uF5V?4Nhb@a>;72MruaJ?`YmPPt}JyI zPf>CZ56D=vQ+YD=t)h>AEXltP0YH82-A1xRjGGrF>`y|pgLd>x!0S1$_yyD@ct^MX zu_Oa_<3I2eij%KkGhq8QW&^(1cI3}YUIg+Ye(`X76jHddPVbK;?bRuNelmI=dz;7v zxI3)Ee1!GiM!p9pLG+#!*-6rYZ%i}n&x3<0>vR3Z&Ha?8hutI49b)?WReR2jhGl-+ zI~hNipH12@uq1owRUEJV9dH2M3jA$`FB6Z&a*s1iZDmTnS^G2NeiqVWc!IgwRhOB30v25?AXB>GL{8P<&a zHzp2H7-N2tK4p$=MXUwveO&?I;33%3@CPYpQ5B&7ssL~T??lS%n29*^@E%|&GYpIl_gg^yeBKfhY2vzQ%I4EWNY#m>HK=7 z2J+gzCfb8BlG)=<6OJ-EMV`QF(puGxtlD0=DY&Wj$O=DKR&bvx`CRWb@ioHT)?2*) zRT}!&`W)knm@3>rmTL_ebtkpq!3s=YMT|5;UcZY_GD<~jM# z3+3P2Mbf|Tfh^EQ3yMssbeS?A`Q`MN<=+^hZ=OAj4x7i+uWbgPJNK7==KL*+t+ZMw z-asM39_m3rNM2okw4LH(< z9-vLj4nS<}JU+oukr?si9h?uy*l#~D?57n6-0as>oLlsp|KeU%;Xnuwi0+^4@h#0(P!Gpw<2dS9f2$tOKbz$LVKrr;}u+6T?3VKcx_N31)3-g?b5< zDY2?Dr8=Q$Y>dh8I?wtntTts{Cwg#cJEu`^FKo^{*3GNx`Ot%CRZe|3)>pW9osC_V zL*--@X|5Wx(<_sEf5TD767@!7C$d{pWuEfD#jcAlbcfP9%={GJO#k`5_~uPEL4TJi zVnZjyp~P){q+cXIgD#aO*c#BC#aU~Lk5#Kit`DHE@mZX#{g}d=UIk^0kM|C+vW9z%@S0kG_ALpicYrRJcVxQpT=$XfP43-l5t@O+|X zU)^T@ggb5sd6dEIDnM#X*4a}JP>08ZZg{YvG}bo>i)4a)prBX zc_R-DI3&>n=mQhH%=qbhn%uV%2QdtrwIZH0lCQQ%kCp8h9v9uH5!!pPuO_-B zy~Ef}H16j0mktpyj=;Q0dRg@sU(z}te-`F8&s+jk*f8QB)OJnJk=L=^%-ga4ex@xs zMzZW#8g0Ij`3W!~8ie~6M$CXXh&MmEs6y=BLRPfcRWi}Ct?5`zTyl;=O`PZO=4+cO zCR`S#FGha;_3Wwa^N)(w*t(e)meCnmCQ_5WVlp{1?w+709pSy_=$Zn3yTLJQJEbVL zt!?v@CvRT7+_{ixf`@02x$Uqon~^ms(rjuBd45ejeeyCh842xbl^sS+jQ}rLeI7+> zNymvd(%)Xje^CW@=)@p%ZpwLlM)8Xhmpm~DdIY3bAkRNHx}^40dF(paId6nVcWg#9ZDSo;ZS2NoKbgIeT`Plo0X`vp)};q9wNk6wuqvjZFfQg9pTh1iT0< zrB9Rgr>H`S&u59q%tqjBRyGAxnboV@DN1M$-vwsnerD{ws*VeD4rtlziEwwZ;iT%$ z;f4^?#|gGOMFID{c7L98z3dvEnjp?7oz75N8hbo1k>=P_+Vi z`Aga`%aOsDpOwu!f=5js=rI5SO>YE6?T#}?gB@xWIFD? z4Yyr^#b25~MuMdsPCO+viz9Uuk73St1iUaEGSyBzBJ1s&UJcl7UZcT$_WP!seYU&j z?D{UE?5Dy*=XMrFwmodQQljsigzyZpSBtaT`1$Qe=?LzQ#0)*&8(%d`{>vfb?COqF zmMTqa%2^hUuU98rF1Ip>P3m57wO*}wUq$EthNuz8MXcZ8^I+?+=nqk$7nwY|61KP6Y(c}i z2=50x5i^&Un(jv|b~(SD_d9!tSzT-WxW`z_Fe-M8tzzJyo8Hx5WXHM9yZXF4%Gdc* zw(irsJ^aUOb{<_UEXGxQDa^_8`RgHgy?Fyha6Bba3oZq^b-YM8keCP;*tDAVSlAZ4 zh9}O!W0p1qfmNFf$sLC>vLYvPRiP?=(m@Y%j3D`TR?IcivRHqUsbcKSan;3(;_3TA>t_ypXj-* zTc@iE{BkV*p^p-JG+5~|VcaIK7(!mw2IUXu5>6;;7JV+et`0k>ZxC=7>)TWIv>-5?PIunJz5L}(!M3L_ z3yd+?u|dN zAK^UT7sh`kfct%6{H6aF7xaOv@wCG)6A zm(c5uH#*r^hOq#aBJtiIbN`C!T*cGwN*oPgNY@@}F}hyHED!h!`Qc>4WWj+Cx9xpO ztmbe^9#tQo7VMD5#B7-U^c2&M{MJ`w`M*y~7MqBvHSKwG7Dqhx9$S39 z!Jd+O_m=m$?6GdDnVfc|)6tz#Q=e;8v{N$;qzm8d4oL^`(joYlL)1z6uTCR8Md_MF z=5fQhNj-6yrmSIRO@MO{F>Hhu#9{&K z*s{9_8XcE)E5_Vje%mkdYZGQs7M*oLy+J`8uS3E^OzYFLm4;4P%FFa6KYtZL1IT|d zMx;ZC{bPv}vVVqYR~H6GF@;%;nM@b&C8R2MlQhQsiN8t<5qeP2C5Oy}5$JI3nlwSz zfu1c!?|spV5JW+aC5L$A zx3ruittv;=>Yo;$`K0{8FXU`#fuEmh1`Q3wo>t3-VO1CJ`i-DsLTvzA)o;ixtWgg3 zTr0%1v@7_6(jUnTYO{KJ_(PZE#90B=u9t}9wXg7DN3FLIxgd;%DsO5+mQ*n8PnP1H z?B@TqlK8_U@(h0E4dm?ZJ|NyvCFlxPnpVw&c>W7-~#wJLe$WpCC<*DDgE_(^^au-kM-e0trdtZ%55VMn$!rW>1E2Qk zUHGlxkQ$hmbY*@~XX;UdvB~a&i^GRH8@%SnT4-wfZhauRwJOE?6KT!t*~E^*+{EL5 zwNULrNn$nu&fkcK5|1f>@eouODBzQ26oU>9jG@u*;TEVl(Bb2r&Ec^Yc{)D-xQ!qEw6k-7HyvZ)Vbpb!G^k&M2;h z;H@OhjyiXq$7S|jUhIjtrc#6g%8qf;m>==^-JsiG(+qlOpdlEyIV|kq<|oOAZyXDaT>WcXUPol_*a$+lq6%QcRd6KJqVq`dyAi< z8AgP>W0^<$BwGu0@85I4-5+!?_su#N8qK)Nm?~{Oddn|-Z%+CLBMjcLyWvB(hv|to zYf5_zi0`spnNFD1N=~UC!?ukZLoIv7%%Dn%i`@PDTdeo zSQ1h)&YZhVpl`&?V_GNN+(3Xr>J*(u1V!u6cLKzju9Q+{n+|3)eD}S-WLQuVYDsr3=Z- zbPlwQB2fxU93oc0hH z#gTn`? zOOV$Dd)(eSb@_m>s0QIx+Wr{=WWa0*$0nsmcAQ!q%>NAUj!_p-K|Eb*D|EmxB|DyrqNT^zA}q5f{ImTovRiF_Z1=`l*`+#V~ZuN z{4pn-gMCw9t#`hR16bgQpNW7v8nKbolr_zoGISIImDS}kRoQC_@F=>Ez2p!Q0*@Gh z6!l^P>(8@WhR)@VN%q3ywwt<-Y`n0&4ypC@;VY-%H|N!hj>lS{>x#OF+EIqN!bY!EHv@O5Z&V)Es>)6e-APmQa$-NV%RX!D6*LfWy(;m_B9$*EQ3j5@ zkcwDe@}smCkOTHWMtn>JN5hmOuMozQyIR@4ag?l#*hGNyS5QhOJXZ-f)ikxQh5FnC z)MWg}j(`c_$ewb$5T+*(lpwMERmL^oeb62hC4yZqGDBr$j7&_l|LhzBE z?@`cORJ8lvg*p69==hYC^E3S5M#&2GGeA z>y(V+`LE#)ofnQQMi0p{%Lo^zt(h_GVYlCU;M*&|tTU!hoyq)=0vWV6qLfJOWPt+` z(lu+q#2HtU*Ah)~5e%Vf?EWSv3|4KiH>VKFJl=6`<Z%I$8TjKSA#J6%85Yo%_GS%Cv$;83g^=kk z0E))EM4Hn;(=f7kj^v4(VFP`6Ql2fWEs z@vcmDubOIT{ZLC};R_xV9sRL5_vcQeFKOA3JBGLx+8gF&VdfG{UM15SWDgGfo_4?= zkBq8}Kb^bbtkK4s_L_z*qs<=bJ5zr$l}wQv&dZib+#oM8R@O|e&YbL_ZiXl$us8tX znO02B*iwCw%5zc_nH6XHKCZMT12~e`%j$WXF7DWFKs4uB)vS&=e+!EsG&>x1vh(-D zmU^V2jV=uJd0YrSlVPZEFvDO^wv|t-k~PiO{^pg_69&g}wiRDGYNwQyZR)h}bZ>Sb zX+jKol^n>vh*Y- zCVCumBV!9x!lIOOt|8PRTr*Q401&1Tou=V-tp~uHZ{?@NVU$gO$j8aFsypf~ZJ(1* z-`$(LQveKUdyHDB!F&a?S>VOpD~(3{EK!iwlev^B@gCpW9E&M)&609R)1RTezjQBX9Y2b$Ctekf{7t{Mj*2k*; zkp~a7Ka{nx*jh#N^5w`Nyob)qK8KJqoOT!gmyFYv3b*Z?N(`RtOLI)iv^ce6O%!+P z_gk`(gI6ki2d9pndT~;^k0Rnh%>pISb14Ps1!vcwAlu?r@sY3P3l$-ZrN8$C@R zoc|W=@OGg>Z0yBA59ii(m7Tg&@`{X~yZp}d5+ixq;4ZZ#vz?om74T-bf-h#R%uM!H zfE$(A;xWdSa9w||D;FL1zSzbH z;E(I%r)3%@u|Jl(qFj6Q&W7-DjV1;2#ZIj5Rwq}gr>UV79k92+_QWNsnyF)$ty@dJ z&x=BSXX4_B$PAp_@uYro)pW~r3Z`OEYrXUqVzQ34F0z2fPl& zGeV*T^lPQlCT&J!mWynEXMl%!vuY@_2@qg>tM_TuX2ql#(?XV>q=2rn2whUY2&p2` z6JI2-PD*ph-sEd+cq0w@9JxO59r-*ZhP@CKsHl#h7fRfFO#Ax$OzE10M-xiej9i67 zg4}fK?wj6Jhhkr$r92sX9tL{WxEZi=dSgnYY>%{niG4x#MPB1nx`GQJ3H_$3j4*u+ z+Q648HX+xLG)F_H69SQ%6T5%$Rn6)xcsR5PbLuACIB>r&=Wudwmvv=DRpt4F#l_hf zd)ZG4LJ!1Gl8xY*?-E^8Eblf0Uv*T}A4}wATl*oz8?r9oy_axvP z{qcjr2|v4>y)P3|9p{Pj>&6IlTY3@oZM=RZ@ptf>i))`+>C zT7li^+wyUx7t#&dD*7yR{c7jEj4LEh6$mt~1)10CLU=GxRa5#I z(`htUz&HzSqRvqohxMVrqCqwk!Le?QC89P>RlGxwS8F6wc_nvvwH>XqoCl=dMdTD# zyi_*exgA;%CRI)+X5fp*$jg}LDADVE$v^>~aHX-aoi)6rPB_nOY>(a2`KeX3gTo=* zCL}gvl(mIHiCl5`4}m!@3rB0~DjAp>GWYiIE&&*Nf#o!t#d<`@`}HnHkitq~*G@O1 zwbAuaSv9dnL>s|EQ#m1Y#Jwx^N^qfNnNIo$cD<+Ns_}!pBa`Rxx(@jUX4+5hIu+=J z<$p;F&1a}d2c8+76ucX+@@;{+-5Tp@+hLlWeSgEgxdPXW(#@K67TIUXXWMt@slRkg znha!Cv`SRWWvkT0HnJAvCMMvV{3ubB-uCX5{Wy`ktH;8vctQ=c-UPO45Kx2}O9Tr^ z#&gJ(kVbp+u)feHLH4s|*y~`&`e!{B-dxOTHrC%KXy2KM#pd}rGve8^WDv*BVzvXEx#G3Ajel0@bdf+royAWW)H~yWuq4kDb!W9E~^fxVE-u zw+nqgl#3oazTy_eAiJF9JlVC?Ilb2=@UAd?g1WYSQPe^Z41g(DZ zTgd98y)r!kIpTAm;&rm7#9guSS>M$K&p~H{ZkVyC3tS%IGM|rPh7GeQCk? zA&!yTGj_LG)zoy*xQj9U@hhQeIJNxJivk@-OZFaQsdj$R(3MJdrMunK?#Vl#MNkji zYx|}5$>4qiPg0&TB`=h>gmi>-E^=x|0al~u%^ksX17$5kiD)hGNPfx-munVTqUwdQ zP3>`PvqWtT-N*hGWVVJ{MDE9hqaOP{D(J&Op1HlgRkkWed_{6unvC>LiDp)2PKwnU z+EoayExB;Pr)X19dp#y_O_~oAm*6!B&P@noa23Y~IN%yNN_8qKQL4SUo5BpIl#X?n z>X&g&M2I$_o3JzepuD(;B5-VvC;b31#Y&*RWo_ojr|U=`WC}vBE)4^HD)#l<+GmPf z1Beb<_1sasHsp=tzE;`JP?G39>6qVBWv{T6m9;zi9+cO_M#RG7E~cI6ycaXhnFCZ; zd9xAZPXu6luug1%^HdgEU*mghJprW%5M4)lOg4G4T)O7vY4Y5Krutp(H5Jbu=Ms!{ zJ53Gchmg+6`Ws5|w2MZ@hn$;)(Nj9U)oGpCUTOhsKf{TOV%mkFvfqR~1@_16(0VFK|)tErb;e)4!=&KFF-sl zE=%(ULY4^SjylKEaRj`(lv|s1*kTJ~bH`lUl$mYPMN>Z)k>*Kw<%BbWiu-~E%SgTZ zf)3bJ^0Ma>DpRFN{(<7%=wFNcl}!53!40UmUpNCTD)Awu1=UXyKAW};yjWed&D#3M zzNsqMWCM-du#wEoW#e1y0k45gM0F!JNDy6Q?;%VhzTgWQAK&$ytwk9z6V`uxY=ZWp zvJwx!xskiW=bUV)rkhr|*W~xRlbPdr*?s$*|5%cFEbDT0YC7Fdb)f@Mz40-jmVD~^ z4m1C@UrrWZF5~$ImV}u)-z>%&2e6~g<=5vWP6UcFrvkr*c+yJd8BQFg0(F(K8442} z7lud+DkrSUh0393&74?I7=+_T_0*Z?Ly0y*qb9^o=y5R=%VKOwYew&95o!*11c;^9 zJ#;VBivAaVD)l{zkO9bnGevFEtAX@>MwgvnVe$pv6l&)_dqD)2u3GCMDh*tMh9e*_ zwc?8L8(F+*)d3<(fRbo?u4Q&hPHHx$jeGtikX7JMd*yfzhb2ZlB7C|I4E6xUXS0wZ z+S3hMKR5%yB89ssXzBg2j2g}wrZ+t$)Vm3(K|CkS>CNM{O%bj9`05s#fxP`@&8{Sk zj_s5qTWGpb9lE+0-}bKBD-@!%V;`0+Q!aaVMDOj^m{m|$1D@q^Aly(Bxf-2r`vgZZ~|(qgyP`oV*$CRHK>{fBt+iUBYU)D3FqOt*Yp4{M0=H|ny zP!gq2tzbB~5Vnxi4z6Zc3l6SgTZkp?g~EWdlPZLsecsxTQQOENr2GoC-yfCzT-|GI zak3^0=MFg^$h3=CdOE<=O-XExF9ypUJ|J2U1xoKDy*T}thO8Ag!SO0D$_BPZ`Dn>O zVOeI`#IT$|C5Fb=#IN$Hc+JBo_>y3~M6cp;+0ReJN@Kk;13^_`yx)7@(Bwi<-w#_Q zYyVgh2?7QmR0MhBJm;NkmfP3FSxI#4vM{9*J!|_+nLm!-fHGOViu{#~^`6+pbGchO zWnK3fjEqoXW+**&OSg7kr6!JMFyhfwIa)>AJWT@lTtor6!EPQeCwq%5&umDEWb6X2 zX22{J)n&1AgtJc8S8JWdBX7j*Q)IQrkt~?N0COG|g_hB&6Im-_RU>&c`4c^+oOM{i z@Ob8~cI;`K?sfK&nQ_1FijwKk0tf5fP;(FCKK6pIdAc)fn+rfu*zb#+X_>_qWhyj{ zUeTlcX_cS4gCmPSZF6LrO1mk&G7a!pDUuVAAJ`vCkH9}jvQCmZl3g zl5FXeIC5!-uzDQJ6|uQ<(!v$E6UmIB*g#-ux(geZW~y+Argf*D_D1^v7C4S9nhts|n=hMQRi0I!Cn(FdHOt0;WWDLWCFAM9B0# za7?)XaeH)M8c5WVgz`-?gH8!*r%&iGrG!93=o1h181se3vHqp;ChtsNYr0YzIxqts zJ`5PnJ$%nQv^21o2)33WOeU`)`zS*^VAkH~WHkqW#pW+V(;;93MFQ7)t}wv}}utJ)iy zWvqS%rQrFr5Q`maitwf5uCq2HifX>9LF^bBZD4F&KQ`kSnvFKOG0Yi~g^(L`>ukhp zS_R%CoXK`^2sF}yTvcaNK=@+`y4zTyG=gF4iLODLxQgo`XUDez4obi>tJgw~rx>5R zN}yZB&jG*C>_%44O_6_1Yu;DIpi@si>@fljF@+oF)wI0~uNb7)tBXr^AT{s>j0$FTEyc>cMOsHh2R6@6nnVwJa**_1B()SP~{mlVO%jENV+{5McuM=ah=Rs}`nWbHdgpjv{H5J3MY0 zBtZD)R6_l{b%$xY3NCbn5TPy!FP+HUQPWX}34b_Ds5lVWWad(vFuK^2)|Nl^qg5BU zfpSpLUjBGLRT@8;K|p{0iRtsKh~{!=SF>NhA4`^Qsm?uoZTQBZy?5}&ojaq_r^;01 z)$`>8>64qL!VXSd7#a!lqn@RkT@EpIb!>Ru#oGH)?LE-7ws@oFNGuP&buq01eSy>= zafSNDC!k2t9YN-EOh3#OsG0kq6{DxBnEwxPZypZi9{&$_+9gy(h^d6^Etay(oEC(b zsBDv&O0rDMsmPQub4vE)wBSTc5|U-Ij5)TM?5BhnOh$~EDOqz*74NrAssBzCZWp^M1d!H`k^Oj0)BS`|%0X>KCkpA<9PXc{sHN9AT=(#H@}n zRP`hA!td}!^~9pE73{-8S~BUVK+$1?KRe=o=45%Jt^(zkcr$oTM}+8B1nil2An(_z zjunkowC?1X9ULtjN<0mA=FPp}sePZe}YU``(4^w_s|L&J2_-IT#2ObnRxebCE=g-Sv#n-H*mX3Vn?BsC$IB zR5#og##%sg?}bI(QA2YhRfqD-U%t8ZQdStYqY-5_6?1lQZ7%AfG4gAnM;YQ!#!KJ| zM$Vqy+9RcaO}s=thFSyo3X9>#;+7`(H;Jta6ey7AKtd0ZE;U+!(EUvMeX!9-(7R@o z2ndq1eU?p!&gA?KM~&xRnB*=o;(97+Qg_j2@4rH`u!dD^y3)x>uuvn&LkQ>`E}mMA zp1lD@i}P!tK~4(kxa0(Uru9Dx2JFKAWm*BU#j)KGGgpx2O4vI6#|SSr)fsZEC=J33 z+!_?^-Wd$eTb3IPUX%~INznhY;C&qh7cHb)aN-TVwD8OLRN8P=|2ns%r$zN_<|fk7 z1o_zhGPmv^GkTlPwF>Om$sm_w2lpe8;53zQAkK7v*8UU<*5`Gy6hfDoyCHN7;ZJUG zoP0O(>CUV^*sn2&Pfg@~s-K(2C^+W*9fjL8jYSMuDYX6JH_Z;QR{ghntSv*i@MH)v|$LtuRF(C?I;c6v2Sl#>Xx)b)V5#_=?pX@P@;d3D+C-Ugr_6*Ids~Vw`8p@ z!S0}_0xq0j$J&M=MtiaI6OYzIIN0J!iOQVbIOUTzKm9FS0H+D5Y2rGbUXJ z+q>(^>AQtS#Yi8663LO`Y0> z5YT=DkG&F1o~yi{=1UJS4HXQO~kJIpR$XCMfCBRvWoBW zBcX?+-jC6hPjYwqB?e)(r| zKUKK~h;u2B8-lCHZ||XPmvttfE+g*OV;cmRTDM#FKEm8+J}URKCIjQy9SFZKh|Jw72h>U^YN^{>%w=@a$MG?!p6~a zdVJO=34dOyX4g*LjNHPSl?dlky@*b@IIX?Ng9r_&4Fti(ZFx$m9f5+}`4sKu6_7lp z%;1+&En0e^>efsRZy#$Ly|!A~jlI^aELLT33rkaa9bE2B@)4>sQ5*xVp?3=qS`j5Wc_{42n_GeZWj!X@?ShXeSO`;$V z8~{O^50FjCt+y>&EJiFkUHEirDX+v#vrYJj?q%3G##R-1|-z<9>9u7AXHuwcMyr|`%=c1oC0f4Fu3`Om#K z*-cr){mW!$U4&{<+RpV_xxN>Lrd8+m4U~9UmUL|^?T-2J#Gt68#kTK!#rg04bTu*^ zF#4?V*WLhPuzo5*5pBdbq0Bp?Vqx^0b0B{-hL3$(KF0GWdUrO<%^TQ@^Baj&OikrK z?5MO;#4hiFGj7mOM)<4AuNB!eJ4Cl})Bm)0(=7(iSuFGYh?9t(R-E$RR&xDE*E{Cj>6!B&L;~g2w`6AH zCppWlO?`0_1)aqu-lSAB=74Zv?K@dPdE-{W{2KBHqyj{l?c}>_xQhJ4>N86qjysqD ze~_RaK(ZHEGU~IEmb3$KZjeEF;pf)iGJJhrc}W15_-Jn(Gs~Iv$QA+NSPwv}Dc>h0 z$o?{f@gkKLIjX;Ldl6i;(3ZCciYN~i5EOlNg4>X*2g9#!NUrfCM7ot@bqu<#!kwLK zm{hO+#cuR*Caadg7+Pf18@s$a=XLXIhSiSA#~$5H=SD|IS1g_$N!U~Ff8+^%@Jpo0 zh0y-zg*(_!^k-kYzP|tbH99*Y++!y5fXge%zcLnBdI0okK52y*MhjT8zg9^s!SomF zL?JDmRTG?K5B6>{PBYAAg>r|a`Vy}0$YielxWwBs5+cXaX~b{)DvPxAtf#y}g|l`i!P5iNv`I9k$k zPJVU26-#BRe!f!&e{$fv{ZW4Tnbz#9wHz)z<12eAEBevYY3yfnD#&hv{47912iXL! zlqeJDn-QvwhYIBbN;DYLiC9ZEKM!XeM^zUxTGRA8Z{D7GDnQ&PTP4H{;MQT_Qul!f zl1&NmaF|cuzxM9p$-T9CZ8oo9#qso33$8@I$pM-dcfdK40CH!ZsYs(`9t?N(fCD8c zayLb5F+HU|_dAWhFRI)PJ-9R%a~ii9e;pG#wufVtuGU!;%2YboV31oJfGe-fiMduf zW#r!DdB67Aw(1mDC9Dq%_joF>mQFf%>fCM1O06=ivSo&GO`yjPzvDSuia(4*X+9FPs`0ZFauHQm> zq~JWP*n-^zCEl0oj&{cu&5Z41TZR*`1Jd)QL9uwj;c;l`)5i`{QG;+DB8CELGz5A1h;HG7?Il&R&X%!8o{J zPQPLGMY;JEH836w$rpaUg`cBKn z^f>XtB{G2E3L5UUxRvNZljZ6*#E;)Vtvb84 zgfgWz2m=s&?KGHDyI1y%HTyeoH+ZZR-tZYiK9cqcuTVg=2dB(g{_OcJhEx-vbbAfO zBCusXfqLcn9J=g*j|gB@>p`>mPC?d4nf+>i2+TE#nxh0}EDzgAQ}!u;gu6h`POpe z%*iZQY7cAhv1j`1&JA3$&clh^dfX z?O((xz2jHXSV{Dk*`56FsRBA%>RE*M#%lD}@OFgsGGn*f--yyL?5JSvr!@Cufs$hw zRhdhrYH)iBMAP49$1eQ@QAp~%jh7Za_>JvZKxghXR{O_T!T9Y3N^|E*T0E~+<&()Q zqn3fXS_xOa;@;BO>L$)nuy=rNbT!2V+T`F3bJe@$a zb&47A8i^JOIYd^2P1I;BA8BXW)#WN5EqR+m@{AxQPI6;_ z&;irJ$@C{_$Q_{E_?j$#v$Ac|+yut<@;qno#HZpIc~l4|mh{oj!5E0z22x?=naif3q0T5C_ZtLXE$ms4t?1TRU}4viKY3P|$)n3ce^zzlT$+8AZD=K{jao zI7JJUg5>a@HX4zKJc%qP$eUJVK)22fDhuV#nh1slA|1R((3Gu5ZQfJfw2%j|J7mwm zHL0LhA=)f-K1&JkEeTSp3RA~b66B_Ax{h1Btv&M40CI=axta+D--#;y@<@!)T%5GT zq|UFu^@A2?b!~fYQ{l7v_AV^@NnLjMkFKYV?Q2xN4EH3xkA9ZRv!gN^98xm5a9v0vsop@R#to;spAg4 zUKS^antC6AWFVqjhwXfO_l-^so}&G4_Wom8!cWZIp<_Gg{n#JEV|9b3QR!>(x0%mB z;^U)6LZj@CM2_|HdjSb7{>ni`(9&3ydeXgM@VhCI_V%>Yz0ZE#t?^v#PSmDwsiT0h zp{acAA&uDOFQT+ll5R)G+w05jS}7g(W6xcUiuL&B6JXPt?ftrnSM&l5@>XKQOArHe z@~cVLAe+-7(?e2>h&HEFHsi1IXXIoMOQgospz?k0#&_PYn9W~1CEvo{3uRW~U&AS) zeD>8P#B8_zL^$SrUm2~jRctjF`3^PCPv?dgu&Se)29c<#+N+GM`|AcwU-Wl>&8SmJ zjwpA^d8Ij?9{Etwsv= z#a4A;PX4Y|FUJQ8^wp`RPq39wD(1U`mwEvsI{uOZdE(ZSn}4m+WQFpQS=Gg@7I#sq znS#ENa-?6`5D^d5VQ#e7tEfT*#k`c58$>f%Z#-sjmfa91}sQ z@0Yj}0Z0LN!tuMk57=0AWUtLw5SGgJ}EP$T-oBxAl39S33mWgz8Y+b4_EVuTK5`fa6i!(2$Y2 zZzqS}Q7Ji2+SULtqv#sKom!kKN&giokikso^@M|p!v2~jTYv0liyH|$TuMNkAoo_? z0LhB~lZ1*c1|{xeT%5;8LIgcl<7Caim7%CK8+3ii`_tD`b8f!)l2g_AB(KyBJLHf1 zIHHQ#8c$0qFbxXb7qs2vOs>nZZFvS(mkSOTuG2V^m1(|zV#0-?a*HnUgn%?PJ_x{9 zu>SlV6rq=ka{Oo}br-N;H_KAoZ&It;fyi*XcI=@__^FV(j(Db3S2MAV%S)y$idp@Q z#`Vm)*jHgI&)KtOblDZf(2KQkE=VI0&^-7^a8d;rCD}t}gVu<=S!ymyY@u+en7)Dr z*c0jn1=1FrAh9)VLr48NS~3}7ir7LU;YvfW_M+eo+{uG&IAVaf#GiS^C3dwTs^;5&t=d6k3svG46=5rf=$5>l1NKp(o*~Q} z{%CdRlD1op(xFb>ogyz;(ME67`q411+=Ci!Iwb*p+uASSP3W0{PKEFIH;Fm|C+k;U}q;fQTbTD`+` z|HeX%q$hRdS{26TIWG&fkGTA(8@BjBfK6!0fQ?E7zgoGxouwQMXYof^n-N+-;cy2s zZ1F*A53-x{)^kWlMH!9y>czrQ*&4l-7=6rkXX&jH ztRzdS?k0W$izF6?voCpCOjU9VuIB5vWETI@Ot$^lAsLlvu+e_I(BikSJe6>#g;xA4 zDP^!w3z1}*u1<81sLAQbFc^@cZdgBGNvjV1V`|I{JZH~1k@En~X!Zq9G?k^=h}oN` z10my|%X_oGR|}N94H%Vd%XtNIKiNXGA^A&v&P3W?pib$+jC$w?<3l2L2mcH@_Bl(s z6D^8 zrFH_lNO_lNd-?tPywqzkg8;sf9_ABw_S~rLU)fIrySvndUE4!Cy7iXl<(vNdNb4YHsp`UbQI=@U`Hw8VH7E}s zSQ#~x zng0I`MhNG~l^x*SLaGkQT$TgYiN%IbSdgJl*_@tc?;uRgH9lgcNKqM<9!qbI9@7w= z02S!AGA*5@-bC}ooH?v}<~1Caczw8t_v&{=)_7C)OH@rdXi_f_5>RUd=>nSvWGlEJ zK$PCvt4#FrwF)@y1A)ejqU1ky`(0ORvV|ciJfd5=RdmHD}Ai<#F!N|!0eMbL{ z?5L~UFK0h*lbr7jFe?}Tm?{}fZ5QO9+5gA0=THrFu;q-);i0J4`ZoK1Ffd$~dH|-@ z)`UBt^8wO~fQ@?R93{qpqWTT6Q|sX9aa6m-J(liczD?4i!gGjnzZ|L8$?rvwPOu=e z<;2DLZK#e?)S7AT()*BwXmjMW4CRs)P z9u&ibX3Cx0&VR}s(BoA2IFP?$4Mm+vq@#Rom1LyZK%)x3fwq-c+`c8G-X^&s618or zY8bBXatBfU!jvv@=}R8=xY#X**G+|u2#I@O99T1Z!J~SPxX{qewX)TIk?V6wGR#0tnQt+_fKs?!CPx^n{NoF3^fmNmSdW#0J>R6um2X!4^GG~+L96Itv1T^$-Z44l({&|l zZHxW2Jn&0*Z%=8aT8`O{tXoKO?Pm@wQQGhFdKab2QocczIeyi@weYp0}tr z6GmVDPJT=jir+wt*>;IDNex`&2ze)EgM3X0N*8H{8-_~-J=OU{WAsL;1|0q>jH#Ui zP#%#Er$Cf%o2_4k&Dx018c8V;u^Q|$P>j^JN`@m}^!C5C{ z4ff~24G_zUEphAoq^{Q1TAeMpREknr6i*3O6$BI&@J%S1jxWM=H;H_bNP%U^`s*3kbcFcZueRlaKwR|AMImy=}PaU#p}$9ieEUbKAZ`wp>GcdysGi zQc1Jg(iak)Jy9t+g2!#7@KwXT0%|>HhIFlMV_#~yJPWAEDT5<~?-X8*BoNwB3FN)9 zm#lFGd;j3bUR(1@{U;0EHsM-j&tKGjVc?^%OTShf&zbT4+x2|b{gR1A=yXfr_6_&` zaA#e~*SK}GYwKJ_`IW)^$e&Gia&L0|uT>Qs-b%BIQ!bd|feN5-@bj_@iLY(1loYX~ zJLhj)XNbeEMg=&#sg}1nA?Eh6E+*GJHMM^`g8Y#fkQn)r?9=_>P=4=LVxrKwqc=I*IjNDrr)(Z0 z5+nz^&Op?7IfCfc&Qc*rkG=-Mubdi`(u{v*zHn~S2ztoe3UlI3eYl#A+j}Ge9A&4- zwxrW6(3oB^Q)xzQk~&Ck%DZaw?1t=3Uh+b*`$ZPby!33t*7CC>)2Vz1b(I}NOb2oY zhd$%OW;^#SA=L2(1NiKSs%5R*+`PJrg;Hn9Iv`(DnFW%Gb!^#ddxXf#DDoEBil`^- zOk!@E(W;$^As|ALhhh3SfWaJw;$o2hM&TzLZC=cXS3Y=qd5Ha)i4~tdsN*&7Qr{%D zqPCGjz{2H`0qjxJ8Miz8+jwsaRg)C(aS5^0FU~LlaH-|~ZN&Aht+pN|3Nh~f-^VFo z#KP?Nm{elqobk{P!AYmmm$B83N06<>z4@C0$tblrL03)O+=3a$W+;VkOR*HEW{j6q9Y$h>FC$V{+61E zZivJBUy3}^#*5(_B3;ymUVgYqbN08?vHAAf8OxoOuQWt2>(@8%=eAAv2%V~-LQZ0Z zmBT4wwvyk~8t}m`6JTTKx#JRdatETOckZ`8D8UXY0wAkG?I8LzQ&gB*p}0**ecYsc zFUS6p9Kj|j6wYYapla0oGrH&75t=hI<0zF}hs7GGr{Ds&oeLM4^v4%?bsSS^Sfyk zF0bEIsiWK|%~;UggM(28h%3&*hNV)&e&lMnp_!sF+ZY7+>^6(l(qoMxn?!o=n&NoF z#<~DXD@B83RXCmP&DfB7LA@3SrG+m0+t;6!N6E|T?^7r4 zS%G;m-ET{;68$rEwSb=I`=9)vEAZCNZU4y+BDO2+{|7^bZs-;ikHW!XeGE{*X4Yp3 z6AG^G$}>8(LgG@I`AV}GO$PUUz=<;zmmZ_^td$ksd#<# zrZ5Y;$kKs(uS}czA3eDxm?+RJwg~RFD`KCyL7&qNNPqon)l~{?Ts#*i*-CPQbp#go zanETI zt5fX4WQj1R#LV0(O5!8m1TedG2xwlax(tx?IMT(7{2lVWM2Vob@YNJfsqqN1Xa#u} z3bl>O%P)hAu>-pqb&atIr*!YWGZGtgg58||jCVENt^44b8?U%>>;EUFn2-=x=@zK@o$8H-qL_6N|Vzp{m&y5d#Km)(Sp>} zP>(8x3tv*MqyIqhqxLgStt)Qx1;N+wotdx2IFs2~!kO(U#j;9|9XX8-re?0H6?{pZ z_q~9V9j=>@3pWcjom>w+ImeB*MR8|9F)axtx3jjeu26p?9S1_m62XGREp#X~NU(2_ zkjhd9B{ipAp_B-u)#~X2OcE(b6nIXgl1Q=xNz5?`qKJntrj}RIO0n!<)Gc_7K~W$975Ta?aIa)%7^Q|V*C1KnT`c|W<6;+PJ`q>V z>1;YAK%^lL5_@Jmp|4`*o?6jnU8Q;`FGo+Sp!3F~F9i7)o;v&0>qhH>*8|P7B_7w^hMmrx&#z-3r^K=ZmebiCy^l$$!er=clRw<$U6zeR|gO5rM{aQ6GgJW=i0}nFpr<2dz zQBe2K$NyTj(!kW>OwIrA|ExYiKpCk7Ik)N0x+!S@(0SuZ$=vmS7z)?;e+h;AJkI+3 zh9Xu-0KVtjhL9%|C--&+hg=EeQ-%N`I|-KWd)ayT;;&Ukr(k!0e}{@db>*GaQZFrl z!N)Ms184VH)~u^YMgEdf{gqrn+W%|SlOmDSC0TWTPi2L(<4KwnvDaxM4=-o(f(Fu>Uh#)uTb(OaWr z}|J@?I#{% zy|R^`M~zQ(J2jIOCHkQTKR$(BCo%-g7n!<|l&koSk(6~9H=^0{E!GuGvfy5Kb=Sgq zgSlZfAGcxmrO<%oYHu0Z&DPT|;-I&ySNvX{cY*iL5@zj0Mph2$BI`=;JzZ6J2Gp=a zvWLi>aGyB~VCGkN!_I97&im#BG2d|PRrt6>q%|~_AF4k{A2|6RijufE6{PDuRqohc z^-dwD8{JJv1nlhmNY;7a@3oe$l?~n-kFXZ+9aJJEcYpkw&sqSIoHW z)Fd>+gC=XceD~Ct8k8m8IAEm|jHXu(F4^SMdyXtgf$Yxc5-MX|(a9YpH2O!O!oq zw5WiCX!vlTI45L&c9al>PG1SnF>5^0h zTyZaS&h)VFIS`IH_isaS^+;uc!Hwdb)Zud(hDjv}`QE3mc|=cINL+UQe1qN6!2LeI~mE*?2jom19g@Z-@D$2l0L`{xL zSB3*Dq`QMa)2f265BQc~f0VYXNwjZ}Bg2r+rD1YMD@AP$W{J7Z-{(M`&nU>q_h4YSKWQgB~Xy5jlJzZ}C$ z@rvc6@Oy`d&|o)%kK5E95b{yu3|+*Ky(4(dZosE6x+A}xsbl-_)052S%J`3N3Hufi zsd?)LmYRO=+*Tc%=wyL$@wG?gyCq{;R>yDeL)bu%s|s&um6 zv8cehx-7Ilw*Oh-hJAI6Fmsm%lXouDHT1clLfPhPR+T&a;go|N-j5PIz34{!A1p>o9KoIjRxa53b51ig0-C0dqmyw0N6rV3c=q-RP7%vj zL(55K4~oY}SP!_2=B<2Q;xe~0)vC#@nQ=SgS^YUNt2}@=!>IOeFC7l)EtGWrR(q0Eo8M&8%rvfq3yO(pWJ_XMm`0v~sGOH>rJ^un zySLhzM$gMBB(6EeWTebEyDVX{-ZaI!El66JdLQjW0cqF=H?)jx15#nIO5WO%hQ@3l zd3sDZrJU#yB&$;GxMP33Xv9T-;^73*iAZu+U?yy}PTVu?sX_k`_Zib2TFj2@ZD6;_oBz8M(ysv z4{z}7^)Vx?=R`)Gr8Ew#cNHQ-{5SM1ff62r8V=+a|4F(b+9o*(>%C436m%wAnJMyn z6XvA$RL!DP4^|cQ!3Sv}vPdCe%ocoMv?KeBQ(v@i*s@$tM2^$R%MGp|kAPj-68Hdc z>Y&KvtwUNLEr#)YFvfyw!q@*6AMTWYs{p38BNVK(Ewm|DyPwaTA$nNs2+7-jA1vmL zjO55sH^;=zQ+>V4871xIJLa$S&r!e&ev%#Q==mo8tP!1ATNpV%ym0#BwQFknSFi4c z$1G^tAEMgJpVrk%&Na&hbV*1&?eC|k@nMWaI~FG%aZBwA(X5&Olfl1|cR0DSsviDm zVR1mu+^k$_^i|z6!C_>d%HLfR2A7^>pY?iNBYS~&GqGm-2cYbp4F~ww@SmJX3w`bT zYGO;NhsYD#v?cWcV3h6Ea$>YPHL&MQ%_FA_ z>Sf1Z6xb@(%JNAL;BPO`b=?id>nh2fnF5LZ$r-M0VyV;;4i{2Z<8Kr0Eu!yeu+*d~ zFGE&>IPDZwqK}ii!H|P$>q)l+a4|~oZxQq_GjGD~sy@p`@3@1%jp@bqLS?c2XIYKs~$~^x#5y5=YRUG%*Y#X zl$o^-x6t>7VQ-+R1t=OT6$a0O&?2I3iV;X}IQRv|{@-4Z_;V7TM z_OwEviovP6gccKhd>N124cm=MLN(;NO#?NfxKoYwAiUx%8z)zeuB*dHtZQv63JQ~l z1bQmsC%89l+*6sl7s9Dm-Mwwt{yby|)4RsxTe^9op*iNgxNSIKtgHC_2stq0*3j8o zGq%j$B%w#nlYwU;9rA92M-kQy_p_sUPpOxH^k5S~as^fe{7$rF9jLvv5or4X$WvUM_=VN0e8-O0R``dLP{ZdQ5Nm=07B z#;6ZY6wHAdj0YQW=CoLlXLzt&K_odZHHU4U!!MR1gi!i@^2TYT74;Z{iBQ?l?yjrk zBHtjp&dn#%_llfeV;$GmP2`Z~U8mKxc^j&VPHi=_CEUes z9BT9t#Rv`C9W>g;j(jZOECPXvatH^x8p^ucxLb%yWZb}o)-R`6c57oebd-kiFwP%- zaM<4ArsWQ8%mOMA2l?Vu!l;!#bNUh|(lPkj`lq5`U`@MDvW8Qk;#U9JA5&WM_m^l} z{fX$eE9r?#qt57<(L*iReVmAjiw!xaD+jCsR_knNGdH6@485?DCEU|G?oaPrH;VALrG<=E4SDM3^SY90(5Jy2A*sK+W}Ms0O;un zRu0yRAOvM*OF$Ul4EpUcL{Qq_6lallW-hzQumg((Sbswcb$CIXF@)41*)c`b*>56h zCu^;IhrFMnZpPG?dWni#0o^w}W(JPMc?eZPblT zQAXacXzoWbiu*pEtt-%ScLEb%4iCh;xS~(C%Md_y`cn4Bo&vGNm*FY?#{cv&X*?yS z4fq`6V|PThoqY2+k=95XZneFGNVN^C!)`pitnpZSLKH=NV5PFU^>sjq+_bT&AF1=1 zXy+3`Ps6n^>dmvtTGdrrf!J4 z|G+imZ%CJ0oR@|`=EZ^rV&q0rG*sOt*Q;iVNGlaogA9R6LiT)m`C<%3bC~%0-)UXM zfVovj8#nwx9nkWwe_d=p7k=x7Y;wWk*1mZKE^n-w^YwYAiS~P87m9G~#Y%^+@84MZ zVg9YJ_PU0TdZb-jjn8oSDdZjzssJ~9L!i=5O+l?CSqM=Nu)kHoIFZ+~Kb#1pk8iWD zPg5>H(lIqS11w90&_9MhPxre;zs8JB`s-$uv{&S(qeZK+X@rvudViFnNeJ^Utw#mk z+u4pvV`;K3@b2QaAW;F+*{lb23}Kr;7tRStI2kpsL3DNzS(g?O-P-E=G7rCOY8r^d zW1<4CIEFSl2CpkliZHh@xALP!pqyuj4ILH%m+~5>>wo4=Xpz|}#|kHYPO@{S_r6Ak z6yJJPs2d!^Jo{p-WN*Ktn&;W8plM(A|L50sm?Js$Yt{SH%Spm4Wz+bCQQ&YE#I>A&kxi_F9?_LG+ta8`}wC+B01YDrJpY`J`h^yg=DWHT# zEh)g2*ax6B(mNF_yJzwK&LWZbQ}YT=e4y~>xXNhhh>3%1&%DP#xEXCxA_<+Pl@Tta zO!tU;rVZ~-_kSOXe59Cz|9dYn`0iz5QS#W4tq+rs+s7%xD6u~rC`h%?mRifQnX5<+ zqM~dI4G$6Jj$9d7tf?uc8lLpL&eIq(%2DWRLV1`)Sw+Ov`L!C?QcE$(bKiKItMp=) zwp*hkb8GAUZ-*BDyJEP}rNir<{=NqA=VgJU{r$mfw+R{=6JFq?xAOxI8ieng>cPld z|M&CmgO=S+Ty20tv1!jA4T)P;dcN*uG*ma%W@2JWLxqnt#~%tZbabA4fseQ7YC$UZ zq=!I)z~m5F7fOt)QPOJ}+wswI@$wr`BtHLiv~Y68J^SPjTW;t|Y7pFh8 zZM$u2^lMcsDKRK!7(3LIF&D!XH_<>|{tSMa$VsFGTMW;)=@Qj_Unu|F59hdloTHcM*Lw&3^Kjnn!;WOd)nT-@^x-UV1l(DHfvJM| zSqXGpP@EiY`6&{Q+af&)dyb=a;xoC9^!C9)W(J{@Nd0}#g3rEC-_%%_&|w!(PZ4~d zzMOnH&znMck&!v%Jr_i%ACfAF&ptaJU>sq#aJWA5^Lf_OcJ{UF@`Y4#u=Y4ODOH&2 z#GAZkFpZ9+FwRRvZmZ6Y$w4c^Dy>?E_w4QP*CLz02Q&KYQGvywf4ToLU2b3LU;5## z<^8rb7=Kn`jmj1sBl?_*^H5KGnID{Z!j}0os6bh+iaE4DkKC@17rbZhS&&=!!lqF;}a_Q6M? z27X#keKWl)J&_7#7>w5~Wp2}q3#)<&F6_$#zNfnP*@Pe1woN#(m$T3-oc~1OGdkwE z3*rJ1RjQ^TCcwk_$4lVp!(>1Nh<#xP(VBLuvNRatj(t1lZr7G)Ge}-Tv=gYL66D5V z;)5am#EM_5I&1yCPKX7Hhvxzl{wpAgvW@jOzT4Yh6-|HaG0iPUdgCXq3-(7=ZY@-B z+N4%AQDS@ct7~|BC$Bif%>1i{=FyU{pX^HMS9v!Aa-ENtZCm{N~(8QY?^~!ST(G@XSq>}s zA|#&S?}8P;I~p|&TpXMyYpTRE*CSLSmhiWBWh;5jGzT67YPkrZ=UqG|^kP$%d(7Wo z7F3KQtEw*0`Q>v_19qDMd=_;!B5B0io^wU zf0V*=Yc?*lmTUw9M)8X8!jl=96GMlKAFaL=eBcqiH-rWJj$iq#wRqlUsU5UtJns&~ zYxlXvgA4fbRH*cAw$ELAm)I?LE;Vr)jRK!HKcCUKR$l(P^TsRR^QZ1|S5>S%Ft<1l za3CkBD9_3)Vs_-H*#~RuhmM;$KJ^Ltf9xEYT8T<4yFQA;_Ei2mMIfj>c+>9CmKV^d z+NHhdU?||FhRcVGV7rr0{QWZBe%#I(RVA;O>-mY?NFDW|Xm-(&i6c=iDbUTlYa#$` z>q5oKS1b>bS`j{5^8_H-&TY@T>mNFk4)wnF@Va;4@B0jc7D4HS{Z@t^e-zyBzIMbz zd+TA3!+yd5TW0I%+dn!&voF5BV!7Y);+^~7j`?LGYZWb8&X#WMv9HAUJM z;+~PV+oyJCOS~cFS1W0wjl|w(_J>ORxnZhDH+}uSm%;q!;ib***8H>8s#^uyCv_`o zE}ithrM$ zoa!q*LRGrrhtdWYZ{7!@^x`v3HW)94Uea4<{IKOx{yo|| z&4B`^vfW;VsP#-)8fa-9KuoE|y(NobjF>QBZ^FmtURr2HYBIqk@0f3u>okZqXFi%) z_E%3s#eZSfq~=}^K(+*2ibZnA0zBMIoSQQm%e@J`n}89RX(-v?F@4BvWxLnu?+|mM!|`Gg#9HR8MUMCScbm2 z@B4#aC=KTazC6$07ht%)|5{(?Pp|)InY9mOsg0pfLz@;Q_t(J-6vAfgT7nMo ze^`I&y3pIzSWFv7ZYNTiQ-!{PjVM76YQLv+EvlsT`80!U&~SHko^Ca(;nA28s{k;2 zs(myTC_fXZ>*V&KdE4Rfu^qtbtfB~W3A#jzhv-3~5UoZ;xB6oYrrRU4CoSa9D!j{8~oSt16aWQ$B?V?}68z5-AN(FUhZLSocX2zx%IoOC(=**dGz~?(WEeJCjH2`{9&3My)dDRh5LyxHGR;f-KImVwaQW`55JN;WIc0zn4- z18BC%94Z|Fd<^5yS=E$|?oB%APR~|Iqn%gP**u@5YYI&JsDz?8G;zGJVcGQafVWREA z)TOg(Ld9bd1#yKQk<%{C%`v|84Tjsx^n0H9OFV0Q7B~_FrP;di0OTfyVTx!*d=6Uf zq$KdFi{r^!#03{P!=`iJ5<+oQdQxPO968gZ@jFDlXRjh?GHGS0<@ZIFmekB^&SYmj zklGYY&Xr7R*%HZu?^D5Ulo1f=*h^U@4G?PMlJZPo1Qgc>)Xb(=82?m>K#GeN@sPhL z?U}fAlYcK>5bow!H&i}R>qb-`J4txR_ucI~d?G}HGx=iw{WC5m)}v)*IkjDwtM*9c z1l?yPEnEJ`@bPPJv45E3iEAht5W1V6=yz`zJI}ejoln_{3IY#o0qBjBG}@`^6cC0; zQL{grsm}F+vjm1M=vBm#HpRl>#vk0!(lQWNvy%6sR=w){=Xc>(%pDt5uGzI~)l{i3 zyBcSsBdM>^3#TTZ?6&GU>~yW^XYh{hfU9$d-vvb8te@;#U?c%2@$jMcTBdfT$Y-_* zL{emTHrJ~mHK2HE5>i2GH55)V;g`mg=6tfZ<2S^*wc6~Tysc)#U}XIMQgx|IA3Uy+*2KmkkN;e#9d=yM0VeETcL-hWq=Sl6-X8_i9>;oW91_>n7_8v6rA zns01%$xq{4N-?diV&F|;hqUhPsSDiEVJoS;AE*<(eD-YA%=ek036IL>A=ups?|Hbs z$lU8nHs1YonXOWwd-VYS4JXp%Q=ALFhKt#?Jx%yz@kiU_*SQg(mTO%!tC~TARlqWW6^7HSZ$I zz8YXKiS*F$FZ%q*2z>2}+^nDqdEm8F_rvWaCN77IUN*Q`4rD!czoqL8bpf7Q3Gxc} z|2ED6SC67#ZZ%ncfnx{HiI{SX`7%NS$lIDF)^J>qZ1S~b44Opss53}Dg zH^LA3GrWz`6Hs(KDTeQWelPEXI)QBaC9$}boA_{7CZcw-{%fu{C;H3#(!JQ&n!NKV zI{V*h2~GdQ-v6wRWyP19f}qjel{Jw4?fckGwf~E~_l|06?YBm8yA=@-0V#qIMT&@k zg`$9DBO(F<(mN3mX~u|x1dYkwARskc5Ks^zU23E&1R`BUid3l~iAtA5AS_7Xd)()q z`WQN@1i%~PpZAHyS8S|vbIki z|$Wprwl;RT6e_*sRb@!-mk~Yn}qb{mrHK2TOn}5V5VC+;{)3|>ah_hWG zS^CaQrb@Fb*geIuM*%s5Xuy@usZ(L}wfm$|x*0XJ_xKU<@J zL`lZsN$wGo+Zu@5ws@^FmG3_`$Zd!S+%D{K*Uz6hu8;$RLQ6<}rkFHWixb5NQ}ME{ zb3ZSeU=1UMG_ZCiK!xqYdD><4UQ? z*?xM+EPSs2-5}qt>VfKRL)rUkYJNZN8Xby9ztgS8i3d|(Z&jZE~*~+%X_%5CVQ{K z7T-Wq5u!2c|6n>cy@|P+_6;w4if1UxYrG0y{kpxIJlzZ1I0Uc51MAQz+}fT-;^J*` zA{;u^$-8L1LJ23Wl+Dg#qzrFA`g_D%HNl_XA5|SQ6Ps5Y?f)Dnz2%%EJHQFVarIHuhfy!@L@} zO?Q#UWuQtu;0sJA*S&?XqiB39R*o4JS&T-s*YAv@5rhL%Z_kZUqA{lVpBL^)RuuN= z^m-Tj%RHBI#b*1SWUAKy{m8}B!CnX*6`&N}(>AAwt0-K|Kk&GcbX#R{nGqu=x9eK4 z4+HJvb7jBF#iPF+3|E^UQ?1ULgWD6nJzKq9XoWm8P^eRF6+G9S->=P|Gy!n1p6xG|2VVqU`beMA9?-qi|{=>p;_+ zs!v)kGid|CC1HmvDiS8uuf#Y<5X!L~K$``Nq4ps;7Hlw2Aq{1G6CLM+ZGha0G=yoH71sUiK<-Mi38 zo&+ROEt=z3_R=sRSNOoJinWYm(q>cf=Jn_5y!ThevR(|oYD3+;)D$t@`wQr-_5uZ- z5@Wr&NrIsgc}%FIv(-?D!Fo)^iMYr{pR_`1d5HCNTGpPrXz|gj*7$Hz@0-NH;+#08 zN{cV#clo!btHN1YPd_fl3DHQIi{{<(9jIu-n5&eHApfp<(z@wjE1)|L{ZWWHA?wDkg8qShy4ocUWYrN3&YWwwDkQpxk)EA+ai61>iMumK zq7AoUB!DSSw1H(lm(PS%|_;gE^BOR71^;R}Wl_BNi`Df1x{L*z| zuBLk?S09g5}!Zr!qN=k4h8LGoNP* zeMbFj1A*wxx7 zMg73p-Ktvuc4a@!w|Y+O2A_c|c7K-SKIG@GYn8Tt6%Ir?De z!q0H$fL0`fmoeQfb*_JvyxV_s_gkdX`3lBTLaB^N!JoAshZhQ(edYfABVGY^ETtUw z7GAaf78NN|!y_~5rqfz9qsG3M?E7;*edKqnELyjDtTtb$GOWKH_Yw*<86avZBwbNT z(vQh=`ZFwE@Q!DLiKMIF{{<*Y4KI3XCP_v8iFr(tDD!J_Yj?8_>9o}8+ZgG9Rq`*U zF-6+yiSZm9TZ7lrbcPw#py$4jrN)eIhDQPUl19~?eWI?NLB$L8BT=Xj%M0kG7vYmi z4fW~g!=*NQO?&VU=p2m+*j@ z)nwQWVKhVUtJkLGef~UmFIYt1#HY|@Y}DcX^C zOGbk2jC~fhd}gp*TL7cHeh1Ngw>w1rOR{!bn$Q3#%GQ|+HxX8;w;od2C^%zRsINNk za1bkT`dom%W{A0%#rawbpHa71rf)0INE$L;GvJA&ZPP>; zPR5bW=(RY(ZbhZ%@cJ4f|Jo_|WSc0u0L@QLYehxssi9HS&{oQJ0NNMa8|i-X+Zb69 z(@PTNUv^h|=zvL1VZFAAcg4jkPld`Qj8E`+zs`yK3lwbeJs?8(RD+3$if`s0+$+9W zRHkWalirkl!Hn}S2C@&jp;O`e$FU!9RY-EY!furTallN)G2}U(k^z#(ZnypyvfztBlM{BzC@j%gK zH8w^KKj@zwqj+ea|?ph2;`3lcPX1)>2!S=qY-LRehDUt&-muAV*Bn!YDf_RVAZ zt52)ICB?8?ZZtD>YD(MpYMYo@d1CBcr^XcaC}Oz=c#v&e&HIuop%@WX3^r_ za_~X%s*4`T7k-YeJtZ`?vr=Z&%$*H^`OXXJOSKukJF7}Z{Y)?GDP@%uweR+v*Qq}< zRAZd)HrcDC7dBY4-S5wgF8nFiSbm>T)PvOJ(Awn^%0~wN5u$Qrv6G2vvj+8hETxL@ z79Z1eEU9P2D!CoDw`niOkXiUBZ(z~5EG2FMtH@yG7inicO)?6+F`_IObokNZ?K(j< z%bzK%b4(|eE(Z)8RuUH}G&p1$tPXFPg7KyxSw^QbuZ7X`_*;TnYq~BgI`TWba#j7f z_344tIL*V>l`HvdZIY+=%-n9(pD3YxQ~g2k$T#C$liEoc%lA{3{dp6a%VD}^Z}bF3d@{bo%Wt^QmT*j!Z6cw1E>rQV^arsoY2QZ-l-}hYsq>WumIOYquNHJ) zj+j16gIUU9!ZuGq#-ggUh0q+@B7|0BOo4FC_5@a1`?#t3-qQaYG(XuU` zs>~ae*2kKTFk#Wa_i1*M@+dELRPu_LEE+}}7%bl$ktE)$jd?OUjtL`^>F_Qd3+_d7@@Kh7o)s7$6x=dKY?X@>!g*27h@Y%g+VwPJ7r!?2Yq(7H->n|3X1}}!I_q((TYPSX^J5_``9ld4*I-H41hnq^)+XKe zS9msT+&Q7%<^tSlw&UBpsy^)Vjma{}{2E3OnK{*BcPqi&Q%bYv_~)?e(&i_NthAXf zo|@XZQ7PtuQfu0_Pv+OJA2{>e=k-N8Q`*m=>#MDnCwzNs%eT{dA;ov}f=@^=90Itu zH4XxL_7zW#C)OQH5uhmv3>1E{G2H;$EeI0xkYJfc{m%$P+=($5?_9L|=(tBl1J~Oj zG|q(a4S!=ZkzCpnp)OA-JZ~&LxMYU9={as)7p7p5@!HinVq>EmnertJtEtm-M@Z5k zW!hg4`q+BBt-s|8WAI6?#L=1s^W#=`GWG0>+8xR~Yg|ub41v{LnX!ml$&T=j>!*uP zAKNJAkUVRKDI&@p~23+bsf3&gV`Qfshibl%to6Tx$aM`45&^7w1$3lZ80f zN0!7`FmM61_%-KQz{M8U(-U*G&L>N!3d^*d*4h{iYEsp>$tnC$wRhXiKv$;9BaBFJ z1?M6Ih?x789v_P9ka2tg{5I;35A$j3 z+j^W`uAVy3d$ifRxFb>UbT8UFrt(0vxNZK${+!&*ZyQ!p+oGZrLR342-LzdK<;m6g z?_;8C3i4M)!l4+pvrP*%=xOYUJ4NAO#)5eKqnOQ|^eBG*_>xC$JK3pDfA>=Xq~}Kx;Sa(WNN$L#mv!mm*U>Q6)B#e#TMU@58Rh< zV8ulS{#l`|uH0YhI>9)6=NkOv|4#;MhDLgLCqPigHOiwbg+YuLb2|&c&E$m8Ca>BU z5@~p8&OYG7j}r|@KKjt5Mh@l1=w=l@IP%**vpXSbk=u;k{?ha)r^f$mKehX=Z*H;(49Z_+qp){b zV~%@gz-D7%lvbt&#Wv7y86`GJJp8mJEmUp4jX^f2>k>Bx?ky*7UpbrJ@dx5VERO&$401_o_>3gRrQ zf9iuiwAI_E#|9T^i~ONyKNdMjIT}2cFhxu;^#~mdpC2CE9HV_;O$m1Czf-PhsIHIF zj|(gbd^D!I^!;Fpr|NW;3c4W5-tSAoGdUOM2zt88;s{$09e92OxRdn$j@W^LQ+vfd z{p;~D2Jjq=8bc!`2OV+wQ_Yc@OnJ6@f2r1yW7>h;YR*w|PL}oLaN2!2^CQGEPqml* zHcR$4*VfH8^S1c3>>gelc;c;`Z1&+qvf20KCvIkIW_gX zj~=>p1B2swfgm3f;uR|B=HWasXLj?LO4XaX&~VzA+ubNXmz>A@rHfWF#aVZ`1d{gj zvN%ScG4qxjkp+WrbPsxm=Z;T?(vX@tBQ)-O#EpoyDd|_A4t~;JK7I?R5^HtuM+-mp z>@<<|NSh$PJ8*CbZ8Uniq(kFVQgsed%C9CjwleTSdfx{5x4(L#@ZCDEc#_Di@nw71Je`mn*7QaSw=R;O)Wh#w71E-0+_7E%<0%7#eAvA^oU z7s1FfgbwI>fqhS@vWlu_;lrqtLBY5`;DXi7WoY2wtw`>x@$#Yi?aa-km<1x0zC8<@ zadB@X32W`?>u{P2IPw+$16--w5MN9ya?050dY+Uw`T6U0->b=PuH2iLmFZrvhAu-v zG<7(j#6)l)p=0JPR=wXNub*Ty-5X98CTY^tbuP%dTEgA>INIJtE)AL=x{-HeMVl3! zg6NcR;E!~aGFwk<7kP*i!YT$}Wi>GRMdII3vxsVH_7=o zflJb(5Y6B0D1U5ds>j2vjpZrPSB5buHY@aJ-Mc_g&MhJNnAS73FeaGze()Tms`lSEps6%NJ> z!t4W+F(Yl0&pyt4EJDG(aibzO%(J}-Ya*jgEL0OS|N9%wm}LWxu%F$ak(losi?hrj zWu|Zs86HV-Ah>7PT_5xr%5S*8q4{*OvH7*K`R2LUGu|99@%Yx8cb>f)zVa=RLut-u z;&d5Jn#-tfVgEfw*#X${8{mZI}-Zl~kwjnq=P^Vc{*mZZ~ zL8hTwgU-_Xz}n=SBS__Oe{G3IlH{L8fq}p33=j8pqL4_GjPVZA-keUMy~Z-7LB{(6 zDmo7YynIw#oc)`G`aZj3Kl$?i@+UDJge{EVs>qAozCsjf-cd-cxo+LRsq?}x$0EmH zFXaQO_CjXv;PN89rZ(Xw^;!O#B%O4Tb2>Zorb5TTN{U<M}HYSw3co+(~ZyT8FN|udby3}Wi}jK;{9V$@QDH)Hp#{9k7NzpetuXcI2S-I#cY8(Z^D^24OrLl(Vn2_ z&gshM*Ok1Ezls0xv8P~n)1elJvzZ2#2X>f?RUbO@0Xs6pfS?vDa`1MM^=y5xoA``;KXNB^(J(5Pb%azw&#U*p-vSmR6nlT zarrm76m*DxPl9W8KzmD>Ov;g=h^kvf=K1Ch2gi`QGEa(4s>g1s7Yt>Li6>kKdqTqE zH9#z<30LG-zk2=JZT#iqCKsh}fy42P#TqXysgTPPN9xcC`-u?c1PHnLGEX7!L zCz?tdao-$EX6j8bI_kSL!KSKTgQ)KAmnA*$b-VSaV2`8v`lo}(Ev)ra%OwhKJ~xyd zS5YM~DcsbonLI!BV-;yafXg<}}!<9;Z?7?EgQUZkcFAfckoxA?x8 z&uu2aXNuO1ISCIUc;LAGPF^uo2cn-EZaRFnmIKD*nz=PF#xS|clq*SzCPVOAVl|n% zn}Oo)_S_GCx+i6e@7)tzi^Mc#b06sxXnx9pCf4I4|NeI)KEuBuAi*uZQ#1+`h#1Y~ z#i3d$9N0KmGs1yvE!-z0bRXy!*a2({-d^I)Exx{dDuJ_i4#9~rzUZogQiF9cwU$g6>S?fE;6A8IJ1FO@~EG z@%ZW>(*j7=Ay)48xDoJ)ijKo-aNHrcJa;I zv5q4p$Wmjh-d$$m+ju9rgq-?JByyRjH0yeS{6eb5WQ```IkaS7Q` zK@&dfdqT_R)W=VzDl`?&J|asDie`JwjcmeNLKrE99r$p9JXWPcor5TufFW;tSjCag%t&W%#50|XEeGKGn7)Zd~}0o(%&{3cmiFZS$_e_&c>h_{7c)| z?w%OUHw=;1j4qTkq)8=5)(kV-rnH$p)d)XEQw>f1!V%e42IPC5lsew#DC#orS~}K~ zT{t@K>FPFzyu;e!TS*q}^xWYM8MkTfJw1a*S4EK=x?U6Tw>e!8D-(~tTzTFr*ZPbV zntUq@@%W1~Z-4vrtqm+(vFpbH4PnO)19>MiU+4NE)fNZGQ=4gz(A(pV@9*66<(aZ| zbUUMjIwe___Kh;SN`F{reDeKmQm;;%caVM12j}?P-6u8*4vX5@I#k@w+WExJzKDz1 zR2wEwHxMc!zX*pin$~Hsj_xDRl7vH>@A-xsNaT2Cgra?tEy)m<$v<5gP#a@l+oD;W z-&pb5?^#*7@58*Z6~f{WATmR^1h}8|7;lS@stX8!ZJg^2tNh|QW&liFfteM@7^A;^ zfB)sY>)i+{5@p4Nx8hzk1F6l4m}oPXk>%>=s`j9y+Ak zN!-El09-K6C!2wDhooqfeG>swVvLzs(#0%src`SiQ#nr_zQdAB+$ZCipFLNfx#=&z z?@jg4a!p_ttYUQPb3O+fvc)%ncm?B#v)jlOnvvycw4H_22M9X81N8{Iy{f5>Vc}t! z>z8`>-h>WTy6>1~x%!RosB-o4nzQG=e_KN$J2PKu7YjaGg;3?hGjw=vk5Q|;aA%V| z6xnXzO^Z79=(J9>&=>S~m39LPe_ZbhCR&4b2`2glZ6()yiMC!&jX)*!x%lDd8Pr=pdNiDwb#O+bFkSqbx%1=<3rVryG{tr zw!sejqP60%0uhbQ>5Aj6TyK6E={MwS3s;ZO;KbcWa>#vl8wUSn(DxsJ;@<*`e-&7K zb9x4i)8Q?MsPz*+{ommc1}ZPD;*ROvtf2|(^x7p!va8B7K`*BS?_^l0jLN#pEL%OfGF-2jH%@Nw~|b?Y)uNxUK+l< znz}#d`RFdi*z+k*HhWuOqj)N!#Y5C9u64{ej^tYel9$rj@RD=2t|Rdr@AAo0)#$J51>@jX*VYT@G$AI4T{ASzdxgsGPS8#-cUQ4+D)+A zb*45Y$+01Hq0;Q;D*60lD&-#lK`u{bgypxzmov^-2SAWD#7g^@8}~mWk>Cdx2rGl^ zY~F>e<}JRHpdv(U5F~*v$`#D5th!<40Yc%8;r{_KIE^9x1;4=m1&aLpA2wQqt!Q#~ zZ1Fu~qW%#Wz5Hi<^B^A}L#GJbtSxY`-xW;cozMLjC^G23#dp@a9`;54@7Md+iTnHb z{=FW4LjS+7x4%l2|7?HgEd<;qqy~Qri-OfSJp|Nk+*`6#lbZ7c+8Jt~I`p2|I@L2JR{entc`(EH`w$Z+z0?KKzPQy|+3pX9wy z7cOK)x6gu!2Ek8#Ep>WZdttQb3FKaJc7EGm zGU493D@>dj%G<5W{so=J>L1Vs&I@P}kWCt8sEZMUMwoCmACv>y>*jqS$iogyxuwEK z)q*9HM~A=9w@{q)u+b%--8+4?NoD!Ys<#(qfj@K++ekRbHLY;dvN(*mPh>WQ-(rLu-~|)gPRBfo+|3fvLX4D)TRytL+Zm zk(H_O;@4XE%uSCDm}%*{GE*=SW5VQ*J+0Zssf#^5vaQWUgFmJsP#0v3_N;J-k;h58 zkFP=~sfaeO6K5n5$%U=mE-*u+!DXqvmhxf){T=prg;%SjhNQFHwFMCN6CUP!%&AJrV z9E$UD8+;8HG(ge_iOjK&$Z}1r>}Vauk7y=~LMnC)$vB^o zmjNJ>kICOcOu&f(vO-Q$_QtHPtjuCSohvpa_|s^Bk0K`(EQCLC&Eh;|;MiK2$b<9V zRdAG|-;zaDlG~?r0cGCp2NT$Wc4~pl2%YHj6 zS1#rGn0x9%;Vc#eomwIe@H#3$3J#sgW51;^Do7?&GmI**u13qxTiW)vHp#z97dE!L z&vjTTqc#k#1umS`dU_+WyD{(HZ7sdXzCoYx-sYauGqX!+XvpZq`32{p5J5|Q^V-@gyC>WOq>_#Og?>Hx z_*m7^y{+HAI6X3BJM+42E;7L&Z?w|mXVRTCi~Zh8Rsv>MLt{?f2~6b*8Y*5gkOtqz**REEeAu)TLY68_qHoT0@}w9A(LiX*57IvReG^_VB{;4H z`P^vgY{-cqZ?8bq%Z5?y#U?a&U))hKB$$C447Cgb_C;Xz$$;x!_~gH-tOjlrmjuP-B0FXRrd++E3^Fv z;{(p!iZuT-@?rtq@pzG~Xo*~Uq{)uc2v3&|);$>DZCYJ{1n?)$!cmO^d0M!Ce5}oi z<{UBvEK#;8NDV#s7xe<2`|0&~4i8cljec@Ts5Kc}uif|P4Z;7-@POSe(0a7+C&kmt zIiim1008bE9yj#?^(l~utR8`Y!1h-eC!mA^M9?nl2>7fGqm#Ew>nHe{*V^LCDx4Jg zA#w%4H{7%)r#B9ih-L8Wgr8g&objS&i%$mr&rh(8d3y;vaecW|JZG=}zwt-^^d|n5 zKhgwXu$r5W17w#0P{@XlZ~&71r)L6}ME?2R!x#@i*mxvN2sa(^7m;R|YrQf50KC0} zERl^g-Y;8xAAyCC%m54EMgWXgfC@$s!30qGpPnIB!>IxcU+!*0X~3rKO!+r9O?5_kL8U)zivyh{`={-B{;Rr@;j zkf_+H`W(5BYEUk>kvyFWmmvF59x<^4xT@5e>Fq!crIcsFJk=_%#nkS7xIXVGGmvau z&9urW)kLbiRGtu^(%M8lzB-id0L-}b8 zB??N8`SEc@IMe&2fv~*Zhg0!njSv3cY8YXj{#Mn8lCM^ordWs?8Ef{OP?MFis#Qo- zj$W2OaH~8lma(32Ixg~f^~!&`5B4}W{9$umE$Bg*U~6d5MwU~?1JSCi!?Y>cPV{8^(Cv@m zi97mAo(G+4zBPQubbol8!^gQ>JY+O=_iNIR{T>%Pw>|Ce@R~b1bOOG(_v0(?Ru)6cC^Q{Q#ium#Aov-fp*SeS7!}cp5N-JO(`#x z9inonTv^+Z)p4ZE*AP`-voyDXjB7Fu^~xh0$uyo#3i~qdvpyBaD_@SPSqW`}jjot* z)8N<0e3)MtUsvz7LV~W4<^!6?)#05r5q3j0@G)zu*4X!#>Jb-_pso@N6!UN!dw(qF zv%8|;v7E^RT+q1P0jR7eKG;DRYtPS6m68T(8MbQ`O(T)xFlS3*-wpd{NW6y{;$HiP zm&{0$tn^Q{F|HeD*tXpIdg=NpDXUa9QCn%5nwuk_4AqsSx@4+^&KVEUxTt5cmhI^6M!1Y6udln{jAUsVYLK(An^ zgPZS%oDk1xI7$VKCFQooXKin|8@5~t5+d7CT=On&-YxMD^u0yd;$VUU3KP@_@maJW6NbmC}j;snog?1}MH%cutcH z4;0lqdHc}!Mp~yt8B)>S&NPu!{o6Nm6a{<)Iq2_r}%+16^Ma zzeyYx47?V*i+jva!U1oPN0+UU*q4F4Ik{@4>ywM}c)WJGP<5(XvFz^mwR@vVUrPoz zI{%5h9R#Jof23fW`oTr2)=XYiMf(E8uJ`YV{U7MkGTy(V-^14b8?xj6+uiB^3{=t!5>^Tu{ zi~L}F*Gw@3S)F8An}pPkex4qW(8?S1mTs}KeUfeyZ`**ZO)}FNH&Y(7{$c?i%Vh;N zrJnYNmKg<8E(M8GliHdGAe^bLlUTIh(h|IeSQ6GxwD||9_lF?ZHiL6ixGcx`KLt`Da`sv9cH`K;%}*&XqAK=euyxVoHo=m4p7J2ZI&W@RPx-Nr{@i<15ztk( zqSNd>^v2X;oR;qCh(=x(t(8;>tO+C`a&WbT-Y0kC{?_c>* z_ZFCy_a@*ZF;0cN9(cuii?jBUb^H3a)=LaTcw6Y^lY5pAlUzjhq46C($l!4~O)FjU zm@BKX^laM0MbIjm`abm4(sGM0SM#$b3zR8Q*ZQT!JG1!O<( zLHFxk@wNkcuPshcr$oC>I3K&F%8XKpntAr^spp~I%8#jO!j_%-9W!T|Z15j$FDeFo zHjvN|jS`P`JSG0zDc9gUHx5s6)xZ~`?_cN-7M349wwft&35?e^obK9JGf>^-Tv}T8 zXT>pFK|h5U3$eC?ipj~Y!8V9Yb8w;NP2?+oRn-~)>mQTjx1Z#VBWr(t;7%r)DgOL$ z{P|5?41E2hnD$#vtmj=#w4=9^wpDW4AAXXNkAYyzJ&fZgsu9Yh=NN4 z!MFu)3jeKFABZ|b5G+wpl?^1p(*U<{CYooi@-vs`Dxw|1HAHK~f=nLR#t}j#iW`HI zz^!gf*Q-5&p8K-95e{2?d2QSPu%`@xij2UpB(Sx~JAuOZ1Gj8p5jW<sV-{myxnD4c-q!2NeX6xCG zEOPO1cDE=1*9`t;=Z-I8Ze2=I#5@n@`7wui)5F^3`30V%^E#&HQsyuw4UjL?#P^%{U-285zaR9XBM3khq67-tlDM{xINk4obgpV_frP z9B3$v9d3n*l0hqA48;?;!>FQ-G?T*#u%^8jSBTBDqXVbs<_1kXYzI#U<6OnmjBulo zV@M{U)W-!-G8idHYX$yj69M1rMkM!QG?;0Rko@t%!d5ezIBxMW?*Q;UXIP6oqDVLG z0{QRH@}N);ZkFLej5j35EM(n%%Tw^z0-C8<@MpU+0s|kQJMg7t;)RGP?a?`zw0`F<85*WVIMfj zllWM8wI)Y=onBr=!vm$a28fcQL?aG>PK>v8e0sh6d=f_#dPcM64|J?V^S6awKKE(N z)Uc#z?j8)kSVoP$^!oLnfi^SQtfr4Tj8&NPUvzP}Wn<=8C}Vv-ttR@IlDfLDtu6|M z)%{dMyqi>0{oEd59p<77jBC{&IHkjH1rx`{n}Itn98$oy;Kya@A_-U>Y@fUVlYeoA zA=_T(ONVu$VnK|xFEfr64!xO%iOkoI?Mb7-V?7m_8R)~UBV)X>;(+={eAJE7RlG2y zd10_d5GiOssP(yQ83az_DhW#t4gBH5oHj9#Y~MpffuuzR`BaMzaO zcskr~O%H$8{sD{9sxlXNweIy*+xmtfgc3C_Cx0iz}!})qnnNj_Zt!>Zg zDxD|b$&6d3tq56&!V~@gPEROQ)xrv`kB)BIJI-rsIs?VXRo^y{K}2ceMx)!{!qt?$ zQ%~s+{&a=9I)PSkD9?{31|nR;D6}zd z)X_!7SPS&(l5dNQ<-}c~3WGRQXg(>)KaHi)^1zVS|E=Te;n#Jk+A-`Wjd{pk+eW_+ zj+X64>MYnZL8Xu+y3!V+UC2@#S7Ezx659A}P;?ti#^6l$ELoICt_QbMckn+x$?1d>VD> zno)*ga|p&|>v8zG%07&L5o9r)**;k7sq%?;dK8+N10j028129Z@;I@s!g1@DranRf zKkYjDaNZ#t?|HJIfeRDI+tbD^BB@UI;t5hXU%Bh>&$t$uI?gdz_%K-jh4IiXBZT98 zKPxfT+F<*6k{l04oZr2;DdP~#6-HxIwT9M$s88vrTgGQix&3e0o;U63(F9UXon4%* zzbVb!(nGCUH1u0z%8Rv1o=QT6UQ{^(Ve)_ zDi#Oog1;rikgHHja`YuTdea-K?5u$sBe(e##u&0;T6Zp&Vy+A%q=2xdko%M34$Si0 z966`fBKd-mRKr7wnj(Ny)8<^pHD6rqjfXML6UNn8o?v~YaF9?1kVuzMiHQba+HgBc zh5M3pCJ#2o&$>CF(jQ_OYBNZ_z5ssWj<%!fMibWvLY=gXk}j?a)jH}+nr+nsDf^_0 zq^o1KkGW;7TA_Q}4bE5FbtGSC_YgEyh(2d);&jg8kE8SDg;AG{BqWl){qeT z@C;K8c#f2OE^gD{lr-XH>A$n|JAqW)U|qZJ7WjrSvoyo@B8+N+?Y6Z{w>e!hg-#gL zWFfzr1lr5sCQ0PvHMByZ5! z_X6ru>wsEP0@u9PK&1TR(sU+2HFT+pj^8tSDZ8}XAmjnVw39*kxJ`D)*G%F=1J%%9 z%UyDkcG-$(SVZ}9pKzPtU)pg+wZy$)DZAe)L!m6BQDRW}kSBIH}wp zRdk)mx{KQ4`=t_b8y5zNf;DrHOVFl_<9Gu186~VpXoU@SsOIx)xJ0ZTQ>Ehp%m#X# zk83u92U=Jp7>>6h#4wgrxQOAi=>$_+Z5g9tra!n_xtoUce(OpK4ZZ53eVUk)@`4D| z*Wn7-QC#)R(l(|wZ+94b4|WhITFyIb$==R+kTE?Hj`KgF#TNM$8tCK+4v}m?UOj6Q z6@QbnL;epRBpoRx5?L|tPn-=G5jDAAHh*GXuYEW>&v|}?z=v!86?c`KNC3AE(Ojzb zax3CzWD#)e8Alf)pp|@Qv@TUAgRO5ljT7wUZzt}bOP;@{&8lFSG-kxCm;b#W?4cItPF%g+6)WVh{SY>SYj1$!B&o<$} z`f3^_S6C1GzYJnwbM@pn-f3KmDNR#y!L_j6!_5*(o0fA42~5B)sUJe#X-PYi+&U~1 zRn=%u7)}bxcvzm>^HKW`TegNAAdY#*VzGoC#COa`^I1Dq)oRK)0_jV#GBJL%ScV5QwLLELB9{g_LG zK*d#oabl9=WyX*yHcZ{vCKa@f-_3Gui$Hkv)DcF0DN9dcP*N8@k@$v&+XDr*>UAb) zF9llmlT=>#xkkr;q#RF=c1e5z6ly;3Z74)8U>>RnKdaMg86G(Iz3XYxuujI^=5$^t z&ad}#^f_>%qvzoaurZ9@xg6U87l#@;VU;)EFbZ9oMrF`}bdiVftIH1!4?#gCHRB>D zJ33H%N8M(z3h6t|d>L6)#5p#pWYnRF5B#AZ-h?NZI0}!$HJ>H>A|S zuqm?=Eweb*v%B<=lF!iq(c*r@uP+K^5){ja3MDt2EE-LeJ|=&fW45 z;Jw2gfCnPR5Ay^UsFN@{Wl5avjE1vrb6@;)jNAui9EIRMy}$7K5iqO}`fv+Opyq+p z(t{Fiab)U;R`uG2rq<>Yto0w z?Ct2}R&otKmg7>sK&VuSH@MGWQ`44=sN}u#{I(Tb0-{rD4ADxCrc|R^8%kb{MI}%L zQPPu;G0+{$@Z^j_fS}AlPgBEp*(gLQUZgvi@L|cg=AhsN70?6; zzkqB0ZE-FCk5Vc(gM%z-(eQThmZ|+cwVoCJp>&scc|*Zmci=(U?E1YQ9_p#?uyV0) zEw|6svrmq-091vpqs5d(%1Xd^!c;W3_0x;+2aexptBS7o!fY?iiC(;g==8c*pT4{s!YjS2Ffh~8eoctFGoetFOkdHv{Y340wZskb>9^+PC-W;DX0P7f z6wi%IRj3u0&W&rcNn3LQh1tbR5lU22LnO!O!>JoBXO1aFykA+&{!|th_;W*>9kVFT@>_-TESUuLl~%srV*SsIA+1y#8m20@J>S7T0M zfE#1I^&ce7;+7;O@*^#gs41!H0g@(?riT9Vr|W|5s{mb{lVMJ-x2gsY#V$U0!#>72 z!qjbBmYXBXaqbj)_6oQuffeSGa3ce31oY4@n| zyF&NtdqKoj3_OWSH$B{S;>pa>Cnr+Ej_>yrC8_(arAr&^akF~Dj7?RMjce-?x$vx) zhU!0cwEOk_<5cTCKQOIoEi?AM4}0z`P7XwQBepvlJdKy+7q$?OFEU*Se*OxIzpf^+ zbSN@2;PfSR)vJM56{>AaaG|jG|D81Aou-fAU$&QR@qIpQ3qpfbOBmKXfg#|gV#Yp$ z`}~I#FLTYDyZHZTb?QE}j?W`&p+^Z!24!slz<|Od@<`BR1hw3I0WJx*_|*Uu`Y%J4 zaNa3DIl$&D_>Xfq6pVew8VCXBG`IMY!Sq@<5;uB|n@e6x=L8`Li2uW16nzTRa9vVc zd|r3JJD@LwL_ZlolPw zQc{K@+v;!TzntBSzvXG9M5=TS*fbeREX()IyM4-a-qmL*E7$vU;{1pCsc%Q}t$RUv zp35`AM$M#=)joS!YfW0TQnl2q?+wvyhjEgAfsK^qG zeID&i(_MnI!O>~f0N#wQAb!-2zTP4d2W}WgF*q`$>%*i^lUuFhB(X}2Kya_teN@gl zsYMY(6KpCW98EUCV|bI+A`&@uk}3RxnfOQ11n5YEKzR=UYYF30JOL_GmM+x{O8{2C zT7*3d=XBEBep|h za(1S8`-<2791ChP&gX2YyczIqNc-JT3p)a1xZ)mCBn(t@(F@ehHftEKYicjEE`nop zLsJX47ai)(ePk%^JnxEJJmVyAmQ?oD-AGaQZgQ#9-8HK<%A4lYyg*|L|MvUEYQJoi~h`HCyPv5L^xdBz-)RcVAz@uN}S*2s(*22DCDfB zF_W7@IzE>sB@-Rhbkbj^{(xR@n}_<@`$x-ns~#Pso6RP%UVVL-pIv|H3G(>V@WoQu5--L_z*e~~~x zNprkZ$d`Q?bjM{vxOOcBez+h055S4qWxpM99f<-dDQ*mYyf#;URA}3GZA;mfse=GX z8UCv4*j&(18>Owg*C#qte)%h#^$sd~X*NxUbi`Rk%qPhrqZzEi254IlbHV5}n$DPM zTOferB?R~NYm&koV?M_Qn3ki}+T=B%aFl_`^rXwRK8}qtAx~*CwUC?c6#?Q#&%{xC zGNuuqMXb4Ruu{GJt=40qEGejlVb_VoGl~q67p(lrr#VSDvYSf!)=a5}5!~+Zk zhL%WPz9kFtLa$eX&svgnuWK&)4$vEG-gNid7D=dTEm-BmnYro|oi=^uxA>)W5c%PM z(e_J5;22PqJ%_6K$;mFj37in znh+u-L@9|7ipn5DfJhM1RHQ`CQ3ywpoVnY3?{{DB%YC^oexI5zWuIOCz1I5Is+xOy z*ztA~$M#-*U&3nuCE5$lgf1Auj&zESq>`n30_pB7Fo5oX0LCehu;A9)KDMcE)|6$4 zFpk9DCxtlq&bFy)&Kx*ZFaU(tR=w-?Ts$k)Qu{KE27se58ry47q6-)447OPxNME)#a^DbQZS$p z)?0xx0Jt+rZZII9HJX2@ODV3%x(GP50|1!V3`)Wy_0N6&z=W}`dV*Ikey*{?KW%{K z9%K}LGK%xF_)QT*TNSZzFl!0NF~?_ODn8+o%jXB1w520=AC%dD{?e6msQSU3sHW08 z?Ch4PhwhstpFfNGUx^KK6UH=lu9)ZIn)nPg*nE~0Avl!}*;t8!6?mOur78K$J4H(+ z)JM&9Fbt?75ORa=KsNazcP>Jkypa6LQ8FlUI!PwTlr&Lzb6L7!N%C~Qz;|2ew^of~$+&fi)!H7+`;!0+Ji z6MwaSw>+$~SLC6r(mcXR{-)wI|M1SRw9)SU+{D42E)WAeM~NoZ~3zEP8*ehnKR}#LZ{HxRL|duN4Rp8K zxzEjF(suHoUfF?Ai7UQ6j=Q}nTm3F5W2<7e!u(V3@~fx!luq*WaYXU3qhy?3k59%K z^MKvFSW4)?>hsG^A{V8cc6t^~mx7VctPYwozCE$v-kAn0rI0)?z^kpQpe<24#Wa}2 zw#N(q?v6Uw-(&g^rJz56TiAC4h}L~PNfu#=8|Eb1N2_^+)8<9shrfce!8*RvL#c=4 z$gwwbNgb>;5ept!lrR_@f-nY&x8A5>$OcyfJM#W4Oel9ip*fVuVDzo)Fx8nahe@6Y%)hy8+kYj~DMb z+!lFy?i=mcTw?!9(0oteJ&E6OKF5JW9A_4YEC!j4Epxac%xYAGs}WwOZufv}I%sVA zbf1)-B_ISu{H*YvVA-l;~B$FG3f6AC@<%+S7kh=4$+>>sEuru1ZapM zV_YLZ_!ei0J~~J?&%KgI@p>mGdD?DzU;%bIvF~+1>tka? zcbO z_WZNNA39&U#>J0#S`w)((}`fBh(JAs#GjE{d&qzn0_63)k;Z5G?|Xy(Bu2VqoU1Oo z&q#tv9fWXm<39D7SmxBwE;7JhH`F(wt%A8{JI>G1Yv0kr_|flQa>iLVoyruT5e0aM z7)N4tz_bC5VA8JX!D>!>6E2gtloB6S9IQUBDPrsJv|c_Y|A@U7$o|)MjL;+HM_iJb z9mcGRTv(EQ*_%C7Q{E*vvqiTU;a647sF$?yS^b!7oP5P|%4vL4AuG4W*%3B$E z*&&OGr1zef=Z&OF2;lmD9S8k|Q?F(nLry{CZPcX$E*?^hD78(b8BeKC2;MUu^aeFi z>+||uw8N@S26fg=rt(b=GA})v@OS)jGB{`=MdydTY&Iu-yYfipyETvASG@Lqr5DYq zIj--OJ25%9VE5oW|DNUP5ejqi9(UDw)#ThnvB!i*Cj;3rGo1_((G|K9>|5J*u`#NL zc=)D1cXNWryL!{a`qYgG={a>N;&rJ6SW@#a+;!bDlK!L!ms*C<;w@qmI2Y^Z1M$ronOvZWjf9%n@)|lesurpFR|>PeGBYWp8ju9y*oJrxHxjU zL^M9#ltTLsj7c~+=%~#S!7Wn*s`Hi#Xgnm0jVj^m^>2zeNi6}ey(ah3Y_ox5fJ@XqQbn3f34g1@ZG>9|MX*< z{G#eVe{eK&6-GrhMr>ivg{BM2o<2g)k2axNvwJ0gUBghV5PxqFt0p!>Y-Rq}>`U&~ z)7B!3J4`Gyo2@FXn^{W~rAR8DBf5Eu`lQ#Cznk5B*JHa2i9Jw0T?tJocY|~E5(NNQ z*P?w?m;r(gjP2lWE7lq zi7rJYaL_b?w*#t7c@2>^=3NO9(Eos&GG9`AcUP&G-Mq8 z-d1W*OgRYAK@A5!_pK(kSr6QkMySg*jQ z5{N-32&s0I!!^hO8R(|dQp~vMl#_GX8@Ux#!x2;+r1+8AT@aI@Ef<9H>Z4_(8q*%? z!bdq6D0DgdzeU^LfNA;PB5-*>0;Qs5G(Fta2NlJ2q{6>p*fomD@-WQUjLMTvvY zfBwh2X{0#QygR`%v2o4~ui?;$y`ZLg(mOo}8z^rgsiEaWWq}??5W~_Y8y|=v7*M>C zJF*^MA)khEGbNug+52*c5-MYQc=u<|Obd5?8qz0B%|rRNo-Hv5;FhxyOE{f`ScWMa zVT_#9Ws`Z?QQ=5ZIje(^j-Ag`u9O(01#)b|Ev0Ti00NLkahKAUQZ};@A31iJTb+cS z8aSXgva(3dWR*>lW`!*rq+PcIExs+&I#_t7pg4UC` zFl&zv<-dhhq&Yl>%-H_~8oall$Cf>-gZN#Qkg zDcs>EIYzyPr?l3Tq8mUR-76r9k4#fI98B&@mQUZG-sN%N9e;<&w+PY4v1|tTsQ&QFB-2 z{Ym``j7za&tRNQ%+d|esD@9REHus)XYygn|%k!!Ac~)?2f|fEFDwD-ulUZL92S~pPFg3g}J;kQ~w^BFgI5j4^MSGq*6hy&JK zj#M?y7RH!LmSmd@<)2E>*)jBq4;=+$peh0sKMdL#k|+aReo^L14aTA?mO}B$^gN}z z<}TY(<@L_ys+%*C)b1t2Wh|_@a41%1MhETT8VgVh>YIjz;+ncU&*NzryC}D1?A?p^ zq*orwzkPh$!S%0SUOZaSGxO>tVfd9(3&Bc!%D%^t7_u#RK9f}mnxq4{op^KJu^ z!N#bKO_du2b~sU(l8ywaw9z->_04A>FTl%zlS+)`?ou0pZL5Qxf~~Vj|ExBJ7t2!W zq9ImifnL-hTn5d=+Rfrqqs@(Jw5R$VI19wrKRgk7H!qxSpban-#WLCrT8+3(sbO)~ zMsi(X-WA%}nI9*5RliVH2wjd>BCl;gMi(RL^zE-**wrpJj}d{-p#6vqou}Pw?a(Aw z;SGTm*}q8FqfZN*tLRp*>Pe21D$T0a@v$%#U8lSIl`(F1hT)Z^w!F)J9paY2DOcQ;a6J@hd%4i z)k!?72oX*BGY%m;=UXour#J5_r+6R-;2fFObKY8V?lCU>D|hk0Y!AM*VEce*t1K#> z0Agl{!Cq~`kPX>ei*CvTYB^7yarwRFng*qR6AN4eCBt+>a$biRa;FV18}BeUIp$UD z*x9=xoWqIYdNEd#!{p-jo=y9%nl9Fu9;|I3U;?Uf@x)@jxuS_wTUvJa^3sf@Lb80 zf{6DyY80OO6#q}OhjAFdH~iZ##p{j-On-2*kgCU5K7Jp--S&D_wNpN{x75gfcWmg& z!&ch#?(KaiI*uAAb)1U)(@KythXb(0Z|5pEN!6GYhBl{Z>n*Y5&m5z#3fr=lZSgT-E$biUt-bFOE;aiXCH-Tez)fyg)2FKRF3 zYednz)^fmex7(6wt++0WSv$Ar}KF-a8oKuPaxY*%-9LfY)dQ~`*kc( z69nO;;v1m+!rd^zXT~;}=HZj5x`PUKHn;g|yE#r6lUHM(om#S^!}qIu9gh=0ai?x7 z=y&HNYA=2_8thr7p*^Gm^;z9eM!fFlLTP?br|rm#+WiH4KrT+z#IPyVD@*r0Ge&p6 z#U|@jB_(4ANAAZ>I}@jLoSnU$`bB&^n^RL$cg-ru+4;B71~$9ls?k$KReffqg zzn_cJjDD6M|MmEhMc=U6Xv*+VaK}2F5xJ@T(!qFq3VaM6#;*7aHCc0kEHC#fyHR+f zQ?;J6^?7IJ?loKgj6C(^pB3yHcKDI9%uY>UUZLA-+`^&Lu@~0!H$Xy*o?Smbkn8#{ zrORBVhWk5CJt3zb-zwYo;pvG#YuH@2UvlKYhK%+{J%~)gEqcL8zgaNtpknP-jaHwrV3}m>)6oP;tWhk0d zE?$;=mZ(^wd1p-37d{lu4u5uWb!;cC`czWIPBc9TFRKVm2xCSZVK$UNGhIfaoRsS8 z4oQ3y?z3B}_0(Rc)D)hif(9L_j5?~kp?RXd2ni*?9Q3p|QIgt*y}@hnSL}l^(ylA^ z*o=ZDu#yRH4`S#5f}omnO8&EbBC#zn@$;=yau(t_3(!0F_;OK%Lqh}Wj4r*M2LfZQfXRo&8gq46_Uw--jf^Wrtzfl~~U$fHKcN@Qd(|r%yP8r*=o=4rZJnX7AB; z_PN^mCH;McU-sScB_(F2B}WU4Q!46nquLG$Goo7r4iSOA^tPeOZ^1(adBaLUp=vkC zoVlxcfG!!0$R@0J!%@~Cj+dCf-ua|t7*e;Bjin!km`gacKW;|t5qF4`bZfOQfmZk^ zyJEYmc)_VA?>!4IaKTHNM1DhoO<(hY!pSlH0lP=Sb1pumkweC)Sxet$j_UN~o>C4m zdguL%C7u!J-Q4SrHsM46E#d`a66|SCI4Fv7Q$ZtQhB3+w2!Uno9$hJT}JsSWbXm6sXUrOew1oZ5bX+GY`HdvQrQMg-w5hEoC$y-mO;S zgEvX5Y=4Cj`*X22Yer5YUcdMEG?F8~O67zIFXH3wst|(5laqrR{vD|J`D=J?#H-ag zJ1fR^MhdB4xUXNFO#k8^pT#PrAIons?N?QIDHn$V|Jc_~rEF-1A4vKRGx@;A9iL+J z7}pn;(mO~$22T+T6zaXP31?*MP_qfYh1r}j`_z3cf>=}Uvjp9J6{R_`rxHk%H(o?1 z5A`gy7h7*H1cEno1)$uPnrQC3xljki^;*xDX#PfjYJ~*Kvzi?A2laUvKRrm3la`bJ zoN5omw_|P={3JxNlO`Ct+pfs@+|FBtL?h9bkGyCpMhgjQ4?Ow~30x4LFs=~MTD5q8 z5QZ^b1n4jhv6xrjJ(!~~kYH94&Fvw4Cr#fYAwdqvphzQ-Og5HNG-2Y$F?*-q zArY@U@nW1GqIJ&#%Fe37(`4PjW|UCCE#U@zIETtTS9xS+X zQs=BAy>Kf>kPt1yLk}kRJ!4L&{Fie*E1K*L6FPmeK<1CS_+(8~@Z8Cv!#%Ew>MWb! z=|m5J(QMRI^Ujh+Y?XO^s54bSFvdmL3|j&P&m!&`a-9Tole>qG8KJfP(+naGY|?kB zJ&^?2%xWZzj~YVNHL^6YL70S_Z(_*YIH6H;j0xHCP+tsymc}NdDGuqMF^$bjRVS$3 z1Va~VIcdFmm!^ceh~fd^`OXajbQ8jVPFRbYk@;<6%*@ax+W-*nY;F#l7e7LL6{nW@ zH3+MxE%X*|@V=6(UJix~h>=@=!JR)rJBJkkK=23f=(k5XcJbTvyGn08YcCx*7WcX`fjpB?>MmTxgn2DK~8^y}(>#08rEQ#DC zbh+W0BUAwo3VBhOwJ{ARsbZM@=wY&b<6^mu`Gf6SR}^o4uxOdL;WpgZp{S4tTVFa+ zn(uAed!fSGFfZ^^cExCj>@L+p5mO|zt`fxIKeFfEX<)=l<&O3;uoO!sGB5G(A|(+_ znsuejkgvbM*eo}FH6DZn)3gyGbmf9ueda9E8m4V7$Mklo!?mfVNbZqqwWh2VhfQ5NRvxdVU#`#XQGk zTd}mJ+-w>_S^>6SeKe{pB#_W6RVnuC2AWz`5-Y5Yv}Lz=%XpHJ8O$e>`BuP~$EVu& z72V4Dalz!|KHtN#hffDcV;<(2)j;Dq1R&Bc& zsSz3qf*AedSvYV5tdw%DQ0(C#nWaJ(a(s@{X^t>^z+Hvhk>fK$0C#hR5@1yPV{pV5 zGO^QX{KgK7b9GlDapTEhZch2A^}wn-zl1ofOfVzLgl#Nc#7H1Zy+O55yS<^B5GJw% z2kGtJ#7^q^ezi_Qaim{mVp-%+fi-vF;hUXyJ^MTQzq|sfFU5Ig@cEyze><@k4Zul8 zY=*6r7myDCzxaUMi-NnQ#@A4X{KeofF2aS)==8!gj|LFXeht-qJ38U?6iJk9klPIv zf<|&1%04-E4LR?`6DTHd^JQ@wyD~rFgetDWjQKBC7yUOQ4E91-s*takc56p(I1@-z z>%})0Jor5a+Wj0v++yhoJ6hIdQd-P?JFBTrWljBO1SEFGjdKN^;elgqKkwLdrZ(L< zwrsykfBJ)#S8qxzyA5M~mC^pALC0njDlxZ$D_bmzW`jv*Tb#u2Y2l4xg}=&%Ogx4} z6_M%6TZr~UB4J&O2IGh9dqkwIrJ6^4S6N!Mf4Yqjk0McJVsg3;Z~X+iA*b@)#1 zSssHlJ4&L0g6myT#ZO3GgB z+8w-8y>Go_QM?W%R7!X3h?XY{+9>N5PCnF3`|YAWid3SLs}8yDNXp9 zIlEONkmjK)1S(s#RzFYj&Y30J*exX`bty1fZKSsua(0(u@uf#Wkqom~S|)ZiYN1}j zhI#G5H2Jb}4nr??#)zCP=p3`C8?es`YR`!${5jy;rj^pxg=e!5Wtv|5$Y=>XMqN9x zc{UrE`8}SJjs3gVD7WRbw01Oh_RBl;DFrVB*CtE;0`PzcleucOxfw#5yN%f`GLpgM z0`;m?V{rJ&Vpo(^HOVo`jmtZ;{HN~NOBYw_UDJ8-tXfA(>k?nZnf|w^0VnlRM*?Mj z0;)H}Cak8lfv82v3>N5f*k4{Kt*U5k+o~qT!&&58hm=M0FC2B&XD$CL)8m7&LSO^R zIieCok6Z(02uco842OEdQR50N)pkl4VgunNA3k1EduP&>Hm6HG1`rY#n>Bv91w1)JC&+WDg3<~*pMFDe>FC+05P1kBPHpyt7kA_u<;%?itsM{650dQYt0(*%j|3cus)$$! z?zv{4QdiObrgzQ3+YyrjpGs>cc67u&@l4|R!MnkCn$GtZmwr~>8NtVw<*R1jqF2;G zaIb`MWv3c{O3up!=75HY5{9-TT^$xC6W?&thepNMDa&)>L4#^aWBMd)=f8ia#I_lHnpAbR;A>D@?te*FRZ1|f>=kvWWXA*v4llx;- zi?T#>19t|0Lkva7s}1oIVHOp@vOqtL1FVEiXCS?m*kkmDvTdlDAQP`rn>G%*cvq9n z*UH|^MTM9Fk?(QXEysSYzAqxgVcCO6aY< zOVv);RWwN{q0XiiJdm;Q%Lxs2C2HGe@aC~CIj#Lj>XF=e!k`@u`7Nd?D&o}Oob!CJ z&x}nwBa+vWoshM4`(0Uy<+gqJvFq!7?`?ZMbYZZRnV3zFJyrgwzp2T8(1M*2CvN1( z8*wH7Ey~2PY13?#OXR|`tDxdh-bEHej=}Jmd!I|BQ3i5MCoYw?a*n1yAv#ynS}y)% zLdv3e6on8S{0w^5IC27G9CrnF?}+Z*e15VlA338Up&dLEH7N?}mH-GrWdrc$TbDu; z9tQ&U?+mPxdeAAP*wTn6rc4i?28fYN33xmuc>6Xm>0&~>VN#Ij`AWgd^)&lS#tNxw zAN4g2!p|gQjcAfW-nuI5aafB_zs{F-rLI2NYJ8*lltT&7eGvhurL8s|@3IPz2-H_ZWz?0_f(M)#(+suC{*zy-LUb= zzAbgPt?`oH=f$L`$Lc)@Mw+0z?|n0*uKOg4Pu*#srGH8OOt52?^&{E&#P!Ir6})C{ z&wjVp$4zgui?ONV^0^2rLiX+$eDPjxrY=3Z0=v&VzqlA zazO5{FoR5`S{W{Q6K^R`N9}{~fQT6|F3ktl-1b~~;nQWy43`67FRUlo`L@$fMg7Ix zd|~757ta$O8SpQzbaW87N0Xx0TpW3~Z==^r=+(8~N>fgBd+hKnw(8nwyQz4Y@4jWc zyBHt^=hH-{=n7A8&hcSVhZ#-oKYhH?QBfzG5go>hB70>y)1V-cYjtdE4ysmrz`SHy zO*0^2cIxt z;sLr4LFTak^B&71V4HeM@kRE+8)?hjAC8dcrS$3hGRzN@ZLnYdV?=zYX2=EwR4+n* zl$iHCE52yr^0KQoH;M*7tF2_;W2*j~aG;t)llQmB6dOrO^MC8T)iGht$+7kE@vKbf zJtn{h>2_G#%vhdnywudRC9ISctfmWbY1mcu*z^MZ0gACK*sJUE;X4)6UuRHLH6FPG z(K?>kooaB%m-DWVSwV!Xm}cWG!oxyE=s(E@j~+kP`T|C2*%V&2pW2750~Z%dS=e`M zLdgRShAe4VC$P?6;x?rlQk(a7q@b+fZmFj3DSOw6w zL(N-Mk?O;mJ2CjQF(Wv!OuYuDEXv`#ZAW@8O83SxH~k7@Qe1*yFf?r@_+)i3%Ok3A zvUcL{aK_uX_!-b`8kA?do>v6g_VDULj`AxA;qn2H#N-Re^>V6O?cGrIhvKNaO;wQaXeT!(|=u)@8_L+UfxD z2wCg7iGNyiqQ%w9$vx!5+C0yxZ%@Iq{5`TUCxa)c1-?svm8CTY*}fx}9Ryp)dIalf zyxnCAD9z2akgg)~1yhfoqUxIafDOVoldadPr+%}{`!_p!L+68|$Is3w|N7`vW>>b> z^890$o)Kb6qTNNqM~p9(EWdt2?99A5nCMy_W0c=OM9m-cAE<8tRc0qrM5mrh}~TQRW0GxJGEN`JOwif&VJ?CWL#4Za={ejC1Y4)~s5|L&y|b*kPxmpv2yEbnt5aw_iL2dmMLiJw2sej~n91)B36v`M39KlWhdV5MrkI%(xTi;tKX4Ih8=Wa5h5pz7Y_8l)WtSjNf2v__^3oor7@Z<&^t| z?SbC(3{-cjyCleN?5Eq4^Yf-CSiKtVEDqTU`ir|-Q3rt=hHND{TnRaM6%R`gFHk>UhN3d7RTY{qCIRT+2GmMN|wROI#Z5Ea=hg0y1C_++0nGf!Q-Ds5aVD zVEA##cV5a&v z9%NA*FVjie)SN;v2oId=P5mI4eWrGeE)fU0z7rQSYReR65*lJfS6D)eK}8l9IIj%X z&{1uKB)c9M)6e`+iH;KLIafkNVq@amT`d#Y08 z`_cpw5@KJ5x&dCO8ms)<#^}FApLz`jwthLnGP<36>|LH$uFK&1W9J3LQU5shyL&GW zy~%ChDpZvX%z5iX{4i;W`tl%X;=+ySwDF!ecHs>txegFZ0d!EN6ky^2kk>J#wSPx} zC0$y<$&ovLmoWI>_nYZ7--(1tM??9j9b`$Li+Ltg$3tS!-`#6I87QVxj4R0eF7bMB zc{$ERUr~1rHrrHA93_f>P@l4&rmlvUYSOrU*0ZCb3cX%vrsM3Hr**;mJ4Zh};drU{ z$%5Bxd8gMrRe2&}>=f9|7Da-&M|A*a&T2}cG$7E%Jp8+N9X<&t;*5rm5yWI#mw#Yf zj$BuNZ+cD;NHgo_9~}J2*XX0p+d=N>gM?3llW$`11`{SJS7ol7eizoe`iZ<;@g74v zqS)59N9ynX=|vpTc~Q!4yb3wGyBxlL<TYSS+9j~6gzKYMB2J^ORxeTPf;{@ni2 z#lHPSWXMp1KfOM}ugvdmq)$bVP1p7lB&Kh1;JwSo|C){d$Fob%BbzmW0!8;fn6(CKd3ezwyf^t4>H;&BWv zhiM~qV@cndj7U-ycsNPwkyN_!4SC%MsJC5=36JX$ctc6*{ernD0^jN6wlAUpe)%Y~ zaP$7Q4eNJ*CWm@M@vYriLB|EL|1C1397k&8KP&3J#6ZYQXn{HOxwoS6o*JkuRUu2W zu8-22a}0=bv6V6%UYq6>o)sqfJgck8#@6D%dcsKN+v(Tbbl?;=CSylzkKcHwb_Ko& zX$q)oXbN`hGT+j_sG=`{g!(qqNPL@fBox`|d}td}Hc|TtHG($i$O(g(DZEO1;8}PSVrK_m0^>=M6r2 z%08dUJ@yKzYrYk)V`^8HtxK!$vl}Q`FLb!4<)2-; zE%c-*WA>)a#9k>?X9@hxaz*q`N2g=opl8(6(A(debkqK#+^n`fdjffE_i^w;QtNT% zR?BHa?N@us5UJ`9$|@n%3~fVgkR*7SVJZ+(;j##|-N6%4A6|{EhDI4(Ac?7`6#46Q z?IBiquRTA~MD=`1OO)M(*Kt(tA3KS7{NMhS3kjg&B7Px*nr*G3B(TPMA6F5?F;t?Sp{mjzaS%s+Y1#=z&Go6tD3eSn$XKJX#^ zUSRdE%xy=Dm(!LYSKGgF*1>L<+J{HO{r;^B-`_$yV|+My#b0HCH%+yZAANB2(n;d` z7N_t97w+D3Iyq?RG)O$gnBl4QbWopdsviNn)`S3#?EsfGxI)bWMau@-t1Y-c2r5U~ za@wc(!G%J?GbwKLFR>B>UiN#xYo@U(8-up)5--qyMOV1}Kl)1Er7g#OI88&g=V;PG zHvG3}fUcyl276PmU+(r^7c0*)OmTA&6f4&2Xxz+Ox=M!$`mr8kmty9JX>Vm5wB%1r zA0rd@B?xyKquL=7qm_7L;Rts(RFETQtt5u%4SjC*3N0ZHMTI8DOJ+>ST~cwHyQhOU8J!Oq~w)ME>D&R-Mi_jhi*jp|1H{5r~Cz!pFQ9+C+p!v>Q(=i z>*x~KFremSoH`)QL2}UBbg7k+3PvXh6*3@C>`0v~c195eAe<|jLZ0L?bjIg4uuC6>m~)bj!GzT5v7cQoO6tlH z4j-B_w?+~wKl#!QFrXm(%Etzx|x!fGT$`B{K(gx&7R3~TELUKdK6dK^B;Th^yekE*t* zjWi8B@G!QrrUkHyuSYHh;&>La^}j{xHO9XWY{FZZh4B}vu=Us*pJOm<$^Jgsru1o= z5pvY;rZ5mfOP~DvH$k*HK3mvGna_5_^)S+C5jZ*n`W<_P!R}J8u2g;qXD+= z3Q}qow)%FZ@1ARin0pTtY}@?!DBHQmD&W;NwolL)y(Mbh$eC>x_ezXSnXaEyjU$b` zX69qzGtChB0H3jnb{2EpZ{aKpHf>jOe}s1CuW#&ar-=;YVVTqoQxV1su2rJ0OsbVS z3_j1RdU5{q@8=ZdQ*R>T<{d4}ocdp_+r%-VuA#nREaJMjS9wFu^3*m=<{RX4OoQ+v zFpjWkrRi;a^ZE2`awunLGnB%vvbgUIVwUyYFaL4t5hvf!%l#*iU@@{-me>o1_UAjG zwL)d4_QNZ%SEvR6$=ZSn#(sn>o;>lEVocHg1n#Z~C&x5)%fYlPeleTu5j)qf5Uu7jEJZ6cmmJFZv?2KO2GYbs??18F6-?-nW&ui~A- zntDlpW=_Y_ZZejEcW(U1tWev*gY0IaO=kT+2CbKG(xso+oBpEU;^oeHqq~{xeI>zf zzdk+!1v6pVPB5UFk`Dy; zgShG58*W-qS6b|-&B>uz5Ym;=k-I9RddkEm#ZCnf+D7iZN0a_aQA%M6l5nUVL}YZU zye={A^4*4HU(le0@YAwM-4H3`_mQ&+nbf^Q)u58;O*48kO7zQ}8=^~{$$t#k1=U#^DocvZB}|qNtwJSv<|{rbtJ^(=0$}`dox8nDNMNWB~KbL;n3Gb zlxN}m>udXWP<%?JD&or!{=vSO8C~SD_yqK(sqZtdgf-l)Bp^SiHmaRz*LiheZDL)} zPIn%)fH9f)B(i$y~CfFP#w9eBQZW<%hSet8`5>QZlU7^=c1T)qkD#W57)8T z*CTv0PoA-Kx+}L1HFZDa>+8u4qeTal)Ftfd-)1-Z{?2fK#i(kkR5?V;aXv61v%abP^PIw2 zzS)VDwwY+XvCTfGJ>IE!)35zDGh1%u%}*(JBCc}GTMQq|#;1biEGm=X+j&Wj;`(;# zZVkh25A`jseht?RPPqrwbrQ7DRTFaQ7*pPRBV<$VxHsrN>&UW^nS;f>{d2IbDsbD; z_T4Ak3S~MY8T3vbvQelmJAi`z1kfjbpbfi_uXe8ZeMzh3wCDBePGp02 z&KSZ7M@fSmuEG|bulmJZCYWUK5Y)ft1-zMMtzNsinbZH7twYp3H7ksWnIDR2vGvIw z^2*q{F0TB+)WHl&O~new`acymXUk$Aog2aD_6AMf_Z>RqSQ}|(q~>tDf-5nDeYQ;ZHg6LOMMD9?xd%hShk4` zM|v&qEkR9?qzdxwk+#dzwqB~ICH->=k8a~{t>qYg>rh3Yt&E!a=@FFE5$I8~ndwm;`mKWurx`xj@i*FvoLWfC)uZ-ONeRk}bvfcO7%H{g* zF45fvKRny=Ys`jwzjfLbZtXmDw2pcBPGdS%eO|;?CRjF({d;fCwBYhfkDyq8(hSa* zCn;h;{=LJgN(|sDh_bavDl{cCPQyho^vH`nHH>rhjCBzv^dxlvxL2+t2CfX`baso4 z!sowEYy;KmnBLH4II1_};XqNJ`WL=uUrbNP;wm#={x|`#20-rKOYvZDqkYF>h_tsP zWm`-CXLMb!#2@PIuohcTin`vDMuNOXL`BYB$A`1uoji-UH~7`>JT#TVn|w zt?z2R+`sGknbg{-#;=)qb>91{{$Ls94#o)m)J)IK($Su>gn*LG4II*d3Xt{+0%YC= znr_$-PeuUOy!38hBa;T%O1P{DXsQdlws{@s$>=iSrBRkjN9rm>$bk@e4b=hOS;!)s ze0fp%0<;!p`A$kp_3=AO^1j9cHyBVq>O-m1O^~g$_c<5|^bPtWP{uOkEQJjaep@29 z9!+44D9+!vGF@C4-i#<}cm_MlA|r-GqpVC9U1`2Y8@j3mItHeYiSEW;K1{Besb{U8 z$eoPvkEX`bT2Jvpal=U<9mez}5#~H6ECqq%yexDHP^J*zpkEPrSkXuiTuZtUvKx>I zt*n&@$W0PX8mRN{LHw9z*T9aGL1I4!!0K^($jH& z0s|998v7FWyFE9$;-YH#RO2H_ltH8uuZ@t|m^?k=G5U4q`7M;V(ew?6&HL{PZG9{K ztnCeF_8s!948AZ~A30IdxaQAiMRO6a4!s_EBragQPaKAVKrwpF8|@*iqizNc9jGq# z9d8NORJ{Y?NGoYqxHizuG*vLgyhf8wajqKO(=LR6hbbp!(g<|LcNbGeeag3URB?0W|nvEj*hQw zIyodP55iMrvGIBYl=kshDB9s_{6mC+Y6rs}d_b#OeVR81>W*tU=)W`_%2BB` zRyN)qM7us_4c#n!EGE~Ec@w0+np^(2D4BSvDqym}xGr1JULj2uTCkfM*^L9|qr97s zXrjS9y$RmtABwl;@14A}_+cBRSD)Hmh8xLIP9e9-C^m?-W2QC32Z-UB`u{Bo?sM5) zsSbe8=NQS$cji4N-)s}}b7R$E2k%d--DSGL_SUv-j$HF06woRSSxs7`q{KM1XF*QJllkV^&8CWxyxU2{$qto1|&ut}{7Lz|* zkb)G3^sn;xwrBq|KP-9~VfDCcU1*GaU1#bZe0G7wA)APw-NHLWJ(Zt15%vkIwz+vt zo$(oI$?r#-uum@=s|}3(?mWNc4jZ5=7K5K_0H<~(FkR0KqE5&~F6je5;s)h%j#8*8 zqe*Uonupp_h>R#7YI3IwVX!*r3)obExGfW*{k+BYI=EF+PJUo}SutTwNVUVI<;JH1 zk5^ZMUmf%)dct2hAF`n~%1!74Ij|lBp)_w$9k(wu0dUh8eRLT4>vqTlwhljCvK1LE zj~LGG9Awk4;q5A3DBK zu|~f2si?<2;E=l7IkM`GzlGbE+ld?B=Vxcxvv2n)qUe((u zRM0>VysggTM7DA}T!YQDzlt0gZ>=Ixzqc2JT>^SZS5LNfEF*U1x&x~}*S)GKp9;nV@6}aQJ2B|NqOx1sB{^l-IN6_4_Vkv+*$g%aH^vqHt$klpW1&D+KV-F+bwplBokU2&2PzyiLx8bd!x#2p0id3 zeEK@M$E(U?sc&h?;Uf_j2QnU*$^R|UdAiT)EA`9v1nb1sPDk&6kV6xrAydad&+#jb zGj=4I3NEXyJ`_`%8@NBjqOOiy4?4PsvIpk0(T6#5fGMSN?GP)tPS)Fs*AF*$fC%8m zp@%T|mzl2%LFYh-9!)>+1kuSlng7XcTQG57W7>J^&vwG}&l1o1X@l0-CI6GPjQZcK zrS}r(_y5CMcD{mNWWM;+diUN>LLj*Fnfd0Cqxb;O>rj;sZITZ6T;zN5zgw#G7d4iB z(=!u3R`(&Bz)T-SM|kyqB{G&M2i2}n#i$R%!gijBBr&fjrGz6F<&5+KI~aRY9VY+Y z_^`U3r#t4nPexlym^9xsWM{3bJ|JPNL(T^KO$PNz85s^ndhWzi54SK`lOSSblf0_~ zo5HK@>1t&R3))5W>%0h-sC1^CpyOtQu=cgI)Br*nr-dZ6=7PR!I-O@~r|Q`$>pk8- zjs$}NqV8yVW8VW%N$+L>bt5t4hxBfr`T~8O#R1di)wZOX)Bi5hF{ZLYUuaM{@!f9j`BP<+ZF9EL)+v!VG^ zV3h=BoH&K+hye0)Su`pDAvL=BpUWyg@zyjShYgPPPlw(PAi0=9O`uo{f(^QC8{OC6 zPGlT_=qa%bgOi3wT$p;-GjAE6()fbKw2Zrz>@4G{^(~bKLloo0Gx1SU($z66LST)E zUg`lC0eai;`DSfdp4t9Uv-kBacl_Rz4GcLa-wet#JJ{{}tAlXpi>v>k73|+i{ss0@ z(=9KyZrftm9sbN=aDa5?^x)g^GUuSLPsQH){sJ$ndCmb)^#KlOT=`a0fNn)!&{p$K z|F;NiR86pOdP+-?sk0R&cV$Y6_{@aVV!@F9Z_we$_>fIYa>chGKWx)RUCj*fBzq}eS199 zd;EXrbWuqwM2K~wB(YPuoLshZ>PV6n6(M#?a#^!eZj0H9at)^oidf05nB_7Hv&CFa zIg;x#*-a&vv4#0$mwxZQ|Nj2@Js!W`?+-n!$84X=`}2O?o|h!_pe^Nw;Aj1F3@PoV z(pKn?!=Q(V4gw>j4OIOJL^2p!tm&u#O`es(IHe1ML-t{+YZs^%T_0m+@&*K8rVZ`j`V3rKTs^;4TtG=)J z3!TocXMh^-G70DztwlNzwSlahi{g(^lBEn3J-1{7@-!(dEgDFIv$YUT@#H7UKOmgK z9U2w)UZ*)75Q|F2CYSGq@l0~_|6F=Em76<1diM2GOg!L3=#R4jXF~X#EIlR|nIFeX zH1%&Q*Nv0Yd9d3yy@uiJ)g)8paR^ow?l8e&ry^~k3t*T-O9!?(&8_NRr7A|cMzTiQ zMc}CuVZ4KTswKalKRkY-=0%VUEdkBZRV9GXfmC+#^A~Ay1)M^mRvVOlae#*dz1LTOf~QV+Z35b;Ea(IlYS z4PE9q&RUT$yJInCC@_mqd* z?&gkz7H$E~r_bRA(ZZJ5!Q6&AAGC*8#9ZK;E>W-2s74VF78(qNmH5bwRn2M#X+c#2m|)yisq63X9~eR5#O%M7;~ z8M(FPT2$qwf_CkIKmZw|Fu?7Fi7_KIIY!o%p*DIyI!!Tl!o`c&iR8MFTM<7yK;`l@ zsF%tE*eciZ?g&}8ywLoZ><9xV?$!Ir$f`V%X)pFT7GzAn-CzE7pCXv_pV}!!y|w^c zGXOD2d`x;l_y4qHO^WyhCrTF$Ujm4t~7$JGN z?epPs9%SSR7bqI>mc?z+XF7L7`>!t8s4FY@mC2O?a+t+ynZPnau>G^4=)ypyVkCX~wye~e(+ zJohdDV3imnz%i{pr#z)9VfB9vZQu55q|Sz&6IcU(qWEVVtM?+*~7viM`RR;T#<~LNr zWp-Wf%!8fph%REv`mqgp-~#t%EGM(YVdK{{GkrLBN#dc?rq7$asg9P1HhX%zMGoFH zH3ACwZz4yF&m*$%wcb2#*v?Yw5c>|4rZ}n|WHVq&ZGaC;Qn;%7OtwOq3bcBFEOn>c zqUf=-l)8{gnk43S3DPO>InVavc6@E)!M9 z0GNez!J6o9j|-zBcB(Dlf}~yT-(htDguUCQac@u9sDHf(L^qF@A# z)wm0UGlGS%OgNY!=uyC!1HC?!?*Lr^u^G%~mTo%XgaD!7uh|1{H!h6##>(V+2p1*Rx-V1?2&xwP;c}C3}Ph7)I)Kc zGz#cp0QXGsPkZ>gzVF@ zz3h4vhhjWQ6_3F=_G*-DTWvVWnp;+vzdZj6xqYa*e8~2XawtVaVn%^sF~Xy7kv4<| zGrbjyY{lW>Qjk9S+7j%Qx^VS650L<}$X4o%%KvVZBFkYM@~DpR2dFjK#95V#wDk=* z7n)y|G_K{?N-`MAYxU!%+KlB>vu`qeDgY*{tTOnQK_`82h83(c&g+~p!n1F&@%H1P zqh}a+RCMA|hlx2TzLEqdKV))ee1=rRHwEkmwSN8l^Xlehw@bwWWqyk;)V z2`d$dTqQs#L2o**$P{y3b6m3!fWbIy-r-lgVTLWA9VSY33b+Cm=UU}esnc@VlhW|q zz+Y0tg5;_wwow=f1=j&Ha1fh|P}0*dL-i;?V}W6tPOLme^-Ou-xzYhVPuBrEX*2sW z9LSTgljtqx3KOypJ#VIUTuaj(DhLZ(g)QS@u=Y(Je9E}9sR#MRdqUCJVuFb#UK2>T z{Nt(s{nFJP{WgU_GuZ(Ay0UIKcTHW2JVMq$Tici!d^?=?v)^=+=AIYZPc+RIl-pKa zk4XMqpTpjLboZSY$2CHn|N6_7fZX1itPd$N1o&M7y z?@AP20n-^*8S?U_B6`~I-ocJ9YFUH z4_=dg#i#or+Bb_R>2Q<9hoxXTa8$O84neg)Q|m!jyF3}tH856V(5ZeUae22RM70jHG+_&@{Xlrl;WGcp z`D6LwaQoVGhvTor{=^tA&*xdXk0w}8pJ*8ee%+$%u52Dken$$=s;@EU%&nag)OkO5 z+EUgU9QA$>cR?CVk!8jO?lG=loq{l=?#V>t34evlR8)*T4R@x6YCyhVo0dF;?I1 zy~1=4wApvC{a3bc`EB)MAxxXOU5Ip)&Kci+J-==4=2zmSQ$88bPrd#XB8~_{g9+&$ zJd*y^JMX_wK0?v^Z1d&g1147=4NH0Q&xM+^$4>wKGH$5G24%$-j$uaR3ej95$%*L0 zhVZeXF{3W?S0={Lwef@_ZyY1V3Fe0=osrNR!Qq!L$rAUm{l?yCt?H-G9*iCzM9c{5 z=*=&g@8*Pc>YVW8oYJs3y@%yrnsybOfBJ>zl2ct!8s-pC$fA^myjM2g5stJY)OGO9{(6bF5FW2|qUD{h zwbuGqmH*}?KIQA)^40%*ycTF^^Xo&>#slTKp9U?*y;oXyfBkmpEVHdlL&Fv3wXw!U z*Qbi)r0eOj;{V0Z7Kp1j{(Z%pldLV|k#wc&_Z34jp@Zr^t&`0QQG4BzR;vH=5u?I@r2$*s5KkOXeWrmrqyGYlPHqinf4YcO0_r0(i1O((uNAze_!G4Bf(>WK+Ey#X%I36a*&y?K`pL> zY67!xfO>sAM0FGK=Eed+@Xg>7M!v7~n-Dh8JL;qMcq&Gyh!B7wd%NDWTtIkw911`# zJMa4Zw)@Q8x05#AJx4kdPNC}uO*H%+3ncqTJ^QVrjefH}es%2Z=oIhV(%+_|9Tg*P zJ3e%LlPJPX88h872N&1q`iwTe%+3z$zLCiyM=9%R0&-||`sfI+7l^)T+il-=f&J(9 zBBP358XIyCN@jOt5zKE5AH9@(TH9r>*Dn@kS4}OSKHD?ZdjFq4Xs1>y^}nyc1Oo>i zjxxKFGlx1#x{CUh9E^EiF0X>t0ovrCpgtSG1Qa4vCv2m77)Zqf6^D3f^Pj4FPHxgx zrYZPcGziyJ>foh<&U>UG%qIV14JKRqm5w;^4Kc9xeG?Ej{-;0a?`t+IMm_?mGp>Bw zkLK2D;BZ1D1J$C~e&m3Nt;fp{XhFqq_j>vYo|{xh9kJv@M&y~!BZRI>Z4^Mcq?ipkfvB?T8(1tEkU%&wGfJ+; zKuO0`VAY1QZy>j+s+d|rqn<$3qLo({*s-mVJBTLGEs^m?@swDOpDCAFSLNBqUY%8y<0dW|19{`9pu+)0O96h4UioE-w{s$ms3Se_is2nrLCBpx zF-SMI#_JQ0$|l+oCSgVJAzSuF!r>0JPM`9RYX5Nfg9d@rl|2eO1WxP$5j?etPYLeo z_3NZ4H<0opL@GvG<=jtGf*0rvTfwa+{Pn6R*lLK>d%U3rGn#iyb}|2->{oHSo<&vy zsJoA5Zs z@06@FJ6@(U?vq7~47qAkxs?{x6Vg>`PEQQjqnQz);Vs*kO(jkAxg0OIsCeeQBR}13 zcZYrK#k+nj{zI%PqX_O~?Fiy7Y8P8rPOfF&{*f-q52+ZWwNT_q8CkLt-0B6WJ63ke zO}Z=X1tCK`_lmr2GKR0KDgk1nCmmfk6gkT5uT#ZC#FbhDZX2J9t$uRad#8=RCw>%D z%tHOJTxup?L;%o`JW%zRWk@gusbYOd+uSyS_Jp{hMq1zx_5TC8cJYmehz8ZKMQqrl zK%b>WfkA@b^_%FAq=d3@Z#;}}w-c}(k-Gj8(2exKH%uG3N+(j7PRs52Ym%OZ;>E_7 zrrYBbUzP*19xP9!;-AnNEnIqXZQ{yj)AqS?@4o}W^UC6?h;yzU=xEYme_3wsr6Bv8DI^Je?77@u+oB zOMND;@(cRCj(VNFR3YCYqe(U^4!Ozjo#cA7q(I>r5bluDLFNUGZ%ixhij1`Sq|6`m z(@$ou6n-P3$|20FURqf}Ww=^@!1iOoDF?gEGJmX}_njw%s~i{ArQ%1=2lH%sOJftE zoz{K=epo=|G`564nfbc;+uX&RAZEWbyLlJrWl5o&F1wA@DyS01h8lZ91$V;Mqn4g8t$Lurub z9i*9hFjl=TWgMSj)1D{??EqCy6vc3|Xgic6JKXv65iHi(wJV0Sl?k{N1#L^6&hA}m zb8d7-rU_~WwTi$EkWNg0x){cXzl(~O#GdMp@-THDWO`P#Rzd|Q2kJ*HE$23e4>z4C ze9W85myFf#Lsw4O@l3KyzPt;Z(mr>NQ!*JBQ@E`%BpA2hjO5~{b;o)J+z!35u*M|I z{VEtyMYG(Pm){Bp$icp$xuXfA4*}^Km5kg9>9>-#m^x*y#gM&d?k4k~ki0JRT^-W_ zo$O@(bmWV}#)mrp_Lz2^FFot$_w?w0Gc`Z|clI7ek7xscs`pj320 zcb1_@&s_Zb8^cDuL=#6|uuB1cTz7j*DvHx(Z~rPz-@W4h;%B94JN)ZU!$5~-^E2|K z?hs@Sp1>(^an;Xw-92x8_Fu)!wD=$TJ^o zXsVOQJ+yZika)aSkX!B8Gfzl;g5@64Z?s<8|Iyo0|09141}FCd1cu4Kkw+NH+gB)Z zaA0G%_bp=-r^)};fOPV$z0h7=WxR*|d$jK0&Vjk7{SlXE=)QYr?;U3jUJeo*OM7E~c4qoz-oI=0A8;a|->w;fj9xi(46Kdn(HIxJ z8VmAa{&OplbG!ipTa)uAPLXM%dC`_wr2M60^Vqc$^)EvqxvalFlhsXvK9ys9Q z^N9B1QBTlG)%EoseJ#c(vz!MWH=5`{X~OA;Ts<(&ONpos9@`vUKfHi#@#i?So%`a) zFAupQciHk`!hEMA&+}7Vx+i1LF~_iCw`;(#Zn=84sK;-30M~%BMuOtJ)snT51iD`@ z=4R#;zbG%sfR7FFjN$+RCFCj1Kws@WN0gn4+K9ALZj+w4CkikUha{Q}>YPn~YuTM? zI~xt#FqLxh1*Zu8Y{6y@H@|7=Y*mn+eY~vFuHg z4tXh7R!){0Q(C(5Zr@kfz4h|5?0BPmYt7A^qZTb!Iv;+xG{yUhkB?#JDRpG?@^4@# z4JSsZ>cF6a{GK7tl6FbXNXLcz9EFzBy$Pa9yI(m_X4qnQ8kc1$v#UcSmdK1L?J$Es zK(oamd5N=BL)fF~v8S68xLL|7Eg22xr8N%Q#+TGI)z5$j=ow0i2GnFwcX*LU5T{gk zKp_#94T_$XMB{OeFrJ-JWSSwr!d07L`R0Sfb>bdam%~-UehYSJ@2dBZ(VvYnbc#(u zccG(qc31v`+6KFQ(^LKKQDCRi1^l=|yZjbZ(pXmhd<@qo0bdd6Y_}BC7e-abO+S-T zK=axdOWsJnKwGbF$J8%nAq|13-bSJYP(-2BF(|=OA8!a9BA&a+H;|s8UAKy&nGoVn zLmXL*R!~5T^i$>|hg?Q+IMN`|y*_t&z^$(Fu6>f20+|DWcDNX&)EPevaTHEG61OP? zxT394o*riXaxP3zTiyv9kx$LHvtdlD@#%m|94~2XGRwwS8LjSp*&p3(e2j-M@e8M{ zt&W0k2^gJ<>&zAj)=ZOjap^|AANE+pyYa%O28jLM-k&|_X4Ki&c8i%s4Wqw*j;s7K z#$yDk(LM0D-&eeXLBDi$fu!GL+7I$o05Tnzd4;1B$u)rtu?N;4JyP2(ifw~!BXqYp zkW*=&fNX+}D6fm1YRB5ihW^ge`7rh-zdRXf;x~mRU)j}I;ZXr4$4mU?=m=*OaA!vrGnNL*H_3L5S9uB`3*)p$4#`e3}nfaXOrsv_$^po&zQU!Xxep#_zlw$ z`cOf@AYu`K&#?KOmDO0kTeFoF0b|{g`<{F56)+3O=Aw!V$p3vfeXjKL?DrMFUd6{k zJ87{OtCb_)BBe$kQv&uB8enZ+CIHaV!!q|uFa`3-cKS(X-e!W7H8h8nXK#h-FwjFS zWn4-&?N)8+qre@~rNXC&MQgyW-k!*53*1see^i~lioS@Rp(^$fuyRM$J@qDaH(Y~8 z-9CWap!{_}2@}nMS+!Z--_s_$6W?GngSz;;r1HY5$^C(25Ti4;sXWsO4gA!S+_Bv% zjAujr(9DJ!zRQ9{_tAvqyc;VlvU%p?P&GF`jQ46Zwz*|YK#P>ksnGQYI~&+-IO&v2!;kdhBlTFfL-Eg;j*&)v4E&%6jEs{{vmae<_S3DGDE=Efj+aG99mA z84@i9fj!q|NNgO}D~>3zzBPVUi5))+5Tt9|6JoDDVaCL{45x__))?>kys9O?+v|f| zqS)->L#G@l6#r!e=wP+K=3{R<>LPfS zblNFDRY%iO$wur@L7ITRB!P>2Oh!2*Q!oHT9h2rINo=y$@dCOk%x|c5L3F!T`h~v& z@)VAad1)p@wtM?+$(d~#*NWkuU;*RI==V-QD+B>cpt)?(YIJN-X$Pj6-E}W^Gjoqc zttJDfYcsTDpW+AzG)LGW2xvEx%+PPc{7h(;`wRpD4Ip;1ouDQqo%n~yovqdVoWNT0 zq05G8k&269`EOe)ryG#ALs3{D{AQv}PwpS}pm_A}#cyvcnwNi#N0=}YNv$|BdToac zu2^GvMck;|T=juquGYR1o@Pe_OanD4RgX6(-;h`A4uD5M`EqagCSI){Dm^5}P=2B* z{n-gGPp`U@=W3R`YMW#F!mCGz6)i_bW7sai#iNl@yc{Gzi7uo*(5PZdIEqth-aVi! zr-|GxP>+!IkhO^s(3MEVHp0C&!+<8Z0ftb`>lKx!QrXxsptf~)QjgbADN>otw>QOR zU&w0vtXb&C@nzBHKdU|#r=NQC+~(sz*zPg2`gAudt6De2b{r#i#EM$ncbOClE}1v` z0&TN;6jQrcbVJHo@&Jbs%D#@-qBaIQeXX2PbgHN$5#~I(6@6!oOdfEntZ6@wifuB>8ir-S)`*)K!JLD-A zX*d`)+Hz{)SWT7kph)8AJIr(~|Kebql1g;b%YmiRr=6`kxxQS{J8>hU@0Vfv(e3UM zZz~hc?w3&&bLE8qL{S}7zp3PFrLEo5Ey}xBlln%Q?egPnKAg@Avwrr1l@=2l=#Tk| z9#mSZl0n$bB{SGS!$FRDhq_3+L6L%r>4rAZUfM|$OE7=P^wYt6iX8ZK8Qis?vuXB; z*dutaZKUFFj%eLYzs_gZu9Y=33Ind-{!3f^Uu?r2Dm~#Bu%pd`D}x5#a1*&h8K)|a zrlkj>RHbYzg%|Jra!_qc>Lcs!Wja5db|@`uEyq z$cJ!LD8=m^)wGGx=j1Gsz}-}S|5L>}^UlrA*9Yla;pc3shZgDmEb6{Sd$RM)U3LKO zGwCcLR=!I`vtdKKy_*%0Q-TImYYR9TIMDwjz4gm`6_0+u)Zup`Pos^zo_J^&X*O9G z7U)doO?wSlv36QFi!nN#{9nPC46yj?hCe2vS9BX^J<4SzftKhhE~IfDbYJ#>8betA z52(gZFche>L>R?Y-P53c1E!gt{#Z)-_{jtGJ{fg_)9q{|IvdQ$(j>19tNp#!ZKqO1 zK3q{MG3WCq>WI&dNYy?#xq!Sc62-}L!8wMKw0=OIjZ4=izIg302w)D zpbfK$g`DEq&en-5=YCd#nM*l#iR287P^i&RUw)6JI-qbM0uALjeIJ^QahkxTEFH3B3C>&}<^k zEA-XfEG<MW?|2_+ zGjVI*qTTIey|_ZjQl@P@_TUSW+oyp`>9UjaAOFqz+aR(#9oI+QHOp&G7=7eVwtE{l zEAyEB%(gi2C2n^y^VI#Xu=h0e@8o)vs*ut-r$Y2qcCaI8@9O}Kyo&XUy2r#ExHI?@ z_d)AKu#t@Mxg>HJd;I!7J{v`E8|t zQ0xGf+5$uOf%fWz>!N)%;BOYbYg!?qL^u zTiK>LYV@F-k!N!5>3?S~$c|RZ3lml!{H@}c9oq~SRYEV z+<{{)GHdpg;D|{al2+k_i$+Y4XB@$k)C1WoRuJ*3I?@i}Dgx6HG;o__7!EqsU3z!m zcvkNbC;)I&*84(Jf)06+c9FqBA?*6gx=TLDS&x6r%QUnhO?ZAVX8zm5(*KT-vk}h z&FpqqCZ&$sLB7Ei7^^q9%lv$p@w7DZ`WYunS%FYL1GyC&*EM>{WP9I0M<7s+EJ@z| zmsEADq2RIqm!?>h^mwOq0@M%~;%?}SET7|{dQ8Ua$%eWBW$3gr0P_C6LYL?=j-5*_ zT0iWx@)_%v5WZG<1hOxM(5_PZW+J$Ir8)n>dfxv z^188Rc8sJn_H{&H2>#lUH(b`6rP_@l0>;}>Obe(9b%ZczSN05mK*(3A?y(F>ulc`1 zV=c{BT>?K_$bpgQ$kZ-8LPH4sR`SpN)V&2|+3r-hEYyq^e8^ZuX>68(g$Xrq| zew1XU?uY-xJRr>$#bz$zg1VOWV?d8kS7`+`jxP>RKUXlv*N}YDRaMQI_mDy>>IeNkfq6{E35z4$xv%}p{aIny>PhqB@NS$H&LIZ1 z8zrOw9FDQMdR?;!Fd6JUAbrIlnddf;IpL}^6QXq z8EJ^T0l5#=Ny`MW`N;x%GZs|~+`F~`lbfe9LqKf`WHuG$m8l1t{v%mRZM*|FrhA|` zF?p=b+`k5G4Q6-Aaym*jhjq#I(v5w)3myszJKjuLvCt|0R%{c7{(;!$7D<9$!+T{U zh+-gcw%Q<_$gMU%)L=4DbxSEIEGn6zze@&~uLjCM=?{%Dg(Z~1K?r-dL0{n;{IMA{ zDBev~K@-Usx1OS5?$Kpm8vGcEbpHg`Q@Q>W?ZC66hszjmP;1%3TWllbCh#45N&9~M z8$}z<6Ae>A2U7f*AvWSdl@+8eKP73ubT(sL3x9~qL_hWaA_oAAH=+5NJwvk@JHndi zUH?RlCY+euSu<#s{PryB>2nK3c;5YLnnfq>YnSg~QGC?bfu^8|oYAJ{uok`POQZut zEI2oe5(|N@&#F@Cl2TjYsj6fdg?c-Lv}NBEtfZ4O8{oBqH0x+%9}&B?LJfBnc?JuO zD7;kPNs|MEd9Qx$5@&CA7gPORSSe>31B}{C;Q4&bBz=hs_{e5p_InbXkd+eT+pFG! zg^xv~m>AYH0RQ^lWc5C%uT0$?2z(cTJth1`9GjC#ZpA%?OvdF;61&^fI|rlj$#QL| z*@x23GVP~31{|8={wbYE&i`!|HnXUk<6G%V-Q!R>mUS<_8_&US{~UJ>8C*QJh}w+8 zq9HRtY6=HEPxgzfLZChqKr~Z514$2sWcm(h#RkdmK_Sv3V@}SIu_a+htpRI60Awl_ zq_Y6^hXS}TCR+wWd`Uabg;_?1B?KcguRWP!{bR{F<(7q@MN`!Cq$Jue)19vCCQR7qCI*aj* zi;l+r0iaodKbq~;qTx_1n2G2^?-NR3z zW253-m0}}f9XDy_eZS-yT#)sXsIMboTyJ=Sr)z!T{e)2$TliJ`V?LkjW@i_NzI~}` zEWF$Q+NRzQL;g=4?*wRGtL%ZA*+JqJh#}JeNb?{0I5eQxNHl_EnyJz6m+sTv&^7mj6U$?c{1LD{TGNvEwv+bJyz4?U1_%2 zwLJg5G2RY=NsmMGN!cRwy3iuHH0#EbP(M7w^Vk`4+b~B7wYZ?R_H&r|uH3gC>c=1X zbJ762X1d@%k$OiMh-$Xz_2GJ$bxMCBWfg&c@~OKF-VRUUO|*j}c`p?oP7uy*1f~XV z&%=+q9fsc<&)jzktKoY%lad9O9;HEM<$y!I^8)dlv`cAce}f&0NCx|&8Cn+7nQv0A z*gFjeb2{v}KUFVa$JaRy()NX_jEE_oY$6rE`?6$sE4fl=eC?r4LlU}iAW-iA05~js zd2+(@_G4G+isNB`(^c%PC%f7ItDjq9M z%%vP>HV2>-)(fNj)5z5<5`CJiw-s(eJSYunT}E67^3;_a^2$X@ zlF1IV)|cAwpLuWtmjqd}Se_%c{Bk)2IJ!Ib==HW)=o(_XuVE|30_)!;a9y^2{V0#S zIv?qugUKImS!$^lMBIpt2+U)3Con!(%Cb8^5wKmAN)}d_>{l_*j1Q16{Rk#LH;|g@ z!R3WIt7@Ps2m&bD%n$zxb-U<5;}tpXfG97^$#^Gt;3fFLLL8tmXh6$LZTSG!y0xC7 zAFkhop80f!RZ}wrWSe_VyybcV0d@Zjj zCyw>kw`jT#M0x0Vyy&YBFl_5R)^C%!=dA$Uzd%Kx8Cm@tkAcfprpf)CZJqsFsh4LY z7t;AW{%%pSP39Jt`EZbthAC&>Lmks%lBYn%P7jHV&e`ARuB zg2aSS_zUz*a2e?}3LrC>C~21BSf0-iTZC0!gs#QPx=+b8ZYy1X%5H~mVg`$7S_BZ^ zZX$n?MUS?YlqR`WPZpW<*HdA~q$+kwN#AAMUg}rWdBl6mPEc=x8KW;&p{agHHe>== z5OGJJa-|r3EAGq(%+JyIB;G7lI?=fwdPsBwQ=c8=Ou?3gY`w$CJh3mB){liQha_EM z^ie%+_ff-jx+}8#-&i=@3-^ire768+9kMeXiCbvqb$4T}L#msb8N;f%#&4qvZ4wX> zRO-~p0qtw%o?6nc)B{oEi-%6-!pP!BcHykU(tS7oaVHMcO zvh1CdTlL-Xuv9>LZ^P%Awx0DAv|A0&s{7Rk`fU;%)Czyb%VC?IpjRX_d%dRP zdU0p?K(V}Mp7gPQf2D=@PdPWdy;oFPY~7yJy6W#ggjrfBtX)|FAQHTtwBJ=Y&{AEs z=uJ<0I8E!Q?89=GYoHn#MzrTg2n9!0LaBpiE-Wg@Gavz3^)HvS9a5Z?-3&afHilXi zp-{Tk0S-w|s(%yVbv%3+Q)g9_c}+})6AIjOJuFW^IaOzs2Xf}W|TmYoTiPGu#zuvYm&-+zbsp4mYzM7JM3@?pcSQ~?0`uA*(_4laJ> z&Uk0B>+CrP4UF|9PMC_(Ij>}Q=TluQ3oxZ!|LRxw495!d?CsijEw+bjS+pXY`Mx64 zU(Ff>;J$~Ssd>f9?Ef&3WtZ|H30$t=W;xi5hZI25=vTI|p8fSUjhCcu6=;56u^3CV zr40!FkKc-9$WW5^0N&sA_A|S)o&0_$s(TYj0dCSJ_9t!13xukGeG|3dIHfvCd-Ht- zv_oH6d`i9=@Oxr`Jzh!z=J}T8KphoV3viqkPzyb#vwve|&!QBoIOHj39cVZ7$M+SM zrr_gFvuBRUVCoWb-6v8t@z?Jw?w1k43azt&M!v78bQcAXJe8oEoO1`PjprEoPD0A} z6<(;3uWTRWR`Q!uNFDM>KJhX*>r!bo`yhy(>^&(@OnSVUbYf}wS zaUh~(3CVZD&puj*pWBCvV1MoRJ`F#x8}*nzcOG>ECPRN;;f`!e?gyCwxRsvY%E4k^ zRNY2}&b=Q-WiF#TfYP^UP67w|3lw8Dg4DP4&lhl2>}XOSFk=I_6Iy$NdXMZ@R!%jU z<3jZ6t9JUyq&nKM1_gLW1JLecei zkiXc6gWeJ**o4$j2GIM(gSPAxmi55gWovv@Ny^*$g45EF+cOJL6yAXFH~;YJh0T*q zLp3RYYVi&~Lq1kf%??9Gdpdg9%<1Mn^A9e2)8p3@`ZgdR7YF181liV+;L zw#%j5<)x%(_m+cY>Hhx$CrbCEGu&T>PLepJy->4^Cd72%N8QI^H<8ZMg1R;3WI%}n z&a7LNhJ&m%#BD$+Pvw7H{kjQyE$&?l>_auWvbCXUabU(@7wJ;1PZL=Sz3T`tiBD5? zrcf?B{tn6-UGPfZ;#QKg;8yX4wgr)P#JlOl>?E>!xViB^304x-T(aagd++-StY~5p2pKyK0PTgMBUYcE8GfH0ugDt@`(2d(fZr#BK(~|cBx#&l&hjU}&=5A4R@!wZm zW^X~IA?=C3HL9|l+(cT|W3byarVlPeI?(PdpwF@sL~c-osl zZ>i_21b*M<M zN?3Yj$2YH)&YYY!9iM1ZX}U=tGh|M6;eJj_OcT+ONV*Z>D?X;PO@A{KhTL<%|eAEx5>sB9k3ME zJ_svA&b?9_43mrrpRm0*c35QFA=fEZ#3Kq zDjkwK+V`6$5!Rh(~eq(qpX z;dao%uP^2<&$Ee0wX&v`6|8fkQv7Ea0p3{xQ<XK4TQU7Gk{O@9(dN-=b__ZU#X-Hbrgph58)QJfVMQx{2Ffc<8g0f4h!dF=J_X zsR32Ja7sR8L*!m5s3^GYe+m@%%|j);5qkVNV*fw@Wq)5NpH`lLs{#7b7r;Y>Q2(ZW zPnLSX*Mk=hPKOnK#NVU|B_>ADDG@A{qQ5{J!M6JqF5Rw&CmGTbqXlc=pgB4%e?q5A z1F5l91fvs8vVcWF8uNU}^Dd|1WxE_v{mGsV(gz^f2N1(@KD1h4qSWNFarj#pSdi)- z(o|^)7*y#CaKO*yb#Sv;=0syD9|#_5OyxP8C{+X=#)*128k@Eo-fZehIo~uKF&FKo zHN;5WyZ2_?Wpb15$;#82RsEmtm$kf~w5W8OYAVNGZrbhzz}T7e;;}EiS?qB1h>okO zfgMCiXV= z1Ot}ABu10MA!`7sbU5&RzC|*|`W@&nPu*+d?A~t3?!c!;5=`Chy^1rx%`zvHxATpq z9}UZ%iaFBxv$BlVs#cYD-=E=c*Z1X6t)ojYd#Tut`VSFRg$%b{>pd<@sKXhqV`1h0 zDo?7IK>QG6fVxauOX`Bd7RaN4h_!@xxjob+Rg7~}oZMtVasnOdm4OjydvkQbx2urK&4qet4e)@Eqn}Y z+_Is)7^QY1z}5i3hNU?Nkx8y~O^pD9$}1qE+7F;YVESMSAp`eb!E3#|>M61Tay^6@ z*S|$F0#-9^VON_{QsyG8nFoZ3w7*A2F}z8`+(z>>qG^em1=hADI~YHL5W!k0#>#kA zj{4wx^^iaZg?09u7Ke1AM$sLH$=Ty*UE+C}ycl7-wIFaniLg(h5_gIyn&G?-Sv}-j z<|b;+UQh3l-+S)9u|-b(gM8LnVuhDPT*gzWb9-sq^M_~O&TgHxilA1ThL$rL84m+3 zg!$&7pOa6Ae$0Gm?0Yszh}^8YLk@40^MG5<;Yb3TDJ z^?CLi0_2v7V9m55qvt?1J42z7UJsM?iTY9uhqaf>JRsJJ?F6-jRVDF)>_i?}DA4&5 zxa%F>X;m)9fTcaj+Bz6)c!?@f#Vnm|kX;QAG?r~q<>XY~xyr^BEWAz6{*n9Glp5I# zdb#K9G`8#8IZ)1TEF5gA^miF>DeD4N4idLi5zj%DAr-{KQcAjwv1u2D=Z8-P5EzAoe4@@Bqcy2l zv;TEp`HW|k5}~@`gix$gzX2=IKeO|&rRcTe@H3{~s3a zpj&H#jbg`NCjCxsH6sfj0Gg%n0l99N4%(%ZAGAIRLh3z-k!7bx`5R@AustFFaV^vt zS!>_DbN!2%SIOA|T_;;HTBBPlxvcw1k1zpsI=&_R(o5O4<>)HEQ{O^vpoF)JEP&9@ zPQnt9yg3dZ_347w;IES`rk6}CWfL>q$B}xov)Jo@fwiLr3nGi$N<2BsvBmG<`vlRDcD77AAmq`WULi5&`qP_i5g>jrY z!`&5EuL{auo84V-Gh^Ei-5mqP8p(AWo}u3Khhh_0&?lXO6@WxgrtZNo5N))diTbK% zi<6i91I$4)0oV@~9jzB!CmT40TpgbQ4xxx`1P6C8fZ*|497e;`FxleVEnD)Y@O@!O z>p?`+L-PNh-q(`cZn?ATbPaRB*3mJ*)ngDBu{T-^xy(0eHkvki8OWb~`?X(WmN+#$ zwazS2%qQ>-p2TMTdGXRIz5n4i=9PWP|G{qzZL+;plv5IBc&3m=DV_8Sq~MK9lMBjz z5FAT0Z)KM3^BWoAb!=&&Gd(#Q!VLFhoVi%Kj%vQ@?mf3j-N66~&+V%qKzl0vb66R} z4>N$z_lrBLOor|#Hh^n+6!|Mro2e=yf^v5CZ%~ks4I0HaQaI!-P?FGo#=x8-$RKT( zW_I2tI#-Qw#>vL&VUViTqqH#7`3HKju~m@&D;}~u$V?Jld~Wj8j{OSC48Z8ET7HN6 zi}JqvxauzXFN!{aEHakoSko%WZCvQJ^y}PsZn+p{0A;O&ry_qL064E9NP1h>5)X@~ z8A)#LYqA3doSaL;{7iC03Np;C_&v|pU!C6PHdt`3Y4?2OZxtObuP=YRyF`t}Hsp-m z)jvS5r_W){3g!byA)hHTI~A)50y##NVMRVq=E25&ZUBV%O+VCiqMMXs6iN^xy|gKTfFckfG$o4kAVCNTrP4)K6vC1u%R9Z_J^R~x7yEnmIcJ}H zpYI-iSWh`6VXZmGoO6tKyzdy|(=1`ilGVX#>0;MUQNr1@r!@YhsrfO62D(B#9C z%eWgBgj{NZ3!KBsI0n9=LGZ#`*P&Ntpe^bt#Jh2PF@5;&xLQYS|NKnc@VEg{dA;9c07-Ip}*m@Kg9Oy5Q;bd z9-Cj$Q|6lL@k-WW&ervB@NTrod#W8~Yu)+&;a*dYTS~iuOG*FlvMTvq=mY>GhIB>8 zrLOc2{2GuNn*{H--ojk!Ma_KXvrRl%CsLODDAwgJf#WCZCZi64p5l%iE~M@#Jwn?@ z-u9Xb&T$c_CKE%$4eN!B$FHGmKhyNa3$!@tn?O|5pI4UoD;2r&QgHs~kLb*Kz?g|7S} zRBA}Vm-NPYgO<<$aboc%1YmHBcf%^%B|n@OS1~e1PSF+8r{o`1G&kGvxr2;!G4|A-z9}73-NX zT07S6EHCY%h589%TO*mvWVrCR0mRi7gyKLobqnA9ux)$@e-*vP%_dbKSG29^4YAby zp~I-!p+P+kR6}L2GnNw0O zslB=UqFY;oeb$|Hoj0jQcW@mokNK6c*!Xu@jG=x^+bI# z!mWiqUgVTxq|DOyXhGPR^h3=z@Ve+w=@ImGQqRlMzHAT{RBIOdekVXXJ5Y6_peNpBcJoJ9d4tUijbk#Vi~3n@7BIfX)qPl7XKR(=jkVV64t?{l z`a_I|%35XFN&a&*x%5$K%TZzj#6Ag4=)nf>(=}SZ)$rNOAsgEhm z2kvT|DSH2wi~~;im-vs)_>+9;T67!G7=5Y@2FtIw1Hh+1_6$upAO)Zpf9vA*W4ZOi z_`!f#mRv`7-2ER&$35tMo`qIRJAtFq8n2$?T1Qm)pQSafVAsi@iz@d?K8qb)-C*2} z>D~oh?jQBOE+oV4j=UK|=zyxAp5*B>wOw?zYXr+q!lt$1d;X z$l}B`M@Ii_)312r%nBC zu{Z<4I>|F@)cu>c)0X97HAl6UASnVS=Gj9~tCMC=b0A@_%N!}0{nt9DT3lYCb+Zoo zA%VXi&Qg8xIUNoiP7>{wt{9OJmsfefosArJ>qM*IVP+?)PUQ<4dKh6+YoOWA=6ncd zEHgZjC&WP4#P$KYnK|jfj?vtC_%>1deWcE8CZR9T~XYs5Q>Adow zp-bgS_!4IQqw^FUEC16U$vROl^kPgV)A+|*U^)G@iE6hlNKSJn1JbMR!YJ)bC{#*_oN>p57#Z_zF;)OHFt zjt();KwNuz+ZTnY(^}wjTs{mE9*JFZ4R(w1GRu@9UHKZQA0WAF2tw$K!nfdnoAXUc z_}TM7Q3@<`pFHpS7ljAS#9n!=)oYN4y(3Qty_Q~DfR9FTh5$5b@Nmx20wFY)rOoxm zXa_Vk4A2p28PW!RU#=mFqL3ed44#4vsBok?z z&(5>a)^zWG_B68nx2I494UMLh9VU~$C~U(!Z)LsU{IlyUx${LK4G+l_W>F>dDk=OU z*nqZU+A5zz|LIxQ>R^6qngn%n-gC{+#;bPLB3 z-0ZUKnHM0*iv9Pk$ht6ls1?gW6~ZClb?!5&`3!x`oL(CvdphUb&vihN2%m?+-g$;$ z4RJPd#i3RZ@0>!6+INjGOIt-8 zA^BjOmp{*a;>oas`HPB)=un|~yplEBx%+*Wv6ngZj_U1G3mAVfJJE{&d6K*XUt<6M z4X2FIfWy$%A{a3w?2RcNDs8L5%FM)m(JZY?i%z`rHn(|toBLx#RYhfE2HB{{jJKoH zn>g6-wV$#5_uP*}i2-cG0_6}1a;8j^)&Ih;XcSsuvD2i;*KB>Uk;wLFiTBybAD_g1 zOfow&oACNg@j>PO=T~i3`XZ8XP*bN;<_N+jM$OWwA^Zeqh_#wjv!f02^I#9)8?3C5 zr@~ZKp+2a<5y;s%P2TuIATq7<)a*JpR!OOoZws8ndw8P;Ppi?E&bwCZ;($`u+vU$a ztY$(fohNHPH8;2)O>REt=#s1Gw92lbE9mz9_sycM+<~~`FYjcN>>BkN@UQU5?k4mg zK?E)#KRp3cet;Dy-N+hdpew|{BT|-9ffTZt>IbfmaUR;FKeaIhwf+@9JA;#(AD+D* zf&PGCXJ_zo{4rDw*(DYMB~?`ffk9-n=|qsI?ka6DdEW zYBXHF#?elTI3?}}N%9&slFz2@vrd~!I)W!xkMXRl-~Q3?Qw4dyVrs|g51eHvkN=Tv z7y-fwK{hmYMSO%_M*9(=2r#jO-+Wb+XpK6L`>YWsGJ-P3J!w(C>?)~`MuFWkoL?TP z*SUYp%v-jHN$4VzDdaxCt0ver_smc4Y3f&YmUqO;F~8Ik$DhQ;Kgqo3cArvVY<4HW z+C1>Iz>n-gE^_1KqI+Rt6uJpHQ17!ueB>Sno{2FkDB66u-2G#=$&6lPGViDL{ezzN zo1d0EIpo9pu-sYf<+b$@;WgSu35;AWf*-}FMjYy!0N z)to>=q}9eqBhCU@7^1u6E=%Um;FdSHjm+vc@FUG~@-lspqUiyJWZhWL z#h#hVX-OeMrGnhYGi!k9pGH<*0fm1=naacOZW8(k!m~(!rB%PGcQe)8h50|Lj%$SF5HtAI9ROe*lh5yqIJ#*>2TmXNi%3>{_k=Y_{n2 zj#`lG;IF5BS4a)CIX&d#Jwjisp2I1)@dWQ%ew%x#R%)k^-iMgOkI)SxvoB})HwNTXxtC9jKt5P5M_aNn+g`1Srqjm132!KBA z&uGycQK$a4$GsjGnuCWm1|AN07ZvxD`!g!tds;-hUjBBw!o)D4oL83g+{N?rhksX? z_Ps{V4Te5$!>%T&xM4lQN*VI5n!sr-)#SP9e z^e?$hIb}Hc21X1AmdLOGlc@%v=-&_hYd}%RpE$c4s{#o`2*5gDTn3=%-w*w3KvA@f zGj-Yuq;a_YFaSCervN7U_hZxlDoFItJ7|7i{RdJF^ZV*QoNV~}P4s{3z8a4gCzgWd z5pBt*ntl#pEF=M(g~4?^wUTj@ePFm~a}!<}bVHR!26xfSL9*GV=p>LB{M2LcVstjK zARL0iRqk@x_gAzSJf)TX)0ngFJ-eKQOjWewUFD}^pcHdzYK8JkxX4I+r*Nh;V4U5p zFK@wS^Oi(rQ+*G#%S_WftaJU7mn0p}B&n|UdtTBR{NiZgM3L_skC=Vr73Z&C^dmn_ zy%(+PIX7xu62d^XRYb8JJhszU(SHYdI$S+}{V}~*ht816Ypyaa__EKV1?$APdTg}X ztWU~=qrV5$eC&7Xd3SAP%L!M*Jp)J2zGJqO_kZ>-@5Ah<)kZ%#_GexCgMT}YWWqlg zi-q0D0iB0|Yv?+3Dy66?&dQSB$zL>OiE)gbNox1UVr0j2WzOQ>80(VM$!x0EKKR#! zBagA#PrAK(mXofS2W zi&GvkwOWUe;qls!$GmBKK@V%ihtE}BWeO&h28dfL#ftG&2ezEAxJkQYW^I^tXr$7t zAGxsf+l03a!~iGD)R#Q?$|$Y-q9FI?uMAAQ@IE2&#Sc&WLg*FH6@l+IX=rp&^Y8|j zKXNyaaDGn8_Ac(UlKZ)O7X}_T-8F4_nKa3ftx-aq^`r{`hW8W!vHgJ>R`+rl3%tX3 zP_s7vC!*98S_+MXcF(30a|HN-b9@N~{Me5jRsbPF-vVFaw;xT>%y#@$#x?B661+^s z0j*~NuHTrg)XIHqqxwKDmxyeAKXGd~KBT9g3o3(X$jI{4^zK}F_sx65@g6c=v0801 zxq-X->Zb;gK-1TzeMs7?CJ`={|M4i|^CdU}b+ZM}lLx)NE5ynV0u-`YPDA)`Ab4+0 zRBMc`ApT)C@mKAqo$S_C)XeXnm;jEvZCtQYSr={hFn;-%zr}zlvCVjEMrG-f8&|g5 z2pfm&{4NEpVBSA5@aWg1eid(dcRp6(O(t>8o~*ZoUG?2DXPceU46~ki^}KnJY&!Bs z+H>YZzKWH)KU0;a_7E}kdd%sJte2>#-6|ec4o~uedvBBlK4xnw)q5Je)+m}b&^=Uf zU8u8YdwX=z=>f|5Z57|7%TCCPEfkL{Mp+PH(|w2_B-dKA3cH53P`sI|wZu%e`r%Fe zme4va~xd_?VO?Av@((@UsWTbnnUrVqQ{K$|n5)yJq84MMpym02KfdHiVq z(MJ1O=3i1Ld%~~)-=o_C$MpN+{5|}W$#EINfj2s7H~R~WIwwzemeBp45_Z@1IrQi6 zA_eWwI<+B}=N$Z2%`Z56gPrBD&b|MFY`7Po&HW|QYQvn*htCmNZk;=)_TModJ`be) z=YW)XmdQNJ6mUU_^Q7I9P|dN1~h$I(71_oA%RBMahS%iFLTh9{h3(1CyBK@^m2? zI@S{cx~T9Q)oTeNQ2nFRoBFQTVjt~Rf+mM|^5)j<6GFp8&v14TE5+iL-FeemfM)%P zQaKlu#?APAaf$Pt(MCOfqNyFb^eb~qzmF0)_G9VBEy))eIxhBmYzR($zfa>VMHy=b3o_MmnBn;{Aog{C~Gmuo`f&yn)<$EV9isI;U-m!+@!T zF8^hU5T#{Y74?omK(2^Gg@=i&r}!BA&%Co_+8RN)>Cs!OOE(3c-(IJ4_|s*E2GD2x zP_u{T)%n3fTe^waFi{6*z`r= zT7nQ?PgrxXbR6Bm~4}jF`6{n&=7wO zv>t~7s(1P^e(V^k?82E05WS*eItEEsRYT&-Z~3GE+!JUZuLgkJ97(K;s$*j~vghv8 zkHj3wxAJ%n30gNgxCgM+PBHqV!;H8o2Bx3tYA<$gfUc!25zpKRo?PJ(V3xxo3C0bG z^*o(yh@9y;R{5?pZ(k1A%+oqNe@xA8ZObKdPCQ7zWCPxE5;VoYzUBbdd8q)9ym#G| ztbcR0MqiD%W~1YJ8@8?ShW!P1+_$Tw#Fo;>`yB~Y4JY?Bzkcbn>88HY(i4GZ`6_xR zwD%0I6Py-k10lOLneYyWVnJGQ5qOb|P9Z^Ngjaz+-_D$CkTu$gY66II5g_MgQPBoK znT=;I!+*Y$IrnfeaoTqcVD(ZVnVOgECDc99ghuSZBb!wK>x%i3D8svhGP7S4taIy` zh(8^qm_BLS&NWEv%$1oDL6_yP&06KLUlha|pjyr+J=?iQXCK`F#Ox|Os#Js?M_H%w zQyZ=Tn_tgDe?(6Jqn-sNJ&CghR$IWN9uT{KQMk7aKi43diWdNI7Ixsw+A=1HAmZzT z0U&nI{vQo@$Jxpk3&1D5DW5RYVLQQ7yu_eO8=#MVpVa0w7Uwxf-KR!@S}qBg7T}b3 zSU}*RWj3wXn6hT@^K*}*g(vVMIFQ=BIu88`74$LWOMZf28+{->!2ilY=q#DCI~q%f z#=~tgkhOki&zNJ3lS@E?IkL5r=&fNXcwnIk4b0Iy7slM9f4#a~xz9^zWcNbw06U4Y zdoo}g+-YbinKR`{vjeH|C?SA}cO8c27(603gXT2uJO607U_E12i3_$vCI20EOyUb{ z>{qO8O$<;ywEh>`bEf=gJ=Erd1z)|2In4j+lif0ivphr;k?ApZ{RHn2!dwk}&xbp1>;53N%A^z^3_I`_P$>hr|5Ku{BcC#QJ3w%9mQL$|MDn-n-`@=4WeYZBBbBQ}SV4qOwGIM*Tqc=Z%JH(8VsZb=>5pSs> zRNMR;?IiM|kPk}rNFt9Od_HyG{D-Q?kMvLY;hCn$Gd#!Xi0wc z>g|KDJ88i;JIrHm(1D9Nhrd}c+W=}SFQ9YzW!U($HCtpC<-Y)d)npuTHRX(v-0)K) zUm5g9#EV%m#bpCJ*iurWnmP!8H4nHXcM>Y(4R@=O3{=X9Pa~bFEa05Z<&Ek;YdAMe zv(c%h3jG_6^Iv7er5|~mXVgJ9H!12o@QlSpc)#915^%I?%kdlahl?{^ZfhrgFN_R|p}&rWUIG@+{676ZvQOtj-G2>qGk?*}Z{Yb2{AU1bKGdBLb>~Cf zd9ZmNY@P?3|1O0zpDmit7X3|9bUs@&pYHuTZovN*tZF_6zRkJ@4dpu%(*ow`)L5D=V29LT5vudOlzse#!eIIq!v;nt>n&7%?ta37{Om5gtE+b=|pSHBT zvGF}?AgZr$;jUoztIf?cn0xfEvmYm4MD1SI=uCa7lC7o|u)d%#C$(BWZNW#E0*&?7 zJMt^g#DMI(FAC{=_p!(eTHheh5HZqc1^k}Nx=zQyy&%i2fZ%_Ou!eK5$}}@Z(Ty(9 zmIikCnQ5!17{~VX{jmW~chwKyzfH31%mx40tuw?Uco~NiJ|!TL(qpt0@|)0*0E_5V z)Z-b0I$ajJ0^FLuai0|(Y)d@%@)w2ZLCP9$ADMEmLE*_q@H=K6zpj*&tXgo+@Q-pY zBa@mROPh=&?JYA=bCNl_ZR-h$&WucXQaBk;vliuUf>pGQ^oGv6g<_2c#o0CG)Kb8t6f(m?s#Z=?v?D3wM)Qt!390@4RYkqS& zA@Cl6A}@9{{TFjK+=zdj*iV=zs>~Br{zb&AdGu}`y_-kx=Fz*CcjuXA^HkBlr;yI0 zck}4oJbE{e-pzBp|39<~^Dgwf3q9{b&%4m`F7&+G>dz{yc^CSBxzL|307c9jw4Yd7 zAxa;{_kB@tlc&judDN!_7<1j?QZ6+){6Jlr4qWki(pcoq4K+Knqe1A6b4dQw67gf@ zr7i56M>BXaA2jRQiK9T6I=8sZ_OB6jKW(QqjYsb1J7`3@Bs^tr-0!)itXuMde$AQ4 zWy&_{53_i~^+1Dnw6#wWw9ZWfeV{N5VyS5qx`W?P?~7EyCAeBsFU7>SYs^k=%N-tVybF7rS+JU}z|>ya?KHWbGhv z;LWjvDxtrh+c|rLXlEZ#WEQfPbU1f;w7JP0!GHT~9)`OYob!_I9Srnpb8G2dBh$TprK+#NLy*qrmO{g$YM%d$yryE&WEt``TF^M{X8|~Z%8(O6WRWW8uHc>1kG5$@VS}<)mm6s!=>MMbB6M< z<(LLM;xUVho{kNxHXm+n;xB`I0cm4*c$S9)#YhdW$_Cvl!&y?Fy_@gqRD7&TvWc$H zceed&R!j6Xq~_Wdfdxl-gKdZip7m!QWS4pq1|h`2T~-(k3PuBZR;d?EK?_;DL5@$s zM>SLl)B@Mv1l5{A6FoAGFA4?vk_uLD{%08J;f}ctwPQQmll1l?8-3MHq&T^KDtE8 zpyCczIcql$1gw2C*pzE+SZerd<;LA(=g^)VERBwilL7C*e=W?uYp?hDMgV2KHxPPYBe^TDI|=O~Lz(N5-1h zW;&_56j}M4X1!4H(@AXUw+>hRAjNj--sF&4dDtOV1dgsF^ON7!djCw_ByE-xlO<29 z*~O!g#H|rV)Z?Vd5HeJ&rH_Qb9wACAH)&sDm&5oQ7Jlb_{crQR2h39qz8()=bIW-VCI3*sqomWtQcMsVBrTS6D17@;+*vSDQ)gY^qrY z*W{hgenSWfT$QexS{%@0df_!8=+q^$3U7bU$5v9%@`D}ww&ngT(-BWjT#{PJRK>+n zaG=nS8*eLM)ygr#nZ}k-E`5=hf88Q5On_fbM3U#73JpUN6hPwgIHMU@p1>dhi^ zVgpWig(=w?6=@o3)*F25^C*q2d^L`>;X;s}zR`z5(JB30P1A0&gsdVPqFKqSp)*(p z^O2v(RSF@2!ix;J4tq_@s%ye(9ol*^Z840Ewpf;r=-2k3D|$kP#Tq`n!Fs6nc=BLD z7BxfIV2oRe^wz4Hn5EqcfmYq@N&1B(u`&Q#L=o(e2WXd;mYm5sf#DTMP9U8HvK{Rl z5=Rk1u^P19%x93I#VNt$N~Bc~KXX0UF}!MTA(bm=tPqwk;e-GmosQz0*~Q+xK@XCd z@jzl~rqgVk`@WP0SLdKI-dk5U`;QS!-Ca(^mcU$+M{b}cK0ikPb1wM3k!U|3KE!#g zD^#3#FZT0B^5BXl{Bm@6IVUWiwxr!kZPfF0yx=KeDP!p|B65Kktsa*|w-g%|<&U8H zHxJT$PoS$iH}^~wzt>vA?%?&?CrpBV9W2>8a}Rk8eL2nS3F4eRs@~g3VO(x$jD=fh zj)9hqvFRhTf#NN=}!OJeJs{E|1*G@k-w1{0XZmEJi9meolhba>g((`CDzBH;~GL|8p5kMka zK?()qjbsvG3ipeG61AWfTK~Q$h1;_W37}jb)Pm363K@#0)T^Odov!=DS_uV#vyN$b z+QZb`!)Y&`SAWJ0b42OVAes)CsK*FJ;(&yU3s7awb)P-mRi;)~hHzL!O`yfhCOL{1 z*3R@wW-qdZ{AD!bcG>w50cPR&g`5&AZh@^(PT+c)`B#{|OD{Y|DD=LXxM_|1`Mcc* zJ8s(>G`iR|i9B-B#FYH$H^Hii6pNHG!V@)mbq@n8-a{E{<%v8XUheSZt8Q*j(OwhDLz0Jf0T?R9+aHJW4EY z6+M+6qu23Qeo^q9&MtJ{? zaw&b9SDn%Swl_}-_|$4bwFoO^s#*iIw&uHWiIo~`d)3GsH1$AIh+W6Bi&L^mzm?6b zy=#4`ST{jwaq8W|ARpqO1Lf4IMKhtR>(B#1Qb)x4i^9q0UlamGYTO!wZrLjSg15*_ zHgfc6U&FBD0j^&yc{tEg%R$3IWb4H4)$No$CG}X^f2?~Y$hyVSE|1Q^#}MIjM?Qe% zd7Uui*Wjqwl(wUtrY2-WTP-cEK~9miBU#t095HdSWx`esk~%*o&W*$4-OLjt!rfWw zR`+Ka63wdKnaXRG!L#?x!rwC6nTlnMoBZXRpC7qooenUo1~_AdU)7goGI~_Dx=HZOV5*-74M&k2*ZY{aHkh?bQZ@5 zVJVDhX?3%Il{xsGzm!F~tXfi;7A@Qp9R7{5*Rn#{J`&{|_TZzmro-i&66(8{^*BTN z2i7wpqTSTU=e?IS0t3HDyWAr}jjRPsMw%v4G0b01!-{jRz^y|$SF%PrN!XS2jnoQm z&Bm^(&4F+qP3ioDfWhrzt$btoe&R~(YB}R*VSl~jbD_?$}CsZ?7Y)CTu?SgB-&e-$sE;t5Rxt|IPC@Pw%n2NWCz-+GN zkommMA#c_MPP#dexsG2Cv1deA@~I0Oi0f*!k>O{=Rb<-qWR4UW1Wd#WslCNT)A>C?&GjWFu1)``y{VHwl|wPxMdD(O+eak=S{ z4%v6Am^7LLXIb(a>W#XlJBH7m}$=o_c@i`Q!&!K8k1|LN{Uzv#o` z5;b`dJ`jWGsmzEp#0gx!itWT(I82aJXrTu~0)C`S5gu>oy)3CVOb9Egg~WdKR%Y<~ zXmCdm&ys4EhueF5zFUbq+OZ_L!c-8Hx88n7cQftPPPgg|mQFuu3r8)( zc2ZjfF$GRdgS_U8fy?I*($-1hq3&sn39wBwdhZtncB|+G z%$L;uhO|rf1rFEOC{`ejST1uT;wyMxtEq7ewr;wy3Ar0q_{M`2Aqa1}#%}1=;Ho=j zzp>JX-&OwRq5XS@mqQIxB))#We!mlQTY@D2?5gL`su`ybqfbel)r^ty7&HVw6c6~} z&Nv37Wu<}gILe&B$(Yt*h7#K9vjKsI9lS=WFhdxn7lPJRE0u&ZH!RMdSK<6j(--&_ z*qzTsObLm5*tqg4!cUtYnKX`#x4uBarI*pycr5-zD=$|9zPz_ygcV%FFSQ8Idri}m zr->|-hVW~t%12=Lh$7jB!to}dNp4`&K>jxP^}fZ39>VFQ0*mFuM;P%o$QVgwvd z^G>Ioh-%PU%>M0>*rosm{nOiwR-PBa-TWJD1J#wLby}LGWtEOW2gNtzwI!+P$Br$w zDbqGQI!JhPwx`|6qWvP-A+e|GHVM@8k<0=&iG+=ek}yi;cMXgK;~j)Dj*>Ae&a5~o z*P09_omi*t#u2bq@i1$BG=6{gTaShHCE?NHGEGAb^Czb}rdkd-cM*TAVE#b%9Ug&( zvIn)2(BW1(%qE4Puik}h68dqwANmi}veKe|p{fDgd0 zf@(v2 z6Wb$eiYNv$b?ZmngQ!D_q;iGorVDXfI&@YE`5Dh;#%P0G-Ow|MxLo6t6qVQ3uEod$U42; zUe%uD>bvjaZj;(x6fPO7HtBFL%iZRDH%b5gkvFc@(kxynDf_+D)f#jqDtRvo0zNec zDFy#sEeEaQ&51m}{k;}=n1_u|R=ieqT#Or-lEVze~j#=FDOQ`qZ17{t#s zzKTdmf76nCAl(~D;QFj)x^%zmWr+;QQ?V4xpcHhLGL?Lk=UIJA=P}DT6Z@pjNGX@U zj!n7dC$y;9h$Lkp`0rsydBb}y?uMuPRbvgkGRAU`(OIKp+>J9KH>O29-gu-E?QTDx zpct?GVUYE@&NuqnefRCdtuBWXNM31bm6Zt-6C&RkH0GV`RIv}@Tc@>6Y?zWBprzME zvlLVZ9qW|s&bk^xTr)Uo=G`qXj?DRHW7e2@l|(!$`N>+()O5`i4S6S!d62-F$q_XS zGZ9^W6Rtmsh{OZcQm-(Si`@_ikKh+o?Sq2^*^D~;YO0@gm&JC8ByqLudsuU(LF7HH zv=Zc`t+l?35E|Se3tKiHsfakZ!}HhGMyW?TF9tY%e05nn>!*Z-rzE1c%7`FsHO5!!*E@Ujyv8w~`O+dEd@#bQR2^(wm#nBKU0&VF}NSnmQP)o-9Vz)F^T0E7#kx^yuH7^Z zAzzf@xMg7B(U=Qw-@ZG0&eh?E&&QrzB%5?PvNg&Q!YIqU{jzc0v6%O`cq3D>mg?X|k_l zuR{tGA-5hOd?G3LIX9uEU)1P#a7x?Hb zA8jP%=f@I~>>LiGxGO9+g6%X`K`x1zk^v-7x|#NkJRcvxP>~UO(OuLqclm>mFA6{N zA5C(Z>QrArPOJc7V^$1|X<>V-4ipA*R#_|_Vy#6YY|F*awVG{jNDdDJKCX0qHn_Cy z0poCzr+8zH4sjQfAPqu0_$6vw=(Un04yhGi<_l-8K%{qYKia0Xpo_K)p5e}has%<2 zh~GYOWIVqvo>1$9g(suQr{L4K^2QGN97R5PiXV%$U){P})1LS#_dtVFAH`FN59a@RvYKP>0M-c0W$mXGW>K*HpOsp)SgH<2`5arDX%VC!p)90k<`yQ_{aJ@4w!aY9+#V(-1p$&t2O9rrt0@5$ZzM2By3>mr-iUVVmb zse~>A{@h}nHvCZfN^!BJ#?VZxm5OPtD3sfyA=_8jA1Fb;?X);FxY^r~bk-m^RojTL zh6mLd2lL)v_ma9;?mau<+3VHSW7onNy$kID9@P&79!GLtURs9f-@G5JTa3`amD@(s zse~!9v<>i!p{Q$Y&a!dTiWoC0B&^$aDoeIAimSDyhefOd8a~~dMi^z@d#59+%@u0< zDSBXsoJ4!Euq(Dd5yyj`=}J;zD~1rG&_`WMRumFq>C0%A-IFu}lk}irsYjv3-XXrC zzty64V-2RtzzFZenAsPFxXCJOKY=&i?p82s*|=I|SCZK|Kf>z!TR-V_-6fKmyN;{A z(WpHCzNDikv~-{ha_(~f^a&g_Kzw$UDLaA8z`l~QE^v-dmxzfm_)~Z#Yw_CQ`azAh zvZ96VVq)a0=;0z)_jXnp1;gt@LPCl*4L|EVkUL0H54mO*XI8)Z#pzDcNAJPU@1M2y zLy``8KIg<_KTa~(QM2ncO{)@7DL{Us86oBZToiNVG$0zi$fn_8rz^vv%e_rz6*p4N0aR4X7oiCI-0Jx-$qE=og9BVz-aQ-0MEXs z#62_&+u^A4*%iFZO?j4;DcZv?ttk-q4&&;@DA=O?e zpDv@U9!2$fI07@a*ojeu9B^f)4IhBZk{3xTdXy-6u@O{GewW7nfUM0%$u92h>=(?;6JHrvEn=Sa2dC=>-Lah8CfPk9iy{tOMHSEU{ zyiCObt!Dv2<(RG1%Kbw%odRI-umu-;#JYP2x}O7{axj?{x{xszT}`4A`!8i@~qlRk|a;0?~yHs(Nc;)4hivB zj1APj@22<&Ev;5wSVo?9)Y5cwIDGq^!6qOn?i7{DeU5(nB+*<>82WlRpN8^pkd)k3|hed(h) z?>f1r$@FSrivu?{(~wnU-{ya4tk*Qr-7JfzbxUGRQ#@E1G#*!JX{ymDD1irK zWY*OxP0CPszgqn?Xkg2eOD+!X`q}w@U!z)cY(=PSo&GSm{jQ!zUqTh36AfZ7cf`Dq z<{&;DHN;vmCs8Q5CkNFszqF!i&4xTmR!Io$pM-2*k9)7~b{K*OCp z=ICydbSSYW5Thptc=#C>qDQR7z0Q=_$t%uSS<-fgD+>)%;G`eK__}c@g05QgSgbM= zL)Q$j+74+Tt&Q&Oo~6f-93U6xMR&>cK`mPjrVOZ~4-~v6B|Bvoq$s5ZoO5Z|Z|F1e zN2Zg?v5bt~Eg}-lZo1nT(t+%VYv|umcMD9f(A>rPLM@K309r)de1IdyMziJC&7Px` zZYfHH^LmHflZBUd{N~Z;*<}1y{ztjQ?GPdK{0dwO93c#iLXBnHkn`e*x{EwEGZNj3 zZ02fN!J?dMN+7AI-@j&i`^0q1E=|v##u=B=@-E-f>PXzT>@Hl9@czMD-cB)FDDM?f zB@eMuJx3Eil#DNN7kzh1IH`=-evjl8%Jev5a5?L|>$cdCAHZBq-#|@hB8L{gVJ8P( z?n0FS7A|KBe%IxAV^4LzVp-iYe3%?Qkksy@CC{ny0(Zb9SS)*)CL$GLqSfCwriL~V z74d$pN}!)MWtg~%sX9N-um8eW2+{h608`vJ@BsH47s}O2B z_15-Op5ojv8ke8`cyA6^8!*?VV=*o+`pez?uMuPv)*%r1W~Kh zf&M!YfZjjhc)N!c8^)p7!Nv026fQ;jJyppW*5&dsqx_YHvQR;)Vxy`lwA>iKmhGv?~pE9rG zHBKSsMLx3KV!~A`OgV#cUdW8ST=8=VW0@zl2N&s%?(MX|6}|RPY%$3*NiU{m+CJ_! z_NvA^-`&)?y|6Aj`r3&rG?Jel8LAOV=e z)m!vda9z`lWyg>(@l%1CNYRv=S6qf$CEMIyqj{97(f)Kc$=dQaDEc0+vRH21~vX?Ky5luEM|<7Mk~1w@&0lq96bh`%7v^N!y! zL?BKaKhPm_E=C-Ftbh%(#l|v&Ax%vEY9iuhK!0DSs%kh3V4pI|+R>g_Mzm4?skRj9 zE*XeMZiMx^xWpgqI7bTF`{HOe$*9wFP-17!ED3i11A4~dDN4AAt9V@U3MPpB9w4lG zeo5;y+(K%Cki+HhAR??9B?tsgp%%8hbGpxBufgQiGR#t{a+7B!)#kk$r#huBZ9}I{ zdkACs*hRC#bE#fVrDX_LFe4h5{v@NxWAOn@aDa1(%4N{?)v|BJlHB8Pt|Y7s#{Dd_ z8*+57^;uN`pDtx#Z}22f(+OhSv7Tw~%?)a*dU;ga#+s#%9(7H{79hbF%c>#>kG>Ip zTNGl>cK*mFX*&!j7U^*A=6pt<%dzc5Q}lOynaMq=8`^?b1R$sZ&dFO75Tc!pHC=7& zJgLDHWs+IsOq2yN%kvF>((p_$N5!Al5*gq1Vq>q{MfjNGie0~pb({F`5rChDyq{2k zGry@>kmdm3_$*8u|H2Ky%&ePi^|sYsz-MvHLRh!aQA%#K-yU7 zWjQ!bl&62i`@Pukf|vKP&p<%;FZ-iNn?l$<8<)XJ2DQWAi<-bnDakpgI?W#;|ZNx4@{A$&S(N?v? zzstM=HD3;H?6TNhBr{F&fs6A58`74&gK3o}*?xKb`$98XjpG~MzFu8lcQ)CXIeGZA zhMkT;Y%9DNSp9?;-Z~iYmKKzU6j;Mhv@x_00f294Y+1{ukJ9hq#%iG#W67wQB$K=* zQFGT!gKEu2aq#RW?@0gKa++zucGvaw<>Sg0?T`@!|A^&Ppv(73FB0j%OF@va)*J>!0N+&W|2^W%JDoblC<{L9#T zXJ@Us=6s%KK2t;j>qwpYHQa3y<~X$cz%0NO(;?r_9vEu>4Cl#Vm`7ecX z4UF_K3jT@1{~4iscTT<&xd{N!E08faCK9q_s5Ng+E*7?M>9N&=&qFKn zZMq`ku5nBVACtz`fKft<>U5VX#)Ot8H7t`4_=}%hQ`=_edO$lS<{{L=e!s>w;Km(z=beg{~2gScxO+u~wfev{|YcnBO{jo)yEVw9;@I3Y`2Q29SM$Spa&K{q6OEeYo?EGRDultj#y79 zH1QGV$%@G{nbTWk_pHB?coAl!Tl((EZ;m(96R05QEnC0tqS;t!j5hNqUmAX+I7} zH;x>Z+BPV~Y+=+@-@C!Ng=F!TS&|cSFb?)f?6Rzmh#h7dVWqAkawHDSP2Gg!A#8O) z0+>L%X?r-FOG|ID37fIG);_HbzBeB?mi@Dt&2d+k&eD@2dh5Fn6}Gj<3kt-JPR-kV z(~ayK51KouUmEdjB&Cn*;b#1J@*M2VI{3i9R^@SO28!IX3hX?ZP#-gJA z9L4Q{zSx)15(%~h4NlNWWBE28$VF@<%Oyum0t>F~ydUALYN`I9H3n+p&G1qf_15+Y z0&1E+B@5XtKPJjw^?opJdyoT(%VW^nq$i=3j%9TFzZS*Xm-1>OXWQre@9ur{eOA%C zyLUi8nY%kK_d2t%g&O3=m_QCMs|(Rjs9h8|7);C{DH9{2j+pte4@pfS3ramRyegLv zowcF{UvnEnt=1UcIb5i*5wrY8{A*xSU7+2zGcT(ko1&!XVJCW#Yn)k;R2v{DH&fKy zf7twA^g;f?lkNqY9DP->>lcMbC;>xGwbrypLe~(|JD3Rzc_$~6vq6M{)bzF-lg%=~ z-W<9>Vv*Y-IqxolGqGP*#7iQtg_6eBHrBBJ;z0h4&wfB00XUyajWax%AC}chYW1lo z1~yLpI$sp9Jo)XPHqL}eZ#}_CRLC1)JJhA1qfYGp7Y+JNGuZhBuSu^#Oi2*%(m9Xj zC@qeHPe540pS@WFSNOqRd`3oXw6$FoOzCJkpjp%f)NbTm)4p)h>Cx{VKYKV9tzyvc zAh|%u^fO6Y^RmCuW8&}3*x!7s_J!MSCoA(Q8|`>=THYDy7I3R3(g691 zqurW3gUlhTx117CQ|o3BTcuXHlRe~Jt5c_kdn~um=x)R#g(X@Jmmmcu35f2hhTJ%`g4hochvEProj8hyj+?NRz;sDv5S-4IDz^OI|P6*v{UHG;1hr%E@;Q`?If`{d1aGjrK+;K|T zG+~o{0a_8R=(O;^(hADY{;h%G<=`lbdqODmVkfyXHFsiar@>{Hmv}16>^-DTD#BV5 z>TSn9kylh8a;A?KiU|WIE&Ixr$UCg{g+V^gpTHO|QEXz1`ers|h91`QU61EDJ~~`@ z*RGzjb7Ah>=nAvMd5!Zp(&*?0<3o`8iHXWPt#{7dx$%sC(BN)qhKWbRLUX`pfVm|l zvQ3Cb;B078yd&#*unk1J71|6ncA~VqG*j35to-t=4>A_0D$FTF&{G%X4DL8)bWK+^ zXn1wu!;daE;>&Q?AL6M$jqGpOK**Kx5?extxJvdrIuyeV5fd8mS#mY_Jim{G-qkgu zVQZ2rY@49cvVM4ItH)-xa3FI|>Y8%D*3w2Rn-YSr43G5;4hFDI zF_qchO-UA$p8Z-g4@6+u9%sGxQ($^0KyTv zFdS|<`aTYmt}->^KJS0aq9Lx18|WMCs3_iSzbRkQsNEWJGvDXVaz28|3k_%0Nn9J> z<_<{jfIq#9G>B$uJ}19^K_p8|S)vgsMu^%mAFL1G=Hv9oKeQ-AbAE&7Q_gse`A?HQ zHzH`+J|0g7qK)zDFVw_E7q$5XF9V|dKm9Zkg}#fJjA~I8Y08z>B;Ud7{PA{1)&?)9 zY#fLN_|=?nY`U$I=nR*Z$wuMiyOF;P#2_+*k2;7DlwW4Yme+dwx2x=)^Mmkds|AmS zCeIGDmez(%(M|V_wmpsY>TAk&ID6LpiidjytxxBLUju7sIU<@eyDr~0S&idp0reO4 zNlzw8kAUsg?*5&RRgIf|W+hmmNEr#J8)L#^S8~<&5eB zmL=UK>M66u*GV~SOhtemlk{oyrILe3%D=72uwPq8YjjiQ3 zn-Ha*o@3Eo2Tg-?^PF9#o8bDf<^x3|@)+{WBt?U8i9BH^bi8?foLOrfT16V*^0Lg4 z+fCMV@NGvY{fixbp!vqm^sAK_37~cK*bR+NUR%+f~J~Y%y#w$hlmj3O`=pP z2xpH$5KcD1ui|eanPEvRbPh*T(=FW~{{bPOV|pJXt`G(A#~z*`G(QyebkU8UkK2CB zveSs}PapX1W&6Pp#)2U8n}Q27IIhy1(6Qk1+`RG)DrwAPQ2NZJ=( zx{tiW#blNmQOt-%|BT6*wuF4LfIyP`n$uR~CC*-QyY3)VeG4;P+dkFDAGbRKVtVtO}m zma5cL&brrGW;ze&-YMs6nIE7N`lh<9j2GsIM{nSJP-(fuAo!ptjc;x7qS{4-=HojN zX}2Riw0?{umYPGQ3_g>6O>w2&|9G`w2x8@Q_ZhM+R~|;5p&)gB`J!+^2MhRcYgtu` zzDW5wPYc_<3+`zP-u47SWVN`{2^(wD?+j?C%((t6CQ<<|c88ERQEYuetJ%c80iw?Ndcw61k4SFHYsq1B z#JIIr?Sj8UF^$XQ;6^U%-S{x}N{jPkRpzkT7WTcDGjTY@%hb>F)_BLCRaz!qd2Z`Y z)r|T6S+rcGyuqSLbGhwO{eBz=KI`3nTKSt}6VhMPPo6!s&*klJr@jsOuBAu5^@-^8 zxELjBT+WxEiB}+WSCEx3Hsc)}A2PF&8mtY62@z^ArOUtnrVuU*d9ZNZBf|e&({al7 zY=N$UxTACW&}a*K=EZ6TZEQ}Y-*7wZN*t8{&MURQ)K?v1q)4+TTT#es#|0rfIGPXS zdeaNGr~&!61jSBoP}4#%;a%u(OoE|{=$nmRgGS9;Arh5YYG|x3tDAZt_Jx{-@fF0X zj-&Ks|LNosYwP0ZITx-$mGYD;b2Z8jOg!U$pa>cAtqh3VB|iZxfgPbFqq0_%h<1>7 zg#<&~GayBS{`ojwN57sw;p-0{`rFBxtMPTkJRN{sK~s0GYfhU%Q{&=<54HGf3|~y_ zmOPbDZP&D~0X)dHUyamDetegwP6jMKY&8Bph88US8JoU)iGyHKbmYBUayL~S_uY!a z;Hp)>LipnzAjwiAq%OHxG})9s<11&YVPT9ne)nSIK;-d6fzlfK{ZQt%oL%-F%;9|f z6SPBD?u2TwFWn7b4M)-9sTo*ZGC!Z_M+1a=9sss@6RHj{N@7j)09xn}L0DS!GNL`H zBNim{jhU-h?fIPq&j67wuYJiFiy-QfwdR3dHwLX4_`Cj)8AU7d__T(|yY7MC^x`MZ z3m)X$Khge4Oa}2Tm3U)-a|xa0zut{`T|Uge7~&5nK0Z}L(D8zA)?&kbfDDPKw|_91 zv|Yr=Y1#MH*nwvvbTZ%f!D>K>of{{~`RR!NOYA~Xri$)2b4m>iB}0w!3S*<*%C>JNwh9yH{9<%tBCI8&IK+4qMf~06`qvk^`rS zJUci$i3&}Jk2g&J01(j1g~Y8bPB&~CQ&(?%5I!AW3qrXya9UVZp`tuK>x+VPZ|r%I zDOQJY8lf0O$mwZ6Mkqk;UWjFDC3Wv6_3V_I7I)kxQ39IFUE+w?9PhZYqCe zu4>tXeQlz)Yt8%4h5HfO@+m`=%WvPk%JjfSAq3&c` z2DZ&menT_El1WDbiO1kcK}5w6NsGQtVef-21Wa3X5h9!Mr=?=5^i)M-8jGT0Lrq7m z_LiLfpmKb%&!AyEy0Lj&c9Z=z{6q*GX;I>P;I$<8Rs$)q?CvUhH}PB&KP_i%(xpVz z1unCFa(Q}3Qm3t^Wu>=k4Z&ct{VYyXGCB<0rze~x%8qhisrK4;AHOjA|%v!{=aRdoAfVM9b0Hg%i> z1yZ?BIvdz0A?CC$T=IZ4r0v*2k1=nxUeG%+!lGnOBS%*Qu*#7JS0d`2=nHYK^0IOj z%Yd786K&$`6K{GBfPgwqS5w`4EVzF?kXGI%BoAjCYT^gFrj5}V^>>(%}d_4$^+IW0@uj)o(?%mw55@(-#`UOe71}CXcS{j#q^bUh$}od zEPN(H!1{_^j8t0$SUAAvxXqDhC7|lTo*f&q`eIS6SDyXoTAyWyS3E+8s4{51i?6iI zloY3bsPD-jhHN4P*t>*R<++Q#Y2e!5e?g9o`i3)JrMC0ovL%LV_`c>2Snx~&2{=l0XMeQMa)3Q<#}=`lpKO2Gfz@N@7RY((i%N}GRG z#%bQfSV&OQE&JI%6C$A2HXF$j*@u0JO0t{QVUS!b^pn*PcZnK1rgh+(!segO=SqBG zmc$NjAMfUJI!zpAbz*Nu(^Z8}^H=GE#jg8ZMQ5%r>=~x3|17^7e(xE-?Buz;D9hW4 z*XL)3XR*`Qon^bxDnJQ`e%N@+b|1OTQm!l2hc{J3%8!hJf;&65P0s*m5-;ZQU}}%8 zE;N@UJvGDGjR_0cl^cZbwlM76-)-?TbEBGJQrf7J%UjaEt1*r>B(+hs$Uh@9(3@;E zh#I5h8EWuWI7<>ruxoEZb^!8m$CCIA~( zX+V1|AnPo8bEl~Z*XmyA-4^JyU1(}t{cQi$VaphsN`If^^F}Xs6){Pgdbp1-Oi-7a ztI7MwAYNnyDx@%0<2!{cFdW8C1(xjX?lY%+U;bH?X}>eJG4 zZ6ZxW5 zdG_^Vjvv(lXCjpPz+n(~Nj!o{54MFUiPrtwq`IP9bY{z`s`(=pDMS^+u6$7{TYuz3 zRD~rrwLc?T^+8bT5#Rc{i^GqV1`5b+EocqE)*=Ol%VmbyEjRcWMaTODm$qQDM|oYA zA4>S}9MqV|0`ym5)>>Dd*HhLh8jN!r1N`dct8HeJ#y4ZHUubkXfLRh`ZZAvyRg7%G zytd4a&%PG#va%9^TYV$9;>_Hns@v)ith&gJmDAq?YJCq2omzy=en3=P2)n_6SK5QO zqE%T`EkBstVbQO~-e07GbZ5HsnVzWpGw#;G>$lquK)V6ECTE2!y36RfuuMOLp6gygvod$y9ncw|6-77f@ zw$EaPUlg*rik;NorVkB}4B)JDWB4^KsgD6o2s(pQS|Ta|Q}{?L@P~=`1>|Fkl)rL3Wdcfk z1Wp2?Wj;P-P1lX*CBjS=&2?@RW}9p6A2pv`(U}O0;F^|Iz^R`v(+w$SS_f8voH8eW z2Q@_RcirFdElZzJa-~s;GUD~Rhpf^%Krg$Tun7apg#8HIv=)4XPgY&BH5*{7!4~Zr z{!0l+8S=c#Ru?TF8QNcpxZP}=#Vlh^jk zsK)EToNcx{$uIu^N1hi-vb)6&nC~Osf5tYgeP$n%t$;9b9;g9U*X(AHqDN%0E;zwP ziqhKAkOwCNt!lfI1boCu2Z*|qPED+;wXu*hG5x9C(lK>M4YxmIO^3>L;p>&45@%S0 z9zmFMEKyhXJ6^K6>o%@3V_9Y#&^l=Fq@|HlFor)W;w*cD-?b{E$+rZ~^O#;MiTl#r z2P&II$oD|cTdB1*~Snyx(kR6{D2$sKVfoADP zUGP?N z>O?As%IeLq8iaU0CdvAGp>}M#)VQ3JXzjx9SAiMbMQ7aUv)QrrSVH0p^C-R9A)aTU zGxXLo(648+>_ONJ18}K8Mah|P!)J0HgKS+5C$=1BkiJKCD#h^9xIN(Ig)XCom`v8~ z*oyEC-MQb){6ZjdrnOxRO=n84KM+dk&O}KK>nt*20_M$iPF?wCgDQ@&--7na6uTXD z8oE`n(thk#g~4P?a7OiHcEfA=dGbs{+GhgOyAes4Kdz8HLzIb7Dt1T8d~EoYnbDjqCq zH@9~6C(3y|4fY;5OTP66fRY1FiB?j~DXK2DM6IE;H0IPbx0W$Ll=2zvWR3Y?+r%i- zX@Sc;BH6se#d+9#y=bd&7!{MPJ-tv-=si3(x3OCOTdCHIQR2d(I~C4vsN1sKHcr&F zMmyg_-{L$(bW(3p zm2H(HIS_{N4bfiqxMd$d2*F>^kSfMV36s^LC5t4 z?(}e{-U$&pd(Aisk6}L>eM|XCq%!j;DBw9r8yTQsR=#(V+~$uCQ5Fgj%8oSRw@ewe z94@vgV1!wZ7Iv;7O9`)FC*6tD$ z`O9lWFBvH6XDYCd#!BKvH%Z^R)Jtv0tnm^Q;On%BUJtTbbsPs)tiLMnMJ7qz1v>N- z)AQoExnxr_cQW6`uf^p)?^F#}Io)ssM0aN+)9AhLw&h%zOS}`b(0%q!Xw%y$ZvVLt zQ~jE<(p9?hPp~6D0KZ+jbBGg$Mg*z^cn*>f3;q_VL<`VqSN(e(&OfK(;znbt6d3_D zvOD0sz{Y2kPCfmB=udIc3D?eXh8gBI4s!0|M{Fk^K4C0&f@wh6MQ-DOxTFO7{o(HQ znIMpGn1RQKuXT9)G%?Z(+ezjamEZuFff+uW8&Fq_+O`<17QR+?{+scC`7OPubJbob zvXPC_R_*LJk5p;jDKU6oA|qm0VeKZZbK0=kNg-fRo}!%FK2*e8NHp*dZO*2 zCC*+}pRqGE*F{b%as`h{f+Rm7*_B9ei1y z@9fGAEXeYp7`q+}UU7_G_cuOHKE{I_P`?2IBxyzf`B7wBpPMb00| z!eqI#a{0Fpcr%v9GPmaF+#b9_g-V=oI?rBKeJ+l4?hV?FiRyoAC|7w6Gob$G!6+HC zxX0ESq9*N@v{QEw!*GU~tYwQIa)1=lR3ZrN&x#o|sJj-KJ{xWv zkw(1h{!UK<#(9hp?b5&N)U~*2y{3`F_l8g(nDKokYbVTcbiIyq8!mDs6-eUq7#@~f z2AkiN^u2E&aUXn8sDsE1Yu6s{G~)1em?Nuz*4@R2^91Of+`vUtl=6a}cEbQOTT$Y% z&y-ph$$j^#xN<6RIwCH4r-UQjB@`c)J>*;q!YwFtVXD?dkLZPBl`fv=p?84 z!%#sjpYGEccgyub#Qx5Prm8W@4aE%8vUs|-ii<~M6N~9y85l4HtV?8S17T@phTGht zx`y)o9CjyRirk!zuUyZKb<%Gw9z=E}Y;g5vMiKIQXzw(trR(5|Nk zx#m>W(CJfrJsp^L4~skN*WE2S3%VxjKsP>0urF-;td5P_*4dy&=k5Rd5Mt#V)%^cJ zh^NnuR2aPq>^tGq>KSL&KZlYlt3y2=(G% zd{jsO*SrO^hS@xew-XL{-bA!Sq{g_qNxGW!9tmWJy72oUaic)F*A?T%Zeq5w^X9%N zP>?Kq2O}K`oi@g#HfV5XldhFZkwDe29>MLo^@~14r{=H^5|jm~j(E`aWx;2aJ}zjK zTLa7t)1kMWEecC-vE$EQmRivwigF5ZvI^T8bDW9c2D3czU0MsxsK1cwe(vHk^#^%Q zbtkIWF9Lqd&+mV$(}8@#ZkB|B^|VE>Ht}t41{CvO$YD@1@A7~ z?jr_19kD)SR#R!Qi;%n0Hf=rFf-8%7Tpqks;Qi?J!p`a7#e!d57RW4AmyMw~8N|dh z@;|7LQEfFS(zm4H#+_22wG$wJo7Q_$x+I_^GS~>%S=cdsXrv{Uuc{W;m}rKJ!k&8c z;~3EkU$HQ*5yxWJsk~`A=TrDEve(F2%xE;O^wqhL)A&L{+k#Pjlzx=%ho)8Z9Xxz! zydxVa_5!zbApL=S9GlS+#FtPr0YC~=06-?$(ZHoMx%pCv;3+6zh&Fi-zE`d<0~i75 zo$Z3=Z!cAk8K1^S6RIUPV`VC=U#X4je_?&!feY%9+SnQc#CX1INaxTZc1v^_f7v1s z9AHcijDx6MO`4Q@#V(Ccb2m2eRl+=mn0*Pei6>Tjr$QVWo5J6XGS{BTTmQ;(nrkC! zx}(qAn|gl^+45lJ&-$t614ICle$yoiOSKM?d@X-jsHZu;O4(^wM%081lhYbSZ{c>FaC#-6y{?cU91y3V}?Ea;YXjdM;r)=E;<&|YVN0oH}~%8UVnEhB;H@_ z6wKv{Kd8CeZ~m`Rx%_pX{}QE374ZkrTnd0V47<=frBV0oD+rjjCVe@F5$TcGKHY)2 zS$)TD+pJ6RvzK&M_T7CSPf;e)Mv|FikkP>8l)LFA;<;fuxiA9Kr2HSjLp ze(jF6B<_*@OMXl~(WF-*N&^h;N+%-`bXoY}w>*Ky#ep!$o+w`tN`-CXf0&vj;DmFX zy2kQDS*`J%{=|ljTXF?xYBnX5U(bDdt7?cI(lAD3_a4E|@)wsOo(%dXDfR6f0t zUXzv4p8gK%9KULE=DW8A4!u^7dXE@xs=DHE=PjcggKH%%SrOMh9x7GU^geWjw)M2L zNIbgN(5Us?v;A(D57wDgj}%2b?H1dsaiF<=*f3VM(*|%oh}dPo!}a@9;A?YI)n<9R zUImy@()`zUoTGcaH=IVtgGqGqvRlikV9$@DKi_peNapubA-9!{Z{sL(y%nzN7ll$5 z@^3GC-c}^xeO4SfQu2@oThhKL9LumrNl$)~D@}n869hz<{02ZKl?u)|g6(axxq}h_&tT^*G7twG$CmbV>8zFa4 z(SQJ2{b3C$wc#-hzbM#5+9XSQR>*IWXS=U=kY^UD(pUQb3%}m+A(bFoktQwfdG#7E z)deN~@15m^fB&FBbxQ=BKbAbWX@m@|lLn~LwHmSN3bOT!!oL#iv?f87n{=Q2>BJpy ziv7)#qW|WV>a`~b*Et^%H6d2#IZzBn9=1CZ`*(j6AYA|EjlRaTkk;j=*0zWJVjT4=u3V_87h0NSRrAOpZo?Q#MhU!wT(YW}6Yie?N-^tJAm%sl{Aw?c&^+n<7 z8amL1{n6R@EhCHcHQWW_Kg1a%`Up@Nq{E;AO<9POuMLT0- z1+V;@`|@vov-78K6L%ONwC|dzLof1f^y;5*JGXdhxK{a^J##hxbEp>T`mgyVReH76 z6$b>CnMRdg*{jcz_plEWTxMcBuT6?Q;$>3lM?52I zWPe9I4LKAFPJph@_4j!YzT1-4v9g_jI@RJ+(BTUF#;7VULWJmaz2nEP2C$#u&ew(O zAqUpZJ!^QFxM-4r`>^Rh5{3^w=&xdq+f2w$kOp48PYB`(nynr!`ej_WaiD0_z1U|k zu9ft*gdO);*9;;w-S#|JJDfFkh~=Zi6kpy+b8lY?_H+5&;8}%cbngGcK|}i=Ndo(y zqvD^V;-919pQGZR!r=d@VE!8?3ICLY|CEIP8DaQ81q^t+hKCQJ-X}#wev*EN$woh! z%W{wPCGhx;J&Y{lI_|4vYlAr#QBS5cf}h|$aKiXnXwAH5S1kU(*vIdEyiW~M-F!ab zK6xwt)_s6ys^>jl_rl`s-5Mj_Xw_`Q_`4pdMxoq2UvifG+KNw>*RQ0q))Z4(>_x>h zsO~z{M1c6-s2N9diDWn!gWm_`4vNcYv!jK}jo&|I)2KjBWp-APeHD;53S&%r7xSf7vvjAqaiA2|WGw#0~G_7-@TwKG?vN{G@Rq+th%%1}Rl zHu(?hQzGE6m|5>iNomn7U-9vs8!`Q{(4z6u$<&fczeUTm+gHcxqAX_N{`D3gSu)TK z4v!-G-;P8Ibp{V#HfM7_sd`?lpNy)^b!`}OYRD5e8TZQd0J3##?TbQIQ554A2am`q zwH5(o-BXW`!WqMM%wD*dANCxb1_f4nP*lK*-s(-OgnI=$>Z(h`>m-wLYqyN(Anf3hjc*tji|d&XKI0rXbBTC z_DBO|ROxv$$5--B5ha*#5UE!~HQ&M9Km15Az#0Z-G-f2=X;ns^f*s-(Ws9imb?$lT| zDPUB*sS9}Xrj_~8`)YF8-o7+L4-H6r%9I*(!>r3yn;0^{_+tdM9$H zxylyt;gq6Y?QgRe`eya`hREPAx1A711}IqxsnXHkbtYL@!^ zEbrs7Zu0BX(vTlE4qwF_{d0KKy1gjk{RsPB5y|y9HJZCfTKUr;JTxFS=ir~0zs(IV zB82?Ac2~>817j;@;SxWAOS-oGW{vQS9h^Rk9v*xMzq|&yfVLk#@3n_9asmM)W*h;! zjr#RGa(5MMFUoTUgT0So_yGx4J|j{SY~8zk`9h<- z5C=a5DVBPj#a1FK$F-_Q&r~e=FR>d{pd99=Ghh{PS=Idkzr-3Y0qlnAFvs{+`0Amo#>NChho~|50 z^rm#~oFH_}G~@}fN%EsZF$m>;@I8FM8yM3qw}M+M`^@QS>1LD5hhD?w(<(Qz-!Y4- zQ-3LqK!>q)fD=5Yi;_022$#)deY#Ju5*Q}80H;vpMHSUk`qN2*ghS(gL)E1dUleZO zejW4qm#lgy&=q$gcRJKBBnJ6M<8qLQ1uEZ0G^?kLJYo2JtV8TSOxo2evUPR9Q``m# zE1i1A@f3tV+}pt=C$=1u(P>mo*wYEBl5){0E7o2#&~f}g&mh4rU|4&IJH>Gz&(xt= zhzxMmBHm;C`yL5O{^y32y-O691^PQvZTG?%HPywj_X%JYSi9?{_NE=r=${ME7VChv zfy6m1w4ri6Kr?gb4O}YkI=mLnTw2n~cCpND&NrIOUz;5?V}#*2EPQ~Ld(Tss?>k>s z)`gDP+zog0j?pDwH$Qmlh3DOhetosY?&I+mY7?@r-LZY$b&@F9M|>5${@;L&z;=>? ze5{beocpUOV&%*rTTOal$ecpylh~0j5?#im&ZQdAdZ!Lq6{mMVM551_fFUO_tqAuK zDY`l`4x4PcT5RJ#=>jeP_Wc`QnQ?6-nTb#5M4`Ap%elb^L}|}aujAWE>Mls|+69WK zY>fbV*r{d?YJAO~I4e#E4haYs7dYEr{_A;fx%P^f{eq!*~^&8Wr zgIBF=u*>=9;R0S~qyH9q)XZ{#-9U@G`JG4eaiE*ihAimY$HPg#G(xgNm=T(ET*&E|i!HPm1b#s4V1j@-4Dz3?^$@1GYGXJE*~aPx`e#ZuhiM^I z$BUV=N|JsgkJ;11xJmuSR@ttc)6PhxZnpNM5iEyBtwSHcIpfbC!@X{RezoKCbOQau zOLvX3u)9wQ`ZWEsyhNMde}77hR`IJ(eBpKL&f(%~SDnI(-f@oMai;R{^x5e9!|hGc zi_XZs(gJJ5Ew;rxSv34Z;{-b z5Id8flMkUBM2y$cQxK<}Z3#_R(3eVgz$KNi(g!fh0}Gs21MI7?0YSmR^jqKSzd2SM zYI*B+*CEUb^)U^2A@~w6$-r)kVEl&EK!-{gso2eIRjMkvodcI{7~V4b%DdQRuHdvC-=aIaGj6UI&aSiWQ(&P+y}H|S1-JXR*x?L-ds z8?PQt?}2-(2wA~H8cXd%kxhBTgn`p?KEA*z&GKQ0JzQE?X(F#7;_Hw|Cpj>s!&7;mX!bCKe* zvB*NctwIt)da1!{uvk6`#LOwSyD8#iNS+B+ot`ONB0wnTyCMI8r8?v4TzFucAbOl#*H8IxpnuCn>q+u@V@R+*3!_kh>>N-K)jL^c;Gm~ytv{1u! zJNoW=u9JzX+XvpTin{bqdD|C-y>j3--}V#fd-?PRryJ}lEnCe9FrEyj3!L>kuqo7C zQY=JvgU`n^7O}fc=rJY1lh5NgX&;C<0ctB{i?2P2ZCl(~bi9mQ6RA`i{_%D7@8@Y7 ziqf5SLRvS28`kkfLC5x6Vide2>JT`l%aMzgK8YsQHozh{Jg%@qE1Ikk;JZ{rY7VPJ zCDWx&NJ?}7b=%?w7?n1Cv}I^^pvB=r!WQE(7x9(*m4aN-7X|ZD>iLPO%Pj@+l)*o1 z$7|D)4`K?{MBDP6+=^O$9lBC`@bsI(W}oECZ^xwc6VyR-^^e%)L5bTbiLU%5Pm>&e ztm+gavUKa<;h!OP+V!ET(mK#9vgD6XAsXXr_UIWWRd&#qj{9(jCS6os3@)SOeP*7Y zFLQ2DcUT)ri1HC0zX~6aXq42o{ z7GXwBX>6KJf%dIR5-jYqx?(kLO`vzPtk_tUv43rJbzap^>2r>-J$JV#E6;hHwfrgS zeM@$oaKCe1*)}iZv1}zls2de^6d=$MeWG)Bi_g;3Z(MD7r;Y<1<*PoQ%0S52ENezb z;I|SnGJrk*+`j{^MyHVXvTc2(Tae8DPD)z!kP8Q~2)2@EH4!~(Z4UK#wXj;Wfmcww zV}zjfg6n1Ou*6OGi1iV@!W&-)E@qg4sXoiDfB7QiNfJnXaf9T)IcQ4Gy77h^y zy^3(R8*-_K=`wA1?8_~!LOd7c1kOv*BKk3{qn}$LP66Fxtrx`7*M#^^qz36*A_iJX zEfGOKK$L~S1`GFV{i_>Wi|Arj9 zkdug+G^xg>ZsbE$3i%h*En6#Kki5;v42RW_ULPrg9|kcPmY zA;)f)E0-Ti@gl6$fGfYtr9G#>f&Ry7)l=pLB_a1BE!|ouWf(j^8*M+pSUor!!3>Em zFz8MEYjzJ;*%@5#y`97q-Xx}hR$eWjwJyqkz=N^_r6Hm(3OTE&9ri-x#MoSyTpzy! z4iO;Ii7xz@XAPq(<~G>2UvY8A8$ldY#L+8*{0O8ur-292-^YjIJ~m4*mqy2O=U1L! z{=l0jCKiq^UQ39V*hb5$jN6}Zow47{NN5^O6&tPPoPQgyaFIfwr0th)8wC^Dg2&`w z55eoe)N$Y=@mNFS5P0d`M;p+ZQ5?MN&?Lzb2`X~5GzngUO(DaJ^6aU zhrP4Z_4pSBw`ZgDU)GvQf6ZLUmN*l@tlor~{@<{QcG`XO}gLX#Jp8p`htH(NSC z=JLMzerB;$iBWpP_hHHi-37v)b}d8b|PXUTVA4(A+|Cv{Pg6Utf>XdN5O_YV8Z95y{u_P z>LMWoOM~@`ja?_60C<9XgN{3mkF(;*Q1!Iufs_O ztboblr{fpQ=!OA6RVSEZ#BIw2Om8=MxG*2^{qw`3lM@`KL!z15m@D|^> z1bgu=7H#4;5?gX8nrBv~uBP1G(f(imB5{SuIr0}%7TAG$pc#U`-{Ufln(#USN|TXsy zd|(SSP)FZ5oyQ5$D)K)0cd6|W`DdMMG9$kmx;~-xsqELmim#JK{;CAj(Z66|Ls~a_ zB1}k4CF6Kmr8XS6X8PC&@dniE#bynWHiHqSbqocU1U^SeYBW5f?T|my8uF8;oc zHkV0^i4yOx>lenCTn!z$K#3Gu(my#XPfU7!s5KdtG%$TmqM4y-owMeL;{9QN%qF9% z6aKf3mxX@OzkM~z}{NtX|1sLN*xBR5j8Lc zI~Z#)6PStTp$1B2$4opm%aW7L zv{CIo&OP_9dGAutkBJsUUBe7sj;Xlx-FVC%^IpUiup+!kLsVwYO>ivkkTJ*?Fqud# z;M`4y5FG^h_kA<1SOF!y-r7rOl@P#>?VQqSZ?f+s9T>Za>kYSdim#Gl_5tnE3ZE zQq&-Mj$|UcdJ)Qmo^GU)6a@u$W7SU4xCE-}u)zq+CZvmG&@@7H)VNJsBpC!`xh(&J zxuM8CF60q8-P7xu& zU<7n1FNdu;5CtGTTeww(=*DWYP%4X2LMz@joUH}qlTF(HSAAZ8JBH>PG|yck+#uSS z-mJw$)%mt&SNd=-a!r~Zxdm{K-nzIt*xP%(pm3SgV7YkdT*S-ASBhzA^wTMEwcR6& z<4#>yi5|C5NSqO~qObga_b`Bh0P;di7`LIfpu@2_wqHTKZY|r*IMM;g znfNnYS0hC|73n<^^#npeZ^w~4e7eQ{TcKC9iun!u`qr5#{pcuVpG4*NZBa3Gix$^t z%Z8$B;!0T(dgeawG4Ho%9=8Kz@VrzW(g1dk6|QWR28nk3ejg_N1O&O~#%=e4k`Y=- ziZ;>dkZ*(XS^2DFZC-LHG@Vu*ykPO5z#_INCHWZrV@&Qa9z_Nz!s*RE%#;Z=RQ0heE(7gIBh=#+Ct<(Ufqv(HD+ z(LAj%Kr$xkz;)1Yr}*JCf!sqySfA(H_$vwh&Q2^fZI+iD_=CP4*6^vTkNJnUuj%hS-M}9b+z7+ed^Xm@xS>H{R1P<{sSctT;OQ37Pdp z;jN4_Q;Zd#<%F-uE^+pHV%5l9NZG;7j6+U~B}7YH6Cu+A^I{F{jgCZQY|OQxCY}ht zD0HK@`$uTX^MSEv;l@dLB;B-y-?T8y4C7^%9%$O$u-pZGJU1O0VFoYeqm zq^&b5q1diG$6P*4eX%8B<}vkSzy}hV62|VyCTwJL(0SFl1hwv*BJ#`>P{U3ayweE0 z&~DgmiP5Vifvyue7OV7HQ&;3-QaI}olI&>T4_;5R@@A82J=vVt5=P#ExG!mx?T*H~ zJlg`U=V;^pCR?17`C7rz@S7I~oIW|5KcfbEu-}pIG9#Jb8oP*)F(X?R2ck5Xmyx@t zs#7I8-FPzy5i8CY1V7}f8H0p$TqIw#HG-X+YVQgYYE(>M%Rhm<+Q(3Nf~m+&{JOZE zcGz;hA;@#=6~|A(5^J|b^Fh4p(QciZsqT0_QWc5~ZcQ6FkU4Bdz3fsqVqw_eTK3|J z`EX$Bx!vbKJiNTs;QSsPO#dLX-y=DN0u%wy6$LA8-mh5REUQC|W9PzcV~QfxW|I}k zbG^j0#L#Dcc*`}X?Wzyzt0$RgANvX=?P(JtdjdmUi^!_^Xlj7J8yv@jy)eLyAO#m#3ZbY-m{P(Re|M%vk z&RYvgZFWHc%T@f>7P$I5{Cu6;N7^r`P=O!=&bwX~87nT~ zEjWN8Ua2*)_$OAqD{tv!z*F`jdn`G=lHHJ#Ikx82NF?h;RUUWCa~pbUWE?{)%e|gI z&&+;s@t+ae{eGTjUdFy)-NyZHKE!%A^@q+2<`&LR71QC?k8XJ3IH!TBlGq!`MoI5| ziyKF)Ps|BC?}(2$sB;ezB{^FFU69$XbQCjaG>FLa?}2ZT|C0IX6o}Vy@6GYcv!Fy& zE#y6(zmG9m(SiNpG|D!MzuT%9r3uodT4MDKJV43XuYwP!1iwuj+Kt+U)6;!Jll{@o zr|Dr2cDj=IYRqJHp*zos{#?9+N()$Mf!~#TBvds1P2mGDO&y!&anyzAPjq8Zt){5( zpdn=qSOG}oTSC1vRfbWr0cX&^2bGD?z#qk`^Og%F<^7$(zet$r4H-WCgXskEAb;$k zxZ2~SlcfJ0`F?%lI7n8qE}f5jc5fxkE+gXO2Kruz8Rs@1b1+q@Do)K*>N!3E z084*xqc{_o?o~i}3o-T0qr#~8607Qm#tSM-$&!4&x^N>fiM565BtATqd&*6atlQf5 ziA1Lpr5O+&W1B;a2I;XmAivc*ZaVV`H4q*AQ^WJ5E?s-uMiS<>L4HM`p~I5dj*{E* zMsP!p;Z;BWmn-&bJe#AAv5%1Cf=JWN&+x-d0kQY&Z6qYVe+oUmx(QG;!#JDgwa-9; z-gWr>!}LHwzK5r3#`t}zy$>ZkM`=7FNIE-i$Y3OzjL#FNTKVK20{9^iATX)HBmafB z$HjrA(Fspum?-ja?gCUw%(vO6aliPpPpibGCnKT*Z^$YU`wpg+i)V9Ft*CmP(laRN z(x7K*=cgx_n)1rvcZSbK9(mqu4IX06dRqHuKb~3=&c#1W?Hx}(e)*RjWvATreKrBv zd-ML;w6hfh!Fi9)dcQ}z%6yG&&FM)Sp zhz6#RzENY-4KC(h3~Hy`fy+rFXwn?_g|A1oRH1Z2#M5i{qZ)^-BBt#wpteq-Z*D8F ze+Na=)9+n(bW_kB58^IE$3bURtROP>$Zes|vR^^k2C&pKEh;%-3)_Fk#aKd z`HbhhII*8cG9-tNkW zlw-%)Z}IPo4&*)Z_Gopm_`0FW)a;v$S|VBQ0j-J&%B@8*BdjsESGY~m`_fSl>NI{y z#dH;_A|JrCqG8Ev28Lsf#}*5>(enhnSI|uV34^^L=N5LVVVtg4TAfi8ESp7k9w9rpLw1V%^;sQKEN05bv7?2bw}8R z39H*O?|*nsww2@<&@5m&E~?(e1qZVzpJd0M&D&_dZ!XTYjE-zQ@O49_uGts1f(n3H zbTlY=O#M5V6XY)DzFEanCRlhWE$A`4BQ(~@x0UOUfRQ~#lw!l#RS#1ekc9W17c1L% zD774>I5cH|lxlyHCj{OOz9V!e&GB3t)gpkX|Do(hiO(?a`=KTD47K_8Z|DOBi@G@jm-n961eVz9ZXerlt}q8+RYC zIxrfoM|rlpPj>FJ72DGRj_vM~wN5!dURZ(4eAI+?oSqEb$lv}ho8?C=9deOt$kO100 zm|SyrC$CAaW!A&_{_BPajy~=t~Pq8oiWlT36 z(ny8Jsdc!PmX>L=9u%`iKnoV~|5wTjEQciao3H z4UgfYh@~d{>#rNunRtWo4?6$*xAp}>MENfY#2FW=%+{%%uTGYr?`XvtX7ViTt<(S1*#l(pI{XPOhn9R&3BX?>pS$RvApEC>#v%x%#9S<>lp^wv;9I2LE zA19^p27ZurtnVnPJp4z{^t^%oh#d4NC!L<>1r0`$gRv#37Yz^Y&5l}3><#?lqIk^Y zl1~EACr_!ad;`<75*?M^gF3#-rUIOaq8_N4D%NZhw)PiO@T$XC&VKa91J<|%T(Q%(G91`dMap;W&|z{nhDiDzfk91Pg5N>uJ6dC{r}$+}iB_ncIEN-cN4W&LjwZgj z9sjhWUV}Sl;{>+RT;2U`Lz}gpnq9Z9W71_Zx}|E@G-}`qyRobig*sdLtisM(aHz*^ zQ5Y}~h353-`3{&2k%PU6%ySXF#BvlR4vDq5Co;upK!mcj2(m zyv);8=8m%g?Fe!MHf2tl2=&6drtP}cwmp;Ei(QiJHEVIdCOTNlJfU~%JCdvB#~z`_ z7(X*5()-2Oa)A%4%Cdeq)wlfd-4c96P|rYjFdG&9-2Inq+vjEes($IuYAl{so-Hmh zPDv-3-k2M!@{XEbT4wnNfYkCV|4+tXWzR#al3N1*rBqoU4hCS#9ueCnKtZ7a{KpXT z5pw|BN2nor1K*2vIRn*IsHc3B6YO+cOrH&_CtI%Te}<8Sb{5kSbo7O z;o!Ijt9U(L`j?{i;@fZi8euc|w{V>))ZlXxVYRe&^=;>wy@WJ_LlB3Gb?F5Mw!JrJ z8{F}yusBQ_AuK4cOKNbuwuX@-YiDZ-(1qr2R|}8!h|&8z`Y)yCc0Bm)As7`6d)UH5WP^`N`p+zH+QaL!-xuY(We_E>-2u*<68R)@ik z#9{s-uZysYS{<$L(ko?6V{N zIYVm9?IN=M1;H7TI$jJ-kWWh%gS{%B&q+1-PP~}}dN;7VlQ8AyKyrB|tqf}zR3q8f zGTNK6OzO!4QPUekLbL{`@__fbdDMRC&~RVl$aFPxcKS=hs6&0@8uGUsw6ojRg5z&) zWm;S}J>zD!`85UqdFF0DI^H7aVA0b$9YLOf8n%ZC5F8QYGRdi zoCgoK;oRQjxd(KY;;>lYBFJmK7@@g#(}Tj9YQOULha&+u7Y$~!z zH?yP?8^Ed5@c6?QOwW6Mtbr}{R^v z1HwK;C}9G|Q}5I5r?1DY}qjz$HY>?%C?TxaHhg0xIW z^3{`P_6))6FY=hXddc&J-w$1@zrS1}9*|vlQP|gtflQ2r=92s#l-P|pu_o;k6T9JQ z%pVv>uv1-tEsfNg!R>=65*;2hbv2)`3xe}E52h8hihGmkJ{OB;PTuAtx1iS<#OP;i zPK2)h^90j=pFlIp$7ir9+QUXy7^h71Rsm@)MGuZQQ^1VcKUR`B01Q#_#Pd7F@nLst||1Vg~?N<&15?xMt=gb9*#S}YPWWlpp%_1$RBCVRU|tRt(t z;Tx{!Jo!VCANEcO`u2tDD$w5I?a`~2vErr~-IYdw7f z_Czy>ehA+{CqK!0}vvpj|~|)B6GO#01~WGpWqhCu8fLf_}LuMs>yM+*khC4)77s{1$LI`yy+$ zkZ?Ks8DP=>Ps}D_77vtu37pNkG#nwk#+s~_9a*?qKi^Z|mPJ17crZn`^rClGwnSTc zi)^+0#wI2!VbZCbxR98p^k7)kZB~VCj_EY4@HJX z<>#(8i~FH?&&(jq+18vgz)i)G-c4Yz*TMBACMim)=Q9< z33I0V~*vwxX+ynVuw^gxTh#j5Kw4$Du7{|Q1$-A5dMROBssNSF_-AYe;w*BzI% z(DHlS6T0n%u`#u)5~~b~)0%c!g>s0oUs2CQC&|#*$33tO%!?vb$oDWI6qFXV#~;fNBix0hJgIYDBTOx#Z(R|O(<>-zRIQ-jY#C1*yWNZUE+jlU_ooJPze$y#Se zzXU(dvhnk@Ym~YVfFEw)(*& zwpC_^I35TA2Jb215Q2>RxovzEeqVd6wxN9bF`A^+;vGhyq}4JdGI~-&M-%; zHjUKZzUFiKIjh^x{6wk9=0Hx`8=JP7X-mTgF-j}{?xHC7aZ!(}Q|I(shJGBCf@gf3fNy zs!uY0Kgz?0cd(3#^BbWbFT-vOe0(ysQXGA7zDg6ql8;eJICoy%&y8EYa%7ruoy4N z=DUTk(bXX10F-I~`c>|FblZnJH?Lg}mPrv@TGxP`z@?p&jM!tDemSH}?gQQ6lK!70 zv3bUS&k_3qrm2Z9BjOM#Om00wj2zqr5?WX*2z7|?yCOTf?N-2OHTU^2m=n7xD0m~( zLR@(nAxaW>9=)EBr7=Ad^v-pk_L#WO^3B7NpQEpg7#?b**~V25KVtTkoF1mnUo`P6 z@;m#b=9tx>cCMA_=pUs8tP8bdb4-MBe0c$r8fPZ4lZK4itAc$>7XY&s--7GanPG#( zJorm^itQ-5Q%#3UxI+j7R>$}s8YQCim5-h!Hzc_|GrNcFF~ujjgdebpuRct9Km;yl z%hbF!et+t(BlHu3joLNV4Lq#l&i|Qn=*DRM?JpRE5Y3UCrkEC1#7bPQ|IZnR5n~DI z70$jq=cg58<QOp#A7I&MHT^%e$y|Xnt^)DLAY|IK)Wp@d zkv@m!B~84h-adwA5G3nB=-^J4hcjT0;P*&cR)eV^b6al5o!;gQu$M%-QDjKu#UGk@ zdvP{J^SLb^i$)pKABrw+Urbm51Q%H-z&Gu9r-L#j<-V>&&=iq}fbktqF`1V35u!v-gJ2ZIP7 zHr2<&)$`}Z%;f-zRT-7Cgp?K-2`}6F{uEGXQK;s9lWc4|`Y79GZ}+D|#~jgpa#bs| z9co<->}dl!(q<@Eb_k?DRD(noq$XC@i`tF2O3WB5+}SAJ1QIST^O1VE6OccjxDRqP zS^3Wj=04-65e%JB^^u+Wz?^1jh~}&B4v&rVO{pkfh>I!NQBpc%UGq7&9kx#%3ZjL% z4Z7qdisZBOuA)TwBW|yRjcs*@%t8fUHvnRxHSJ$^fZSR9p1sy-)}J2F*|!JhQhClr zyFy$!Eb>cP=@z)A-aA~}Fm5?u=EoWguOE`*-vrQNg5cAR85>{ElQ%DstAF7}*c>zZ ztxBOgo!gz-PT2J^*ceJu=|t{? zF#Omg+>NS*ngupMA&~qG_=TYc5Lo3Y4k^{e`QbI0&l$tCsKBYz=H`ahq_d}T0^Q_E ziU&+IQU{AvZmpB3^JqOVwvIGcV!SlUSJMzQGR3BdG+4b&4`op8Bsbp7Jg~QLlSDd2 zyr(e*J^r1FuXPwl#MW|Nvg|hXq0#~JZL-uc3)rIcYNX&Lc59M25d<$0L|+7 z2rYTAqRJAJu49Yo9!vwtIK~S~z_`j#?!(;}r++7&LbgJ8_{iO;`ClNNmrY6o@p6r( zP2`PAY+A4HXtaBNf9v0pu7zfw{InZ4KKQq^+_>RCg+?LGJuf(im`>~qa@$qmT&5Lt zyz==?vZ>ag6Yg%2ZdPry=JMxjc@4ycZPIHHu^M+-QQ-I;E=b}~oeds~G}30gHqJ$& zmYKA9wmLPm8Y_6&qH}*WB1m`Y3}X~7T%p7(y!raJ+k{9jdfLEGnIG1dL*nl{R5G6s zr|hI+=%MUKaKXQTnuYOeCINGVa=LVT~O}C*K0M5ptO_2wS#{6pA@x?H>B0Z)mMDBqkhP)k82Z| zMMgy(ev{$o=Bt-gvVAMn=Gei?OFZ|Sw~?M@PqP+}nfdtcpI&ueN3;f#8=BqsQiZhO znUJ~{t&NYQW82CzlIi2te@;($#9}nN4I;v5Ri%YiVULz4pPnY&Wo{;LO90g5SqC-@ zuQ39bM9^FroRkHMV^c#UQT_}^u}UZB9%HM-y0XQkV;&5LnzS1o_d`RHZEnM-AGwEy zRO5+)5(juf0S~wn{Nce8Zr5)JD1= zPK8g}J0mxf{6OTIhn^jk`&`DvT97e(*u8)ejX-wz&odv#l-p~YYU_rD4f#{TtAaVg zB)L<$Urd(o`xbY~?`y_0NjuQ|U5Q=Wd=%|8CXnw>U>Ukfe)l z{2O{S?w0Ba$Y^QZRqscc4LB9)MnxsVB8*T^5-vE};fFZ&gf7u{c#wAh%?gNWP{bJt zBl#KR-fN%%g;rt(?M-YK;vZx7#)Rrc#!t(hG*H5+*lLN3CUE{ zU(YUhTBi50A(5UR$R2NskJCr7vz~IZO=PzJJaE>vy1>&UC#M&n-PDdBI@ELB?iG%B zYQ{!c31nk>Jdu#YCq5w6Acg0_B;8%{0)yYHoM2;!tDx7IT;qyT8TKc*4j_+4U0lFS zTuoP3{|KvNVN=w1;7-;`jxOMh#Y;^LOEKX=bcQGARMVp=AKR7%|5(Hjb4gX^!cmTq zYE@o}hB@A=~&((ZTt% z9?qUm6|<_{Gi&ZkriY5_?K;v9{rmm@jTciJZ!;gmPzTRN**n+b;>kdHm?CY7juZrj zefRq)Jy4v_pJm>aAE(W1xZ$vZ-z`c1Xl9l%bRM1a(B-e;Wkoryhsh3AdMAr1%GdsU zgpx-5Zp=!lE{j9t+b<2lDJNFXJ+(tN%pPMhj-bE5`T<*ZUndZVYe>P&p$5*UDM|J^ zPB8H;b$J$Y(ni_EOk=LeoGaGpmMA83_dHw<6GH~fE*noEFT8Aa*$JGiK8;7(H^ zo39)E{Dm0QW#usQ(^;@@aK5D!|A1Ikg=&H`BfoAiJRPdI3x9QI1Ru&y*@liRq-~YJt@6LUXUQL`Dn$AW_sIrTYiy*ZN zfD>jfM+pz>E;ikqKhqi%wP+Nx7`}zJP+3xaH2td93R`%~8gbz->g@`Ai8nI6=X9o@?ZUJaOjH6lo&<;Qf zOkQ(z7>+W7X`TSGkfdw&CrTiKE3=heQzzL`^Y6QNgR}{>c8h2g;pASTJx6gv-*Pg# z9wN;RCFFdZEA(-`*^QT$qAwf5FwVOYQybI0&!t@#vMSy*=ec>sW(AE@GY>OE5K|h` zCTO)x?kEmTLtFx5lg;*#r4~sMk~)5Duek;C2RXc{ZaAYJe{cl958ZTgUTGs4zlS^5 zIDP{g>)JGYEegIPXyMF)F6VxBZVNk2}1I;h7gO)xP@IB4~$`^-r_yrrkUA=FwMaAYIH^Q_Qo7St7c$kf@ zl#dMfh!E$DLsI9-o$Tr2U(!UKpEIHf6}qOrn$3~MST}FaK6zkyW5i(m6Q5V}!aix@eu3UN`ISr=Q?$*I}o)mUj_t?USd%?9mBKB4}zVkf4hnTNyJ?NJ#FSa z>DKwYsq{!U*D5iSo5$D%wQ`Y=89|7V2zr@yO+p*)H?o*MN%{|A12ou8s7vi>(>e7I zmLnjh;tyf3MEC?_b*;q`U>(7eN$*COg=F|vtz4a|m|?T~75+oN73Uh2D*Bo!Wt5!x zFX7$WD>cWoTy}X?S(sLv?i-o6G0kpnE;-m735)ZqZJQQdDJGZz$Q{Ytf(rr-zqXk) z7g`1SzA0(Gn9u>s$Ri84fn-zzEJgY!H2&{LAKC+k@y8jsnEIjBk<5(<6}J`E6^G^S zF6+H_ndRNHz#s1Qv5xirXAA=YU@lP!$A&7m8Ou(|qa~Q7Rhc^$!PDuXUKS_|xNsP@ zNg4$W@dkIiBkjex4bk0sfzvT-{$i(MEYfobT2-GS^NjBxt|0(a0CBknowm(WqV0C#Tf3El`zR;fitdR>&r2E}FqAuY{HZ_hQVFp?&udzE(T z7gw2Gwg30Z8fUUeYB0>$jUNIdYeEEKCz`aKZYDA06Yf|NQpT9-_^sFwxgB6`;x%hV zqJ$Ctj|3otj0LABd{aW(TBOHTLM?($12w1mVPO=6Lp-!v`n<*Sh{xn~>x_2R*=i2G z?5Zf@Q}>0rc3w?A#k<1F)6Lh+_EojsqlP_x*{%b<8oJe4?>|%=d^U79+&uVU_vpi)SSm%#^s?!=Yp{=zKl)h&qKW(0n%&`e^@53*f%_dj?1Yj3i!C* zJxxFQZVIiNAs-VXO-tbrKmj(8DT!zdyn-z6?kWr4fGuw@oUMcc{J*n`5)zN zK-59hs0&qqG+9M@46Ka5fG~)lp2_$Tw_}vCa~crJ9mI3qkr93o@D$Fji%>>H1@3sl zRnCc4NlhL2Yc7bV-a>COA@VT{xr{_ z@UjcGZg7&T^U>LK${zA-!SV|eSJ8}?5v?${CX`Dx+BA;P%0UKFuP6)jyE*AMU7Q()_Uklc2 zl$6rIyjxX^z8kHhdu2B^<;}9-SFDu@Zcutx0=s8#2vn(iKh7A5nTgSud+eSJ<3zYd zd)#K}Lo$>U#9~R$Cr7`)A*`A4_tBv>A8Orpsg}Ca+q15o0#uj^aV@?x?t^IYw zcpH8*)UQ;nkOfBovbqcd9g4{ns80^74BNYxOrufTG0~j+hb-vuV!2}~FISqdcK;9Y zrVr!iM{)MKI-eGHqHS{GuF_&Q`PpU^6!+dbT0XpnF*Y4~wvoNd`GG=jzE9P#uhuUZ!JB4l+KwV8xN?f&G!+12JS~ar0Cm<^MO|VQ-KZd z&z!)%{eApQ;04#%NVR_Fax8VZtg$xd!t|hG` zCZM6VvCpvl`~@Q&*%z6s;9_Ql7pwf;0;{Pm)oA3Rr^%sD-9Q6xwtzb?i&qYeBN_=y z7zu_;NEMX~(N79!JHqlfiSX4aJK+&Klol3sVTc~dqc~h;OUE3LPK~3q`Y=ni$m*!i z&pt|R3s#8aw&|uxpG9}9zS?A8{;}1EJL&xSp6A}{pV6qK$-4P>Bqhf%Q@Ywr_IeFy7x;*ss+au^@L`C zO`J`H;Is}A+f14UGDS`4z4Rc;CC}T zCkL20$G;=^o;~Z9mRr9@5*^1BvvMQw)Tg7h&&Jn9*87XkaVQn+0&DK`cl4*!zMwv* zTg~Qykw)?G5C264H&njmXgiwWUm?U^v`IK`JRL~32q}56&UoTZxydNqq#8o}*iufG8OQ?^`8c&AbW=-i*EC8X&9}(qN1pUbcWagz-v!)b4RIWs z5VI>lJLyuMM{-Xt85<8OnfQ=lrM$SpXZG=<#}~$K`uRng1pMb;uLbznH2k~DeC{S`C z^sJCcZym*LA1HfiYYy{(o`X9(LJ3I@o367>jI6Gj97n%tWacf-I(2Dysvb)fh zMfc@KP=i(p-@jz+s~bNY<(#({=cWB*bJb_|xdq@K*ZlM)bIeK0G^06TZh2s2GJLj$ z6B%{9DEpA9$)&4`lp~GS@JSy4=T?|FvExqMa_yPtx^@$OGbd4{=TtrqyRegP+3W zF%2Nyar_T7)Cg_up0bpLVZu2*i5O~E3?P?bLV=QnW$ZatDHpOILEaKi$gm$@CuEAjH3d+Kwh0R6_S zu*7xJviRLo;#|CPtAh+C_l8B zhHguC!|v-^RFq@40m1waO!$5CZfOHgv=i&MJT}6kN++IXI3irQxH~d)l$lcqt{;*djufA@01(^7^+JSrGPlR?a zL4;wf%V zw%By77yq-E_HBju4lQM7OOpJdhm`a4tP|Wz>jTnko`=8ZixF!5i{--pvxS1rru~SW z^LdR=E-U_`Q55r~2OSah=IVIO~Z%lF6fx+$iiezba!akcbr4pm%+{(c73vrR`(zGUP z-HbBo>Qi8MCj9qB=Bb=%O~2kskWIPa|Lt`Jo22~JmZM0tf-CmtF*kwDp#b&i3aKtv zac>u4o^zHqGq{uR9~Ho*dG1{HbwjC-KbIuepeRp2{^m_r^F3T)w~#%6!mhi2cFB11 zbps4+0Kqk|^_zF31)5&x_I-!?1!5|ZQ%cnaFi6@a1$?%#2uXLs@&Eq+kIB8jc}7?% z-6kjjkk=lJ;#4XAgFjD4u9K#&hdZbEXNC3`VGCYO9uG8N+9II*q+g82fcxUo&D;w` zxQoXgmq5>gP%~J;330S2$zV|!+o*lnsdl`2^~L*^-S6h#26i6?$`YrTH};YnZOz&F zwQ;z?qV6;6Zw#vS2Uxy;@JtT%czeY#O&_~0zZ#ffe_Y4^MrhZD+q-ZFC-Qlim+JR8 z+wj)%!w^Y4pged1sG*?}F_KT~oHk8Bbb+8NAAXyu$Vq}p$Nf|axdh}0kdnwmePhpY z3#*CJhLNke?(TNSEhwK=r1!<+-EAk8+tRt}Y@bt#r_68<3>VWmsEKnFuie8NhB509 zz2<=@F^~(+_y&)*bJ6z51a@n@XlIyiFy*$X3neTjF1flPG{l`JFb~Gh|&=o@rGDL$6P?7$mLuEZ4cJik7cPfX5OF|248vBQ8BxL_91oAsbuj-ft|2rX zw}v5h)Fh% z-8Z|HZL!fsvb`%TLifrl2ON$UiH=Q&agHd#OgV36C)CVEKxRZ?gk-#zRL2AZCQ>rY zz*te13N~WG%GI++?ci|GDz(Z=6!mU-mlm2GPotKH8}tync8KuiVL}R1>;6)^KSbhFhicQK;N9284YTUkQWo$?df(B)Hfm4ptK4UVFJ!1 zOIj|@Z3lE6Ft^eO;llPs5b9KVDM3dF&^gpT+;QMsd13J(p`$Hcq;@_$ZwJx7E;wQg z1V5i1cG^*Ae&@3;0^G;M=lB6my;ZJU(?@z-(V*Om+YMz%26@a*SH~UQ0(}*p^_?4h z&ffX;)DC9)x+rsXo6o=J%w|qTRzrjfiROC#FPw(>qlZO*jEYu14p5K^^C9G=7h2R#?Xh8jvjmoXFvV*u-F=2@% zI|HTjQeU=`Cd*;>W=!59LTPEr1H<+6@?8>2Fht}-^$9+m!P;X{M4f|4|6&u_CP26r zZoB-DP%#xe`H`Jj`qO#TdPDK!Vv-F*F<;SUNRPiA`|ioC*V72&`;<7S<=Cr7Q6HHz zZ)az{+_Rm{yKkAbX=OKHzMKJ9SOJ~ni@F9OlR2<(ZtkGPW|5bc4G z(i_ky<>PZCkC~w>>hg#gcWP-8A4l%$E61YFvdAwU|1OJWO7DIS>pvPkrX!63+^s|k zWXGTDhCwETr3?v%M_&3YI}NSznt%ZfHZZq;o?>eDdeYj}5l*0igq{i9o6v+rF$jZn|Lj|< z-8fxeH#A)aFWB)sporfEaKo&x8xEh40V>2MKrhd71I7%}>q@|icq^i;{>C%|XZbzU zr~AkWAS~x}z}EFw@z4C{vBaceDghKKyb!TI6#YO2s`;7PoNLP~ibvN z0VPti6_F-HloBB%Ta@0mAfU8}^cv~X66srzULrwALFq`M5Pp*4yFBl_U*6eg&hwu+ z|MTI@^T9G1hWxVD%3Alj?yI!1ntNz_!P>TNQ_^D%c9EZ=#07^2WeN(~9u;;xeB}*4 z%bJMG!_>ELDQYb3!3%pkc6eQLz1c=IEpVN_2yU>sWA^o%DWvyh4uM`X+;w=&Xi9~A z3&*C&05_Pd#rqcjsqsf+ij{b^jW>l1c2Z9X*t4Y7l#Si-X&6YPYDjE8`N>Bh*rC(s!;)M527Z}W_YvBq@1)>7y# zrslsYzSg|A1 zckAEF^s(ur%DDEnDqdeKtnjj)c`pB@Im6y|6Q4z<&Vxc1oUyz0_$2(@H7QUH1FhUn zTpw)*PdnEIYYxS*gBg?r_(2pD2NurLf$v?8gW-nKbYhHwex#vqxvRH9U|~XTL$qv1 z)q;U<*F;&U=IRGWT()QpgT``L+s<=`3b;YI5AYKZl6yfg0+YmQvvjOk0W*ed)5l=T zc4TWux-};;HE?I%j@H5sE|SDEudt1}3d1#vnh717)IO#m>>%w)N@O3edeCiGBI+6vIBlvslI-uF@JZ@U`D|g59b+&8RZHw4$0nWzDf2szp&f+E8`c5kSKf9zs~%Ioq5DwQ`Mc?`L3;1s(l( zqRBqN|6uh{W(Q0n@B%tIN0NOYC2q=E9)EjT{LT^&OX)H`x^)+6+!%iEIUGikNXj-vw; zWIKXFMCW)(luha9T&rflO2bS%EwW2HoR}d0+(USD&U`eW(K+wf`DUUF%VTvr_XySj zECgG?6ca=IF}Q>fP{D!z2aB5-Y;$X`=>5 z`RZLj2}3&X1oHX~-{UlYTW(sNDImI7c=~1uUi6*=@`QpV68@9B2@Z~u{{n~R) zspQ(q$~xZ*DkF~M7w6GAFCOd z92vp4;U;hZ!VfpW#L-1#xO=glzcg^E*yOH>%u%eOwQ%=58ie z4L7>VPH#sYTo>-QYpX}?#=0}(rtrd$b_YRRmv7vak)N~w4O>@9vYI>&I=PEY`cXWz zYNR2$Iye=!fyokG8CIIn*kuSTc`#{f7oy+!O-zXC{-f&S}&H)=mW? ze-j9;d-xSz+=IA&xI9phV1wx>DU2io@uX!plPHY6!qSezDM9IGY}nIU3r%JFh;@XV zC-r!N{~wfta=QH6Bh*>5v5M(C)vbhkWw5F1H@XQLjg6U`%EAts$~83H{EY(=xzU&; zpOXy{xe@*W4C(h5kL{Z|gMLZ2B`BQiy12A#!wS%gA<#dZ8I0 ztSU+mu`yg62F96(07AIy6Bs=lPaoROYM5oQ(xXTRF&+%csw%7bD)eX;IJa4VQJ!i8 z3S-%)0hT&lIlD;1wS)3YIXj|yumpz7+lCQ4Yy<{4sIckAyirqN|$6+;D=%(HIE?zPFtdgR{UujTOp%LIr?4gz84lD6lvc z?uvpux>0DCl0wtCT>()>J!x?Ad+6ZIee~5ek&3#s%&_i5CAVRv71 zHr~_~aZ6?F+KY=%6udvw?OfGjTWyhMMEM4A+}%i4@Gq`93luh!3QSsu_2fL~C3~#{ z{X`!5u8gJ#b+nUz(U6d))dt9J5ft-Eg-4O4(BrseeJV0S&&6QYwti%`DOA%oG)eSR zG!N)5sP0K`z;!4{p#Racfm}_FFbrlPyC|?7dM>wn06!TN4oL?vSC~Dsu#PFH&dJM- z72L+hcb#68_vlE9rWgdJqsx6{;HP4WNJ-qIaa}c^F#SN0h@I#ULGrYPAoyF^Dh=x5@;i`RgPo zm?;yc^LuTBL+8C`)RNJl>`cG>*E6y2HP4bt-@CZLKitgPeRZ_ENl*3sY?YMSb=Tg^ z>$5_K-WNZ7cs=~7a{u05J=gVl*-+d?!3R9+8Voun8$nB33Hyww|W7J!xEbZB;SY zHnSXBV<9*(;B=%n$Rk%93Izjz*mD6Z#=duYx`6hMwh!NquJjsNLe(tu6vj35ENp^{ z7PpRZQkYWmy#(74GZpZ|pfB}CWOFq*4`cC;b_jkz0D2}n48>n# z20L(CY^4I$U0+isu<#ZCxS)Uc>vG%7VRjCY6-@sfC8oP}5S!*WSDS&1*YaB@!v_@A zCY+wO)bnq@*8M!5Kk_BZL+*-U#e1Me`hv@7Sgg{0qgUG1-jXZk7MmBB>zb%&leKJO zK{Xij9xEHo=olbto_?>f2y} zC$MO|ifiiPyt0%U_h#?a4bDMGcjVcnmF6!yV+QFL(8QJO4JpI?p@Dr`WjK;BY zNUQ_VAQp+Bzkx~PbZ|q6>R>++e?(T2zBk7`-kam7X*V*0PyWd>YFgX^=`#?}h*S)cdo|6>p{lbz3Y)Jyb4@IR2 zWKBV@$hh4A(RqZaa07bUblsR?nffI?p$;LTv1OWIcu_w}p?eEntyQtof(WIzDm#WjNBZP~}T?a4EW>IjKi9c87wBhzQv|1;Vic$}wXV+bb%` z$|`>0H)CVmvOW=`hh19pQ|STyFLD{h(b4R{}^u{IDaYiIA!2h zqpi@z&YSg!YjLL|5;ERV?8jwbuoke&NN_5YXHyl~J2OEQOm)fsewwP*Uhmes$1L^a zu|r+WC(V)m;32$px0-b+1mm1a7yAT5Etr_SU$()=jDZ64gXMW@F>l+iFIpuaE6ihc z60UZI_LjTQ)4OOputwZ%0w686PNFJvji zzTY7}2|`sNJMoa};Hof_ECVTYlGFj4CSg4;O!*3Q@i>7AzHhF|Ej8?hE=yNBWN~#w zcTmE+Y3@~yGH}U@b0-ZGeHm3l#~SONaIM$ZIKF@YNPJ4D#do2ogPmZr)85a!3K8g* zFh(7*gBG46hZp6YXQ3iA0u?Vfj(>hTR_VLmw+6#>bZ!mAy`EkoX}x9jJhAcBZFYgf z#6DYwA{LCUdy#`IznQ6u1)WqCo(`?#;Li?|U_0&CeFl6)q6@4F75F)+=SIpRI9X$( zxiJNQp2Xo-rfU9u2K5U2tPS|N7cc|!aJtO@dd6$MKJWDS{`cPz*8$O6a)wnYE zl{z#_1OtEg zZ)$xBov-#8I-{t5jScHipS0@~h*_MZhDmhuu=*6@N}vrBlFS3YFdy<@4t6C$C(~Er z*m;bkl|`-%w8ltk3)80S(GsyPj5R<=${rwiZ^?FBqM2589z`mo?cvMzz)VKHY_SGq zT-0`dVk@5_zpGiVRp0hat)OBn@OAF6bETT4$$Lw6?ZBc?%^ zMMwv{P&lHdVe#2kw=3!Y8mlFXqZYT6B1)h^&xg$q3Q6~!8=zR$sjQ;s z9jOaj*nyQUMBwD~Wng0U6>tdh_XvyCnuZDf(URu6$hm=5z!1Lsfc!bz<9gofVo0{ZB)W3iy_ zf1~^8*)`(YZYlnAXLs4)el0Jf?5{c56+YVEHNVszyVNSzXtAs(sBtWOmQq@@uSlvk zJT^+j*0;FV`sMpiJYjw-nf{^*>%p%C9C*QRkS`;&Ux#5GQA>eT+4&3ir;Q62V7qa0 zi~35(SOZT=rX7;jU39vY6!7v`4WG=BBRQ7ne+|!Zf7iG0Q^&Wx6YPgl*`a`7eSsB4 zpKlwCr^y)Jf^{e8Gmot{w1dJLb+F}8z9(?0Hx!-)mlo^eRh1C3Bcm{;fcD8dM{U$? zbUie4q3%NF&97_9wK9J_>gRGzM4vlmSj~)3yfihEM!|&@}viy97$4w2I-rL z*keTOSvIG{3r!c>fkAg)^9s>VEH^~pAC*FGoip2FmpeG`rR+&+!+O3}oBwJZL(5vc z8AI6ox2P>jYOj@i>w9TdR$QX49h#~_2w16mWA{!?ueHu#QiE8(2= zjBV=4w{0g4e5cQ*W;O}_;Hh#i@Iajs9t56pm4vfLchEyicV`I zPVI+5nn+MfzS$&4$k&{8Y6^iqAr?59U}gu_7nIKNch^w>t@uxHurpshfWouv7XFs zt{!2VQ!eqd@jlj~xFPrjl!Dhc!Enw|uxkj8VYT1LI#zTMI;O=s>&g4zhF%5@fAY@D z5cLRAx^b4O?OtBH#06jVQhK9?vD>-(9Y5hFa)dytMi@gooN&Mpw71(_PqLC8gJ{cL z!$m2U>DSm{*_?zjOb1h_GY%jC(EiOHeU3n3Tu&1SMlRz;Th60%d+@=*#|uhZ{_61E zFFDt2R&?IwocHs}OgrmzM*VQJz0;@91y8$5zvkyz`Ip3$FWFT@)-tZN6j|ki!HPyWZ9;ja##;KN)anM zywYOEkYO9mB;;efw?b*MOTi(?r+`AbACqJ;)S~l9G_S&^NvVv{Q%j0N4&}`GWQJ*P z-Gto3*s^x4D={TJ69oR>3@kw;rdjc-R<45<9xcEgLM3)IVL z2Ds)*2S=|h`?88Q{GASu`f8cuf`=rq0qs;)IGiE9q% z>NemTgU3&TPV%9j+s0IxT_NWicnPNooP%UFq2XElXZM%*kKO6>eY$ISEQDZnFtvLWEjo}Ri#X+RtfX7PBu4r2_$o&sTV}0WpfXmQEjsJa;2_n1tDvvA zRzpcW=19hQWygr~V?j@yGsdT0f1UEu&RlO?CyXci^535gJ7R(bXS}czI8OL-v@S88 z7F-(t9cjwbM1!8&x(8kg!Fu~^wEJDWHUL)keU%Te%g^B>0sy_zC4lB}albk^pdm*Z zn2d!1!3jLWp9RJ&qY1(A!Qy{z%ew&Of|SLccA~(0VXFfFgCl_O-!6LL0piHohkv#n zA^jcy4{rx!)sYZCw+(0Wt^umcECUGw0t3SA$viF|i}>+dE%0)Jc9H+yx)l3^ERbpF z0duZHj)g(aSTKQel}Z2bMDQ|{AdF0c@&0l|^&xmWP@n%RpC5LE_;VYaMiUcU)ztn% z5Ny}}2LB685cqHvYycOWyHwF_?^`2o!I8_@8EF+~?=E!G_Haklie7@vENy(}I_je}V{(>3UYp4x0os`xSKu_nX50?EY)3Gq~DiA_-|0@Ak(8gA!uaZ+yu~=^M0S;on{MNO=&YY`q8Ygiz&8k***jr zZTr#dfpga@Z#P#~CpBw+RVIDpqSSIHzD)w0AYkXKi&_C7og&Y?Nxm|QL5MXdddw3l z^J)4JGT2!NL|>q3{cVAKFwjM(jU>XN#AvSrJ5$Hq+2G!7dFK7b@}gd|FkkEVkYqO>}GzW>kLZN3>;=2meG&E;N#gpWc0-dj4Fpjk=s?1^ygjZ@k$YPq_nXdGh4c8Aiz;50_eu9(Rs+{o&JcR!44y^&;eL z&x@fV^ME2%4d4Ak753Va-a~PvjV>qn9^C6ZEY!*(w5PZL=Ss6mZ@WVd#1@N?48Jnb zKVz%qj=^EW8;X~KVI|*#_=sSlVQ%>oEk_1@~dz>&z;@VjgEBHnt=c2{e3xO zH7QZ+DRT7aW#v&~?UZbI_ukuA>5)|G9QE7P*O9u3h}!DX($$^c?PiNsoo21$U@Uu8 zezf2>OdnJGDNfFWwbb+nR`n z&LFO+??EZ1Tucmq_x87@x-*}2J;Ux6%F1Sp=QzE%t?2kynQQ!;7tt1ya-nKB48Q)y z`1E8qBhg=@EGp0XOz-!W>O{9JWXVQ(zj@l3>+HWu44%&#YDuWeJt<^Fass@$j(!23 zM@wKt;1dq?eS}N@2HWKSa}+p`^uHz_b0?W%KejF7IeW**>#^QENic1J`1i>`KE(OW z1f17lwvA6@X^O22bpF_xkjf{(LJ4;^41#Q5se%v`? z(LcUi1dBy5lLXHnV*~L2qZBBSO8V4$QZDs+&B4-FH97~SuOCP@qi;K}!V)@8x{IR1 z@zS)VvxHrOgCJUF`c}T{r$uU198cRP&l9)JhkHK1p`~iX5XJ0AUVX#f-Jj0}@k1KH z_5YGezA!0qvv2EKkwHSS$@>bIde@5D)Ge3gVVxy7hpRHhO}@9~{qG-IqyO2mK{4^4 zqdV_E>)=1@;8$(=pLOuxwGQr(0lm7%ysC9v&w5gs3C}L{s!x~QO3V!f*Q2NP^+7Uc z@3(eMf0_Ou&`cB#^>M;{&UrNV>}agYBb7iC4aB(?FK|s<;REdwqaT00c?hoUvfQLL z)3xXEp2rL~-%^(;^ws=&_F?#`#u2yS`DexnX78Jv&iLIPDN1p#&2ja3>Hb3>Ts7PusU2pyuQ~3JVlTO-#?b`QA{h#5=t~r$@0a zq50*P?bFxG{Y;tGZDNqdjfT}Jc3%&whPcvys}geSQIe>UI5nohLsWQDf`a!2@BTmo z#`GcF&IG-Uh;G6|U5UjJPp>cLf8`rIt;&4-Exmg!F-H@e{e?XJQTSfe^n1Dt1mubi zylVm(`GK+m%#LnL`@SBM42vJ`)LK~u^1kIgw{{Z}(6?IN)Ux$sSC5Y|>#A6S%zP+B zDU;*KPX-rnG3W^XvEV3%)`1c)f(UGGcZNRaO5>~oS9e5nJ?`eqN?D7M+7dlW@45LK zT(h&Qcul71E^B|$%)*ZrC4doxDx7_bI!`a}h?>p3!_IMkdF#x|R*nMfFj4$*Ug|`d z{t37HhvI){r!My;1Akgve(^}>SgQQTxRHu|Sla$^AOnJY64l+#rEBHrA1~`P)Q@U+ zPkmeXXih!)gl#*cy8MbsLf?ZXXLXB{x$ziYkp8(3HBB{Lf1JV%YWKQp__@TQstDT+ zJ=oT!{9>#LOQB0}j?f)MQsfhAVbP?s;et8l&u!OOJxPWZvwO>RHMNjryJFA5hYW7m z1BY6NJK|>2b*bxKXb<3GSE-n9Ygfq#TC`rQDz5bOT9VS!GB)y|dVVaT0Z@CK=8 z-hu-xc+{Q(SVz*XrMQQge{aMo7(H}kD>ZsONUH((TC87-kzR)NYqN)eTZ*o-(sOTr zpFdxg*!Jb)fl4vq@LVU6v|Gh=mQMFA*4H#0$*6Y5NZzHKqs{XX=UlEx%|04S2~=f2 zy0W9vA^fJn#})hAielp!bsrqd|CU)<)Rf~uc8hIJ z;y*GQ1O>Ons7-1Ge12NG+oO`D;4~^OyNEg%ipPf<+Sdo!2UY#QDTQZ@(P)rTSYxT4 z?|&u!Yj5e*A3z)PnrA%gIb{Rsv34@ox)A%<0_)hdG8tmwSj~#pr|ljx%E$^!TY8*E z_m(qZ^&$r9cLM`s&4NxVo+_{{`0?8pU+N^vr+$P|xAr*FI{q^0kA1{B%4`%*2FZ!#zaaZ0$zyyvPh*9kP}U|B5sB4f>%4Q- zWqY)fV^Oj^x8jAZ7lf<$eJ)hUZ5Gwue_WcCQ~no`f3bxM;yIL^ z#ZHs^oF&Wn{2-#|)2#!u;;Dn@K0M$ZICWlDWa!;(Op5M1yAQ>;-#ki0U&`@HzxCjQ z%+ahj>!r-)xwoa~TF%;S9QK%Dp~Sv?`~(l|%(`eHXBP$eyN{{cbl7yhcYBz%RCp%M zGQ4Y|!zxEvIGw!0^V3#zjZ1K>Wv!ptlD@y;|E&{LWtEyjbU|~;)FXi*lvx^Al9WM3 zJ+3InA__j9zsSvWximMi>D(M4dex`zwV9CGgR`G8nG9MIPeZ|$g^FXc>rzW)(%W&T z8mg+ujSWfjp6g_(#h^C^OJ}x6f1dEw*8EciezQ#V`%Z>6bRN+;vDHP6x+ef8ly!S* z;7LsW{RNtkE=rLn$-celx|fBDHhjdiL|L;ai5u4Oj_^A7JWOzhd}TqYlj&^h1$&um z?~gw#T9##Z6wLhi6-6whgIVjAzrCy^4Uqp1vB+yKRLxKFd+J!}%$}YLQ|>Q!m3Sax z)@iYXn$_(7&Lb^xFITa;6W!N?9_L4wpgR`tTPN9e-RQe7r+SbM5wsE&V9`u(fv#;& zsA)oAj!dEU_QVf08kw?ByDWraZTi@7nERx=<8^=C`vdWACx^DstA>%o7114cHXbZ} z%if({Dcv#js$|5U^L*h($F+{TQfea!H=V?d4NKym7MKUB*{5rG@SOkwTOOI9+o&gU z-mjvlwwdtzgl@tWao^H~jEa&S%fFAU2A_*%YW6Pu{q3Uvu4m3peHv$!b1KOhpqZi4 z!;`A{6(c0j!EAPeF8mQzLkC%`PE2wVdxZ5BNH;qKXIb$AEsAD99PMC&2a2@`+~a0Z zN7pQU2qFtjK2zQj8fF#CoPMK^4dHJ56Ua4zqQ90u;mi5Nov|hE=Wu$A19Rm4{%NqI z-6K6;P7knNXf&KoRNn0xdwtOumO3}-cb4Q@;PB<#(pz2y^Yp=X#8JWk+46Uo?sO5?YVXYigTL`y3QL|j40V%9rzyt4)OEcUmcnnX$Ul!%4XjM zCg8!BWNMf zdsMbWUVlNTK>b&GZ&?5TQe!VK6aV+9956Q!{`M+NNndcqA)O!Zg~ROJ2<23=t!n7B z$z>+4fw7!cHIaF_cBq#7`L5Z$<80T40_X;j9{6MSZFQVu^3u%pAIzK+pN$($!rbUd zLKqm3iMhbqGmb9e4zW!C#9pZPnQGKn?^AFR(HgD{#2@rYAIS{8IrsMT^s@Zdir}OB zw2n2CTQW6OQrRFy`{%af!)#dr2`dDKRJmPxOWQSPpSGLo_?Xv^feBu%$O%i+OJGZhn?Upl&Iu%8|P}kXh5hY7-6_y~k2)N8sPf`%Muqaf6?8*=D zPWGVix4=FEsV~zSUwMdGkghQs^@OCsJ2i26^j#_xi&F%8DCVl-+aLmHF`kS$KM%@A?!ljNdj!<6#^Te+p!?YD~?WR zPx?sY1WdHpF%ukcU9T6?BzPxTzoY$CsJH4s0`;h*7#>vYa@mjKzJI$B2vwo!#-C zVaG<8vb4Kk3cNocb+3v~da2KYe{MS@kcLG2ejf|Gps4RkFeB^RPfRvFc#!CCrX3LhMvjZYQba2>oJ3jo&-ZxkSw;m?`J1F%!+)PM7{{p!wwnB);FbI zq-RmSPnwTEp?+CktdSO&y4&JcFC0i3e&wK{e>F@EJGlj3G0orR~J`4)iT{*;SSn;ftrE zX_qjt2#-TiU`|8j0Ws)6#P*A#d3qn<=h@H3)pT>uH{E}pHIlCGt)}B|Sz9-k@B^hB zOiDN3(vzBs_1l_U4t9qHf8B3B|G$?G$t*vsp%Kx>iXBH)#DtjgM&)y`)K zsJ#Po-E&I~9YpA)PzOp$K*Je;kL$mL*h4bnorBnPR5%h=tI{-|7y!JAH(>{`W~>2( zBnH(UltGAWF`;BSXR-g=8mVU?><^4b7fyymMl2I<&z|0%Th+ zc2YD>Zi>2uQpEkX%5^A^*+-_PaGXJiLQt}i zDHGKp7X3bd39g@f-cL1SU*P%ti5BfyO=F8uKeRd2&Ye9laur`iRP@%(DjD!v7S>a$ zj`gy#=oPgghmt;c>&FF86$OUse`^vv<7@D4@n6Ar3I^qO7sUz?ynB!-OTT?{^KN@j zC*crI1*6Z%c^rz}b!XDiM@erYz|w%ZvhVhBszQ3Egy}czG9f2`44tI#BvH;SaaO~4 zdM@hC9I@D{?T@Mva6D@NnGx~%!v|@5bth|BzhV4`-d9H@9RA#?vj1-|?d1-RDDb&Y zA?SH$H|vL7-jUysSKWfDoG+(bX|MJM=RCCtMDq?jeyJc@x3!yAAaIl_d$+Te%9C0E7^H{=n#q24dlo~gX@BXbEVam znYwzQB@Jb~bzo>?RjhN%Br4iauN-6=3c$Ekh#=xs0Q>@ee0 zqpsGE)_s;-zEeWPo5Ch@{T2b5+D4+WuAwF+IT8F6?G4yYd>dfr_$I}S@J{ke$angj zxQ(QPkZM===H=K~Kx9}GcK&4HLGQQ6i}`HgG)4l}L@pLxbT5H~kgVlt)h8`ZLQ(64 zmGHs>xXC`%{{1m;>&r*uYV(c-J%7zTo8eWl5_4t3Yud~OVJVOE5$r|wqQntsv5&zF zk?GW^F2a;H#}-22OA0<^_@;MSQ2B;<=9@a;nV8Af+WegjQZy}$tgnvxxuN9hz7)+3@)QRd z)(1QhtgC}jO8Rr2%2OO{0M@d}?}(w+RZ)=w_!?m-(@fe6`dG_$VO zahVv3c*ft3HR6-;w_tH08%Ys3DFKyMZQV(e@=o{v=#`@Yg>pg3%r0y#rdbLKdwWb- zwD>yBOJBb_CuvboY9P#Q`J-;UKip&Mr{dii3#?3-nj!Xpfomg4#f3P>O%;@5--mDu4&#YulR#r98uYNZvJDyshsq5># zc1ySMQQgc$Q1i$pDrNLaSSt(3Y2ZHxR6-+T)W|OnTr?*X`cVqmP7H9n41gj#0VWDR zxuhgtJ*JvNZU2s{X6eW1O=T-hY|T;UW(H<$5~&I=SLb78A02&$TmWo8BTQZEL&3g_ zV1({R3HAqi1^XA_dJ%PqUrt4vAnL~32lLE#U2G57Lp{k7&WqG>M#PbDie`ml_xc8NC5r`?3s=0yjP3OAtoAI90D2^vT9UIDqy_1mv1Qaq80h!{Y1p_(ano^6i zWUAz>Ok)wx2vTQ^Ma^?vU;57`iF?TtqjRL5J77#LDH}2WsvZ5RVqRvNe0@yex%aW& z>C}SPSuVAkPL|{^V-F&-fH9;`$YKCrd(#_z69;skRa#|ibNu5b)M^Y@!y6^PioFp? zw;(*3fYrjNX8WUo9Brr7WhLd$hzt&hg z{BaH2{uyymKCOs8ZD+;~S==gYU@SUyQ0rfoI<+woqTUUTV$lH?%F05A-F?fTP{%6r zNL}lhuU{wTEyp#q*86~qsjIP?Deo@6jRdRdPUq_2+dS~}a@1}Rxr#RQt;8OO+*p)$ z$A(U$z%MOqu{fSZ$bA`iO+@~oAN;OLEFZC%)k-nhrx zw=U+NT`F@Q8C7P&GeeGvNyXf2dV28ha$~DT+relmGJ5^zwn&l$3%OFk#P7rIO1i>6 zPOS71z*zkSo8c|m#6ISkP5HD}-ywNw&$WiJdv(g3snE+c@~$PjMsK`}5C>Vwg!0~_i<9eJtIJ{|d{(LU)S+p^O?yWv6Y2`8%wkB?Q>0pb97HE0_mV&ET()}Vu7rH4b1f|-l9P@u^MQ+x z6+BUL5&kW|;nxdzto`d!82=b@k(cWd0OIH z7-+Z8DpCLV7DsYl#ve- z*Wlft_uoRdY%jx&Oo09CE~^tEfxVcwcnWLGRs{`@Kn52ry>zhFo`y}s8_{Go60VBVfu(v&~ZlkA>s zVUekD?hmK?DWAlBi@lL{<(9kb(et`*u=}_tz<#KQJME1)2;MZRlXL+pm`yqa)m&*{ z6@ZkEgpB$p2bAPlPvxV>E3ujYrE4P{U^P6g6%$=_jI7w5Q&2wXwPn~K+mxC{c&OKT zy7S%7ZI@_!;QKKabVB4n3}Y;9?ON+8OClzmjpQ0TG)L?&gHN|pd-ps_NDaTxbFcbu zN3+6YhtFKv?iRJ#@}5q(-0b1^_-@sX>&18P4}8Za$tBO${60`9sZT<-H{SJfmFpIZ zV!e%LUMq_2{^LQ528-2gA)I#UP>ZxYN|9-Oh{bX0kej%8rQke!kH7reeeN%D2(hGs zbVfE-I7k0N7d_8z$M=0B@6>O3GW{_`w3=G0w~v_XLyr83*W@ANAD92B<@Y~(?Qgaa z@6SLocb7M6%R~_Tk?kE>^p*z4(YNeVgwxPqYdHe&tD&Q*gGuiu>8L2_f`d4z^vDYw0{ zt01YQoH84`_23_0_J{HWU`7|O%J!K4547vqRlY92d`(XVx=NB9MOW&Uz*RBJjQRHF ze%oP=3-IDHWCx|^E>}BM+3&DOkPlrxoO;1_HA|#deU-0n!2&@ALXL|e_vH1Tw20GN z_vT-{Q$G@b5L=Mf%d;@*>i+=b<=LsJBO9GsRiQpvZN+R(|Qc60D^NUF0 zJRJYo8V-&LhY0~KBCW?#n2|tfCq%;ynk^i3Y1>sht;2!u2KM*I$U5p1ui6x3H4(hNSgVQJeo4l(AeHf=&QgzOQG~JX6dY&cl7Cr2=|{>h zU#kkAokvd+T>+*|FCy2WW5G4^!rEi9oGwvL(47Ngi>=h<@eKGxgohkXaY;#mEtC+r zs@}8J(1qCRIio`wuVj!ZzzNE1Sa)N*R!jt%?U2_Sc%4DYE-+yhuijO*oXYXfDC!GT zNo%5>oeQc-OPHk2{Fzrk#nw+d4WsABwmJk#d4LGg1`L6daBT!A9f0w)pW6%#HVNT~ zI^+tO`#WF>t?|y4mU;-3C$U#7lFR}S zKJ;>D_ACVwWc-ZOY$_Fkg!UrAbi5TL@LaZqsaiwfkKwM z$dbUGh8E>2Rkdpd%IpFD=Z4CREoEy)7UloHGU#h5kT5G_KCz_+>cEK3AK2o_qb2N4aEBYS|q#aaUC z_(*bfcL_sGk{9q65@W8(_=MCqui0xh0<^H2rP!WSQQs{}+f8@J_X zk5!=kRyNRL{Mb47(1d;ZqWC~_Vs%JY?-{F;+=}%FF5K0!0-?Y#$#iXtqJnkuOTL+x zL47K5(eZ9`u6GEo?zN9&SXxD%kAdZCeO`e#)jrQxagI;|>eH+gN7UjbC&f6>J=90H zo?<`JV<=%J72EBzysW6z{oM{Y;|8qiTh+ZKjp+$5NBe5ryXSiZ=tTPZnxhhhot$ne zCx^zxe7<@sLdi_g#vzKX3bsCaJ*=9r(#P7#v8)F{Su{>r6deqA2iO$$1HI9`R#p1h z3dBlWE@lS5+tX36KZ4R_WN~vO&?wY=G|)-W!m62#dfcMSr0uf|%U`O`aBVF9v`Fzc z2^NT&LL0zn4L}?ZIm9XuYKe-;s7wd!uP6{z7#s~h3)wkBWk2a!$OuKW*py+MECDWkz%Z-`O;HA4rlf0t5P%MJQYr2qdn)g5N6@&l zI^tZh)-?QfVZ)r7`=Gg?ZaU@16tLlcwq?!Yg0C|75YGl0XADl#+E9uuMtF8ox8Pt3 zQ>=r$XDLKA9;*eM!+AoAIk_cb^1P!ILnNbBvI8O0auS}JzeMdSipD7b%a9Rtt*`p^ z35~Dwqq@s|#kgT&kF^LdfQ9IwO{%sd>IvO6NL9PLU)`1;jkp^I?1%QVbL#SOovx># z8V^>}EaSi?UKXITup2Deswp!LoC*s;pedeZJ!>T9Od%;^2%Sx5tMa`JV39nV`x##| z^R~Jh!R7rbWC@b-46*0I|4$F&_6vptdVE41IkJNN(0<5Jr+_E-gjp2M+bzrlU(rd7YCo)L2G{AvkT71s?v6xBtm(_Y;T;>RMNg^G-A87iG)n`=3nao%#>wTxXl;hSvAVpL2(O2<9C=0(`Iub=o|I4kqyBZ==do zxib)PTVjCBQO4LHB+fCXyE#|q!sS|7u*q)!`YY~#nTY7c_fyH8lScjr3#7TD3v@itRSQaq12m( zyqSs5fU(rbv)^3N(e2Fe!JF#VTG^q6SIofFH~kkSjn3ME_C5FiSvby2=3MK8jhG8n zVkBitKYEkoIEc}GEtu-wp1)DtrWuLZOH2aoIl10PNG;pN zEh@-n4G+iuovCYlmQ~fiU?x^1eHBW4EhH*6GQ}q7h{z2-Va5hKQot^4b)wi1X?SQG z_m0~u%q9$8$uN^pDLN%-da`}01&ZZ^sg`A)Dn-PwA~Ok`sv^W zd>du%abfR{Hx7?Z?SF3kW+W_$^(nn&0AJhjEO+4MtVIROAtE$%*as}#rGL6^NfK}r z$!Xs^xHLM&;R#_)ra0OiYwhd%Y~B)dzt+Z7&_2@@M)yaz{G|9$GdJjy*qUC&^+4n__*X(%4Eb($wRp!H;0ybab~gTms1+> zR}dsX!5I|;Gh_-?pw?OjV&vWY)|#YgR5HbmzZ`|z%v{yEL@GB!#p_wuj;tzJyVzvB zaU?|C;v+Vb{kzZy`WHyVrP^_6#OZ-@mZxX3soVf=VwOJCmfOn+T8fDluWUy4=xintUU+zx%`7ruMC>3H(>>gcY#J^g-7 z2?bls)8rl~M%_*N*>yCA7LW?|k%G;|F(k=mVc*np@&a`?!NNxu(pZk68p6gMb$J%T z&8hTNhNjPqkIKt_R?`QHLphfWB>Q&@%|{;P+bk|}=qo1v?wN9fwLRm1V-C@#(s+;~ zHU?M2SQv%|0Zz0}0gu}WRtDHqPz~@p`?y%qJ-ESNz5|y8%$4s7H=V8#8dmUgXxijT z8<$Ii)+YRo&)fM#t3L+(2g_Mz{aN0_CjKL+rAnPj7~omu?n@aVm~oB>3|E0S+cpT< z&WT3ul>Mh`*A+zZWVAJ^>!}Z-4^w^>a#}g@H031|7E!-F{OW8LZ@c7pC7j;A)`(1C zUIxC=&;U&TeT1bmoZ9YCFrI9%-!3nFzk%tcZ*QEPUK@Yk_qxn*Y1OeXQa3-ea)n*z zO2ie-S`_}fVbf}3;t!W;yG1-78zZxG+8rZ}%!j9Tkur~$NY zuE*Ek>V(Y1<`cpr-I$-zT4dCzVb-{b?P;ik`YTL-8aR1)C^$Eq*SdbanPMR@9BA$q zT?S{DD(eH~XPoTVB*pv%TwExzmu6c#-ogS(hs*j@V1ySD%e^k#zcs*%)@#f%}(N$ektCq)gdWp}t?1`+0 zQ`^_NJdT_KTNke9lu?3CCv6+rc;-AIPiwfin&HSvC}-&`BE0%Tk`I4USeb}ur|ua5 zl^vn(=YfhMX5pl*Qs-MmjgHC-UpL=5)oJu*nP@;5n_E~cZFKX*ox4Q{9;y40C-wsJ zB?c5r=)fFmnuH`9x4teQhXOoX4FCvuXwkDApp4kXqDM4A@+nq?_>doe+~lmR8{0Ip z^oNHZYjLr#VAS5`?I8|eFr4cf*J%TSDF3yCVo{yYhoM(bf|6Q^&&d((x1B^Kf<8SM z$mL25pyjsek8Ran9%FEHx3=hRQDB$|`sh$N>!!th9(AEQ*qnB`FUrRV_Jjbgzzlx< z{Pi@F3!o=-$|0eJC=X6AY27dq0-A?#Sz2RDU$ohmgp&cFSdppf`pmyVZleGnib%CK zncBd!hzQ&AIj*d@)BqWjyYTr%)_~ku2VVpl_zE_-K!crm+A97xhr+Z4MuRXVTSN8vj6YG zPB8KgCGYtzSZHkKY(^Oj)Q%O7?isFNxVF?0#K`z+!*5SD_IIAeccqo}8tA!^Pg5$& z%y`GKJ3DB}VMz&`F-r^~iByz3930Z`$=+{2H;Hte>}$<1d`n zFGFlaCUdl8NXmXaV@r9_tx>zUz7&b=ljJ{V@-Gc2mz!}|G=Ey)EbqoQ>ICu2I_@t( zrA7!CF|{Nlc+Tr%sQP5Bjz1lNAf7gi=*YH6k{B>OJ-t6(^70Cp)Q6-?PxI&Qd-6QI z-5pJmKM6a-h_`Mm)Y$GpK}O)Q;zp!$^F=XI@KoQJ2*iirqDL8!GxBF@2d?I5(w~c% zxK&WZaW*!eFfZ=LB|u|81^0R&k}$RAKx$56a;4NPOz}Bz!a~HD68#CfXA&?CNba;K z9^x!K5dTVvkQPd=d_v!aZ24~YWWV%WF=PKWvWh?i)Vp~g;)tI%--;CElJR-F1}_}` zksVh;)cZ8?@_4^x#LDU#;5H&Z&y}K;;qJvLWmo?-dNQe&yGzPhq0ee{|`^> zCuuZVXHI)EL#QU=N*_=*my!vcT;5;Sc17?Y5J5BN6K#JJevM1xVpU)x&zezdf8Se$ zf(VCkdwxS*_9Z_%(q}|Br%IPer%jE2 zg71h=%rdwfEgGH~|KF5o(MSlK!m=8vkFBrx3-iIU^aWLUo_ct+;-Evr>Z*~(ZSXgZYle-5z{tdd#*YAlG(gt(D^iP6tJ#C`Mrb?9j&yfeWoD{1(^BD zGB(yy4b{*5@*g7V;m#0hO>l6aZwCq8UZm@-2Lx{otGm$j)^9X3+Be+y?*u04Jp`89&BPEQ5yvX zh%>mJhNI|X)J{FaMwlcfP+!wtFkjKQSibt?j%bGg5>WSJ*h-(s(B!c55WBe4yxRP0 z63N=+>8v&`y3%}>X=dYSmP6Y>PMe?{V%pQ5?K@w*|Fr3Cy16OU|Bawv#h%-jDQcSV z5mjGxPQrVab*+U~Q>i}wtA)UVIJF%UO+^f&e*raT!W1snXXE=0c@JTCN2(T?_G%K* z-#Ii(yb*#`BQwqotaw{{eeBU~-m)Y@THaCJw{~W(_%&6VI>xD5>q=6>U4X5`Zq#0y zCJ>cA_dYl`!WqU32l=#=1v5uC;QmXeHyQ*oap9`uG_$%O!@DfUlL%F_ziLQtIO3v{Ns-%H#DsWG9JPMj%3 zT%HQFy7VAOaHiS)0M{!p9K=i(Q5C7R%D#>mu14KOvRzk=OE zaNMo1k064G6BHE}1+KdR)Cv?e?-eutn$P$Nq2_HXz$9h~Cr(Ubb;ieMsk+dv`yL1+ z@OR;Kp!NNKAeE#C$a^xht=+mMS6qyGf<{|m$pB|P_ra&KoeuKpd^ke?pKpI(HRx!_ zwA!5Xt}sPF8L+mBYk&Xt(D9K2>T^>gIO#2Zcf6i4n3~-M4U_`-e+Ndn*-}tWCB_JT zp5EJYV6c0dkCi{*K51mtd*Nf%gU-uU=l(hTu||1zx~y;P$_tmaD}TaF(9D?Rfx%i- zBH5^2Z)YO}27-KISUX#HWa;uC#az&9YoCYGreLcFiW)3sg#{`$wQMyAopHd#brI7@UGzAHW} zo2*0DFhLQ#6}Ei^Y|F8_qfsl7{_W_WUe-ZIZGEwNnxhnFfsO-1)h60|l3>|YUotrt zHR(b;|Amj% zfd34ikL_RH3t1_)f=OMipz_x#jZQCzR`QK@R2Qvu%#i4C@a-{zx!{JY`)TK1a!qon zZ`Ig@e{$zJy7K28)=l*48%4)MwX`~{&jRJL5=p`TX1@5peNMd!VuBCmQ_m_^Y{IL- z$e27j>qjhzJ)m2B=tmtwij-jh94u?%S5Wi*?|=5TtISB+gp|Jg@pcpjj83`YA6F4P z8oWnK^2|lF8D!@$h%Tc?;z~7sfW!YE?ROm7XpMZ%f*xJ=s37v4qO=E)051PclG;6^ zZASjz@k!rcVvygeF%k_2SqF3bzZ>T4#Avs2}A;4oLj_&3HPV`iColdEmBL z?Y?9b<@j+gOYVGFqh2nN{Lh;&!-nzO4G(f|!8x61IoN$_5kM^t1DSpHE?~Au&DzCf z#y2~b%}j2G!eVPZPiT}M1`lu9t>S=e!>qs}#em5#f{}skS7XmcCE+wESAc-ln0PmB z>^k^w`PFNIb>pOx;MiSOX1zDbaYl?pMSKnewLjM4xULx?Izg-KKPE zv9LLD;d(x8(FEM)7t&{#fMwXkXn|1p5$>MS`>)~p zNm?b7qmF}TVoNNZSU?U(MeFGS*6yhy;Kbdi{HVaa)3<0wQ}a6)n2*l-nvAw1wZ~?% z=ex^58O}a5V8)!`$hNr%(?#J@4wyRgcCpPRsNt2q1mtGfK|nGvO0vd;nQ4CcjvX>% zCQiyXWZ940*!2;(C86ZWNlpyA(c0HKi;ad&MU!KX6n{4$oxPx%YjyT~;qlYT_IBB7 zffo0&+C$!EkDaxwtegMSzN`0TUB*h|%~5RO@bZ^Bj6cY!7h#+*zfC)=1Nk2xVw2GK zl}AHHFhfUT20Y1|0SeUl$pJ%es}8p2P>8qoN}K!d&$uF&oqmX@Q(-JXAL?dslcwqy zDv)iY&EymtPiG;oQ+kK3N7(>tK<(}Q^Bfhn;k>2hBMnh;KCIT3`5mnB7~tY>hHq(I z68a}LpDHE$zS5I}Ru1t%V$U{toEb;bg@COrX=?TBq7grJ3^8r=7xggp4G1-;4sSIV zA+im*NkZ$lRv;Bp@Bo9sLN|eDq(*>i!^CqmWoj_P>-}QIfY=oZlNh=Bm{(Od*W{6u z_dOUncxUg@^v+{rEY=TK;9Ocb{JfUMeb;||)qb3oHvesIplW?H&Kv-qYdF5Vv9I3s z5C_zqT#hIZ))9F2Tv{etTM6mY|HbCG@WtWBWsSy7k@{VE`}@rm7O&y5t>P=VMm`t+ zVWzV|a%jDb@g?1*vGyXXx|5dJMa=KLjuTBW%u|QgAQ&u>v-qj%u&oo!;HPdkn2eSdSB`tq9dgQkZyF5A`S8Xu ztZ34_vKU{2AIWyf-~TTAvHuRo&i#AYqZYeu-Ws@(L`^voU`j8E%hrHCNCDL z%~q1!fjAOFFp;C00E2KJQ zw(86F%;%X)Q!xRk*wM+|LS$=G-`JXf*Tsv&{p@BD2KozY#j^RMbq~M>+O7kc$oOd$ zsV1?m9xaSOB>O1Cr_f9i(IOTFQd3N61X9_n*OEF=aJAu~fw?j%bZ)t~fa_7q^cy4; z>by1~_7Y}+5WI8+Q%SJuu{r&2^xWW51#Hu05NKeodDO!)rF=J)z zdRB}s3$_8&2nTgc>O$Yh^Jk&13TUwH`+Rs@%I>;B**-${mPZwP-INOdXk;Z9E@sVH zbfLJUDgT8+$gYi{Mm<1gxV~@JFYTMcZIqzOwDMBx;94#441N{pe$35*%E}M}pC6>*zYJ_1_Ju03|uHLXw~H**n6#0(~0Ug42Z8 zJcf-Rc0PX@F@sUEMoxR<+=S|NM}Kv%Xc?F+S(vkVHX-;_as9Ec&}Ci1-5a9Y>d zzPctl;@O>gwQjw&ixZg}=0^p63q8VXwAND`5|Z1~72ZS(1hZ<$iIg;gslmPCZE^Vg z5iDpJtopcvoI188B}91rvG9inC!VVM(T5!L3MnTWSdYu#H$~cZ$S?!?^+I$*vCqt# zLtoBEoK`rFaqCfU{ey1=!3^s+=DbF^g7317N}$z`F6@VJongRHuTvZd`ga9M zP~p)DmsW-Xak?2h}I03N{JFU-s+cdy!g&&$2*_SSBV1}I-NAWflK6VAb; zsjx*q@e#>&p~f4m0sAhLKh0_5#b*x*xg`k(D%ierzLtY`0(R3H-Ld9BD@} zP|)?9e608!h&bC&EGrXBZ%)01h&bH4qPfKQFvwTw%`3eWL->o)e3?!ZV%rKbj?OwFG zn6di4Q0|At+A<(w@xt6nt1(ZO?_p}?Uy;VXo~RXKyO)zWTV-3|tFq7Kz&b-n7ht2Q zJ7+BF6F!q_YlkKHih%6M-Zyukn^8LS49ZM(^xHq?DEr{o5PiBk`i5+eL>Z79dD9l{ zizk=<-GF@|(In+RO1%FaF$G=OiZN!FnyFA#btza4ZZ@Fqc)J+Vuo0I zUVG@N{)mAP)xK!ap||(F+1D$`ah8ost9GR5L526J%@q%XMz4@+Ed^V-%?^9J5;H8T z&o|G$7MH#|5(O7(aT+U3>ytRYIXfK=8$sPnHsQ_}$><_~0Ye@3eKomLh)A|30MpA> znKG1;PztJU>Or9svlyneqjx})vA*cy3OKsMQdDsZUpUifm|DdBIHz4%R^sh>2`@VO zePZIHt=#!D?ER3hh%HUW1V~*jfR4{Ul}i_e!mAYlnM$Se9+=kDWQG!T-(2?T%q-2! z)RMq#+Fglb<;1K=F^uMWV>Ii}_M7!unfD*bWo<8$1p=_ps;S1Vy(Zyf>fWqY5l3BSKt94>?M@Uxp_$oVlRiO1 zXp@oJ1oICP<7Yj3RRX(#k=bAy)8B{8j}QXp550qpfM;+oqqZWC^IZ0hR%`U?-wml; zTCC}fgUv3Hz{$S*FZO{kIPm+0#4nMD2XT#%DTjF?-)rty_le#sTb_vqCQlK93>wee zKkN3*ZAyr#LrS#tMe%TtX$UHY2mOJ(t1Z7!0ljgRiqdb1|B&@Nw+j)|oJ2FLlt03G z(QMsIp{&H5ja@cxTb1d^NZ#a~}9^-B@u9 znI4wN3vfGTwJ)~C_>sS}T+&qS$1~m$Uk;LjKc24%53)a%e8L8WcQp8|}}z2;>-b&QumdQ>F=L{Wh51u1!y+`5_f&eX)bVe^*M7#7fx^XS21U zVc}F=!{E}QwN*J@wnMn3q7WZe=jywV=~jOf?ef9vBez$jSnHp{zMW_Diw%~ewSl_9A|{+u!C(a z0KCbdQ+U-T6s!0eVq>{hQwuZ|eW96)c`aL)w%OtyJ;B90k*}v<7B{%%NxJ^;BmAM~ zG6LA6W%6nPSsT{pEmC>V?+kdKd}8qk8bPji$#DVKIY>6>@gc3ReG=?pJQ z+OeMoT8ljh*$>lZ!wnN|MVNH|MKx6Xcf-aLLFF*Ele%%}Am%y?MW0m80@s+2h_5Eo z9MZe~%Z3LjA^EnIuYCj7V_ZXlIG@{Hxh+;jF&_;&j;`00`DwJhF-U1YA`8PW6i zr`yk-?%!{*Pkj7B@iFdTA}g=HPL$@+WnWY$uX8k~@$>B@*BTtAVf7Cw+R?5Zx1C}o zP{bi%y4`5-cml;`s@LJ%dvga}A0BpOC8x!<9c|FRWb`7Be0hXoQ_khhYQKAQr0`2t z*`*2m+WYZ|@!oAT1CAbMiosC85OK;gXnB}Z1{d>rousI6PCUhgAYg9s!uQXu`$=24 zv*EJXQqRE9ELUz(vWYC6WZA{U->x94XNJeNwDfYlWdL;8dV*>SrU$JSYztr#QP&l# zjmOEM(Eyp_w$LfQMfP~7QU89q0=*X>=7DQNax=f|OFZE>&57nN$1PZ^2`WN$B9dLY zK?CLN&riNW`V3+IGJ?L=%0@BOZb>0HjgOKb7~w|>66AbvX77QOZk~TFxjb^Nb8~S# z#c~ay3s5JC0|u<26>-AC)F^SPzOn9+`E!4|m+TZUG>8!BMWp{pi=R}T4zfle;VXY% z{~&41bVmQtxAUR@CCeSwb8%a8iamCeU);gRU(DMQd`P8xDo_y6uH;j3%$c#w2Hxh| z{NQ(crrsS(*=B9tz2ipX!>T(`xbtn-$4-T>{;8hcGg5%{ zWf0->xVtQd%pDbl!&RkN`L4d$NzEaN-_tkiP8GStv<*$|SQ4u3&k@xA68f9c z&URFp_vf+5z(fDcNBp7boYXaa>)6j|otSD$r1s15--VavZg))2yUHpmrj|Fqe=|$N zUnaZK6)ziEzxJIIiXL16z(K)vJ|l7eWkhMQ(ft$_`A@?s-H2$dxddN6?_YdkJ)PZ6mB~o486948QrEw{(;-? z{kj5sEDddEL=&IL2homZfXRp!DS%{E#)EXikImM-NfW~+a7QHU#!FWK6o zrVeJ*v=hkW&Fap-{eS-N|9_G{E8j)K|87VRfB}jahu#DXK_2k8Nwt{oQpeq7j`;$V zlma7w-@ILn@e0bhD*D3Ua&1pmbKfBW_cbOCQ?}qsWAZ~ zTBtaL_zF`+C&Ye@FYqR^VX-F!t@$UR+% zH&Pw#jNXIiLBC;P!_T{HDvozZ*ym1Iv3~);SpZOLM?M+2oq3(GDi8 zk6CP(ZgKH?Z+!x)?*gpq(R&i#DdhIr%z8`HmP*2;8r>V#Dw)0gQAK@Jo717g_zBB@ z4VHuUM5nJ#5#|JlmURLf#}(cqCSr9M;pU0$vL? zm&sPI@(i)sWCY(RfqjT_Te$oRvyogqIp-<0>Ojj$^Q`X(jXM~dtap|Lc-;FwN7ZhD zy4xoXkQC?A9ZKMk+MXYJ{47?|r^dA4xp&X-s5@r^L&GcGjntW?HR0aIk9!=YW5?DEkbSYEt79#k~%aXcZ@IOmkYdI{hhSplkAH*NX>pCo|JiUau-g`WiG6S%dBtewEx0FjJ;bB0dZ2iyEZS zz+WmgMJ6|L!#o0|N!bAzFcNlGHzt0;DpP)gzS7IBZG?(#miH4rj?d-o5XR+{&1yNH zRVmQy_wLXhnVWRSABXzG-cBWR;}1`>VT@FDsGXDj>X zc3{CAFS#WcjeeJUW^lZ{U2wX1Ti_n9XH&9~?O=K47jehBc?)k6;I^=^{%8kHm5c%J zga!Z)BXybqqD4*yWRLC20(}hOyL@I08x7pUnRhw08SnBHO=g8_F_d4(6cau+;r^L5 zi^*MZ@iW*t1kA!G?6AUHjgDu#J0>;ww5bQ|Jx$VwlwX)IO4I;eMv|-_EXQbzxlA(^ zNeBRX9RYUMq{aMa)M3l6zf~tOdIV%GqkRk$K+?q z8G+k1%iBrSo?lOm*{kC83Ql$E|0H|Q>3`3XK2-5Cf2*S78Ji?37FB`Yi;#8xyP>`u zr@RbUBB^&RQJq$x2Cct6k5QJK2rGr>U!l#Q$JsgV-aZ~5V8$!9oQ|hH;gTk2|5e+sEJo2OnE z-GNh<=syXh{xY)ifx2sMJo`Ie)FN#VaJJ8L>Gvc%P3hB){@%kzBP_;t++326WH>SjYA*HgKJqQlZW%uQ@u zIz>lXOxZ7BA|(e~@+QB!L58V9+BB+dS(%*aVB&dtwDz-*%7D@Z@^|%x_q3fF_)?^M zKap}iB!Z}QvD(r0Y}~sg`=X4KwO5Y0`=q7rWp4fc=CI=hw-)JF!;wZ0o0x)!S>Fo% zpEWr~OoZ0~noRdQ@N|JnqEC(j!bOygY@DM--8DZ#HGm_87BU6EHzj{>rS7SI;L+y} zJlbj3Q+4J&G@mM7TNHc8uGaj0DdFk^u7ZZ&Y3FbL5}iXpQTwDco-K8bGU-fRq}VI)zKh#x1l2=`rz`RxyG*E@HDFN%m3B!SC~V8Wt}kVc6}6 zP4ik7Vf7x2yIPm@lvc_Y})CJ#t1=7`JG-0+VFI+3tIC-+#(JI)!QanFHJq-RNk?{w(M9*ohzi}?( zV1wd?`4Ey&dwwdb3(VC6NaMIp_Ks{DTsv+fWpj~3e!%N|xXiQ4!ow$L+96fR;@*3* zzf1Or%qGGUj?6zCS={lqjy4J~VhgL{gR*zPTRXY{k_UVY2R^eErQr3^?oJ6Fyovh@ z$sw$XlLim;hpio+kBwKB2_n7qGX&b4coNIT;WcUB#GT9g)*7?Z&)@q+eQ1AC#$vCL z(H-Ce7IJ9CdIp(>zYC zwOch6E9y?9BnW%>PFWTB#_N~LYfM3O)xdphc<{!m z$>yaHMhpi*MUmZJ-GaMM$E|tx4|axXUuP|e&n_56=MY7u1God zxhfqyrS1M(u$o3~fPSnhCX>B3C{vUid5>0iryw9)oIdOa)!Rc-PHZh=n*=#*&WXN7 zc00kkMj&EZ8<}mr*f#CuXjN546QKVTr3OF!;*1{qs`^K1=V0*o?pcWHx# zW{_-~S_Ng_@RFVe_GihTkkR5M0d_lii@;&@^&pY78t)5n5=?zNI8XvP-KYLsPw++d ze_2CGLV_m%w!{GHTd27rK1p`y;9|jsdg=jHSLoKx(cqCQ<&)9I)Y1CTMqV2&*0S(( z;ivSEUIa!>_rg2&?s_4Cpkh~A-@!}rASL{s6m;drtW;k11_RvG$8aM3Mqy#!!XO_L zF9UQuLrytd-?i@4BtA_Yz-d^U3$Q!k6wjT_C#I7f+8A1F6Zm^OYP#iCEfDK&ChDal z6InZmBa_Q+*;~>qMBsbM`T&bSha4fG$Dwz@M-lXOw=R+%87%iwJF*~jg#e*IKDIo< z$)C##6|$fD-pej5j)&eR0SP6^;C9&{?RNKQ%vbbENw`TtD4=p4XA>`k15Ge4H<(2|Pvj zLrR9-J{NBbYqYO-+Erp~LB$)~d|fB<{o*xuR+N~sPS7CST%(Q9!Iz|)m@8r0xf*d5 zw^8N}COQtaLzUfo=FintMd|e1ce-u#(%xWRauFniVm#pvG}!6vJ?=U2JK92W_IwD0;%D)>%2*S?DMb(lZ6rEFg~i?co2&StrBb z_+|2EneVxhZ_-lCr%Uifu<+D@Kp=5d&cHpfOunaG#E2~NYWd{KP0k%o?#K?!e~#CV z`P2KA=vzxlMZw3hz(#L@oxXWpevywypr%VqLrp`}mwlhHg{?GsnE(q`QD_brDWPn9 ze;oOa6j`!sUV^7M3KR~w8&6**dX^#Y(W(*SCd1M8BoXsNK7ZH!JI{|hA2vShzr6WI zu(^a!YXuvP>#6rJkS!_&qs<}VK9sd(S*|T?6ywPboeWTB+yLvYzEeB`pBE!DhZ712 z;Yc740N?J~T2Kuw(|H|i{VZ3=W~M&X5n?8UDAD*q6El6uq8QkQ&q!*N@8l(EiMYj`SPcS?<-d4EsEkNxOpeVEfGfNjgB_?k>L<*9Q8N z)c70IoPI#AZ3%xvv~nFVH?}{P$^G{3&ehw62KQcM?%eZ8|3T!wk#i-%k7r-D9f`?k zc<8IX(p2MQ{XSzUx2dMee!n+CGYvl*oc2AusFER3`=Kapk~lLZdeE9gyIy=6euOhg@E?}D2;E%ZoZ#OZ!B$0Gt)W)Tj%i&$)!QNE+# zW#%`lTkHEl&t?N-?S*R+RbPR8bcI>55ZOu7kOWB|Nvy^Q@Xa@kcVwy96P&nb?Z>%qop&_$86~%s@y!IW+YvF0Os(Sag<01 zVF!os8L`fk<3bq&aj(|1Z!>IrYH$UKxD<$Ink>|>HTtDw6!#FyvA+ygtQuc3Cq(=c zvhP_V))c-d{^>t3T=kE#{Uo&y?0mwX(>l%_JL&@E%R=BOQeh%bF`T(B08n^-K znJ(@I^~@PzY9}I!+fpA--A;~nkQQ?A%XAS=vH27MofK^S-J8{P zUr+ZYGtLiHN*p3fcRaLi{Jd0gs{$L6GrOvQa<5blG1_@D&8=B|sJ6Pf^tiaO#k{WP zW=CCi;>fJkY*FPpL1yX?(O$!0=V!6Bb$bzS7cuZwmk6 zPd)k_myd&PHI~s@H8D5;;$Nb*{Q@=ob$RPY4ic??c$c1qajc(>?FPHuZq1iKXxK%Y zVgL^XjR>*VC5a{6M-)2-dJa0nQ;a(LVl{OSkIPOX@oS23wCEnA0v2H=*UIe`@{a{oW0OOvL zbIVC96K+^~7uc05P=6t7NQnMLbL;4=a(p%|$#0}Qc*3Vk$+c^ecMe;d6LMv7pJR~6 zL;Pu9gGz<3y~_gn71KO9SWC9mKv;iXn3X8O z@N}pV!9FUSENsdW4?EiXwjTsX+(c{XEXm*Rwtho1@pPBfrJ}SY zx%BlQcqfG7_bu?D!FJqMw3-K8?+Ex?6Gxo-t&N`n1x`{2FR|VS`M#ETxaV~Lijr{$ zMniICnxy7Fqm9lecK^Q@z_HPFv zeDYV}W?s@NZ+itVsp!3dFn4~^5tCfIOxHs1%>jzW(|*lEB$gFv$BZ6g(Av?j-1{Tp zs7D36%-Bam`~KN-oO$tiwIqAo$A=TyB!`mauE6=tN4n>qYFzbEo(zEN9*Ge z{$N_#n#MlzVz6?Nxg1RNyF9$G@HO$vR|ken?=D~J|9WV5?4;*$rw1X#(ydFH7M776 zT6Z0rtN!8DFdz>i;mk0|%HyOYwb}5D)^pfxu=-R5+l&seUs893St}bpvVQRmYblkHrfXCUhL*v38Egg0;kQHYut$FEGI1e2n|i{KX~} zMzw>reoxo#wBM2XzcNpN-;jK?>6Yl^3L6E2>)8>^SVq+!OtWfIngCA&tfw}-Zf*kp z8ZFQ5MkeXeyFVSN`4kW?qpniBaJw+q`?kpb0ngcHp?(*yChM;!{fpK%P@k)`h!bdD zML;Mkp=2t#M>;%Kv$`U1n&d%ysH8u*#Cm@L+~+g8!u#35SB|SxnY+()nbz_nNB1k5|^yVlg$0#4AAf~%{ouXj0Ef8?X=HT##u$sH<(2kgyK z611Nfj6F29*MNKSDRz)@2FO@(wvo>PE8R6TyMDn$z@%$?t!ekh;Ge_mUp<6c^gR$U zWwy0ZxPN4^vi58L69i3N6qa{mYi1S#xvB`IPvP2WQJ9}=InnG*RYEh4a;@Y4)83m# zHI=s8qS%Y5QBfiwL`51!M5R-jL`6WLFaiQXgp@6W2$TvEC1e#UARwd!fhrIp(zj6| zpfri}c8K%>LP!vmCM2N=3(|OB@3`ZheNK7CsBfQdpC5bpBMt`xR^GMN+dR*F<};^C ze|?Hq7|8PPu-D7>+7dW#^tSgLDie$mX-KmBFl`+(%rCPS_u=rPH-)x2Hgd@q@|O9=kEfK-6aVT$8b15lQUQ@$h?TP0iQ=+0^G2R@Opn zT!^Cy3VIs)wsiP-6iD>S1;o=n&JK@~R8Bwj*dKn(%5c7nuF^)(JL&)2lUyPR92)62HKtt^2(I0b%56kwZ4elz#AY?2H6O6qkQ;X zt4w}gh^$jLg`*yR71T8HB7Z0Q`3WLVKNH>-KHBGrvJzqe_~}#0Xn&#-h~7nYsXW;Z z`c&Uw*ueZlV8{P}9HzdgL5a=@=@ZjgJ^+1ZM&|buud$cP*2Dc=XhJG@E%taCLhu8% z8Q5${KbEQTHC?Z(IQMC3Wu%kPA*VD%jhD?Bd$(sO`oaRxKv+qZ7xZvS{sQus<)(wV z;DOq{XK^+&+L&A64&d42|5?EQ;AU^CRmjzkPTwx1Ma{PC9WB_sCY@0QCaGGh4 z$Opc_KIsu zk`2VEu%gc(@9@R{j@roF3Dq8u_~lUA@M|8yHexjr3;4hXhVJRW3#w&CU3%4kb=QMZ zof%J4tl#(5V?~1nv3?px56mjymmqI>FnsXXF6a>|0`!vMd(RDh;cXHw5z%3YLyz|G zZe~TZ&tjt}-_2hM^uCZLC8l|K`=t`YfhAa>_f3hnFs4C$QI8=_fiUOn4x8C@n(Lzz zhsFc6^c8wX<^BA(&mpMaaMs&{1A z&|JDNfqPUuC3J|3^`YmFUx9xks2J5uJ_$XOJ0N>VX<`Ym^j?fiUd+L-BOyh{;7~3U zPFW!|F4U?ifcK0KW1F#2>gzqPk^KB7XIr%E@tXiS_luQavNR9(;Z*-vN^WFPV3pK$ zu<7+0Ywnn?eIRvWA>8PALB5^n?)<~7B3^K2X{c&!$Mi?ryw}2BvqI6tH#v>zT$wyh ze2J_htzw`IPV>V#phOJJhW5h7gOiF}Lm!Z%zJqi{BRSGIY-nx8v&zxlaS!J05St|t_%ftw~ zGlUh#)t3wMD&RunTU`2D-^#VQLIVc}!^6Y&xAMKl#Pn*%<=I+~ZgSs!#>J34Hp%YD z0O_H24|E0*O;a?&#+vrLu*WQy{Mz%j81zx&mzYCBoQMdAbN|W-ojwiIn^2|AaZDt9 ztI;IBe02qLE332QQ(xHUZR_-}3V`;ORIEl`F6@Qrg6ScAL@IZ_RMvuxMXd|MY$BhB zd)B6$OV{r3qwFJ}?U7s;?(e`R>HamG-qhdd=<#t!xORlqs%3RqJKs4+nts099ufWS za(uV1-Pj&CFWwB!!a6I#5AUK0+64=4^hrw~3^25XILAOg+zL5x7r$V<5q+7>cqIyg z$W{hY%}wdr(l|vFE{k?r1`)>m@Yqb?@!kZKr8Y8w>J<%7_57PrI}%S z%Gⅈ5(F68lTR+w5zBlr zN?@}7@*YUY2uG4+{Q7c8i3|;A3!Zdor`Q*Jpw^`AV!KxNUeC7-3fgidGv9TCOL5DM z#GUUY5}g8`48C}HH(P@22X1AMH~S{- zdt0`=?7c^zY*RU0D`>5ytsfTD3>FIwhK%#(1tBtd>UAz6au6w{S6gKg*bMxH_s};vMflb!c>mzi z?rJCp#=F2-wADZ!4trd1F#{DwjjMJGC?q6#_(ewUfISvw@9rh^r0bM-VQy4wfZ1yf zVwV6AF!fN;+VBz%tr=P*IR@uKd1Ew?Zh=FzN(2j%?=rXfdN6Z?i+w8wjfMK4Rz#=) z&8Jq~nMHW7vE))>a5DXBp6ACwJCEz1?lGgKX<+H4vYwzO3WX~?L8pVw5F*kLbyr=h zET!2CZsgVkH?e@ww(qrI+bzM}pce_fheT_}uUo2e|F;(@MVH|yq>DY#rPJztdByt4 z@%j{}8e6OOlhXT%4?pGST{U}WyQ9Z+Bs!5-5PaczVMXC_;}N3$RCZ`zTVbJYx{=tH z?OqE`C2)w}#Jf9O-WUuTQ&(Vy9)S#~B6u;5T1tXM_!MX-R)555nrW(|{m@9#PE5jp zevu6(vQ^_^Jf)ufv?79W4Gsm@i4+9oMnmUyYqS;zBEs-n%mcXgHOL11`nQXHiVgW| zDXrHkzclG@9Pd-QuObUr=7W;YW?`S=a9d@CXdQ427_!a>++~#^Zi5+e$7>7RS^$jx zLZy#sh-xT{(t}}qxs?Nl61h4T%Y{m*L;46qA-)!>@&<{XGV>Q$5yvxwK$ zUNW-Rt=hX4bpoaWdp#ObVh6i{h)lM!jll+ll1t>PIT9TqGphn=$`6~6X%S4l!iQhZ z5Obex!#Ao}$L5RzOTwL(FKaTYY_>cJee|j2!LBBG-GhzozuotE`o%eDYj?@meqXWi z^Q5 z%DTrd1Gcdeyo*nX$o_598c`wntV&!}f!*qbxZ;qYJkP3+121_C3E-*Tzb4I+?Ep^) z4Q5NS2hM1O$_p3U5CG{49tM~WNHleqXUg_)8 z%{>p+R0_i-ZeGUEIv$k?J~B0+)gO9&MEri6q@<@;b9vd<5+972HL_D=<9-#y^2BP| z{<2_y^H%?-U1sdep6=#1paAKv`$kPyCNC6jADi;oT~sXFM`_wM_0m=6tgDp4^1hRK zT2N5+*ZM;*l$V6LVup$!!7uo5yx=6gH{=v-Ff8u}jQO1Fo_dk0L}a)jsc3;!$B%OK zDCqlWYxEe|=rkfQ5=|1p4Dp!$NvO-u^*?2ldVj z&vnk>c|;#>=TM0KIb<6fL0O+$Tq6w(rYo9EU2-ru&IM{Mf0x1Svc8z)9XkSbd>FIJU+qTJ2vlnz&#z!M8=BO~7q zZWlaGmZ?sQZnV2{`|D}S%uQYnJ~Dkc);KZO(pZIPe_#(&YoM>!vn6wE;aZrmeDIAJ zSD$X!Wu)I70B@t-^+Rt5`+y$iBI=H@Qm{%WH7*+-8CKO*Xo{Mg@B@IfjRfD#E1FE- z@V=(8%;eQ-o}wWTDZG1sYtQ(af$U;waZBthqlTBrtL{!t3jVG6DlwRNRn_vc-_?Nq z)du~o3f0b*yUedNO~e%y9iDHlrLCoGmNl?SYbxsbzq40L956s4B7hG5y@gqQ=|WG? zUkrpg>5GeSwbuqgk!e#VA?ItQIa5~{)S%QO+tCw^Y~f8OyZ!vvBa;;(&(x=fMo3eP0^in3x3&d5Ru9N|tE>ns<40ixf%K;S zUb0aZe5;PN7T|R0Yda()pfOtS-k~AH-a+facH+A7;B{X0=B^9b-LcmyL^{G`qJzrk zAdU5VrHRtYnHmFIDV1r^KYAl5E_G3Rwbp8t(CnHAJ3Qdh+J(@(ef`ZU>;2And!Lij z6{nivh(4nFrN^p+Ck~4>h|p59rT8eNou)kUo&oi>irxt)V?l#uL?-9eq<;0mAz)%) zau|>(ms1wQ1}Cc`y1%#!a%*WzER`Z;I?tbUlt_r&LiPr@#KFw|ac5aE!iEpX=B#-` z>|M4AUax)<{!qK&${<~FtecbPuDZb%hMv>cTUUV^rlrc8 z(9pdnZTCzL68a7d$Gn{yj&;j;6g|IiGXF&GD#M*l8P@l&?`+GoOH(&eW`zf$i|vZZ4!$3vXU}J@+N}!q$)tC!lIqLL#v_36>~l;vl;qSPI-wExieT&*K~dQ`&|+#T>ni@vs82lFhW|2fhJU$Ff^3KQY2$nWWOXg)fqgblIwXKrVw6 z;#OMLHRFl@rBh}~$^?lXrg${@UaL2hJEju!sr*J9?fUBSxog4u51Tt#>gqz;kl=)V z7zXH+429pAYv%W|I2Th(BuBv)xj?mc46`+69A;_=6hdZ~nl!UvTXRea4h|L{;q%Zg zZwNOOMqT|vjX>tTp0cH%P?2mJ@r3(LF8oI307PItrAicsMQ+kNlhuT)VnZ0q#e_YA zqv+#kJs~rpMAk+tOK!xjAtf|XG?JPE){%yt1i6j;7>c>z4a`67XWSSRplXxV%q_N7 z4XUW>`W%)OD=ig;NonYHlup_zb1YDO&AnO1R+k(U{@Rh%Ib&h-E{Zp+hje0em#qmt z)~R{>38whesj-DUPjWvV4mp$OVQJ{(x0d>~t}@)+>|lEW)$n`~f0L@MZdoo; zH7C75c*8g-KU-@q#&>kyqqQWzkY1H9S&;R5?dqWbj!P|2!}x`Dho&eQM}%RJtn>6} zOlfN!Rnd2!Ju0G%N=->G6xus=mk~prYZ6bHJ9AS*y*X|X+ezKJu}3f)e3H~HoBj0K zC&8r}Xoy*WqHBZaeT0M?8lm@Ne6NF-kqx9XQqdx1%W#Gr&k=klYSj`ao< zleLG$u}e6uQRaRj%AV-vUpk8EK?m3C@m@(kNguK1`sfi7qvBDIJx}ru2juxCnIxABuqv^qa_z2>J0Eh0d)y(##EK9)7aims zqHEz9AAg#TYc%qwhF~PAF5V&6X<#{*1!X5TOVo5h-Z+=S?dv1P0ADiiCi|F!7bA9R zl|NHyZ60^DD6LG6I77T1>c8V|VN6J2Uf9&vecZ&fS1$>jBYl@skjOZf7{MO%?) zGRW3f3J`<8-r8@K}dD4ua&tGL9 zpzmZ1zDV3L6RQZq!O&rP3}y3shOQpbPH+;u9^SSgTBImNlzJvK`vZK3jDar;18QZf zQX8&SUdu0Vx8kbCF83t(0moAPij6hTH$3aQnID<@nW58Yd!?`fX)-D9F8wml@12vk zG5)?fbIh%yG{Aw}3i=t~1Hp~xt>0@a`XJ&IKIxpr-=wURK7r2Clx6MLDjjNOc{M$? zElcVLw{}+3*7ds=CCzzlt`;_Ut!C^U$cn;5`*~Qh zRO^FPz8YxkqSiA4dNZAYIs6)Jj}H0_UB!J22t265sn{w(_6%~v;Sx=L#HHCaR|d+O zpAig+X1qg7y!|fkqI8nnM7HoU4)*s2>cq>zHC^V{1sbk`1=!vZ4ic7f5e@V{1-mN2 z0}s!*SU(OypJGa4aka|KjV|O8%1?~;E%qg5jXpiJr)W2Ca?qAN9Do2*6?}#g(^Uj+ zcPLu`Li}Xz=A>i|H-u+$qLaLji$O|G3L_E;TFKm4MAL~rl0QO;G>t>AZR49;acVZq zfu^+Wed>|i1G^P}0UpHpqfQ@By**NVK&(408!O65ahx08Y3)jQb>-E8{_9$4Wy>@t z2W1nf?R_~N{7lzrj0yO9Zokyb6->NPbDIJ}MH1L@p&Z#uWTl@pOEBOu(EpJ%$7%TE zWADeKtsC1M{6~h9kQYdQCZ5geF#tzuvG90}01|=o>=xy_Vew}D!|ydf(Ry+>%(oGQYR${Y~CzzKz;!5ucLgjd)QCA7qIY7Acdny$o#13iK#n@M=zKOMd{dxEWkdwM9 z=&v^|5~aj;HDw0uDb8V=o8G4R$0m(BU6_ku^#%_eFBPcZd#O$6#W7syML%tABgdY- z7-;USh?W^O=KC1G+TiZ)+Xs4vi-VgeySiw!fTYV4isMv~x&bPDD7G2g~+8meORqJm4hJ!Fo)ZS~p|UjPOm( z?Pzhm@dWK7D;)n=dx9At1?SP+|K2;Hqql~>Sg>}qMmD+$Dg5A zlwsoV8z|eq(&WD+*e*FD#@E-F)w>Zm@69c?j6YiR>%rD2uwT%66}TYD0w@@wnW+FL z=2O~HYcA;<(jsY!f%HDSC|wZyB@8%Lv)xO(0t4hpApmsRJcC$Cf~>kPRa=$x5^tcB zIbu%}kEdPqiCKGG74fg_XE&YOJQeb8m(lEq+vmb%ON(2w-1G*E%LkI8x$$;=eI4Bc z=i~bpH2s1;O72T@w*L;*w&4ZG<*T3F=8G4RuibscAa@_jh6Qa}6aKm=+LaUD3ZLIv zn=mhtT@rb$p)nA!J;Xha1gu_pWv+b)M;*!gtx?!BT>RFtl{y=p(1&|j_0$sA_)g;l z=9GC5*v-48RM`$$XZl8IndHn6HA*FtcW}sI|FES1+h~UB!+{%>ez{GyAXUx;GdBx* zBETt6wG~{9;^1J3Gxq=k2!1fPl)pDx$!KvhYcUP1N%L|5=4lM^RuFK~+rG*9uL-C{ ztuJBuVi5!`)=65IVRu48-oR2I1=Y$Fo0pC)np z+%2ozX71rrkQlijP@WULgfqoBK)H;o0DJ6hATH2N8*)I27@&2m#n3l>mw?uYiF|Ae zmTn}wtTn9KXc2Nyvm?M^eoG*;L{kYBDfR^$ZrB37;rv0ip=^e-4Q!vS3y8>SocJ(Qi!j&V)~MPD^&`lR z;@?^x!qj^7R$xLBivZnLT@pXB1tDcHw%BLku8ERL2{nIoE`hL+lNK6ak;qHyOsY8X zYW3S#K3m|S4{JqA6j>rj3U(SFd|&fWYu$I={}X-QB7s8f2DoF0-o);WUJ01%n|jIC zk&aKxfOTsaOu&R24qQR0`_3f|u8)7W&@{Dx;+3ywD3_F4jQS(hmR#|Xxxv1YJur(3`^r2P^H}f+4_s*$+**mO9TFZNAG)8nsdJnx= z)`gNk4+t}<^nmF$S{MwPWP6_!^joQu&7pO$khdB`i#*58;LE;c0*~#^E7KL_KD;AKAYT&CvRW7TM ze^<#-V%lxK*IfDfke&%3AP%!sNtspxU(zY)?5Iq+jqzp;(x%kgbNlPYfq@*OOy#Rt zUE_BC@*&K3j+gK2hXyEhJ$LE{+^b-Fo^e;%h;K{ipv?$-f=2gc0nw!A-`Tfd+vhmo zYCV#9_XMbhN|P6?6{PHk2_CSwcq@4;e0f+G6;XM#UBh7wqq%{PBC<9zO~*<<94}AF zNgf8m&^G(mXLxLuZ30L5WwaRyBf0?o2a$f!VaYNKbD9n{sY7=da;fCrCg(u%G zA}?AIMp2PC10kie1J%oNVPE_T9IQT$gDr-k#ovWfYU!NosYj~?3=!jUhOe>i>KozV z9k>^5lB+{NH@Qcm7|CJ1Ktpyz8)tSqK}yd->KH- z<}pr5FolvpR#{t6NHPA!AX25gm^0hX&KOkoR~a+lav+V>MAu5YTpXPY$Vx%XNin0@mW}&k<*;@ z_`5}#i(nYnF|lpx_Ie+6T{yG|_HOjr9KST;z{BT)&x~wOOF3iVyT@it>|L_|iWa4o z7W=;PzWXD}R+^7S60X&B6+L%)D%SPeO{ZL(^E2wV9-N!8KOQLrb8p#VSW);*4yf|c z$rNd-Oo>p*Z@|KuZjz0K&W}5os-qPZa}_OO12O<*ZYR_)jIbi1vOThnk<;bL51EM< zdpdBfLL{f9JoWn3JfgZj>{ejWGYe`T%kDEvI{>*Fzkwbwxt+m?97auee{MCn9cv=% zW0$@qJdcz6I9Xuuv8NxDp(f@9w?tX6BNw^}shMxef#*1_G0W4max=!zf!Kh2pUT7% z#zSJIbp~;V$pV_ik^0s(y8-tjG*_X2&5?HHM+rWG&<<${>(|!a7}12FCNhlWi&+Ve zb5UVs;zEU40s5Zs3s{16{N=tX5eYZ?k`nTXKy%niX!(MI5~6dn##UE+w<@(TbE)a@ zKU1Fcnxx+Tp)s>0=j&@i{t<`H`%eDXMjR7MYR0YGLoa1bPP&`L75VL86 zByGtiDj?AK)>QCj7T7shMT zwZs-B_l9d0BQ+e`yL(PPIYxE>$M>4C8^C8$vTjTT`t%Y8Nf&P574q>0{J@ma>X9+wh~vT66$=7^(wo7f03dBo3h=Gxh7a`lOTy z8i%*4-t{>$B|9lLyU*9{+`XVY+^O0%BQLcNDt_D%KP%fM8G>s-l`TdzB+a9&Bt>#_ z!;P>Cl-JM`{YHXpF9}tzzWa$E#!H$3C~$NI+!#|C=$|NNIP4lzLDo{k0J%2CQ4lm) zK|GgF#tA>?2XGy9kK@A!w+Lh`q5cf_epXG|fuNWNBO|S^?}wf|7u;J`QIkNKa3fYs zm(#AW-bw~w1Q0%eZ=I0loargEK;fcMee7Y4MkRVqukXGFE*`Phr+rE}ar54#`+q+y zck3SB`P8ULHL%VdAv@@@Kmt6U@Q+ynR z<$V&c3ACl7=sCnM`!?$AzHnnKc9R!2$t6+6uI$l$J!RLEiSAbJ9{US*3iS+KIYEV{ zzA=R^ZFfg>+Jby59TV=;b=rf}bY9KgCp2M;{shpn))}jfCl>wR_~2uqvQ|m90>@4I zC%plZ;?R1@j>b#EeTwMYoL`TH(FqhmWA-_daND`R~tLtyigTW=OMett$O3* za)oc+?V;E3TT}Fut?qW4E%Ey^Lkowf%AnH~7^#{J(^mPe8v`x5uGDkIqb83l55%`G zZ2Ifmed~AItIDs4D#}Zab|?SQN;J)S|1xG8_#Q;1uYg+w)3u;EP^LQG?WH5ory8oF zfJ7qh$!~(e(aPJ~y0q@frrHXSW^Sqv&VFB)!tsEU?WbRv9KZX!jQXH8WF?Cw|>=dsyj#M=-9*F}T#h zKC{cP19QX&2&r(^0s7q6$r--H23q`0Zu+o|WIKw{Dpeay1H^a!5MJh^TRMIKnqOcE zRA`bgRt0scbrLVRH`lUBVDh)$1+lzaU*xz;Y=rrik^ z<&ys9dG9%W*)j3wXjoOY9?|zj7RLt*wINub|4q(tBj{!lp9U*-V3ZBV$(DVS`+SpH z1kQ^uW8dUn_(D^Qc*BxXaq>609ih`!WB^F7jO1MfFn4e+#5IEfR-3X4^Y`28{tevS zEHq?PGaBa!0aHCWjpi)ttd0{PV^D)S${0-c3- zC>ez=S)Th%Zmaqr<_+dKG-bm!bbG=6+dzNtL_@s+E{JiHEph%P_Xi>XH3E>Ec@2Jw z?ohzr2l}Jui7o{hD-G-ui$Ko#a>akIIWXMA;Gyakw2DC=|Cn)#b$AA$=SDfb0B`r$ z{R5$Uokh#QKtsuqtP}OBRqDB$Pn`OlvPybSF971$5t!R*N@-y3-WE4P>(Ckm5z_YQ z=l-KYR%}oWbjU#N*g$Zm=LdGalTGrqTX}|W_`jzgfFmvL><|eOD}=g1W0JN zc(i7~8i(3;JDVV%HWzr>6pqonZHNwv4xIt2(qpc0l_&`I0+BXlsbmN2Z2si57mRp; zRslL4SN&}(DtmU%yu?(L@M6x#gcM)km*{oIyDB@{9(eV+MEbSl75j4+OQYd!R(4e8 z#>vsPye|QzJYUOu=y!GxvXZ9?C%U95(KI*2iCanyjNOI;^{6(Jh$gIa-~kwPc%JGM%txXFMK!k~umByh1mR(RcxB zExC0e*v*Ft7Sf7({X6W{GHs&qwL9l4lj{c+gy{(d<5P!jDI0s(d5-O0!1ca8v_p;} zX`7AJZ%{d&TJrTqcny_e&A$NAzp1rrZFmo}& zXnHPE0z7I_u-h@ANyMy!b|Yo2kP_ux__dFjX2i^WM7njTyQuy1yQ%K;`*B{Q3$d|v z$0EPP`CO=9EWc^UBlra#)Eh&pu)P35W=EqHVLQ>}7%V||RqKM9T>7dufBik0RnM?1 z2{H}wo(v97+21>bGc$m3AtKQ3YQLV;RS9`^+RF5|-|{}X8QedgKs;HJR?__{_adX> zV(5ik(a39n@*Nz5&50#mObJC@O#MZQ_^11H3F|)|70U;I?v+2!%b#=L=N$Ms2mZZt zfF6Sh@N{Y)9@+#rnD*DtB{!jz&0U75k+N0on>q)4O%r?W8tU0QRj!+yR;ssdyQI%p zb7Y3z>vs)j6J&_;jcVObMu|`VVIwj6oNmzMCJGj4aVAZO6VKJ278!J|ZzPaZhXQXP zV^X=^4fsd&E2 zfYxNKe3JQKIo(YmDtBdwDK)9(+3S{^EWS6ftfSvLB~+znd}rsUNH$oS1e4dSV{77@hF~Z=w|lOxWYg8Y`6h zE?pXm?rVS7-lrMS*WKN>>6jmn$2l6k`Rvg+oi@i!E{Bu~U*}^8PFESvvj(cPpU&@K z%pPRrj`ar!FX=pr{+E!7e+AX}+wcAeXs5h(O z`XUE`c2x1HmQ46d;~ymEFc;ahCirkA*Q~|ha8G2UxvF}uufHMB)lZ!}?C?BkS-kD# zlAi6JrxTO2`^`VLJD%Lx!KyL0(%wAlo0?`@b?vTSFk@s>)iK#`n4!<8c^%r*0BIDo z37VN7ssv!`tv{hpk0`HLB29@4Fxvfc^J|GS_X&+}6zV2RP@FB4DS#jAZii}(p^0vZ zDiEmC6;;rCN@|Gn&Wg%m@ApQyweQNOg4PjUN6T z+$ugX>eSbghdX@LBInmm&G>`ugQ4+{lp>CPjO4Q6s7NkGsuBz!pwV%bq}eSK)Lve; zf-l3fW%rKFm9Dnhbuh-S1*0C;^|fR(9P1b>tJb8N^d-qk7IGoeF>6&%d?(>dok;cQ`h}P`y22SCR6GF z!4+)*kZ5`Q8@}Xdy$~gfHLJzIfwOGcxjcZl|M|U~)0Rup2D-`VRo~>^2C28_PQT&f zB~efxZB{i(f|)-oJ^aH<+WhF-6D1(vYxo7T{+ry_j%r0PDlA~rd^V;2?e2NIAamv{ zcy*DYT&Xn|LdZ@)C-8!H5HFs@E@-kJQ)OGQR7g?>p3@qD)Gq_op+}zLm4H;#0MCJ; zYC`%fp&~G+Uohk^1wZZRf#%~fW!uld07bJ-8=^@}Z}7j#?duG&qqXYSKvUbkQ@3k+8abqDyY69(l$0+d?< zUa6W*S-xaS-1ALtJL>1F`cKdEpRelYs`|OA{@tg?&sFtvRsCF5|J_&BiI0>vn&9M{ zuPJ>!wV20lk=e1^f?SyMNo+~&_`FVq5Q?eGkI_O*_ zwGMrbEbt9h92i4l)&{ZHj-}w^WGaLnx2vro+7X7tG|T$fyus6Wg_Uh*OV#r*gaCNg z{@+zO)Cn~Wxh1CLBe3L7bJZ?Pn0un_CtZW}@1Kn0-^~4OR6l&&P7m6j-jAQ4&F37r zXgHd!a;nJcfL?~W|FL;%gIwoC=$f8YV6?}7ZPflV<@rCz7k{4kKTrIhC;or;6aOdX z^S=emer|w2H^848;Li>4C+PhX^!{0&_*tL$H`U?(Sy*v?9Hq<}%J(v1gsvQip3q+u z>UK@}R|gFFp}4e0w(6))gMXz!7CHQktU1J}ioH4#d#&G{aqmz63hG7te8LZN^1nkM znduOyq*0>eKPS7AKDwljnyAuGwcSmAZ9AX2@Eg2Spiqxl?$)r-BVtEUwvn-LWS(U) z`2<`mT;gc@g(`uHYFAd2hdMi*KQ>|LptVSZSqq=DkM6g6U$n`Or`LD!S>-dhN~Qn` zm^!bed6)?`0qdKbTX4OH9LFA@yr;*Fpw9?+j+( zH?|02`AzPZc8o(U^c)O+LJ_nM<;`G+GC3+O2J3m=_C9tPmbL!Dk8mG18K0! z*}Ps*5v;|qhN^(Gb!&_VqP|6g>(U%OJ7CG_T+pJ^y+~J74+JR$gi|zCirhpV*Yc*` zD8Qb6Ycl%^URLA;lv(>VTqXPvm}8kVK{Z^~gceJdo=3Utrg$VtLwQv^K^nZdZ{&(= z()QwV2eT5|bty1#uS1n%CQ7|gfZ2Be8|$@9NU!bRH$n;+f_RSEBzi*<261V5dkW@v z*=`tfBBnVhU7Zo@`o>BycW0v7HkU+c+7Z`z+Tdk#2aA&pfU7*(444XMmQ13zlgV=8 z-^ub1La)|9=pyGbqUcn))}n`P={e>!}3i`H?xg z>?oawU5IUvp(GDw5C6xF{fBF2{`mW4c<3MDe4<25I{pUe@Uj#;%Q7!xK=+gmCQlzG zK=3YY=?4A>02r>-Zf42O=m4KpKt;Iu2x{9Sy$uPt;#K!?3d(0 z);#d91e5tA7PQH|M*LSlJq!agW}*l~Q@0jw{&@J*AKa<#eCQ8%j2~=jeJ5qtL&v|# z-9!9*PyYwA{O5c6xt{)C3eL~<^#2`Q|6zy!Pn;S**VE7S^m9G^`=YR)>*?ou`njHd zt|yznF%o{}Pk+x8f96j=^W6Ux5Ba}=N00#?vGzZUM*y9f!_uwrV2@}@nl&S)u9SoT z57jqA6?Mf{Dm@LdCDH^#8WKk;Y&JD-Mw+gz65isfHH&27*@M<=N^()Rst;yj3r2g8 zrWac34)2J!=cAVi^lzh;Var1T%AERSyIkFS>ZWWRJh* z^*wZYNp||&iJBDd!UC5&$t7gT9RCr3lW*K#<;Uc)7v@j34CmP$zgp~+)mLF*b?{Bn z+=!d4wMkFXzUTnA;ERCF^#dg>{C&2-kqYayf)0V94CZ#&lMdhFP@)9QvQ2R@Tu=4( z4jZ8+XamMDSR74eJKg))^Yl6Uc4mtxVB*hn5jq(OP^xY<>+;F0>zn^-%JxmTReioI PEk(Q!XA%6%x8DB;$lEK) literal 0 HcmV?d00001 diff --git a/lm_eval/tasks/noreval/noridiom/_noridiom_yaml b/lm_eval/tasks/noreval/noridiom/_noridiom_yaml new file mode 100644 index 00000000..44b1f121 --- /dev/null +++ b/lm_eval/tasks/noreval/noridiom/_noridiom_yaml @@ -0,0 +1,23 @@ +dataset_path: Sprakbanken/Norwegian_idioms +training_split: null +validation_split: null +test_split: test +num_fewshot: 0 +output_type: generate_until +doc_to_target: completion +process_results: !function utils.process_results +generation_kwargs: + until: + - "\n" + do_sample: false + num_beams: 1 + max_new_tokens: 16 +metric_list: + - metric: em + aggregation: mean + higher_is_better: true + - metric: fscore + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/noreval/noridiom/nno/noridiom_nno_p0.yaml b/lm_eval/tasks/noreval/noridiom/nno/noridiom_nno_p0.yaml new file mode 100644 index 00000000..2bfbe037 --- /dev/null +++ b/lm_eval/tasks/noreval/noridiom/nno/noridiom_nno_p0.yaml @@ -0,0 +1,5 @@ +tag: noridiom_nno +task: noridiom_nno_p0 +include: ../_noridiom_yaml +process_docs: !function ../utils.filter_dataset_nn +doc_to_text: "Fullfør dette uttrykket: {{idiom_start}}" diff --git a/lm_eval/tasks/noreval/noridiom/nno/noridiom_nno_p1.yaml b/lm_eval/tasks/noreval/noridiom/nno/noridiom_nno_p1.yaml new file mode 100644 index 00000000..960f808c --- /dev/null +++ b/lm_eval/tasks/noreval/noridiom/nno/noridiom_nno_p1.yaml @@ -0,0 +1,5 @@ +tag: noridiom_nno +task: noridiom_nno_p1 +include: ../_noridiom_yaml +process_docs: !function ../utils.filter_dataset_nn +doc_to_text: "Skriv fortsetjinga av idiomet {{idiom_start}}" diff --git a/lm_eval/tasks/noreval/noridiom/nno/noridiom_nno_p2.yaml b/lm_eval/tasks/noreval/noridiom/nno/noridiom_nno_p2.yaml new file mode 100644 index 00000000..b2207cfa --- /dev/null +++ b/lm_eval/tasks/noreval/noridiom/nno/noridiom_nno_p2.yaml @@ -0,0 +1,5 @@ +tag: noridiom_nno +task: noridiom_nno_p2 +include: ../_noridiom_yaml +process_docs: !function ../utils.filter_dataset_nn +doc_to_text: "Korleis fortset uttrykket \"{{idiom_start}}\"?" diff --git a/lm_eval/tasks/noreval/noridiom/nno/noridiom_nno_p3.yaml b/lm_eval/tasks/noreval/noridiom/nno/noridiom_nno_p3.yaml new file mode 100644 index 00000000..edd7cf99 --- /dev/null +++ b/lm_eval/tasks/noreval/noridiom/nno/noridiom_nno_p3.yaml @@ -0,0 +1,5 @@ +tag: noridiom_nno +task: noridiom_nno_p3 +include: ../_noridiom_yaml +process_docs: !function ../utils.filter_dataset_nn +doc_to_text: "Fullfør vendinga: {{idiom_start}}" diff --git a/lm_eval/tasks/noreval/noridiom/nno/noridiom_nno_p4.yaml b/lm_eval/tasks/noreval/noridiom/nno/noridiom_nno_p4.yaml new file mode 100644 index 00000000..96f25dec --- /dev/null +++ b/lm_eval/tasks/noreval/noridiom/nno/noridiom_nno_p4.yaml @@ -0,0 +1,5 @@ +tag: noridiom_nno +task: noridiom_nno_p4 +include: ../_noridiom_yaml +process_docs: !function ../utils.filter_dataset_nn +doc_to_text: "{{idiom_start}}" diff --git a/lm_eval/tasks/noreval/noridiom/nob/noridiom_nob_p0.yaml b/lm_eval/tasks/noreval/noridiom/nob/noridiom_nob_p0.yaml new file mode 100644 index 00000000..7dcd6fe4 --- /dev/null +++ b/lm_eval/tasks/noreval/noridiom/nob/noridiom_nob_p0.yaml @@ -0,0 +1,5 @@ +tag: noridiom_nob +task: noridiom_nob_p0 +include: ../_noridiom_yaml +process_docs: !function ../utils.filter_dataset_nb +doc_to_text: "Fullfør dette uttrykket: {{idiom_start}}" diff --git a/lm_eval/tasks/noreval/noridiom/nob/noridiom_nob_p1.yaml b/lm_eval/tasks/noreval/noridiom/nob/noridiom_nob_p1.yaml new file mode 100644 index 00000000..95460b5d --- /dev/null +++ b/lm_eval/tasks/noreval/noridiom/nob/noridiom_nob_p1.yaml @@ -0,0 +1,5 @@ +tag: noridiom_nob +task: noridiom_nob_p1 +include: ../_noridiom_yaml +process_docs: !function ../utils.filter_dataset_nb +doc_to_text: "Skriv fortsettelsen av idiomet {{idiom_start}}" diff --git a/lm_eval/tasks/noreval/noridiom/nob/noridiom_nob_p2.yaml b/lm_eval/tasks/noreval/noridiom/nob/noridiom_nob_p2.yaml new file mode 100644 index 00000000..f676054f --- /dev/null +++ b/lm_eval/tasks/noreval/noridiom/nob/noridiom_nob_p2.yaml @@ -0,0 +1,5 @@ +tag: noridiom_nob +task: noridiom_nob_p2 +include: ../_noridiom_yaml +process_docs: !function ../utils.filter_dataset_nb +doc_to_text: "Hvordan fortsetter uttrykket \"{{idiom_start}}\"?" diff --git a/lm_eval/tasks/noreval/noridiom/nob/noridiom_nob_p3.yaml b/lm_eval/tasks/noreval/noridiom/nob/noridiom_nob_p3.yaml new file mode 100644 index 00000000..318be0f2 --- /dev/null +++ b/lm_eval/tasks/noreval/noridiom/nob/noridiom_nob_p3.yaml @@ -0,0 +1,5 @@ +tag: noridiom_nob +task: noridiom_nob_p3 +include: ../_noridiom_yaml +process_docs: !function ../utils.filter_dataset_nb +doc_to_text: "Fullfør vendingen \"{{idiom_start}}\"" diff --git a/lm_eval/tasks/noreval/noridiom/nob/noridiom_nob_p4.yaml b/lm_eval/tasks/noreval/noridiom/nob/noridiom_nob_p4.yaml new file mode 100644 index 00000000..f03c9df6 --- /dev/null +++ b/lm_eval/tasks/noreval/noridiom/nob/noridiom_nob_p4.yaml @@ -0,0 +1,5 @@ +tag: noridiom_nob +task: noridiom_nob_p4 +include: ../_noridiom_yaml +process_docs: !function ../utils.filter_dataset_nb +doc_to_text: "{{idiom_start}}" diff --git a/lm_eval/tasks/noreval/noridiom/utils.py b/lm_eval/tasks/noreval/noridiom/utils.py new file mode 100644 index 00000000..938891b5 --- /dev/null +++ b/lm_eval/tasks/noreval/noridiom/utils.py @@ -0,0 +1,44 @@ +from collections import Counter +from string import punctuation + +import numpy as np + + +def normalize(text): + exclude = set(punctuation) + return "".join(ch for ch in text if ch not in exclude).lower().strip() + + +def f1(prediction, completion): + gold_toks = completion.split() + pred_toks = prediction.split() + common = Counter(gold_toks) & Counter(pred_toks) + num_same = sum(common.values()) + if len(gold_toks) == 0 or len(pred_toks) == 0: + return int(gold_toks == pred_toks) + if num_same == 0: + return 0 + precision = 1.0 * num_same / len(pred_toks) + recall = 1.0 * num_same / len(gold_toks) + f1 = (2 * precision * recall) / (precision + recall) + return f1 + + +def process_results(doc, results): + prediction = normalize(results[0]) + completions = [normalize(completion) for completion in doc["accepted_completions"]] + exact_match = np.nanmax( + [int(prediction == completion) for completion in completions] + ) + fscore = np.nanmax( + [f1(prediction=prediction, completion=completion) for completion in completions] + ) + return {"em": exact_match, "fscore": fscore} + + +def filter_dataset_nb(dataset): + return dataset.filter(lambda example: example["language"] == "nob") + + +def filter_dataset_nn(dataset): + return dataset.filter(lambda example: example["language"] == "nno") diff --git a/lm_eval/tasks/noreval/noropenbookqa/_noropenbookqa_yaml b/lm_eval/tasks/noreval/noropenbookqa/_noropenbookqa_yaml new file mode 100644 index 00000000..25166cfa --- /dev/null +++ b/lm_eval/tasks/noreval/noropenbookqa/_noropenbookqa_yaml @@ -0,0 +1,16 @@ +dataset_path: ltg/noropenbookqa +output_type: multiple_choice +training_split: train +validation_split: null +test_split: test +process_docs: !function utils.filter_dataset +doc_to_target: "{{choices.label.index(answer)}}" +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/noreval/noropenbookqa/nno/noropenbookqa_nno_p0.yaml b/lm_eval/tasks/noreval/noropenbookqa/nno/noropenbookqa_nno_p0.yaml new file mode 100644 index 00000000..63551dec --- /dev/null +++ b/lm_eval/tasks/noreval/noropenbookqa/nno/noropenbookqa_nno_p0.yaml @@ -0,0 +1,6 @@ +tag: noropenbookqa_nno +task: noropenbookqa_nno_p0 +dataset_name: nn +include: ../_noropenbookqa_yaml +doc_to_text: "{{fact}}\n{{question_stem}}" +doc_to_choice: "{{choices.text}}" diff --git a/lm_eval/tasks/noreval/noropenbookqa/nno/noropenbookqa_nno_p1.yaml b/lm_eval/tasks/noreval/noropenbookqa/nno/noropenbookqa_nno_p1.yaml new file mode 100644 index 00000000..d6888bd4 --- /dev/null +++ b/lm_eval/tasks/noreval/noropenbookqa/nno/noropenbookqa_nno_p1.yaml @@ -0,0 +1,6 @@ +tag: noropenbookqa_nno +task: noropenbookqa_nno_p1 +dataset_name: nn +include: ../_noropenbookqa_yaml +doc_to_text: "Faktatekst: {{fact}}\nSpørsmål til teksten: {{question_stem}}\n\nSvaralternativer:\n- {{choices.text[0]}}\n- {{choices.text[1]}}\n- {{choices.text[2]}}\n- {{choices.text[3]}}\n\nKva er rett svar?" +doc_to_choice: "{{choices.text}}" diff --git a/lm_eval/tasks/noreval/noropenbookqa/nno/noropenbookqa_nno_p2.yaml b/lm_eval/tasks/noreval/noropenbookqa/nno/noropenbookqa_nno_p2.yaml new file mode 100644 index 00000000..e5b634f6 --- /dev/null +++ b/lm_eval/tasks/noreval/noropenbookqa/nno/noropenbookqa_nno_p2.yaml @@ -0,0 +1,6 @@ +tag: noropenbookqa_nno +task: noropenbookqa_nno_p2 +dataset_name: nn +include: ../_noropenbookqa_yaml +doc_to_text: "{{fact}}\n{{question_stem}}\nA: {{choices.text[0]}}\nB: {{choices.text[1]}}\nC: {{choices.text[2]}}\nD: {{choices.text[3]}}\n\nEr det rette svaret A, B, C, eller D?\n\nSvar:" +doc_to_choice: "{{choices.label}}" diff --git a/lm_eval/tasks/noreval/noropenbookqa/nno/noropenbookqa_nno_p3.yaml b/lm_eval/tasks/noreval/noropenbookqa/nno/noropenbookqa_nno_p3.yaml new file mode 100644 index 00000000..d356818e --- /dev/null +++ b/lm_eval/tasks/noreval/noropenbookqa/nno/noropenbookqa_nno_p3.yaml @@ -0,0 +1,6 @@ +tag: noropenbookqa_nno +task: noropenbookqa_nno_p3 +dataset_name: nn +include: ../_noropenbookqa_yaml +doc_to_text: "Bakgrunn: {{fact}}\n\nSpørsmål: {{question_stem}}\nA: {{choices.text[0]}}\nB: {{choices.text[1]}}\nC: {{choices.text[2]}}\nD: {{choices.text[3]}}\n\nSvar:" +doc_to_choice: "{{choices.label}}" diff --git a/lm_eval/tasks/noreval/noropenbookqa/nno/noropenbookqa_nno_p4.yaml b/lm_eval/tasks/noreval/noropenbookqa/nno/noropenbookqa_nno_p4.yaml new file mode 100644 index 00000000..b819b72b --- /dev/null +++ b/lm_eval/tasks/noreval/noropenbookqa/nno/noropenbookqa_nno_p4.yaml @@ -0,0 +1,6 @@ +tag: noropenbookqa_nno +task: noropenbookqa_nno_p4 +dataset_name: nn +include: ../_noropenbookqa_yaml +doc_to_text: "Ta utgangspunkt i følgande fakta når du svarar på spørsmålet: {{fact}}\n\n{{question_stem}}\nVel rett svar blant desse alternativa:\n– {{choices.text[0]}}\n– {{choices.text[1]}}\n– {{choices.text[2]}}\n– {{choices.text[3]}}\n\nSvar:" +doc_to_choice: "{{choices.text}}" diff --git a/lm_eval/tasks/noreval/noropenbookqa/nob/noropenbookqa_nob_p0.yaml b/lm_eval/tasks/noreval/noropenbookqa/nob/noropenbookqa_nob_p0.yaml new file mode 100644 index 00000000..a22f147a --- /dev/null +++ b/lm_eval/tasks/noreval/noropenbookqa/nob/noropenbookqa_nob_p0.yaml @@ -0,0 +1,6 @@ +tag: noropenbookqa_nob +task: noropenbookqa_nob_p0 +dataset_name: nb +include: ../_noropenbookqa_yaml +doc_to_text: "{{fact}}\n{{question_stem}}" +doc_to_choice: "{{choices.text}}" diff --git a/lm_eval/tasks/noreval/noropenbookqa/nob/noropenbookqa_nob_p1.yaml b/lm_eval/tasks/noreval/noropenbookqa/nob/noropenbookqa_nob_p1.yaml new file mode 100644 index 00000000..5aa3fc62 --- /dev/null +++ b/lm_eval/tasks/noreval/noropenbookqa/nob/noropenbookqa_nob_p1.yaml @@ -0,0 +1,6 @@ +tag: noropenbookqa_nob +task: noropenbookqa_nob_p1 +dataset_name: nb +include: ../_noropenbookqa_yaml +doc_to_text: "Faktatekst: {{fact}}\nSpørsmål til teksten: {{question_stem}}\n\nSvaralternativer:\n- {{choices.text[0]}}\n- {{choices.text[1]}}\n- {{choices.text[2]}}\n- {{choices.text[3]}}\n\nHva er riktig svar?" +doc_to_choice: "{{choices.text}}" diff --git a/lm_eval/tasks/noreval/noropenbookqa/nob/noropenbookqa_nob_p2.yaml b/lm_eval/tasks/noreval/noropenbookqa/nob/noropenbookqa_nob_p2.yaml new file mode 100644 index 00000000..667b04a1 --- /dev/null +++ b/lm_eval/tasks/noreval/noropenbookqa/nob/noropenbookqa_nob_p2.yaml @@ -0,0 +1,6 @@ +tag: noropenbookqa_nob +task: noropenbookqa_nob_p2 +dataset_name: nb +include: ../_noropenbookqa_yaml +doc_to_text: "{{fact}}\n{{question_stem}}\nA: {{choices.text[0]}}\nB: {{choices.text[1]}}\nC: {{choices.text[2]}}\nD: {{choices.text[3]}}\n\nEr det riktige svaret A, B, C, eller D?\n\nSvar:" +doc_to_choice: "{{choices.label}}" diff --git a/lm_eval/tasks/noreval/noropenbookqa/nob/noropenbookqa_nob_p3.yaml b/lm_eval/tasks/noreval/noropenbookqa/nob/noropenbookqa_nob_p3.yaml new file mode 100644 index 00000000..31f7b7fe --- /dev/null +++ b/lm_eval/tasks/noreval/noropenbookqa/nob/noropenbookqa_nob_p3.yaml @@ -0,0 +1,6 @@ +tag: noropenbookqa_nob +task: noropenbookqa_nob_p3 +dataset_name: nb +include: ../_noropenbookqa_yaml +doc_to_text: "Bakgrunn: {{fact}}\n\nSpørsmål: {{question_stem}}\nA: {{choices.text[0]}}\nB: {{choices.text[1]}}\nC: {{choices.text[2]}}\nD: {{choices.text[3]}}\n\nSvar:" +doc_to_choice: "{{choices.label}}" diff --git a/lm_eval/tasks/noreval/noropenbookqa/nob/noropenbookqa_nob_p4.yaml b/lm_eval/tasks/noreval/noropenbookqa/nob/noropenbookqa_nob_p4.yaml new file mode 100644 index 00000000..ce7b8ae6 --- /dev/null +++ b/lm_eval/tasks/noreval/noropenbookqa/nob/noropenbookqa_nob_p4.yaml @@ -0,0 +1,6 @@ +tag: noropenbookqa_nob +task: noropenbookqa_nob_p4 +dataset_name: nb +include: ../_noropenbookqa_yaml +doc_to_text: "Ta utgangspunkt i følgende fakta når du svarer på spørsmålet: {{fact}}\n\n{{question_stem}}\nVelg riktig svar blant disse alternativene:\n– {{choices.text[0]}}\n– {{choices.text[1]}}\n– {{choices.text[2]}}\n– {{choices.text[3]}}\n\nSvar:" +doc_to_choice: "{{choices.text}}" diff --git a/lm_eval/tasks/noreval/noropenbookqa/utils.py b/lm_eval/tasks/noreval/noropenbookqa/utils.py new file mode 100644 index 00000000..c74e93a2 --- /dev/null +++ b/lm_eval/tasks/noreval/noropenbookqa/utils.py @@ -0,0 +1,5 @@ +import datasets + + +def filter_dataset(dataset: datasets.Dataset) -> datasets.Dataset: + return dataset.filter(lambda example: len(example["fact"]) > 0) diff --git a/lm_eval/tasks/noreval/norquad/_norquad_yaml b/lm_eval/tasks/noreval/norquad/_norquad_yaml new file mode 100644 index 00000000..c1a168ce --- /dev/null +++ b/lm_eval/tasks/noreval/norquad/_norquad_yaml @@ -0,0 +1,25 @@ +tag: norquad +dataset_path: ltg/norquad +output_type: generate_until +training_split: train +validation_split: validation +test_split: test +doc_to_target: '{{answers["text"][0]}}' +process_results: !function utils.process_results +process_docs: !function utils.process_docs +target_delimiter: ' ' +generation_kwargs: + until: + - "\n" + do_sample: false + num_beams: 1 + max_new_tokens: 32 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + - metric: f1 + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/noreval/norquad/norquad_p0.yaml b/lm_eval/tasks/noreval/norquad/norquad_p0.yaml new file mode 100644 index 00000000..690a10e8 --- /dev/null +++ b/lm_eval/tasks/noreval/norquad/norquad_p0.yaml @@ -0,0 +1,3 @@ +task: norquad_p0 +include: _norquad_yaml +doc_to_text: !function utils.p0 diff --git a/lm_eval/tasks/noreval/norquad/norquad_p1.yaml b/lm_eval/tasks/noreval/norquad/norquad_p1.yaml new file mode 100644 index 00000000..640a8d85 --- /dev/null +++ b/lm_eval/tasks/noreval/norquad/norquad_p1.yaml @@ -0,0 +1,3 @@ +task: norquad_p1 +include: _norquad_yaml +doc_to_text: !function utils.p1 diff --git a/lm_eval/tasks/noreval/norquad/norquad_p2.yaml b/lm_eval/tasks/noreval/norquad/norquad_p2.yaml new file mode 100644 index 00000000..b18ce5a0 --- /dev/null +++ b/lm_eval/tasks/noreval/norquad/norquad_p2.yaml @@ -0,0 +1,3 @@ +task: norquad_p2 +include: _norquad_yaml +doc_to_text: !function utils.p2 diff --git a/lm_eval/tasks/noreval/norquad/norquad_p3.yaml b/lm_eval/tasks/noreval/norquad/norquad_p3.yaml new file mode 100644 index 00000000..5f10683b --- /dev/null +++ b/lm_eval/tasks/noreval/norquad/norquad_p3.yaml @@ -0,0 +1,3 @@ +task: norquad_p3 +include: _norquad_yaml +doc_to_text: !function utils.p3 diff --git a/lm_eval/tasks/noreval/norquad/norquad_p4.yaml b/lm_eval/tasks/noreval/norquad/norquad_p4.yaml new file mode 100644 index 00000000..dc8b42df --- /dev/null +++ b/lm_eval/tasks/noreval/norquad/norquad_p4.yaml @@ -0,0 +1,3 @@ +task: norquad_p4 +include: _norquad_yaml +doc_to_text: !function utils.p4 diff --git a/lm_eval/tasks/noreval/norquad/utils.py b/lm_eval/tasks/noreval/norquad/utils.py new file mode 100644 index 00000000..1a072fb8 --- /dev/null +++ b/lm_eval/tasks/noreval/norquad/utils.py @@ -0,0 +1,62 @@ +import datasets +import transformers.data.metrics.squad_metrics as squad_metrics + + +def process_results(doc, results): + preds = results[0] + reference = doc["answers"]["text"][0] + f1_sum = squad_metrics.compute_f1(reference, preds) + exact_match = squad_metrics.compute_exact(reference, preds) + return {"f1": f1_sum, "exact_match": exact_match} + + +def process_docs(dataset: datasets.Dataset): + def _helper(doc): + doc["title"] = doc["context"].strip().split("\n")[0].strip() + doc["passage"] = "\n".join(doc["context"].strip().split("\n")[1:]).strip() + doc["question"] = " ".join(doc["question"].strip().split()) + return doc + + return dataset.map(_helper) + + +def p0(doc): + title = doc["title"] + passage = doc["passage"] + question = doc["question"] + prompt = f"Tittel: {title}\n\nTekst: {passage}\n\nSpørsmål: {question}\n\nSvar:" + return prompt + + +def p1(doc): + title = doc["title"] + passage = doc["passage"] + question = doc["question"] + prompt = f'Tittel: {title}\n\nTekst: {passage}\n\nGitt teksten over, hva er svaret på følgende spørsmål? "{question}"\n\nSvar:' + return prompt + + +def p2(doc): + title = doc["title"] + passage = doc["passage"] + question = doc["question"] + prompt = ( + f"Tittel: {title}\n\nTekst: {passage}\n\nSvar på følgende: {question}\n\nSvar:" + ) + return prompt + + +def p3(doc): + title = doc["title"] + passage = doc["passage"] + question = doc["question"] + prompt = f'Tittel: {title}\n\nTekst: {passage}\n\nHvordan kan man svare på spørsmålet "{question}", gitt teksten over?\n\nSvar:' + return prompt + + +def p4(doc): + title = doc["title"] + passage = doc["passage"] + question = doc["question"] + prompt = f'Tittel: {title}\n\nTekst:{passage}\n\nGitt teksten over, besvar følgende spørsmål: "{question}"\n\nSvar:' + return prompt diff --git a/lm_eval/tasks/noreval/norrewrite-instruct/norrewrite_instruct.yaml b/lm_eval/tasks/noreval/norrewrite-instruct/norrewrite_instruct.yaml new file mode 100644 index 00000000..e2a96046 --- /dev/null +++ b/lm_eval/tasks/noreval/norrewrite-instruct/norrewrite_instruct.yaml @@ -0,0 +1,20 @@ +task: norrewrite_instruct +dataset_path: ltg/norrewrite-instruct +training_split: null +test_split: test +doc_to_text: "{{prompt}} {{context}}" +doc_to_target: response +output_type: generate_until +metric_list: + - metric: bleu + higher_is_better: true + - metric: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + num_beams: 1 + max_new_tokens: 256 +metadata: + version: 1.0 diff --git a/lm_eval/tasks/noreval/norsumm/_norsumm_yaml b/lm_eval/tasks/noreval/norsumm/_norsumm_yaml new file mode 100644 index 00000000..e6121c03 --- /dev/null +++ b/lm_eval/tasks/noreval/norsumm/_norsumm_yaml @@ -0,0 +1,35 @@ +dataset_path: SamiaT/NorSumm +training_split: null +validation_split: null +test_split: test +num_fewshot: 0 +doc_to_target: summaries +output_type: generate_until +process_results: !function utils.process_results +generation_kwargs: + until: + - "\n" + do_sample: false + num_beams: 1 + max_new_tokens: 256 +metric_list: + - metric: bleu_max + aggregation: mean + higher_is_better: true + - metric: bleu_avg + aggregation: mean + higher_is_better: true + - metric: rougeL_max + aggregation: mean + higher_is_better: true + - metric: rougeL_avg + aggregation: mean + higher_is_better: true + - metric: bertscore_f1_max + aggregation: mean + higher_is_better: true + - metric: bertscore_f1_avg + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p0.yaml b/lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p0.yaml new file mode 100644 index 00000000..8f79a7e5 --- /dev/null +++ b/lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p0.yaml @@ -0,0 +1,5 @@ +tag: norsumm_nno +dataset_name: nn +task: norsumm_nno_p0 +include: ../_norsumm_yaml +doc_to_text: "Skriv ei oppsummering av følgande artikkel med berre nokre få punkt: {{article}}\nOppsummering:" diff --git a/lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p1.yaml b/lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p1.yaml new file mode 100644 index 00000000..e693f350 --- /dev/null +++ b/lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p1.yaml @@ -0,0 +1,5 @@ +tag: norsumm_nno +dataset_name: nn +task: norsumm_nno_p1 +include: ../_norsumm_yaml +doc_to_text: "Oppsummer følgande artikkel med nokre få setningar: {{article}}\nOppsummering:" diff --git a/lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p2.yaml b/lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p2.yaml new file mode 100644 index 00000000..7c2d725c --- /dev/null +++ b/lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p2.yaml @@ -0,0 +1,6 @@ +tag: norsumm_nno +dataset_name: nn +task: norsumm_nno_p2 +include: ../_norsumm_yaml +doc_to_text: "{{article}}\nSkriv ein kort og presis oppsummering av teksten over. Språket må vere klart og lett å forstå. Sørg for å ikkje introdusere feil. Oppsummeringa må dekkje følgande spørsmål: kven, kva, kor, når, og kvifor er denne saka viktig å vite om. Oppsummeringa må vere engasjerande og framheve nøkkelinformasjon frå artikkelen. Oppsummeringa skal innehalde maksimalt 700 tegn, inkludert mellomrom." +target_delimiter: "\n" diff --git a/lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p3.yaml b/lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p3.yaml new file mode 100644 index 00000000..a21f8438 --- /dev/null +++ b/lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p3.yaml @@ -0,0 +1,6 @@ +tag: norsumm_nno +dataset_name: nn +task: norsumm_nno_p3 +include: ../_norsumm_yaml +doc_to_text: "Gje eit kortfatta samandrag av følgande tekst: {{article}}" +target_delimiter: "\n" diff --git a/lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p4.yaml b/lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p4.yaml new file mode 100644 index 00000000..1385549e --- /dev/null +++ b/lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p4.yaml @@ -0,0 +1,5 @@ +tag: norsumm_nno +dataset_name: nn +task: norsumm_nno_p4 +include: ../_norsumm_yaml +doc_to_text: "Lag ein kort oppsummering som samanfattar den følgande teksten i nokre få punkt:\n{{article}}\n\nOppsummering:" diff --git a/lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p5.yaml b/lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p5.yaml new file mode 100644 index 00000000..8e89e95c --- /dev/null +++ b/lm_eval/tasks/noreval/norsumm/nno/norsumm_nno_p5.yaml @@ -0,0 +1,5 @@ +tag: norsumm_nno +dataset_name: nn +task: norsumm_nno_p5 +include: ../_norsumm_yaml +doc_to_text: "Heile artikkelen:\n{{article}}\n\nHovudpunkt:" diff --git a/lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p0.yaml b/lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p0.yaml new file mode 100644 index 00000000..4489c355 --- /dev/null +++ b/lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p0.yaml @@ -0,0 +1,5 @@ +tag: norsumm_nob +dataset_name: nb +task: norsumm_nob_p0 +include: ../_norsumm_yaml +doc_to_text: "Skriv en oppsummering av følgende artikkel med kun noen få punkter: {{article}}\nOppsummering:" diff --git a/lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p1.yaml b/lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p1.yaml new file mode 100644 index 00000000..2311b177 --- /dev/null +++ b/lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p1.yaml @@ -0,0 +1,5 @@ +tag: norsumm_nob +dataset_name: nb +task: norsumm_nob_p1 +include: ../_norsumm_yaml +doc_to_text: "Oppsummer følgende artikkel med noen få setninger: {{article}}\nOppsummering:" diff --git a/lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p2.yaml b/lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p2.yaml new file mode 100644 index 00000000..6f56d457 --- /dev/null +++ b/lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p2.yaml @@ -0,0 +1,6 @@ +tag: norsumm_nob +dataset_name: nb +task: norsumm_nob_p2 +include: ../_norsumm_yaml +doc_to_text: "{{article}}\nSkriv en kort og presis oppsummering av teksten over. Språket må være klart og lett å forstå. Sørg for å ikke introdusere feil. Oppsummeringen må dekke følgende spørsmål: hvem, hva, hvor, når, og hvorfor er denne saken viktig å vite om. Oppsummeringen må være engasjerende og fremheve nøkkelinformasjon fra artikkelen. Oppsummeringen skal inneholde maksimalt 700 tegn, inkludert mellomrom." +target_delimiter: "\n" diff --git a/lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p3.yaml b/lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p3.yaml new file mode 100644 index 00000000..66801038 --- /dev/null +++ b/lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p3.yaml @@ -0,0 +1,6 @@ +tag: norsumm_nob +dataset_name: nb +task: norsumm_nob_p3 +include: ../_norsumm_yaml +doc_to_text: "Gi et kortfattet sammendrag av følgende tekst: {{article}}" +target_delimiter: "\n" diff --git a/lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p4.yaml b/lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p4.yaml new file mode 100644 index 00000000..f2425e92 --- /dev/null +++ b/lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p4.yaml @@ -0,0 +1,5 @@ +tag: norsumm_nob +dataset_name: nb +task: norsumm_nob_p4 +include: ../_norsumm_yaml +doc_to_text: "Lag en kort oppsummering som sammenfatter den følgende teksten i noen få punkter:\n{{article}}\n\nOppsummering:" diff --git a/lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p5.yaml b/lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p5.yaml new file mode 100644 index 00000000..6a367f79 --- /dev/null +++ b/lm_eval/tasks/noreval/norsumm/nob/norsumm_nob_p5.yaml @@ -0,0 +1,5 @@ +tag: norsumm_nob +dataset_name: nb +task: norsumm_nob_p5 +include: ../_norsumm_yaml +doc_to_text: "Hele artikkelen:\n{{article}}\n\nHovedpunkter:" diff --git a/lm_eval/tasks/noreval/norsumm/utils.py b/lm_eval/tasks/noreval/norsumm/utils.py new file mode 100644 index 00000000..24ed9c43 --- /dev/null +++ b/lm_eval/tasks/noreval/norsumm/utils.py @@ -0,0 +1,126 @@ +import datasets +import numpy as np +from evaluate import load + + +try: + import bert_score + import sacrebleu + from rouge_score import rouge_scorer, scoring +except ModuleNotFoundError as e: + raise type(e)( + "`sacrebleu`, `bert_score`, and `rouge_score` are required for evaluating the model on NorEval." + ) from e + + +ROUGE_SCORER = None +BERTSCORE = None + + +def process_results(doc, results): + completion = results[0] + references = doc["summaries"] + + bleu_scores = [bleu([[reference]], [completion]) for reference in references] + bleu_max = np.nanmax(bleu_scores) + bleu_avg = np.nanmean(bleu_scores) + + rouge_scores = [rouge([reference], [completion]) for reference in references] + rougeL_scores = [score["rougeLsum"] for score in rouge_scores] + rougeL_max = np.nanmax(rougeL_scores) + rougeL_avg = np.nanmean(rougeL_scores) + + bertscore_f1s = [ + bertscore_f1(references=[reference], predictions=[completion]) + for reference in references + ] + bertscore_f1_max = np.nanmax(bertscore_f1s) + bertscore_f1_avg = np.nanmean(bertscore_f1s) + + return { + "bleu_max": bleu_max, + "bleu_avg": bleu_avg, + "rougeL_max": rougeL_max, + "rougeL_avg": rougeL_avg, + "bertscore_f1_max": bertscore_f1_max, + "bertscore_f1_avg": bertscore_f1_avg, + } + + +def bleu(refs, preds): + """ + Returns `t5` style BLEU scores. See the related implementation: + https://github.com/google-research/text-to-text-transfer-transformer/blob/3d10afd51ba97ac29eb66ae701eca274488202f7/t5/evaluation/metrics.py#L41 + + :param refs: + A `list` of `list` of reference `str`s. + :param preds: + A `list` of predicted `str`s. + """ + score = sacrebleu.corpus_bleu( + preds, + refs, + smooth_method="exp", + smooth_value=0.0, + force=False, + lowercase=False, + tokenize="intl", + use_effective_order=False, + ).score + return score + + +def rouge(refs, preds): + """ + Returns `t5` style ROUGE scores. See the related implementation: + https://github.com/google-research/text-to-text-transfer-transformer/blob/3d10afd51ba97ac29eb66ae701eca274488202f7/t5/evaluation/metrics.py#L68 + + :param refs: + A `list` of reference `strs`. + :param preds: + A `list` of predicted `strs`. + """ + rouge_types = ["rougeLsum"] + + global ROUGE_SCORER + if ROUGE_SCORER is None: + # init RougeScorer once (https://github.com/EleutherAI/lm-evaluation-harness/issues/1692)--rouge_types are constant + ROUGE_SCORER = rouge_scorer.RougeScorer(rouge_types) + scorer = ROUGE_SCORER + + # Add newlines between sentences to correctly compute `rougeLsum`. + + def _prepare_summary(summary): + summary = summary.replace(" . ", ".\n") + return summary + + # Accumulate confidence intervals. + aggregator = scoring.BootstrapAggregator() + for ref, pred in zip(refs, preds): + ref = _prepare_summary(ref) + pred = _prepare_summary(pred) + aggregator.add_scores(scorer.score(ref, pred)) + result = aggregator.aggregate() + return {type: result[type].mid.fmeasure * 100 for type in rouge_types} + + +def bertscore_f1(references, predictions): + """Computes the F1 score of the BERTScore metric. + Args: + references: A list of reference strings. + predictions: A list of predicted strings. + **kwargs: Additional keyword arguments. + Returns: + The F1 score of the BERTScore metric. + """ + global BERTSCORE + if BERTSCORE is None: + # init BERTScore once + BERTSCORE = load("bertscore") + bertscore = BERTSCORE + return bertscore.compute( + predictions=predictions, + references=references, + model_type="bert-base-multilingual-cased", + num_layers=9, + )["f1"][0] diff --git a/lm_eval/tasks/noreval/norsummarize-instruct/norsummarize_instruct.yaml b/lm_eval/tasks/noreval/norsummarize-instruct/norsummarize_instruct.yaml new file mode 100644 index 00000000..3327a4c3 --- /dev/null +++ b/lm_eval/tasks/noreval/norsummarize-instruct/norsummarize_instruct.yaml @@ -0,0 +1,20 @@ +task: norsummarize_instruct +dataset_path: ltg/norsummarize-instruct +training_split: null +test_split: test +doc_to_text: "{{prompt}} {{context}}" +doc_to_target: response +output_type: generate_until +metric_list: + - metric: bleu + higher_is_better: true + - metric: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + num_beams: 1 + max_new_tokens: 256 +metadata: + version: 1.0 diff --git a/lm_eval/tasks/noreval/nortruthfulqa/generation/_nortruthfulqa_gen_yaml b/lm_eval/tasks/noreval/nortruthfulqa/generation/_nortruthfulqa_gen_yaml new file mode 100644 index 00000000..1cf6d574 --- /dev/null +++ b/lm_eval/tasks/noreval/nortruthfulqa/generation/_nortruthfulqa_gen_yaml @@ -0,0 +1,54 @@ +dataset_path: ltg/nortruthfulqa_gen +output_type: generate_until +training_split: null +validation_split: validation +test_split: null +num_fewshot: 0 +doc_to_target: " " +process_docs: !function utils.process_docs +process_results: !function utils.process_results +generation_kwargs: + until: + - "\n" + do_sample: false + num_beams: 1 + max_new_tokens: 64 +metric_list: + - metric: bleu_max + aggregation: mean + higher_is_better: true + - metric: bleu_acc + aggregation: mean + higher_is_better: true + - metric: bleu_diff + aggregation: mean + higher_is_better: true + - metric: rouge1_max + aggregation: mean + higher_is_better: true + - metric: rouge1_acc + aggregation: mean + higher_is_better: true + - metric: rouge1_diff + aggregation: mean + higher_is_better: true + - metric: rouge2_max + aggregation: mean + higher_is_better: true + - metric: rouge2_acc + aggregation: mean + higher_is_better: true + - metric: rouge2_diff + aggregation: mean + higher_is_better: true + - metric: rougeL_max + aggregation: mean + higher_is_better: true + - metric: rougeL_acc + aggregation: mean + higher_is_better: true + - metric: rougeL_diff + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/noreval/nortruthfulqa/generation/nno/nortruthfulqa_gen_nno_p0.yaml b/lm_eval/tasks/noreval/nortruthfulqa/generation/nno/nortruthfulqa_gen_nno_p0.yaml new file mode 100644 index 00000000..240d5fb2 --- /dev/null +++ b/lm_eval/tasks/noreval/nortruthfulqa/generation/nno/nortruthfulqa_gen_nno_p0.yaml @@ -0,0 +1,5 @@ +tag: nortruthfulqa_gen_nno +task: nortruthfulqa_gen_nno_p0 +dataset_name: nn +include: ../_nortruthfulqa_gen_yaml +doc_to_text: question diff --git a/lm_eval/tasks/noreval/nortruthfulqa/generation/nno/nortruthfulqa_gen_nno_p1.yaml b/lm_eval/tasks/noreval/nortruthfulqa/generation/nno/nortruthfulqa_gen_nno_p1.yaml new file mode 100644 index 00000000..9412dada --- /dev/null +++ b/lm_eval/tasks/noreval/nortruthfulqa/generation/nno/nortruthfulqa_gen_nno_p1.yaml @@ -0,0 +1,5 @@ +tag: nortruthfulqa_gen_nno +task: nortruthfulqa_gen_nno_p1 +dataset_name: nn +include: ../_nortruthfulqa_gen_yaml +doc_to_text: "Spørsmål: {{question}}\nSvar:" diff --git a/lm_eval/tasks/noreval/nortruthfulqa/generation/nno/nortruthfulqa_gen_nno_p2.yaml b/lm_eval/tasks/noreval/nortruthfulqa/generation/nno/nortruthfulqa_gen_nno_p2.yaml new file mode 100644 index 00000000..be9d318c --- /dev/null +++ b/lm_eval/tasks/noreval/nortruthfulqa/generation/nno/nortruthfulqa_gen_nno_p2.yaml @@ -0,0 +1,5 @@ +tag: nortruthfulqa_gen_nno +task: nortruthfulqa_gen_nno_p2 +dataset_name: nn +include: ../_nortruthfulqa_gen_yaml +doc_to_text: "Skriv svaret på følgande spørsmål: {{question}}\nSvar:" diff --git a/lm_eval/tasks/noreval/nortruthfulqa/generation/nno/nortruthfulqa_gen_nno_p3.yaml b/lm_eval/tasks/noreval/nortruthfulqa/generation/nno/nortruthfulqa_gen_nno_p3.yaml new file mode 100644 index 00000000..f8b8a525 --- /dev/null +++ b/lm_eval/tasks/noreval/nortruthfulqa/generation/nno/nortruthfulqa_gen_nno_p3.yaml @@ -0,0 +1,5 @@ +tag: nortruthfulqa_gen_nno +task: nortruthfulqa_gen_nno_p3 +dataset_name: nn +include: ../_nortruthfulqa_gen_yaml +doc_to_text: "{{question}}\nKva er rett svar på spørsmålet?\nSvar:" diff --git a/lm_eval/tasks/noreval/nortruthfulqa/generation/nno/nortruthfulqa_gen_nno_p4.yaml b/lm_eval/tasks/noreval/nortruthfulqa/generation/nno/nortruthfulqa_gen_nno_p4.yaml new file mode 100644 index 00000000..5153b878 --- /dev/null +++ b/lm_eval/tasks/noreval/nortruthfulqa/generation/nno/nortruthfulqa_gen_nno_p4.yaml @@ -0,0 +1,5 @@ +tag: nortruthfulqa_gen_nno +task: nortruthfulqa_gen_nno_p4 +dataset_name: nn +include: ../_nortruthfulqa_gen_yaml +doc_to_text: "Svar sant på følgande: {{question}}\nSvar:" diff --git a/lm_eval/tasks/noreval/nortruthfulqa/generation/nob/nortruthfulqa_gen_nob_p0.yaml b/lm_eval/tasks/noreval/nortruthfulqa/generation/nob/nortruthfulqa_gen_nob_p0.yaml new file mode 100644 index 00000000..62317156 --- /dev/null +++ b/lm_eval/tasks/noreval/nortruthfulqa/generation/nob/nortruthfulqa_gen_nob_p0.yaml @@ -0,0 +1,5 @@ +tag: nortruthfulqa_gen_nob +task: nortruthfulqa_gen_nob_p0 +dataset_name: nb +include: ../_nortruthfulqa_gen_yaml +doc_to_text: question diff --git a/lm_eval/tasks/noreval/nortruthfulqa/generation/nob/nortruthfulqa_gen_nob_p1.yaml b/lm_eval/tasks/noreval/nortruthfulqa/generation/nob/nortruthfulqa_gen_nob_p1.yaml new file mode 100644 index 00000000..6633e8cd --- /dev/null +++ b/lm_eval/tasks/noreval/nortruthfulqa/generation/nob/nortruthfulqa_gen_nob_p1.yaml @@ -0,0 +1,5 @@ +tag: nortruthfulqa_gen_nob +task: nortruthfulqa_gen_nob_p1 +dataset_name: nb +include: ../_nortruthfulqa_gen_yaml +doc_to_text: "Spørsmål: {{question}}\nSvar:" diff --git a/lm_eval/tasks/noreval/nortruthfulqa/generation/nob/nortruthfulqa_gen_nob_p2.yaml b/lm_eval/tasks/noreval/nortruthfulqa/generation/nob/nortruthfulqa_gen_nob_p2.yaml new file mode 100644 index 00000000..ac7f57da --- /dev/null +++ b/lm_eval/tasks/noreval/nortruthfulqa/generation/nob/nortruthfulqa_gen_nob_p2.yaml @@ -0,0 +1,5 @@ +tag: nortruthfulqa_gen_nob +task: nortruthfulqa_gen_nob_p2 +dataset_name: nb +include: ../_nortruthfulqa_gen_yaml +doc_to_text: "Skriv svaret på følgende spørsmål: {{question}}\nSvar:" diff --git a/lm_eval/tasks/noreval/nortruthfulqa/generation/nob/nortruthfulqa_gen_nob_p3.yaml b/lm_eval/tasks/noreval/nortruthfulqa/generation/nob/nortruthfulqa_gen_nob_p3.yaml new file mode 100644 index 00000000..f196b101 --- /dev/null +++ b/lm_eval/tasks/noreval/nortruthfulqa/generation/nob/nortruthfulqa_gen_nob_p3.yaml @@ -0,0 +1,5 @@ +tag: nortruthfulqa_gen_nob +task: nortruthfulqa_gen_nob_p3 +dataset_name: nb +include: ../_nortruthfulqa_gen_yaml +doc_to_text: "{{question}}\nHva er riktig svar på spørsmålet?\nSvar:" diff --git a/lm_eval/tasks/noreval/nortruthfulqa/generation/nob/nortruthfulqa_gen_nob_p4.yaml b/lm_eval/tasks/noreval/nortruthfulqa/generation/nob/nortruthfulqa_gen_nob_p4.yaml new file mode 100644 index 00000000..2d6deaf1 --- /dev/null +++ b/lm_eval/tasks/noreval/nortruthfulqa/generation/nob/nortruthfulqa_gen_nob_p4.yaml @@ -0,0 +1,5 @@ +tag: nortruthfulqa_gen_nob +task: nortruthfulqa_gen_nob_p4 +dataset_name: nb +include: ../_nortruthfulqa_gen_yaml +doc_to_text: "Svar sant på følgende: {{question}}\nSvar:" diff --git a/lm_eval/tasks/noreval/nortruthfulqa/generation/utils.py b/lm_eval/tasks/noreval/nortruthfulqa/generation/utils.py new file mode 100644 index 00000000..5cfe16de --- /dev/null +++ b/lm_eval/tasks/noreval/nortruthfulqa/generation/utils.py @@ -0,0 +1,152 @@ +import datasets +import numpy as np +import sacrebleu +from rouge_score import rouge_scorer, scoring + + +try: + import sacrebleu + from rouge_score import rouge_scorer, scoring +except ModuleNotFoundError as e: + raise type(e)( + "`sacrebleu` and `rouge_score` are required for evaluating the model on NorEval." + ) from e + + +ROUGE_SCORER = None + + +def preprocess_function(examples): + def _format_answers(answers): + formatted_answers = [] + for answer in answers: + answer = answer.strip() + if len(answer): + # Add a period after all answers. + if answer[-1] != ".": + formatted_answers.append(answer + ".") + else: + formatted_answers.append(answer) + return formatted_answers + + incorrect_answers = _format_answers(examples["incorrect_answers"]) + correct_answers = _format_answers(examples["correct_answers"]) + return { + "question": examples["question"].strip(), + "correct_answers": correct_answers, + "incorrect_answers": incorrect_answers, + } + + +def process_docs(dataset: datasets.Dataset) -> datasets.Dataset: + return dataset.map(preprocess_function) + + +def process_results(doc, results): + completion = results[0] + true_refs, false_refs = doc["correct_answers"], doc["incorrect_answers"] + all_refs = true_refs + false_refs + + # BLEU + bleu_scores = [bleu([[ref]], [completion]) for ref in all_refs] + bleu_correct = np.nanmax(bleu_scores[: len(true_refs)]) + bleu_incorrect = np.nanmax(bleu_scores[len(true_refs) :]) + bleu_max = bleu_correct + bleu_diff = bleu_correct - bleu_incorrect + bleu_acc = int(bleu_correct > bleu_incorrect) + + # ROUGE-N + rouge_scores = [rouge([ref], [completion]) for ref in all_refs] + # ROUGE-1 + rouge1_scores = [score["rouge1"] for score in rouge_scores] + rouge1_correct = np.nanmax(rouge1_scores[: len(true_refs)]) + rouge1_incorrect = np.nanmax(rouge1_scores[len(true_refs) :]) + rouge1_max = rouge1_correct + rouge1_diff = rouge1_correct - rouge1_incorrect + rouge1_acc = int(rouge1_correct > rouge1_incorrect) + # ROUGE-2 + rouge2_scores = [score["rouge2"] for score in rouge_scores] + rouge2_correct = np.nanmax(rouge2_scores[: len(true_refs)]) + rouge2_incorrect = np.nanmax(rouge2_scores[len(true_refs) :]) + rouge2_max = rouge2_correct + rouge2_diff = rouge2_correct - rouge2_incorrect + rouge2_acc = int(rouge2_correct > rouge2_incorrect) + # ROUGE-L + rougeL_scores = [score["rougeLsum"] for score in rouge_scores] + rougeL_correct = np.nanmax(rougeL_scores[: len(true_refs)]) + rougeL_incorrect = np.nanmax(rougeL_scores[len(true_refs) :]) + rougeL_max = rougeL_correct + rougeL_diff = rougeL_correct - rougeL_incorrect + rougeL_acc = int(rougeL_correct > rougeL_incorrect) + + return { + "bleu_max": bleu_max, + "bleu_acc": bleu_acc, + "bleu_diff": bleu_diff, + "rouge1_max": rouge1_max, + "rouge1_acc": rouge1_acc, + "rouge1_diff": rouge1_diff, + "rouge2_max": rouge2_max, + "rouge2_acc": rouge2_acc, + "rouge2_diff": rouge2_diff, + "rougeL_max": rougeL_max, + "rougeL_acc": rougeL_acc, + "rougeL_diff": rougeL_diff, + } + + +def bleu(refs, preds): + """ + Returns `t5` style BLEU scores. See the related implementation: + https://github.com/google-research/text-to-text-transfer-transformer/blob/3d10afd51ba97ac29eb66ae701eca274488202f7/t5/evaluation/metrics.py#L41 + + :param refs: + A `list` of `list` of reference `str`s. + :param preds: + A `list` of predicted `str`s. + """ + score = sacrebleu.corpus_bleu( + preds, + refs, + smooth_method="exp", + smooth_value=0.0, + force=False, + lowercase=False, + tokenize="intl", + use_effective_order=False, + ).score + return score + + +def rouge(refs, preds): + """ + Returns `t5` style ROUGE scores. See the related implementation: + https://github.com/google-research/text-to-text-transfer-transformer/blob/3d10afd51ba97ac29eb66ae701eca274488202f7/t5/evaluation/metrics.py#L68 + + :param refs: + A `list` of reference `strs`. + :param preds: + A `list` of predicted `strs`. + """ + rouge_types = ["rouge1", "rouge2", "rougeLsum"] + scorer = rouge_scorer.RougeScorer(rouge_types) + # Add newlines between sentences to correctly compute `rougeLsum`. + + global ROUGE_SCORER + if ROUGE_SCORER is None: + # init RougeScorer once (https://github.com/EleutherAI/lm-evaluation-harness/issues/1692)--rouge_types are constant + ROUGE_SCORER = rouge_scorer.RougeScorer(rouge_types) + scorer = ROUGE_SCORER + + def _prepare_summary(summary): + summary = summary.replace(" . ", ".\n") + return summary + + # Accumulate confidence intervals. + aggregator = scoring.BootstrapAggregator() + for ref, pred in zip(refs, preds): + ref = _prepare_summary(ref) + pred = _prepare_summary(pred) + aggregator.add_scores(scorer.score(ref, pred)) + result = aggregator.aggregate() + return {type: result[type].mid.fmeasure * 100 for type in rouge_types} diff --git a/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/_nortruthfulqa_mc_yaml b/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/_nortruthfulqa_mc_yaml new file mode 100644 index 00000000..590dc608 --- /dev/null +++ b/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/_nortruthfulqa_mc_yaml @@ -0,0 +1,14 @@ +dataset_path: ltg/nortruthfulqa_mc +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: null +num_fewshot: 0 +doc_to_target: 0 +doc_to_choice: "{{mc1_targets.choices}}" +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/nortruthfulqa_mc_nno_p0.yaml b/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/nortruthfulqa_mc_nno_p0.yaml new file mode 100644 index 00000000..7608b46a --- /dev/null +++ b/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/nortruthfulqa_mc_nno_p0.yaml @@ -0,0 +1,5 @@ +tag: nortruthfulqa_mc_nno +task: nortruthfulqa_mc_nno_p0 +dataset_name: nn +include: ../_nortruthfulqa_mc_yaml +doc_to_text: !function utils.p0_nn diff --git a/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/nortruthfulqa_mc_nno_p1.yaml b/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/nortruthfulqa_mc_nno_p1.yaml new file mode 100644 index 00000000..8b4fae7e --- /dev/null +++ b/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/nortruthfulqa_mc_nno_p1.yaml @@ -0,0 +1,5 @@ +tag: nortruthfulqa_mc_nno +task: nortruthfulqa_mc_nno_p1 +dataset_name: nn +include: ../_nortruthfulqa_mc_yaml +doc_to_text: !function utils.p1_nn diff --git a/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/nortruthfulqa_mc_nno_p2.yaml b/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/nortruthfulqa_mc_nno_p2.yaml new file mode 100644 index 00000000..f8d61d46 --- /dev/null +++ b/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/nortruthfulqa_mc_nno_p2.yaml @@ -0,0 +1,6 @@ +tag: nortruthfulqa_mc_nno +task: nortruthfulqa_mc_nno_p2 +dataset_name: nn +include: ../_nortruthfulqa_mc_yaml +doc_to_text: !function utils.p2_nn +target_delimiter: "\n" diff --git a/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/nortruthfulqa_mc_nno_p3.yaml b/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/nortruthfulqa_mc_nno_p3.yaml new file mode 100644 index 00000000..d93316d3 --- /dev/null +++ b/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/nortruthfulqa_mc_nno_p3.yaml @@ -0,0 +1,6 @@ +tag: nortruthfulqa_mc_nno +task: nortruthfulqa_mc_nno_p3 +dataset_name: nn +include: ../_nortruthfulqa_mc_yaml +doc_to_text: !function utils.p3_nn +target_delimiter: "\n" diff --git a/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/nortruthfulqa_mc_nno_p4.yaml b/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/nortruthfulqa_mc_nno_p4.yaml new file mode 100644 index 00000000..74914f80 --- /dev/null +++ b/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/nortruthfulqa_mc_nno_p4.yaml @@ -0,0 +1,5 @@ +tag: nortruthfulqa_mc_nno +task: nortruthfulqa_mc_nno_p4 +dataset_name: nn +include: ../_nortruthfulqa_mc_yaml +doc_to_text: !function utils.p4_nn diff --git a/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/utils.py b/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/utils.py new file mode 100644 index 00000000..b66e1feb --- /dev/null +++ b/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nno/utils.py @@ -0,0 +1,35 @@ +def p0_nn(doc): + prompt = "Spørsmål: {question}\n\nSvar:" + return prompt.format(question=doc["question"]) + + +def p1_nn(doc): + prompt = "Spørsmål: {question}\n\nSvaralternativ:{choices}\n\nSvar:" + choices = "".join( + list(map(lambda choice: f"\n- {choice}", doc["mc1_targets"]["choices"])) + ) + return prompt.format(question=doc["question"], choices=choices) + + +def p2_nn(doc): + prompt = "Spørsmål: {question}\n\nKva av følgande alternativ er rett svar på spørsmålet?{choices}" + choices = "".join( + list(map(lambda choice: f"\n- {choice}", doc["mc1_targets"]["choices"])) + ) + return prompt.format(question=doc["question"], choices=choices) + + +def p3_nn(doc): + prompt = "Gitt følgande spørsmål, kva av dei moglege svara under er rett?\nSpørsmål: {question}\n{choices}" + choices = "".join( + list(map(lambda choice: f"\n- {choice}", doc["mc1_targets"]["choices"])) + ) + return prompt.format(question=doc["question"], choices=choices) + + +def p4_nn(doc): + prompt = "{question}\nVel eit av følgande moglege svar:{choices}\n\nSvar:" + choices = "".join( + list(map(lambda choice: f"\n- {choice}", doc["mc1_targets"]["choices"])) + ) + return prompt.format(question=doc["question"], choices=choices) diff --git a/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/nortruthfulqa_mc_nob_p0.yaml b/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/nortruthfulqa_mc_nob_p0.yaml new file mode 100644 index 00000000..36acd76f --- /dev/null +++ b/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/nortruthfulqa_mc_nob_p0.yaml @@ -0,0 +1,5 @@ +tag: nortruthfulqa_mc_nob +task: nortruthfulqa_mc_nob_p0 +dataset_name: nb +include: ../_nortruthfulqa_mc_yaml +doc_to_text: !function utils.p0_nb diff --git a/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/nortruthfulqa_mc_nob_p1.yaml b/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/nortruthfulqa_mc_nob_p1.yaml new file mode 100644 index 00000000..74b5d170 --- /dev/null +++ b/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/nortruthfulqa_mc_nob_p1.yaml @@ -0,0 +1,5 @@ +tag: nortruthfulqa_mc_nob +task: nortruthfulqa_mc_nob_p1 +dataset_name: nb +include: ../_nortruthfulqa_mc_yaml +doc_to_text: !function utils.p1_nb diff --git a/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/nortruthfulqa_mc_nob_p2.yaml b/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/nortruthfulqa_mc_nob_p2.yaml new file mode 100644 index 00000000..30665f9e --- /dev/null +++ b/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/nortruthfulqa_mc_nob_p2.yaml @@ -0,0 +1,6 @@ +tag: nortruthfulqa_mc_nob +task: nortruthfulqa_mc_nob_p2 +dataset_name: nb +include: ../_nortruthfulqa_mc_yaml +doc_to_text: !function utils.p2_nb +target_delimiter: "\n" diff --git a/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/nortruthfulqa_mc_nob_p3.yaml b/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/nortruthfulqa_mc_nob_p3.yaml new file mode 100644 index 00000000..40893b52 --- /dev/null +++ b/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/nortruthfulqa_mc_nob_p3.yaml @@ -0,0 +1,6 @@ +tag: nortruthfulqa_mc_nob +task: nortruthfulqa_mc_nob_p3 +dataset_name: nb +include: ../_nortruthfulqa_mc_yaml +doc_to_text: !function utils.p3_nb +target_delimiter: "\n" diff --git a/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/nortruthfulqa_mc_nob_p4.yaml b/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/nortruthfulqa_mc_nob_p4.yaml new file mode 100644 index 00000000..f0595fdf --- /dev/null +++ b/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/nortruthfulqa_mc_nob_p4.yaml @@ -0,0 +1,5 @@ +tag: nortruthfulqa_mc_nob +task: nortruthfulqa_mc_nob_p4 +dataset_name: nb +include: ../_nortruthfulqa_mc_yaml +doc_to_text: !function utils.p4_nb diff --git a/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/utils.py b/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/utils.py new file mode 100644 index 00000000..08e5cdab --- /dev/null +++ b/lm_eval/tasks/noreval/nortruthfulqa/multiple_choice/nob/utils.py @@ -0,0 +1,35 @@ +def p0_nb(doc): + prompt = "Spørsmål: {question}\n\nSvar:" + return prompt.format(question=doc["question"]) + + +def p1_nb(doc): + prompt = "Spørsmål: {question}\n\nSvaralternativer:{choices}\n\nSvar:" + choices = "".join( + list(map(lambda choice: f"\n- {choice}", doc["mc1_targets"]["choices"])) + ) + return prompt.format(question=doc["question"], choices=choices) + + +def p2_nb(doc): + prompt = "Spørsmål: {question}\n\nHvilke av følgende alternativer er riktig svar på spørsmålet?{choices}" + choices = "".join( + list(map(lambda choice: f"\n- {choice}", doc["mc1_targets"]["choices"])) + ) + return prompt.format(question=doc["question"], choices=choices) + + +def p3_nb(doc): + prompt = "Gitt følgende spørsmål, hvilket av de mulige svarene under er riktig?\nSpørsmål: {question}\n{choices}" + choices = "".join( + list(map(lambda choice: f"\n- {choice}", doc["mc1_targets"]["choices"])) + ) + return prompt.format(question=doc["question"], choices=choices) + + +def p4_nb(doc): + prompt = "{question}\nVelg et av følgende mulige svar:{choices}\n\nSvar:" + choices = "".join( + list(map(lambda choice: f"\n- {choice}", doc["mc1_targets"]["choices"])) + ) + return prompt.format(question=doc["question"], choices=choices) diff --git a/lm_eval/tasks/noreval/nrk_quiz_qa/_nrk_quiz_qa_yaml b/lm_eval/tasks/noreval/nrk_quiz_qa/_nrk_quiz_qa_yaml new file mode 100644 index 00000000..9e716706 --- /dev/null +++ b/lm_eval/tasks/noreval/nrk_quiz_qa/_nrk_quiz_qa_yaml @@ -0,0 +1,16 @@ +dataset_path: ltg/nrk_quiz_qa +output_type: multiple_choice +training_split: null +validation_split: null +test_split: test +num_fewshot: 0 +doc_to_target: "{{choices.label.index(answer)}}" +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/noreval/nrk_quiz_qa/nno/nrk_quiz_qa_nno_p0.yaml b/lm_eval/tasks/noreval/nrk_quiz_qa/nno/nrk_quiz_qa_nno_p0.yaml new file mode 100644 index 00000000..e94c9a27 --- /dev/null +++ b/lm_eval/tasks/noreval/nrk_quiz_qa/nno/nrk_quiz_qa_nno_p0.yaml @@ -0,0 +1,6 @@ +tag: nrk_quiz_qa_nno +task: nrk_quiz_qa_nno_p0 +dataset_name: nn +include: ../_nrk_quiz_qa_yaml +doc_to_text: !function utils.p0_nn +doc_to_choice: "{{choices.text}}" diff --git a/lm_eval/tasks/noreval/nrk_quiz_qa/nno/nrk_quiz_qa_nno_p1.yaml b/lm_eval/tasks/noreval/nrk_quiz_qa/nno/nrk_quiz_qa_nno_p1.yaml new file mode 100644 index 00000000..05d260db --- /dev/null +++ b/lm_eval/tasks/noreval/nrk_quiz_qa/nno/nrk_quiz_qa_nno_p1.yaml @@ -0,0 +1,6 @@ +tag: nrk_quiz_qa_nno +task: nrk_quiz_qa_nno_p1 +dataset_name: nn +include: ../_nrk_quiz_qa_yaml +doc_to_text: !function utils.p1_nn +doc_to_choice: "{{choices.text}}" diff --git a/lm_eval/tasks/noreval/nrk_quiz_qa/nno/nrk_quiz_qa_nno_p2.yaml b/lm_eval/tasks/noreval/nrk_quiz_qa/nno/nrk_quiz_qa_nno_p2.yaml new file mode 100644 index 00000000..fee4fadb --- /dev/null +++ b/lm_eval/tasks/noreval/nrk_quiz_qa/nno/nrk_quiz_qa_nno_p2.yaml @@ -0,0 +1,6 @@ +tag: nrk_quiz_qa_nno +task: nrk_quiz_qa_nno_p2 +dataset_name: nn +include: ../_nrk_quiz_qa_yaml +doc_to_text: !function utils.p2_nn +doc_to_choice: "{{choices.label}}" diff --git a/lm_eval/tasks/noreval/nrk_quiz_qa/nno/nrk_quiz_qa_nno_p3.yaml b/lm_eval/tasks/noreval/nrk_quiz_qa/nno/nrk_quiz_qa_nno_p3.yaml new file mode 100644 index 00000000..ac9aafd6 --- /dev/null +++ b/lm_eval/tasks/noreval/nrk_quiz_qa/nno/nrk_quiz_qa_nno_p3.yaml @@ -0,0 +1,6 @@ +tag: nrk_quiz_qa_nno +task: nrk_quiz_qa_nno_p3 +dataset_name: nn +include: ../_nrk_quiz_qa_yaml +doc_to_text: !function utils.p3_nn +doc_to_choice: "{{choices.label}}" diff --git a/lm_eval/tasks/noreval/nrk_quiz_qa/nno/nrk_quiz_qa_nno_p4.yaml b/lm_eval/tasks/noreval/nrk_quiz_qa/nno/nrk_quiz_qa_nno_p4.yaml new file mode 100644 index 00000000..d095b6f1 --- /dev/null +++ b/lm_eval/tasks/noreval/nrk_quiz_qa/nno/nrk_quiz_qa_nno_p4.yaml @@ -0,0 +1,6 @@ +tag: nrk_quiz_qa_nno +task: nrk_quiz_qa_nno_p4 +dataset_name: nn +include: ../_nrk_quiz_qa_yaml +doc_to_text: !function utils.p4_nn +doc_to_choice: "{{choices.text}}" diff --git a/lm_eval/tasks/noreval/nrk_quiz_qa/nno/utils.py b/lm_eval/tasks/noreval/nrk_quiz_qa/nno/utils.py new file mode 100644 index 00000000..b9dcae1c --- /dev/null +++ b/lm_eval/tasks/noreval/nrk_quiz_qa/nno/utils.py @@ -0,0 +1,44 @@ +def p0_nn(doc): + prompt = "Spørsmål: {question}\n\nSvar:" + return prompt.format(question=doc["question"]) + + +def p1_nn(doc): + prompt = "{question}\n\nSvaralternativer:{choices}\n\nKva er rett svar?\n\nSvar:" + choices = "".join(list(map(lambda choice: f"\n- {choice}", doc["choices"]["text"]))) + return prompt.format(question=doc["question"], choices=choices) + + +def p2_nn(doc): + prompt = "{question}{choices}\n\nEr det rette svaret {enumerated_choices}?\n\nSvar:" + choices = "".join( + [ + f"\n{label}: {option}" + for label, option in zip(doc["choices"]["label"], doc["choices"]["text"]) + ] + ) + enumerated_choices = ", ".join( + doc["choices"]["label"][:-1] + ) + ", eller {latest_choice}".format(latest_choice=doc["choices"]["label"][-1]) + if len(doc["choices"]["label"]) == 2: + enumerated_choices = enumerated_choices.replace(", eller", " eller") + return prompt.format( + question=doc["question"], choices=choices, enumerated_choices=enumerated_choices + ) + + +def p3_nn(doc): + prompt = "Spørsmål: {question}{choices}\n\nSvar:" + choices = "".join( + [ + f"\n{label}: {option}" + for label, option in zip(doc["choices"]["label"], doc["choices"]["text"]) + ] + ) + return prompt.format(question=doc["question"], choices=choices) + + +def p4_nn(doc): + prompt = "{question}\nVel rett svar blant desse alternativa:{choices}\n\nSvar:" + choices = "".join(list(map(lambda choice: f"\n- {choice}", doc["choices"]["text"]))) + return prompt.format(question=doc["question"], choices=choices) diff --git a/lm_eval/tasks/noreval/nrk_quiz_qa/nob/nrk_quiz_qa_nob_p0.yaml b/lm_eval/tasks/noreval/nrk_quiz_qa/nob/nrk_quiz_qa_nob_p0.yaml new file mode 100644 index 00000000..688c2ed1 --- /dev/null +++ b/lm_eval/tasks/noreval/nrk_quiz_qa/nob/nrk_quiz_qa_nob_p0.yaml @@ -0,0 +1,6 @@ +tag: nrk_quiz_qa_nob +task: nrk_quiz_qa_nob_p0 +dataset_name: nb +include: ../_nrk_quiz_qa_yaml +doc_to_text: !function utils.p0_nb +doc_to_choice: "{{choices.text}}" diff --git a/lm_eval/tasks/noreval/nrk_quiz_qa/nob/nrk_quiz_qa_nob_p1.yaml b/lm_eval/tasks/noreval/nrk_quiz_qa/nob/nrk_quiz_qa_nob_p1.yaml new file mode 100644 index 00000000..c1920e95 --- /dev/null +++ b/lm_eval/tasks/noreval/nrk_quiz_qa/nob/nrk_quiz_qa_nob_p1.yaml @@ -0,0 +1,6 @@ +tag: nrk_quiz_qa_nob +task: nrk_quiz_qa_nob_p1 +dataset_name: nb +include: ../_nrk_quiz_qa_yaml +doc_to_text: !function utils.p1_nb +doc_to_choice: "{{choices.text}}" diff --git a/lm_eval/tasks/noreval/nrk_quiz_qa/nob/nrk_quiz_qa_nob_p2.yaml b/lm_eval/tasks/noreval/nrk_quiz_qa/nob/nrk_quiz_qa_nob_p2.yaml new file mode 100644 index 00000000..5b7746da --- /dev/null +++ b/lm_eval/tasks/noreval/nrk_quiz_qa/nob/nrk_quiz_qa_nob_p2.yaml @@ -0,0 +1,6 @@ +tag: nrk_quiz_qa_nob +task: nrk_quiz_qa_nob_p2 +dataset_name: nb +include: ../_nrk_quiz_qa_yaml +doc_to_text: !function utils.p2_nb +doc_to_choice: "{{choices.label}}" diff --git a/lm_eval/tasks/noreval/nrk_quiz_qa/nob/nrk_quiz_qa_nob_p3.yaml b/lm_eval/tasks/noreval/nrk_quiz_qa/nob/nrk_quiz_qa_nob_p3.yaml new file mode 100644 index 00000000..39407053 --- /dev/null +++ b/lm_eval/tasks/noreval/nrk_quiz_qa/nob/nrk_quiz_qa_nob_p3.yaml @@ -0,0 +1,6 @@ +tag: nrk_quiz_qa_nob +task: nrk_quiz_qa_nob_p3 +dataset_name: nb +include: ../_nrk_quiz_qa_yaml +doc_to_text: !function utils.p3_nb +doc_to_choice: "{{choices.label}}" diff --git a/lm_eval/tasks/noreval/nrk_quiz_qa/nob/nrk_quiz_qa_nob_p4.yaml b/lm_eval/tasks/noreval/nrk_quiz_qa/nob/nrk_quiz_qa_nob_p4.yaml new file mode 100644 index 00000000..0f2e645c --- /dev/null +++ b/lm_eval/tasks/noreval/nrk_quiz_qa/nob/nrk_quiz_qa_nob_p4.yaml @@ -0,0 +1,6 @@ +tag: nrk_quiz_qa_nob +task: nrk_quiz_qa_nob_p4 +dataset_name: nb +include: ../_nrk_quiz_qa_yaml +doc_to_text: !function utils.p4_nb +doc_to_choice: "{{choices.text}}" diff --git a/lm_eval/tasks/noreval/nrk_quiz_qa/nob/utils.py b/lm_eval/tasks/noreval/nrk_quiz_qa/nob/utils.py new file mode 100644 index 00000000..aac29f69 --- /dev/null +++ b/lm_eval/tasks/noreval/nrk_quiz_qa/nob/utils.py @@ -0,0 +1,46 @@ +def p0_nb(doc): + prompt = "Spørsmål: {question}\n\nSvar:" + return prompt.format(question=doc["question"]) + + +def p1_nb(doc): + prompt = "{question}\n\nSvaralternativer:{choices}\n\nHva er riktig svar?\n\nSvar:" + choices = "".join(list(map(lambda choice: f"\n- {choice}", doc["choices"]["text"]))) + return prompt.format(question=doc["question"], choices=choices) + + +def p2_nb(doc): + prompt = ( + "{question}{choices}\n\nEr det riktige svaret {enumerated_choices}?\n\nSvar:" + ) + choices = "".join( + [ + f"\n{label}: {option}" + for label, option in zip(doc["choices"]["label"], doc["choices"]["text"]) + ] + ) + enumerated_choices = ", ".join( + doc["choices"]["label"][:-1] + ) + ", eller {latest_choice}".format(latest_choice=doc["choices"]["label"][-1]) + if len(doc["choices"]["label"]) == 2: + enumerated_choices = enumerated_choices.replace(", eller", " eller") + return prompt.format( + question=doc["question"], choices=choices, enumerated_choices=enumerated_choices + ) + + +def p3_nb(doc): + prompt = "Spørsmål: {question}{choices}\n\nSvar:" + choices = "".join( + [ + f"\n{label}: {option}" + for label, option in zip(doc["choices"]["label"], doc["choices"]["text"]) + ] + ) + return prompt.format(question=doc["question"], choices=choices) + + +def p4_nb(doc): + prompt = "{question}\nVelg riktig svar blant disse alternativene:{choices}\n\nSvar:" + choices = "".join(list(map(lambda choice: f"\n- {choice}", doc["choices"]["text"]))) + return prompt.format(question=doc["question"], choices=choices) diff --git a/lm_eval/tasks/noreval/tatoeba/_tatoeba_yaml b/lm_eval/tasks/noreval/tatoeba/_tatoeba_yaml new file mode 100644 index 00000000..01b1cfea --- /dev/null +++ b/lm_eval/tasks/noreval/tatoeba/_tatoeba_yaml @@ -0,0 +1,19 @@ +dataset_path: Helsinki-NLP/tatoeba_mt +training_split: validation +test_split: test +output_type: generate_until +dataset_kwargs: + trust_remote_code: true +metric_list: + - metric: bleu + higher_is_better: true + - metric: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + num_beams: 1 + max_new_tokens: 256 +metadata: + version: 1.0 diff --git a/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nno/tatoeba_eng_nno_p0.yaml b/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nno/tatoeba_eng_nno_p0.yaml new file mode 100644 index 00000000..c34c861d --- /dev/null +++ b/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nno/tatoeba_eng_nno_p0.yaml @@ -0,0 +1,6 @@ +tag: tatoeba_eng_nno +dataset_name: eng-nno +doc_to_target: targetString +task: tatoeba_eng_nno_p0 +include: ../_tatoeba_yaml +doc_to_text: "Engelsk: {{sourceString}}\nNynorsk:" diff --git a/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nno/tatoeba_eng_nno_p1.yaml b/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nno/tatoeba_eng_nno_p1.yaml new file mode 100644 index 00000000..7dca37a8 --- /dev/null +++ b/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nno/tatoeba_eng_nno_p1.yaml @@ -0,0 +1,6 @@ +tag: tatoeba_eng_nno +dataset_name: eng-nno +doc_to_target: targetString +task: tatoeba_eng_nno_p1 +include: ../_tatoeba_yaml +doc_to_text: "Omsett følgande setning til nynorsk: {{sourceString}}\nNynorsk:" diff --git a/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nno/tatoeba_eng_nno_p2.yaml b/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nno/tatoeba_eng_nno_p2.yaml new file mode 100644 index 00000000..1e84f18c --- /dev/null +++ b/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nno/tatoeba_eng_nno_p2.yaml @@ -0,0 +1,6 @@ +tag: tatoeba_eng_nno +dataset_name: eng-nno +doc_to_target: targetString +task: tatoeba_eng_nno_p2 +include: ../_tatoeba_yaml +doc_to_text: "Gje ei nynorsk omsetjing av denne setninga: {{sourceString}}\nNynorsk:" diff --git a/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nno/tatoeba_eng_nno_p3.yaml b/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nno/tatoeba_eng_nno_p3.yaml new file mode 100644 index 00000000..ccc70026 --- /dev/null +++ b/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nno/tatoeba_eng_nno_p3.yaml @@ -0,0 +1,6 @@ +tag: tatoeba_eng_nno +dataset_name: eng-nno +doc_to_target: targetString +task: tatoeba_eng_nno_p3 +include: ../_tatoeba_yaml +doc_to_text: "Kva blir \"{{sourceString}}\" på nynorsk?\nNynorsk:" diff --git a/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nob/tatoeba_eng_nob_p0.yaml b/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nob/tatoeba_eng_nob_p0.yaml new file mode 100644 index 00000000..cfd9c77f --- /dev/null +++ b/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nob/tatoeba_eng_nob_p0.yaml @@ -0,0 +1,6 @@ +tag: tatoeba_eng_nob +dataset_name: eng-nob +doc_to_target: targetString +task: tatoeba_eng_nob_p0 +include: ../_tatoeba_yaml +doc_to_text: "Engelsk: {{sourceString}}\nBokmål:" diff --git a/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nob/tatoeba_eng_nob_p1.yaml b/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nob/tatoeba_eng_nob_p1.yaml new file mode 100644 index 00000000..c762b47b --- /dev/null +++ b/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nob/tatoeba_eng_nob_p1.yaml @@ -0,0 +1,6 @@ +tag: tatoeba_eng_nob +dataset_name: eng-nob +doc_to_target: targetString +task: tatoeba_eng_nob_p1 +include: ../_tatoeba_yaml +doc_to_text: "Oversett følgende setning til norsk bokmål: {{sourceString}}\nBokmål:" diff --git a/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nob/tatoeba_eng_nob_p2.yaml b/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nob/tatoeba_eng_nob_p2.yaml new file mode 100644 index 00000000..d1362c22 --- /dev/null +++ b/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nob/tatoeba_eng_nob_p2.yaml @@ -0,0 +1,6 @@ +tag: tatoeba_eng_nob +dataset_name: eng-nob +doc_to_target: targetString +task: tatoeba_eng_nob_p2 +include: ../_tatoeba_yaml +doc_to_text: "Gi en oversettelse til bokmål for denne setningen: {{sourceString}}\nBokmål:" diff --git a/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nob/tatoeba_eng_nob_p3.yaml b/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nob/tatoeba_eng_nob_p3.yaml new file mode 100644 index 00000000..66bda471 --- /dev/null +++ b/lm_eval/tasks/noreval/tatoeba/tatoeba_eng_nob/tatoeba_eng_nob_p3.yaml @@ -0,0 +1,6 @@ +tag: tatoeba_eng_nob +dataset_name: eng-nob +doc_to_target: targetString +task: tatoeba_eng_nob_p3 +include: ../_tatoeba_yaml +doc_to_text: "Hva blir \"{{sourceString}}\" på bokmål?\nBokmål:" diff --git a/lm_eval/tasks/noreval/tatoeba/tatoeba_nno_eng/tatoeba_nno_eng_p0.yaml b/lm_eval/tasks/noreval/tatoeba/tatoeba_nno_eng/tatoeba_nno_eng_p0.yaml new file mode 100644 index 00000000..a09be934 --- /dev/null +++ b/lm_eval/tasks/noreval/tatoeba/tatoeba_nno_eng/tatoeba_nno_eng_p0.yaml @@ -0,0 +1,6 @@ +tag: tatoeba_nno_eng +dataset_name: eng-nno +doc_to_target: sourceString +task: tatoeba_nno_eng_p0 +include: ../_tatoeba_yaml +doc_to_text: "Nynorsk: {{targetString}}\nEngelsk:" diff --git a/lm_eval/tasks/noreval/tatoeba/tatoeba_nno_eng/tatoeba_nno_eng_p1.yaml b/lm_eval/tasks/noreval/tatoeba/tatoeba_nno_eng/tatoeba_nno_eng_p1.yaml new file mode 100644 index 00000000..cf69ce7c --- /dev/null +++ b/lm_eval/tasks/noreval/tatoeba/tatoeba_nno_eng/tatoeba_nno_eng_p1.yaml @@ -0,0 +1,6 @@ +tag: tatoeba_nno_eng +dataset_name: eng-nno +doc_to_target: sourceString +task: tatoeba_nno_eng_p1 +include: ../_tatoeba_yaml +doc_to_text: "Omsett følgande setning til engelsk: {{targetString}}\nEngelsk:" diff --git a/lm_eval/tasks/noreval/tatoeba/tatoeba_nno_eng/tatoeba_nno_eng_p2.yaml b/lm_eval/tasks/noreval/tatoeba/tatoeba_nno_eng/tatoeba_nno_eng_p2.yaml new file mode 100644 index 00000000..3f179696 --- /dev/null +++ b/lm_eval/tasks/noreval/tatoeba/tatoeba_nno_eng/tatoeba_nno_eng_p2.yaml @@ -0,0 +1,6 @@ +tag: tatoeba_nno_eng +dataset_name: eng-nno +doc_to_target: sourceString +task: tatoeba_nno_eng_p2 +include: ../_tatoeba_yaml +doc_to_text: "Gje ei engelsk omsetjing av denne setninga: {{targetString}}\nEngelsk:" diff --git a/lm_eval/tasks/noreval/tatoeba/tatoeba_nno_eng/tatoeba_nno_eng_p3.yaml b/lm_eval/tasks/noreval/tatoeba/tatoeba_nno_eng/tatoeba_nno_eng_p3.yaml new file mode 100644 index 00000000..f5b55837 --- /dev/null +++ b/lm_eval/tasks/noreval/tatoeba/tatoeba_nno_eng/tatoeba_nno_eng_p3.yaml @@ -0,0 +1,6 @@ +tag: tatoeba_nno_eng +dataset_name: eng-nno +doc_to_target: sourceString +task: tatoeba_nno_eng_p3 +include: ../_tatoeba_yaml +doc_to_text: "Kva blir \"{{targetString}}\" på engelsk?\nEngelsk:" diff --git a/lm_eval/tasks/noreval/tatoeba/tatoeba_nob_eng/tatoeba_nob_eng_p0.yaml b/lm_eval/tasks/noreval/tatoeba/tatoeba_nob_eng/tatoeba_nob_eng_p0.yaml new file mode 100644 index 00000000..9213403f --- /dev/null +++ b/lm_eval/tasks/noreval/tatoeba/tatoeba_nob_eng/tatoeba_nob_eng_p0.yaml @@ -0,0 +1,6 @@ +tag: tatoeba_nob_eng +dataset_name: eng-nob +doc_to_target: sourceString +task: tatoeba_nob_eng_p0 +include: ../_tatoeba_yaml +doc_to_text: "Bokmål: {{targetString}}\nEngelsk:" diff --git a/lm_eval/tasks/noreval/tatoeba/tatoeba_nob_eng/tatoeba_nob_eng_p1.yaml b/lm_eval/tasks/noreval/tatoeba/tatoeba_nob_eng/tatoeba_nob_eng_p1.yaml new file mode 100644 index 00000000..efb7c631 --- /dev/null +++ b/lm_eval/tasks/noreval/tatoeba/tatoeba_nob_eng/tatoeba_nob_eng_p1.yaml @@ -0,0 +1,6 @@ +tag: tatoeba_nob_eng +dataset_name: eng-nob +doc_to_target: sourceString +task: tatoeba_nob_eng_p1 +include: ../_tatoeba_yaml +doc_to_text: "Oversett følgende setning til engelsk: {{targetString}}\nEngelsk:" diff --git a/lm_eval/tasks/noreval/tatoeba/tatoeba_nob_eng/tatoeba_nob_eng_p2.yaml b/lm_eval/tasks/noreval/tatoeba/tatoeba_nob_eng/tatoeba_nob_eng_p2.yaml new file mode 100644 index 00000000..c8a17df2 --- /dev/null +++ b/lm_eval/tasks/noreval/tatoeba/tatoeba_nob_eng/tatoeba_nob_eng_p2.yaml @@ -0,0 +1,6 @@ +tag: tatoeba_nob_eng +dataset_name: eng-nob +doc_to_target: sourceString +task: tatoeba_nob_eng_p2 +include: ../_tatoeba_yaml +doc_to_text: "Gi en engelsk oversettelse av denne setningen: {{targetString}}\nEngelsk:" diff --git a/lm_eval/tasks/noreval/tatoeba/tatoeba_nob_eng/tatoeba_nob_eng_p3.yaml b/lm_eval/tasks/noreval/tatoeba/tatoeba_nob_eng/tatoeba_nob_eng_p3.yaml new file mode 100644 index 00000000..85a29eec --- /dev/null +++ b/lm_eval/tasks/noreval/tatoeba/tatoeba_nob_eng/tatoeba_nob_eng_p3.yaml @@ -0,0 +1,6 @@ +tag: tatoeba_nob_eng +dataset_name: eng-nob +doc_to_target: sourceString +task: tatoeba_nob_eng_p3 +include: ../_tatoeba_yaml +doc_to_text: "Hva blir \"{{targetString}}\" på engelsk?\nEngelsk:" -- GitLab From 817a2fe7cf6816e089ff2f9cf9ba6d0f82908e29 Mon Sep 17 00:00:00 2001 From: Anna Fontana <101867173+annafontanaa@users.noreply.github.com> Date: Tue, 6 May 2025 17:11:21 +0200 Subject: [PATCH 04/46] Fix import error for eval_logger in score utils (#2940) * Fix import error for eval_logger in score utils * pacify pre-commit --------- Co-authored-by: Baber --- lm_eval/tasks/aclue/_generate_configs.py | 4 +++- lm_eval/tasks/ceval/_generate_configs.py | 4 +++- lm_eval/tasks/cmmlu/_generate_configs.py | 4 +++- lm_eval/tasks/csatqa/_generate_configs.py | 4 +++- lm_eval/tasks/realtoxicityprompts/metric.py | 4 +++- lm_eval/tasks/score/agi_eval/utils_agieval.py | 4 +++- lm_eval/tasks/score/math/utils_math.py | 4 +++- lm_eval/tasks/score/mmlu_pro/utils_mmlu_pro.py | 4 +++- lm_eval/tasks/score/utils.py | 4 +++- 9 files changed, 27 insertions(+), 9 deletions(-) diff --git a/lm_eval/tasks/aclue/_generate_configs.py b/lm_eval/tasks/aclue/_generate_configs.py index 8bd1792a..60666bc7 100644 --- a/lm_eval/tasks/aclue/_generate_configs.py +++ b/lm_eval/tasks/aclue/_generate_configs.py @@ -3,12 +3,14 @@ Take in a YAML, and output all other splits with this YAML """ import argparse +import logging import os import yaml from tqdm import tqdm -from lm_eval.utils import eval_logger + +eval_logger = logging.getLogger(__name__) SUBJECTS = { diff --git a/lm_eval/tasks/ceval/_generate_configs.py b/lm_eval/tasks/ceval/_generate_configs.py index 9050c75c..81cbcb15 100644 --- a/lm_eval/tasks/ceval/_generate_configs.py +++ b/lm_eval/tasks/ceval/_generate_configs.py @@ -3,12 +3,14 @@ Take in a YAML, and output all other splits with this YAML """ import argparse +import logging import os import yaml from tqdm import tqdm -from lm_eval.utils import eval_logger + +eval_logger = logging.getLogger(__name__) SUBJECTS = { diff --git a/lm_eval/tasks/cmmlu/_generate_configs.py b/lm_eval/tasks/cmmlu/_generate_configs.py index f1b60e28..74348ed4 100644 --- a/lm_eval/tasks/cmmlu/_generate_configs.py +++ b/lm_eval/tasks/cmmlu/_generate_configs.py @@ -3,12 +3,14 @@ Take in a YAML, and output all other splits with this YAML """ import argparse +import logging import os import yaml from tqdm import tqdm -from lm_eval.utils import eval_logger + +eval_logger = logging.getLogger(__name__) SUBJECTS = { diff --git a/lm_eval/tasks/csatqa/_generate_configs.py b/lm_eval/tasks/csatqa/_generate_configs.py index 567deeea..1ef34b8c 100644 --- a/lm_eval/tasks/csatqa/_generate_configs.py +++ b/lm_eval/tasks/csatqa/_generate_configs.py @@ -3,12 +3,14 @@ Take in a YAML, and output all other splits with this YAML """ import argparse +import logging import os import yaml from tqdm import tqdm -from lm_eval.logger import eval_logger + +eval_logger = logging.getLogger(__name__) SUBSETS = ["WR", "GR", "RCS", "RCSS", "RCH", "LI"] diff --git a/lm_eval/tasks/realtoxicityprompts/metric.py b/lm_eval/tasks/realtoxicityprompts/metric.py index 552cd33d..9e7d0958 100644 --- a/lm_eval/tasks/realtoxicityprompts/metric.py +++ b/lm_eval/tasks/realtoxicityprompts/metric.py @@ -1,10 +1,12 @@ import json +import logging import os import requests from requests.adapters import HTTPAdapter, Retry -from lm_eval.utils import eval_logger + +eval_logger = logging.getLogger(__name__) def toxicity_perspective_api( diff --git a/lm_eval/tasks/score/agi_eval/utils_agieval.py b/lm_eval/tasks/score/agi_eval/utils_agieval.py index 4381a2cb..6539d05d 100644 --- a/lm_eval/tasks/score/agi_eval/utils_agieval.py +++ b/lm_eval/tasks/score/agi_eval/utils_agieval.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging import os import re from functools import partial @@ -22,9 +23,10 @@ from datasets import Dataset from lm_eval.tasks.score import utils from lm_eval.tasks.score.utils import prompt_consistency_rate, robustness_doc_to_text -from lm_eval.utils import eval_logger +eval_logger = logging.getLogger(__name__) + TEMPLATE_FILE_PATH = os.path.join(os.path.dirname(__file__), "prompt_templates.json") PROMPT_ROBUSTNESS_TEMPLATE_KEY = "prompt_robustness" diff --git a/lm_eval/tasks/score/math/utils_math.py b/lm_eval/tasks/score/math/utils_math.py index 4068b179..3750b985 100644 --- a/lm_eval/tasks/score/math/utils_math.py +++ b/lm_eval/tasks/score/math/utils_math.py @@ -13,6 +13,7 @@ # limitations under the License. import json +import logging import os from functools import partial from itertools import combinations @@ -28,9 +29,10 @@ from lm_eval.tasks.score.math.math_grader import ( normalize_answer_string, ) from lm_eval.tasks.score.utils import robustness_doc_to_text -from lm_eval.utils import eval_logger +eval_logger = logging.getLogger(__name__) + TEMPLATE_FILE_PATH = os.path.join(os.path.dirname(__file__), "prompt_templates.json") PROMPT_ROBUSTNESS_TEMPLATE_KEY = "prompt_robustness" diff --git a/lm_eval/tasks/score/mmlu_pro/utils_mmlu_pro.py b/lm_eval/tasks/score/mmlu_pro/utils_mmlu_pro.py index da46e101..681c76f5 100644 --- a/lm_eval/tasks/score/mmlu_pro/utils_mmlu_pro.py +++ b/lm_eval/tasks/score/mmlu_pro/utils_mmlu_pro.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging import os from functools import partial from typing import Any, Dict, List @@ -20,9 +21,10 @@ import numpy as np from lm_eval.tasks.score import utils from lm_eval.tasks.score.utils import prompt_consistency_rate, robustness_doc_to_text -from lm_eval.utils import eval_logger +eval_logger = logging.getLogger(__name__) + TEMPLATE_FILE_PATH = os.path.join(os.path.dirname(__file__), "prompt_templates.json") PROMPT_ROBUSTNESS_TEMPLATE_KEY = "prompt_robustness" diff --git a/lm_eval/tasks/score/utils.py b/lm_eval/tasks/score/utils.py index 61d7e3b0..9a995186 100644 --- a/lm_eval/tasks/score/utils.py +++ b/lm_eval/tasks/score/utils.py @@ -14,6 +14,7 @@ import copy import json +import logging import re import string import sys @@ -24,7 +25,8 @@ from typing import Any, Dict, List import numpy as np from datasets import Dataset -from lm_eval.utils import eval_logger + +eval_logger = logging.getLogger(__name__) NUMERALS = ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"] -- GitLab From cff3da29a7a9eaecb3d75309d924444710717fba Mon Sep 17 00:00:00 2001 From: Ihar Hrachyshka Date: Tue, 6 May 2025 12:53:54 -0400 Subject: [PATCH 05/46] Include all test files in sdist (#2634) This is useful to run unit tests during distro builds. --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..93f181de --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +recursive-include tests -- GitLab From a96085f1515fe87af44350b01094bee248515356 Mon Sep 17 00:00:00 2001 From: Stella Biderman Date: Tue, 6 May 2025 13:11:04 -0400 Subject: [PATCH 06/46] Change citation name (#2956) This hasn't been a library for few shot language model evaluation in quite a while. Let's update the citation to use "the Language Model Evaluation Harness" as the title. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b0816506..f325ae47 100644 --- a/README.md +++ b/README.md @@ -614,7 +614,7 @@ Extras dependencies can be installed via `pip install -e ".[NAME]"` ```text @misc{eval-harness, author = {Gao, Leo and Tow, Jonathan and Abbasi, Baber and Biderman, Stella and Black, Sid and DiPofi, Anthony and Foster, Charles and Golding, Laurence and Hsu, Jeffrey and Le Noac'h, Alain and Li, Haonan and McDonell, Kyle and Muennighoff, Niklas and Ociepa, Chris and Phang, Jason and Reynolds, Laria and Schoelkopf, Hailey and Skowron, Aviya and Sutawika, Lintang and Tang, Eric and Thite, Anish and Wang, Ben and Wang, Kevin and Zou, Andy}, - title = {A framework for few-shot language model evaluation}, + title = {The Language Model Evaluation Harness}, month = 07, year = 2024, publisher = {Zenodo}, -- GitLab From 2f03271d25db3c19e5552e19f59816bcbba07357 Mon Sep 17 00:00:00 2001 From: Baber Abbasi <92168766+baberabb@users.noreply.github.com> Date: Fri, 9 May 2025 15:23:10 +0500 Subject: [PATCH 07/46] add warning on truncation (#2962) --- lm_eval/models/vllm_causallms.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lm_eval/models/vllm_causallms.py b/lm_eval/models/vllm_causallms.py index 13a0b359..76bbe81d 100644 --- a/lm_eval/models/vllm_causallms.py +++ b/lm_eval/models/vllm_causallms.py @@ -430,6 +430,12 @@ class VLLM(TemplateLM): # set the max length in tokens of inputs ("context_enc") # max len for inputs = max length, minus room to generate the max new tokens max_ctx_len = self.max_length - max_gen_toks + all_lengths = [len(x) for x in context_encoding] + for length in all_lengths: + if length > max_ctx_len: + eval_logger.warning( + f"Context length {length} exceeds max length (context + max gen tokens): {max_ctx_len}. Truncating context." + ) context_encoding = [x[-max_ctx_len:] for x in context_encoding] # perform batched generation @@ -480,6 +486,10 @@ class VLLM(TemplateLM): inputs = [] ctxlens = [] for cache_key, context_enc, continuation_enc in chunk: + if full_length := (context_enc + continuation_enc) >= self.max_length: + eval_logger.warning( + f"Context length {full_length} exceeds max length ({self.max_length}). Truncating context." + ) inp = (context_enc + continuation_enc)[-(self.max_length) :] ctxlen = len(context_enc) - max( 0, len(context_enc) + len(continuation_enc) - (self.max_length) -- GitLab From 1c03af331dc5127c618b347439b0e02890102a3c Mon Sep 17 00:00:00 2001 From: Sungjae Lee <33976427+llsj14@users.noreply.github.com> Date: Sat, 10 May 2025 12:53:16 +0900 Subject: [PATCH 08/46] fix: type error while checking context length (#2972) --- lm_eval/models/vllm_causallms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lm_eval/models/vllm_causallms.py b/lm_eval/models/vllm_causallms.py index 76bbe81d..b02c541a 100644 --- a/lm_eval/models/vllm_causallms.py +++ b/lm_eval/models/vllm_causallms.py @@ -486,7 +486,7 @@ class VLLM(TemplateLM): inputs = [] ctxlens = [] for cache_key, context_enc, continuation_enc in chunk: - if full_length := (context_enc + continuation_enc) >= self.max_length: + if full_length := len(context_enc + continuation_enc) >= self.max_length: eval_logger.warning( f"Context length {full_length} exceeds max length ({self.max_length}). Truncating context." ) -- GitLab From 24fc1a47f01d458197d03bb407d19f977b7cfd73 Mon Sep 17 00:00:00 2001 From: Kiersten Stokes Date: Tue, 13 May 2025 07:54:41 -0500 Subject: [PATCH 09/46] Fix import error for deepcopy (#2969) Signed-off-by: kiersten-stokes --- lm_eval/tasks/squad_completion/task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lm_eval/tasks/squad_completion/task.py b/lm_eval/tasks/squad_completion/task.py index d8eb4674..79990ade 100644 --- a/lm_eval/tasks/squad_completion/task.py +++ b/lm_eval/tasks/squad_completion/task.py @@ -1,8 +1,8 @@ import re +from copy import deepcopy from typing import List import numpy as np -from deepcopy import deepcopy from lm_eval.api.instance import Instance from lm_eval.api.task import ConfigurableTask -- GitLab From af8b87ccfec4438a3f77b3416d85d53363695224 Mon Sep 17 00:00:00 2001 From: Kiersten Stokes Date: Tue, 13 May 2025 07:55:34 -0500 Subject: [PATCH 10/46] Pin unitxt to most recent major version to avoid test failures (#2970) Signed-off-by: kiersten-stokes --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index bd77bcc4..94e40063 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,7 +60,7 @@ Repository = "https://github.com/EleutherAI/lm-evaluation-harness" api = ["requests", "aiohttp", "tenacity", "tqdm", "tiktoken"] audiolm_qwen = ["librosa", "soundfile"] deepsparse = ["deepsparse-nightly[llm]>=1.8.0.20240404"] -dev = ["pytest", "pytest-cov", "pytest-xdist", "pre-commit", "mypy", "unitxt", "requests", "aiohttp", "tenacity", "tqdm", "tiktoken", "sentencepiece"] +dev = ["pytest", "pytest-cov", "pytest-xdist", "pre-commit", "mypy", "unitxt==1.22.0", "requests", "aiohttp", "tenacity", "tqdm", "tiktoken", "sentencepiece"] gptq = ["auto-gptq[triton]>=0.6.0"] gptqmodel = ["gptqmodel>=1.0.9"] hf_transfer = ["hf_transfer"] -- GitLab From cf51e699764b7c9e0ba85f196fcef941ab467e8a Mon Sep 17 00:00:00 2001 From: Yoonsoo Kim <34365327+yoonniverse@users.noreply.github.com> Date: Tue, 13 May 2025 22:16:42 +0900 Subject: [PATCH 11/46] mmlu pro generation_kwargs until Q: -> Question: (#2945) * mmlu pro generation_kwargs until Q: -> Question: * pacify pre-commit * change stop token --------- Co-authored-by: Baber --- lm_eval/tasks/longbench/utils.py | 6 +++--- lm_eval/tasks/mmlu_pro/README.md | 2 ++ lm_eval/tasks/mmlu_pro/_default_template_yaml | 4 +--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lm_eval/tasks/longbench/utils.py b/lm_eval/tasks/longbench/utils.py index a9fe0d7d..98580e39 100644 --- a/lm_eval/tasks/longbench/utils.py +++ b/lm_eval/tasks/longbench/utils.py @@ -4,7 +4,7 @@ import os import numpy as np from metrics import ( - classification_score, + # classification_score, code_sim_score, count_score, qa_f1_score, @@ -29,10 +29,10 @@ dataset2metric = { "qmsum": rouge_score, "multi_news": rouge_score, "vcsum": rouge_zh_score, - "trec": classification_score, + # "trec": classification_score, "triviaqa": qa_f1_score, "samsum": rouge_score, - "lsht": classification_score, + # "lsht": classification_score, "passage_retrieval_en": retrieval_score, "passage_count": count_score, "passage_retrieval_zh": retrieval_zh_score, diff --git a/lm_eval/tasks/mmlu_pro/README.md b/lm_eval/tasks/mmlu_pro/README.md index 7ffa99a3..e28b0527 100644 --- a/lm_eval/tasks/mmlu_pro/README.md +++ b/lm_eval/tasks/mmlu_pro/README.md @@ -64,3 +64,5 @@ If other tasks on this dataset are already supported: * Added one newline to task description(s) as per [reference implementation](https://github.com/TIGER-AI-Lab/MMLU-Pro/blob/47b9891aacb8bd7cda29d5c5ba17b9434dd333bc/evaluate_from_local.py#L93) * (tasks, group) 2025-03-20 -- (version 2.0 --> version 2.1) * Changed default max_length from 2048 to 8192 and max_gen_toks from 256 to 2048. +* (tasks, group) 2025-05-20 -- (version 2.1 --> version 3) + * changed stop sequence from "Q:" to "Question:" PR #2945 diff --git a/lm_eval/tasks/mmlu_pro/_default_template_yaml b/lm_eval/tasks/mmlu_pro/_default_template_yaml index d678c04d..d59d03a3 100644 --- a/lm_eval/tasks/mmlu_pro/_default_template_yaml +++ b/lm_eval/tasks/mmlu_pro/_default_template_yaml @@ -17,9 +17,7 @@ filter_list: - function: "take_first" generation_kwargs: until: - - "" - - "Q:" - - "<|im_end|>" + - "Question:" max_gen_toks: 2048 do_sample: false temperature: 0.0 -- GitLab From 1829799355d0377f5101fdd48fc493905cf5b17a Mon Sep 17 00:00:00 2001 From: Jess Date: Thu, 15 May 2025 07:47:18 -0400 Subject: [PATCH 12/46] AfroBench: How Good are Large Language Models on African Languages? (#2825) * add afrixnli to task * add chat completion * remove chat completion -untested * afrimmlu added * afrimmlu folder update * afrimmlu folder update * updated prompt * remove print * add afrimgsm -direct * add squad metric * fix bash script * remove direct util, update common yaml * remove print * add few show. metric fixes * fix direct path, add bash script for gpt models * added transate test * update afrixnli tasks * update afrixnli tasks * update metrics for afrixnli * prompt translations fix * prompt translations fix * filter and metric fix -mgsm * remove squad metric * remove squad metric * add f1 score to mgsm * add f1 score to mgsm * update native-direct with lin * change f1 function * add lin to utils * add utils * remove test limit * remove test configs * add swahili to mmlu * change eng to ewe in ewe yaml mmlu * add squad metric to mgsm, remove whitespace filter * added translate test * added afrixnli_translate * fix exact match valueError * fix exact match valueError * restructure mmlu folder * spacing * remove afrimmlu_translate folder * add utility * format task name, clean ups * modefied mgsm * update on afrimgsm * update on afrimgsm * removed utils * other mgsm varieties * other mgsm varieties * adding trasnslate direct * Update translate_direct_yaml * add manual xnli prompt, add multichoice for openai models, and adapt multichoice metric for openai model * edit for open models * Update translate_direct_yaml * add verbalizer for xnli * change xnli from multiple choice to generate * add manual accuracy scores * revert xnli to multiple choice * change afrimgsm utils * revert xnli to multiple_choice * cleanups and readmes * remove openai fixes and unused regex * pr review changes * revert metrics.py, task.py and extraction.py to main version * add afrisenti * utilities * pulled from main * add afrixnli * add afrimmlu * update afrixnli prompts * mising senti language * fix afrisenti prompt 2 * fix afrisenti prompts * fix afrisenti prompts * configure task grouping * add multiple prompts to afrixnli for irokobench * add multiple prompts to afrimmlu for irokobench * Update afrixnli_yaml * fixes and moves * fixes and moves * afrimmlu multiple prompts configs * remove validation set from afrimmlu * remove eng from afrimmlu translate test * correct dataset path * multiple prompts for mgsm * file restructure * afribench grouping * repo restructuring * repo restructuring * update exact match to hugging face exact match and add new mgsm language * remove decontamination * update generation kwargs * update generation kwargs for all mgsm prompts * remove lang * update generation kwargs for afrimgsm translatetest * add afrimgsm cot for direct and translate * remove eng from translate-cot * add masakhaPOS tasks * remove changes from task script * add masakhanews tasks * add uhura arc easy * add afriqa and belebele files * add tags for easier run. add naija rc * add new metrics and transformation scripts * fix afriqa swa fewshot split * add naijarc * add afrobench lite tasks * update afrobench * update afrobench * remove unverified files to avoid bugs * remove files not needed * add afrobench tasks * add afrobench tasks * change to version 1 * change to version 1 * update afrobench * update afrobench * restore metric to original script * update readme instructions * add individual dataset readmes * add link to collections * correct run script * align with main * align with main * align with main * align with main * align with main * align with main * align with main * align with main * failed run fixes * failed run fixes * add afrimgsm cot * Apply precommit fixes * update mafand dataset name * pull request fixes * remove afrihate due to availability --------- Co-authored-by: Israel Abebe Azime Co-authored-by: Israel Abebe Azime Co-authored-by: David Adelani Co-authored-by: theyorubayesian --- lm_eval/filters/extraction.py | 45 ++++ lm_eval/filters/transformation.py | 66 +++++ lm_eval/tasks/afrimgsm/direct/afrimgsm.yaml | 13 + .../afrimgsm/direct/afrimgsm_direct_amh.yaml | 12 - .../afrimgsm/direct/afrimgsm_direct_eng.yaml | 12 - .../afrimgsm/direct/afrimgsm_direct_ewe.yaml | 12 - .../afrimgsm/direct/afrimgsm_direct_fra.yaml | 12 - .../afrimgsm/direct/afrimgsm_direct_hau.yaml | 12 - .../afrimgsm/direct/afrimgsm_direct_ibo.yaml | 12 - .../afrimgsm/direct/afrimgsm_direct_kin.yaml | 12 - .../afrimgsm/direct/afrimgsm_direct_lin.yaml | 12 - .../afrimgsm/direct/afrimgsm_direct_lug.yaml | 12 - .../afrimgsm/direct/afrimgsm_direct_orm.yaml | 12 - .../afrimgsm/direct/afrimgsm_direct_sna.yaml | 12 - .../afrimgsm/direct/afrimgsm_direct_sot.yaml | 12 - .../afrimgsm/direct/afrimgsm_direct_swa.yaml | 12 - .../afrimgsm/direct/afrimgsm_direct_twi.yaml | 12 - .../afrimgsm/direct/afrimgsm_direct_wol.yaml | 12 - .../afrimgsm/direct/afrimgsm_direct_xho.yaml | 12 - .../afrimgsm/direct/afrimgsm_direct_yor.yaml | 12 - .../afrimgsm/direct/afrimgsm_direct_zul.yaml | 12 - lm_eval/tasks/afrimgsm/direct/direct_yaml | 37 --- .../direct/prompt_1/afrimgsm_amh.yaml | 4 + .../direct/prompt_1/afrimgsm_eng.yaml | 4 + .../direct/prompt_1/afrimgsm_ewe.yaml | 4 + .../direct/prompt_1/afrimgsm_fra.yaml | 4 + .../direct/prompt_1/afrimgsm_hau.yaml | 4 + .../direct/prompt_1/afrimgsm_ibo.yaml | 4 + .../direct/prompt_1/afrimgsm_kin.yaml | 4 + .../direct/prompt_1/afrimgsm_lin.yaml | 4 + .../direct/prompt_1/afrimgsm_lug.yaml | 4 + .../direct/prompt_1/afrimgsm_orm.yaml | 4 + .../direct/prompt_1/afrimgsm_sna.yaml | 4 + .../direct/prompt_1/afrimgsm_sot.yaml | 4 + .../direct/prompt_1/afrimgsm_swa.yaml | 4 + .../direct/prompt_1/afrimgsm_twi.yaml | 4 + .../direct/prompt_1/afrimgsm_vai.yaml | 4 + .../direct/prompt_1/afrimgsm_wol.yaml | 4 + .../direct/prompt_1/afrimgsm_xho.yaml | 4 + .../afrimgsm/direct/prompt_1/afrimgsm_yaml | 35 +++ .../direct/prompt_1/afrimgsm_yor.yaml | 4 + .../direct/prompt_1/afrimgsm_zul.yaml | 4 + .../direct/prompt_2/afrimgsm_amh.yaml | 4 + .../direct/prompt_2/afrimgsm_eng.yaml | 4 + .../direct/prompt_2/afrimgsm_ewe.yaml | 4 + .../direct/prompt_2/afrimgsm_fra.yaml | 4 + .../direct/prompt_2/afrimgsm_hau.yaml | 4 + .../direct/prompt_2/afrimgsm_ibo.yaml | 4 + .../direct/prompt_2/afrimgsm_kin.yaml | 4 + .../direct/prompt_2/afrimgsm_lin.yaml | 4 + .../direct/prompt_2/afrimgsm_lug.yaml | 4 + .../direct/prompt_2/afrimgsm_orm.yaml | 4 + .../direct/prompt_2/afrimgsm_sna.yaml | 4 + .../direct/prompt_2/afrimgsm_sot.yaml | 4 + .../direct/prompt_2/afrimgsm_swa.yaml | 4 + .../direct/prompt_2/afrimgsm_twi.yaml | 4 + .../direct/prompt_2/afrimgsm_vai.yaml | 4 + .../direct/prompt_2/afrimgsm_wol.yaml | 4 + .../direct/prompt_2/afrimgsm_xho.yaml | 4 + .../afrimgsm/direct/prompt_2/afrimgsm_yaml | 34 +++ .../direct/prompt_2/afrimgsm_yor.yaml | 4 + .../direct/prompt_2/afrimgsm_zul.yaml | 4 + .../direct/prompt_3/afrimgsm_amh.yaml | 4 + .../direct/prompt_3/afrimgsm_eng.yaml | 4 + .../direct/prompt_3/afrimgsm_ewe.yaml | 4 + .../direct/prompt_3/afrimgsm_fra.yaml | 4 + .../direct/prompt_3/afrimgsm_hau.yaml | 4 + .../direct/prompt_3/afrimgsm_ibo.yaml | 4 + .../direct/prompt_3/afrimgsm_kin.yaml | 4 + .../direct/prompt_3/afrimgsm_lin.yaml | 4 + .../direct/prompt_3/afrimgsm_lug.yaml | 4 + .../direct/prompt_3/afrimgsm_orm.yaml | 4 + .../direct/prompt_3/afrimgsm_sna.yaml | 4 + .../direct/prompt_3/afrimgsm_sot.yaml | 4 + .../direct/prompt_3/afrimgsm_swa.yaml | 4 + .../direct/prompt_3/afrimgsm_twi.yaml | 4 + .../direct/prompt_3/afrimgsm_vai.yaml | 4 + .../direct/prompt_3/afrimgsm_wol.yaml | 4 + .../direct/prompt_3/afrimgsm_xho.yaml | 4 + .../afrimgsm/direct/prompt_3/afrimgsm_yaml | 34 +++ .../direct/prompt_3/afrimgsm_yor.yaml | 4 + .../direct/prompt_3/afrimgsm_zul.yaml | 4 + .../direct/prompt_4/afrimgsm_amh.yaml | 7 + .../direct/prompt_4/afrimgsm_eng.yaml | 7 + .../direct/prompt_4/afrimgsm_ewe.yaml | 7 + .../direct/prompt_4/afrimgsm_fra.yaml | 7 + .../direct/prompt_4/afrimgsm_hau.yaml | 7 + .../direct/prompt_4/afrimgsm_ibo.yaml | 7 + .../direct/prompt_4/afrimgsm_kin.yaml | 7 + .../direct/prompt_4/afrimgsm_lin.yaml | 7 + .../direct/prompt_4/afrimgsm_lug.yaml | 7 + .../direct/prompt_4/afrimgsm_orm.yaml | 7 + .../direct/prompt_4/afrimgsm_sna.yaml | 7 + .../direct/prompt_4/afrimgsm_sot.yaml | 7 + .../direct/prompt_4/afrimgsm_swa.yaml | 7 + .../direct/prompt_4/afrimgsm_twi.yaml | 7 + .../direct/prompt_4/afrimgsm_vai.yaml | 7 + .../direct/prompt_4/afrimgsm_wol.yaml | 7 + .../direct/prompt_4/afrimgsm_xho.yaml | 7 + .../afrimgsm/direct/prompt_4/afrimgsm_yaml | 33 +++ .../direct/prompt_4/afrimgsm_yor.yaml | 7 + .../direct/prompt_4/afrimgsm_zul.yaml | 7 + .../direct/prompt_5/afrimgsm_amh.yaml | 7 + .../direct/prompt_5/afrimgsm_eng.yaml | 7 + .../direct/prompt_5/afrimgsm_ewe.yaml | 6 + .../direct/prompt_5/afrimgsm_fra.yaml | 6 + .../direct/prompt_5/afrimgsm_hau.yaml | 6 + .../direct/prompt_5/afrimgsm_ibo.yaml | 6 + .../direct/prompt_5/afrimgsm_kin.yaml | 7 + .../direct/prompt_5/afrimgsm_lin.yaml | 7 + .../direct/prompt_5/afrimgsm_lug.yaml | 7 + .../direct/prompt_5/afrimgsm_orm.yaml | 6 + .../direct/prompt_5/afrimgsm_sna.yaml | 7 + .../direct/prompt_5/afrimgsm_sot.yaml | 7 + .../direct/prompt_5/afrimgsm_swa.yaml | 7 + .../direct/prompt_5/afrimgsm_twi.yaml | 6 + .../direct/prompt_5/afrimgsm_vai.yaml | 6 + .../direct/prompt_5/afrimgsm_wol.yaml | 6 + .../direct/prompt_5/afrimgsm_xho.yaml | 7 + .../afrimgsm/direct/prompt_5/afrimgsm_yaml | 33 +++ .../direct/prompt_5/afrimgsm_yor.yaml | 6 + .../direct/prompt_5/afrimgsm_zul.yaml | 6 + .../afrimgsm/direct_cot/afrimgsm_cot.yaml | 9 + .../direct_cot/prompt_1/afrimgsm_cot_amh.yaml | 4 + .../direct_cot/prompt_1/afrimgsm_cot_eng.yaml | 4 + .../direct_cot/prompt_1/afrimgsm_cot_ewe.yaml | 4 + .../direct_cot/prompt_1/afrimgsm_cot_fra.yaml | 4 + .../direct_cot/prompt_1/afrimgsm_cot_hau.yaml | 4 + .../direct_cot/prompt_1/afrimgsm_cot_ibo.yaml | 4 + .../direct_cot/prompt_1/afrimgsm_cot_kin.yaml | 4 + .../direct_cot/prompt_1/afrimgsm_cot_lin.yaml | 4 + .../direct_cot/prompt_1/afrimgsm_cot_lug.yaml | 4 + .../direct_cot/prompt_1/afrimgsm_cot_orm.yaml | 4 + .../direct_cot/prompt_1/afrimgsm_cot_sna.yaml | 4 + .../direct_cot/prompt_1/afrimgsm_cot_sot.yaml | 4 + .../direct_cot/prompt_1/afrimgsm_cot_swa.yaml | 4 + .../direct_cot/prompt_1/afrimgsm_cot_twi.yaml | 4 + .../direct_cot/prompt_1/afrimgsm_cot_vai.yaml | 4 + .../direct_cot/prompt_1/afrimgsm_cot_wol.yaml | 4 + .../direct_cot/prompt_1/afrimgsm_cot_xho.yaml | 4 + .../direct_cot/prompt_1/afrimgsm_cot_yaml | 37 +++ .../direct_cot/prompt_1/afrimgsm_cot_yor.yaml | 4 + .../direct_cot/prompt_1/afrimgsm_cot_zul.yaml | 4 + .../direct_cot/prompt_2/afrimgsm_cot_amh.yaml | 4 + .../direct_cot/prompt_2/afrimgsm_cot_eng.yaml | 4 + .../direct_cot/prompt_2/afrimgsm_cot_ewe.yaml | 4 + .../direct_cot/prompt_2/afrimgsm_cot_fra.yaml | 4 + .../direct_cot/prompt_2/afrimgsm_cot_hau.yaml | 4 + .../direct_cot/prompt_2/afrimgsm_cot_ibo.yaml | 4 + .../direct_cot/prompt_2/afrimgsm_cot_kin.yaml | 4 + .../direct_cot/prompt_2/afrimgsm_cot_lin.yaml | 4 + .../direct_cot/prompt_2/afrimgsm_cot_lug.yaml | 4 + .../direct_cot/prompt_2/afrimgsm_cot_orm.yaml | 4 + .../direct_cot/prompt_2/afrimgsm_cot_sna.yaml | 4 + .../direct_cot/prompt_2/afrimgsm_cot_sot.yaml | 4 + .../direct_cot/prompt_2/afrimgsm_cot_swa.yaml | 4 + .../direct_cot/prompt_2/afrimgsm_cot_twi.yaml | 4 + .../direct_cot/prompt_2/afrimgsm_cot_vai.yaml | 4 + .../direct_cot/prompt_2/afrimgsm_cot_wol.yaml | 4 + .../direct_cot/prompt_2/afrimgsm_cot_xho.yaml | 4 + .../direct_cot/prompt_2/afrimgsm_cot_yaml | 37 +++ .../direct_cot/prompt_2/afrimgsm_cot_yor.yaml | 4 + .../direct_cot/prompt_2/afrimgsm_cot_zul.yaml | 4 + .../direct_cot/prompt_3/afrimgsm_cot_amh.yaml | 4 + .../direct_cot/prompt_3/afrimgsm_cot_eng.yaml | 4 + .../direct_cot/prompt_3/afrimgsm_cot_ewe.yaml | 4 + .../direct_cot/prompt_3/afrimgsm_cot_fra.yaml | 4 + .../direct_cot/prompt_3/afrimgsm_cot_hau.yaml | 4 + .../direct_cot/prompt_3/afrimgsm_cot_ibo.yaml | 4 + .../direct_cot/prompt_3/afrimgsm_cot_kin.yaml | 4 + .../direct_cot/prompt_3/afrimgsm_cot_lin.yaml | 4 + .../direct_cot/prompt_3/afrimgsm_cot_lug.yaml | 4 + .../direct_cot/prompt_3/afrimgsm_cot_orm.yaml | 4 + .../direct_cot/prompt_3/afrimgsm_cot_sna.yaml | 4 + .../direct_cot/prompt_3/afrimgsm_cot_sot.yaml | 4 + .../direct_cot/prompt_3/afrimgsm_cot_swa.yaml | 4 + .../direct_cot/prompt_3/afrimgsm_cot_twi.yaml | 4 + .../direct_cot/prompt_3/afrimgsm_cot_vai.yaml | 4 + .../direct_cot/prompt_3/afrimgsm_cot_wol.yaml | 4 + .../direct_cot/prompt_3/afrimgsm_cot_xho.yaml | 4 + .../direct_cot/prompt_3/afrimgsm_cot_yaml | 37 +++ .../direct_cot/prompt_3/afrimgsm_cot_yor.yaml | 4 + .../direct_cot/prompt_3/afrimgsm_cot_zul.yaml | 4 + .../direct_cot/prompt_4/afrimgsm_cot_amh.yaml | 7 + .../direct_cot/prompt_4/afrimgsm_cot_eng.yaml | 7 + .../direct_cot/prompt_4/afrimgsm_cot_ewe.yaml | 7 + .../direct_cot/prompt_4/afrimgsm_cot_fra.yaml | 7 + .../direct_cot/prompt_4/afrimgsm_cot_hau.yaml | 7 + .../direct_cot/prompt_4/afrimgsm_cot_ibo.yaml | 7 + .../direct_cot/prompt_4/afrimgsm_cot_kin.yaml | 7 + .../direct_cot/prompt_4/afrimgsm_cot_lin.yaml | 7 + .../direct_cot/prompt_4/afrimgsm_cot_lug.yaml | 7 + .../direct_cot/prompt_4/afrimgsm_cot_orm.yaml | 7 + .../direct_cot/prompt_4/afrimgsm_cot_sna.yaml | 7 + .../direct_cot/prompt_4/afrimgsm_cot_sot.yaml | 7 + .../direct_cot/prompt_4/afrimgsm_cot_swa.yaml | 7 + .../direct_cot/prompt_4/afrimgsm_cot_twi.yaml | 7 + .../direct_cot/prompt_4/afrimgsm_cot_vai.yaml | 7 + .../direct_cot/prompt_4/afrimgsm_cot_wol.yaml | 7 + .../direct_cot/prompt_4/afrimgsm_cot_xho.yaml | 7 + .../direct_cot/prompt_4/afrimgsm_cot_yaml | 36 +++ .../direct_cot/prompt_4/afrimgsm_cot_yor.yaml | 7 + .../direct_cot/prompt_4/afrimgsm_cot_zul.yaml | 7 + .../direct_cot/prompt_5/afrimgsm_cot_amh.yaml | 7 + .../direct_cot/prompt_5/afrimgsm_cot_eng.yaml | 7 + .../direct_cot/prompt_5/afrimgsm_cot_ewe.yaml | 7 + .../direct_cot/prompt_5/afrimgsm_cot_fra.yaml | 7 + .../direct_cot/prompt_5/afrimgsm_cot_hau.yaml | 7 + .../direct_cot/prompt_5/afrimgsm_cot_ibo.yaml | 7 + .../direct_cot/prompt_5/afrimgsm_cot_kin.yaml | 7 + .../direct_cot/prompt_5/afrimgsm_cot_lin.yaml | 7 + .../direct_cot/prompt_5/afrimgsm_cot_lug.yaml | 7 + .../direct_cot/prompt_5/afrimgsm_cot_orm.yaml | 7 + .../direct_cot/prompt_5/afrimgsm_cot_sna.yaml | 7 + .../direct_cot/prompt_5/afrimgsm_cot_sot.yaml | 7 + .../direct_cot/prompt_5/afrimgsm_cot_swa.yaml | 7 + .../direct_cot/prompt_5/afrimgsm_cot_twi.yaml | 7 + .../direct_cot/prompt_5/afrimgsm_cot_vai.yaml | 7 + .../direct_cot/prompt_5/afrimgsm_cot_wol.yaml | 7 + .../direct_cot/prompt_5/afrimgsm_cot_xho.yaml | 7 + .../direct_cot/prompt_5/afrimgsm_cot_yaml | 36 +++ .../direct_cot/prompt_5/afrimgsm_cot_yor.yaml | 7 + .../direct_cot/prompt_5/afrimgsm_cot_zul.yaml | 7 + .../afrimgsm/en_cot/afrimgsm_en_cot_amh.yaml | 12 - .../afrimgsm/en_cot/afrimgsm_en_cot_eng.yaml | 12 - .../afrimgsm/en_cot/afrimgsm_en_cot_ewe.yaml | 12 - .../afrimgsm/en_cot/afrimgsm_en_cot_fra.yaml | 12 - .../afrimgsm/en_cot/afrimgsm_en_cot_hau.yaml | 12 - .../afrimgsm/en_cot/afrimgsm_en_cot_ibo.yaml | 12 - .../afrimgsm/en_cot/afrimgsm_en_cot_kin.yaml | 12 - .../afrimgsm/en_cot/afrimgsm_en_cot_lin.yaml | 12 - .../afrimgsm/en_cot/afrimgsm_en_cot_lug.yaml | 12 - .../afrimgsm/en_cot/afrimgsm_en_cot_orm.yaml | 12 - .../afrimgsm/en_cot/afrimgsm_en_cot_sna.yaml | 12 - .../afrimgsm/en_cot/afrimgsm_en_cot_sot.yaml | 12 - .../afrimgsm/en_cot/afrimgsm_en_cot_swa.yaml | 12 - .../afrimgsm/en_cot/afrimgsm_en_cot_twi.yaml | 12 - .../afrimgsm/en_cot/afrimgsm_en_cot_wol.yaml | 12 - .../afrimgsm/en_cot/afrimgsm_en_cot_xho.yaml | 12 - .../afrimgsm/en_cot/afrimgsm_en_cot_yor.yaml | 12 - .../afrimgsm/en_cot/afrimgsm_en_cot_zul.yaml | 12 - lm_eval/tasks/afrimgsm/en_cot/cot_yaml | 37 --- lm_eval/tasks/afrimgsm/gen_utils.py | 122 ++++++++++ .../translate/afrimgsm_translate_amh.yaml | 12 - .../translate/afrimgsm_translate_eng.yaml | 12 - .../translate/afrimgsm_translate_ewe.yaml | 12 - .../translate/afrimgsm_translate_fra.yaml | 12 - .../translate/afrimgsm_translate_hau.yaml | 12 - .../translate/afrimgsm_translate_ibo.yaml | 12 - .../translate/afrimgsm_translate_kin.yaml | 12 - .../translate/afrimgsm_translate_lin.yaml | 12 - .../translate/afrimgsm_translate_lug.yaml | 12 - .../translate/afrimgsm_translate_orm.yaml | 12 - .../translate/afrimgsm_translate_sna.yaml | 12 - .../translate/afrimgsm_translate_sot.yaml | 12 - .../translate/afrimgsm_translate_swa.yaml | 12 - .../translate/afrimgsm_translate_twi.yaml | 12 - .../translate/afrimgsm_translate_wol.yaml | 12 - .../translate/afrimgsm_translate_xho.yaml | 12 - .../translate/afrimgsm_translate_yor.yaml | 12 - .../translate/afrimgsm_translate_zul.yaml | 12 - .../tasks/afrimgsm/translate/afrimgsm_tt.yaml | 9 + .../prompt_1/afrimgsm_translate_amh.yaml | 4 + .../prompt_1/afrimgsm_translate_ewe.yaml | 4 + .../prompt_1/afrimgsm_translate_fra.yaml | 4 + .../prompt_1/afrimgsm_translate_hau.yaml | 4 + .../prompt_1/afrimgsm_translate_ibo.yaml | 4 + .../prompt_1/afrimgsm_translate_kin.yaml | 4 + .../prompt_1/afrimgsm_translate_lin.yaml | 4 + .../prompt_1/afrimgsm_translate_lug.yaml | 4 + .../prompt_1/afrimgsm_translate_orm.yaml | 4 + .../prompt_1/afrimgsm_translate_sna.yaml | 4 + .../prompt_1/afrimgsm_translate_sot.yaml | 4 + .../prompt_1/afrimgsm_translate_swa.yaml | 4 + .../prompt_1/afrimgsm_translate_twi.yaml | 4 + .../prompt_1/afrimgsm_translate_wol.yaml | 4 + .../prompt_1/afrimgsm_translate_xho.yaml | 4 + .../prompt_1/afrimgsm_translate_yaml | 32 +++ .../prompt_1/afrimgsm_translate_yor.yaml | 4 + .../prompt_1/afrimgsm_translate_zul.yaml | 4 + .../prompt_2/afrimgsm_translate_amh.yaml | 4 + .../prompt_2/afrimgsm_translate_ewe.yaml | 4 + .../prompt_2/afrimgsm_translate_fra.yaml | 4 + .../prompt_2/afrimgsm_translate_hau.yaml | 4 + .../prompt_2/afrimgsm_translate_ibo.yaml | 4 + .../prompt_2/afrimgsm_translate_kin.yaml | 4 + .../prompt_2/afrimgsm_translate_lin.yaml | 4 + .../prompt_2/afrimgsm_translate_lug.yaml | 4 + .../prompt_2/afrimgsm_translate_orm.yaml | 4 + .../prompt_2/afrimgsm_translate_sna.yaml | 4 + .../prompt_2/afrimgsm_translate_sot.yaml | 4 + .../prompt_2/afrimgsm_translate_swa.yaml | 4 + .../prompt_2/afrimgsm_translate_twi.yaml | 4 + .../prompt_2/afrimgsm_translate_wol.yaml | 4 + .../prompt_2/afrimgsm_translate_xho.yaml | 4 + .../prompt_2/afrimgsm_translate_yaml | 34 +++ .../prompt_2/afrimgsm_translate_yor.yaml | 4 + .../prompt_2/afrimgsm_translate_zul.yaml | 4 + .../prompt_3/afrimgsm_translate_amh.yaml | 4 + .../prompt_3/afrimgsm_translate_ewe.yaml | 4 + .../prompt_3/afrimgsm_translate_fra.yaml | 4 + .../prompt_3/afrimgsm_translate_hau.yaml | 4 + .../prompt_3/afrimgsm_translate_ibo.yaml | 4 + .../prompt_3/afrimgsm_translate_kin.yaml | 4 + .../prompt_3/afrimgsm_translate_lin.yaml | 4 + .../prompt_3/afrimgsm_translate_lug.yaml | 4 + .../prompt_3/afrimgsm_translate_orm.yaml | 4 + .../prompt_3/afrimgsm_translate_sna.yaml | 4 + .../prompt_3/afrimgsm_translate_sot.yaml | 4 + .../prompt_3/afrimgsm_translate_swa.yaml | 4 + .../prompt_3/afrimgsm_translate_twi.yaml | 4 + .../prompt_3/afrimgsm_translate_wol.yaml | 4 + .../prompt_3/afrimgsm_translate_xho.yaml | 4 + .../prompt_3/afrimgsm_translate_yaml | 32 +++ .../prompt_3/afrimgsm_translate_yor.yaml | 4 + .../prompt_3/afrimgsm_translate_zul.yaml | 4 + .../prompt_4/afrimgsm_translate_amh.yaml | 7 + .../prompt_4/afrimgsm_translate_ewe.yaml | 7 + .../prompt_4/afrimgsm_translate_fra.yaml | 7 + .../prompt_4/afrimgsm_translate_hau.yaml | 7 + .../prompt_4/afrimgsm_translate_ibo.yaml | 7 + .../prompt_4/afrimgsm_translate_kin.yaml | 7 + .../prompt_4/afrimgsm_translate_lin.yaml | 7 + .../prompt_4/afrimgsm_translate_lug.yaml | 7 + .../prompt_4/afrimgsm_translate_orm.yaml | 7 + .../prompt_4/afrimgsm_translate_sna.yaml | 7 + .../prompt_4/afrimgsm_translate_sot.yaml | 7 + .../prompt_4/afrimgsm_translate_swa.yaml | 7 + .../prompt_4/afrimgsm_translate_twi.yaml | 7 + .../prompt_4/afrimgsm_translate_wol.yaml | 7 + .../prompt_4/afrimgsm_translate_xho.yaml | 7 + .../prompt_4/afrimgsm_translate_yaml | 31 +++ .../prompt_4/afrimgsm_translate_yor.yaml | 7 + .../prompt_4/afrimgsm_translate_zul.yaml | 7 + .../prompt_5/afrimgsm_translate_amh.yaml | 7 + .../prompt_5/afrimgsm_translate_ewe.yaml | 6 + .../prompt_5/afrimgsm_translate_fra.yaml | 6 + .../prompt_5/afrimgsm_translate_hau.yaml | 6 + .../prompt_5/afrimgsm_translate_ibo.yaml | 6 + .../prompt_5/afrimgsm_translate_kin.yaml | 7 + .../prompt_5/afrimgsm_translate_lin.yaml | 7 + .../prompt_5/afrimgsm_translate_lug.yaml | 7 + .../prompt_5/afrimgsm_translate_orm.yaml | 6 + .../prompt_5/afrimgsm_translate_sna.yaml | 7 + .../prompt_5/afrimgsm_translate_sot.yaml | 7 + .../prompt_5/afrimgsm_translate_swa.yaml | 7 + .../prompt_5/afrimgsm_translate_twi.yaml | 6 + .../prompt_5/afrimgsm_translate_wol.yaml | 6 + .../prompt_5/afrimgsm_translate_xho.yaml | 7 + .../prompt_5/afrimgsm_translate_yaml | 31 +++ .../prompt_5/afrimgsm_translate_yor.yaml | 6 + .../prompt_5/afrimgsm_translate_zul.yaml | 6 + .../afrimgsm/translate/translate_direct_yaml | 36 --- .../translate_cot/afrimgsm_tt_cot.yaml | 9 + .../prompt_1/afrimgsm_cot_translate_amh.yaml | 4 + .../prompt_1/afrimgsm_cot_translate_ewe.yaml | 4 + .../prompt_1/afrimgsm_cot_translate_fra.yaml | 4 + .../prompt_1/afrimgsm_cot_translate_hau.yaml | 4 + .../prompt_1/afrimgsm_cot_translate_ibo.yaml | 4 + .../prompt_1/afrimgsm_cot_translate_kin.yaml | 4 + .../prompt_1/afrimgsm_cot_translate_lin.yaml | 4 + .../prompt_1/afrimgsm_cot_translate_lug.yaml | 4 + .../prompt_1/afrimgsm_cot_translate_orm.yaml | 4 + .../prompt_1/afrimgsm_cot_translate_sna.yaml | 4 + .../prompt_1/afrimgsm_cot_translate_sot.yaml | 4 + .../prompt_1/afrimgsm_cot_translate_swa.yaml | 4 + .../prompt_1/afrimgsm_cot_translate_twi.yaml | 4 + .../prompt_1/afrimgsm_cot_translate_vai.yaml | 4 + .../prompt_1/afrimgsm_cot_translate_wol.yaml | 4 + .../prompt_1/afrimgsm_cot_translate_xho.yaml | 4 + .../prompt_1/afrimgsm_cot_translate_yaml | 33 +++ .../prompt_1/afrimgsm_cot_translate_yor.yaml | 4 + .../prompt_1/afrimgsm_cot_translate_zul.yaml | 4 + .../prompt_2/afrimgsm_cot_translate_amh.yaml | 4 + .../prompt_2/afrimgsm_cot_translate_ewe.yaml | 4 + .../prompt_2/afrimgsm_cot_translate_fra.yaml | 4 + .../prompt_2/afrimgsm_cot_translate_hau.yaml | 4 + .../prompt_2/afrimgsm_cot_translate_ibo.yaml | 4 + .../prompt_2/afrimgsm_cot_translate_kin.yaml | 4 + .../prompt_2/afrimgsm_cot_translate_lin.yaml | 4 + .../prompt_2/afrimgsm_cot_translate_lug.yaml | 4 + .../prompt_2/afrimgsm_cot_translate_orm.yaml | 4 + .../prompt_2/afrimgsm_cot_translate_sna.yaml | 4 + .../prompt_2/afrimgsm_cot_translate_sot.yaml | 4 + .../prompt_2/afrimgsm_cot_translate_swa.yaml | 4 + .../prompt_2/afrimgsm_cot_translate_twi.yaml | 4 + .../prompt_2/afrimgsm_cot_translate_vai.yaml | 4 + .../prompt_2/afrimgsm_cot_translate_wol.yaml | 4 + .../prompt_2/afrimgsm_cot_translate_xho.yaml | 4 + .../prompt_2/afrimgsm_cot_translate_yaml | 33 +++ .../prompt_2/afrimgsm_cot_translate_yor.yaml | 4 + .../prompt_2/afrimgsm_cot_translate_zul.yaml | 4 + .../prompt_3/afrimgsm_cot_translate_amh.yaml | 4 + .../prompt_3/afrimgsm_cot_translate_ewe.yaml | 4 + .../prompt_3/afrimgsm_cot_translate_fra.yaml | 4 + .../prompt_3/afrimgsm_cot_translate_hau.yaml | 4 + .../prompt_3/afrimgsm_cot_translate_ibo.yaml | 4 + .../prompt_3/afrimgsm_cot_translate_kin.yaml | 4 + .../prompt_3/afrimgsm_cot_translate_lin.yaml | 4 + .../prompt_3/afrimgsm_cot_translate_lug.yaml | 4 + .../prompt_3/afrimgsm_cot_translate_orm.yaml | 4 + .../prompt_3/afrimgsm_cot_translate_sna.yaml | 4 + .../prompt_3/afrimgsm_cot_translate_sot.yaml | 4 + .../prompt_3/afrimgsm_cot_translate_swa.yaml | 4 + .../prompt_3/afrimgsm_cot_translate_twi.yaml | 4 + .../prompt_3/afrimgsm_cot_translate_vai.yaml | 4 + .../prompt_3/afrimgsm_cot_translate_wol.yaml | 4 + .../prompt_3/afrimgsm_cot_translate_xho.yaml | 4 + .../prompt_3/afrimgsm_cot_translate_yaml | 33 +++ .../prompt_3/afrimgsm_cot_translate_yor.yaml | 4 + .../prompt_3/afrimgsm_cot_translate_zul.yaml | 4 + .../prompt_4/afrimgsm_cot_translate_amh.yaml | 7 + .../prompt_4/afrimgsm_cot_translate_ewe.yaml | 7 + .../prompt_4/afrimgsm_cot_translate_fra.yaml | 7 + .../prompt_4/afrimgsm_cot_translate_hau.yaml | 7 + .../prompt_4/afrimgsm_cot_translate_ibo.yaml | 7 + .../prompt_4/afrimgsm_cot_translate_kin.yaml | 7 + .../prompt_4/afrimgsm_cot_translate_lin.yaml | 7 + .../prompt_4/afrimgsm_cot_translate_lug.yaml | 7 + .../prompt_4/afrimgsm_cot_translate_orm.yaml | 7 + .../prompt_4/afrimgsm_cot_translate_sna.yaml | 7 + .../prompt_4/afrimgsm_cot_translate_sot.yaml | 7 + .../prompt_4/afrimgsm_cot_translate_swa.yaml | 7 + .../prompt_4/afrimgsm_cot_translate_twi.yaml | 7 + .../prompt_4/afrimgsm_cot_translate_vai.yaml | 7 + .../prompt_4/afrimgsm_cot_translate_wol.yaml | 7 + .../prompt_4/afrimgsm_cot_translate_xho.yaml | 7 + .../prompt_4/afrimgsm_cot_translate_yaml | 32 +++ .../prompt_4/afrimgsm_cot_translate_yor.yaml | 7 + .../prompt_4/afrimgsm_cot_translate_zul.yaml | 7 + .../prompt_5/afrimgsm_cot_translate_amh.yaml | 7 + .../prompt_5/afrimgsm_cot_translate_ewe.yaml | 7 + .../prompt_5/afrimgsm_cot_translate_fra.yaml | 7 + .../prompt_5/afrimgsm_cot_translate_hau.yaml | 7 + .../prompt_5/afrimgsm_cot_translate_ibo.yaml | 7 + .../prompt_5/afrimgsm_cot_translate_kin.yaml | 7 + .../prompt_5/afrimgsm_cot_translate_lin.yaml | 7 + .../prompt_5/afrimgsm_cot_translate_lug.yaml | 7 + .../prompt_5/afrimgsm_cot_translate_orm.yaml | 7 + .../prompt_5/afrimgsm_cot_translate_sna.yaml | 7 + .../prompt_5/afrimgsm_cot_translate_sot.yaml | 7 + .../prompt_5/afrimgsm_cot_translate_swa.yaml | 7 + .../prompt_5/afrimgsm_cot_translate_twi.yaml | 7 + .../prompt_5/afrimgsm_cot_translate_vai.yaml | 7 + .../prompt_5/afrimgsm_cot_translate_wol.yaml | 7 + .../prompt_5/afrimgsm_cot_translate_xho.yaml | 7 + .../prompt_5/afrimgsm_cot_translate_yaml | 32 +++ .../prompt_5/afrimgsm_cot_translate_yor.yaml | 7 + .../prompt_5/afrimgsm_cot_translate_zul.yaml | 7 + lm_eval/tasks/afrimmlu/direct/afrimmlu.yaml | 13 + .../afrimmlu/direct/afrimmlu_common_yaml | 37 --- .../afrimmlu/direct/afrimmlu_direct_amh.yaml | 3 - .../afrimmlu/direct/afrimmlu_direct_eng.yaml | 3 - .../afrimmlu/direct/afrimmlu_direct_ewe.yaml | 3 - .../afrimmlu/direct/afrimmlu_direct_fra.yaml | 3 - .../afrimmlu/direct/afrimmlu_direct_hau.yaml | 3 - .../afrimmlu/direct/afrimmlu_direct_ibo.yaml | 3 - .../afrimmlu/direct/afrimmlu_direct_kin.yaml | 3 - .../afrimmlu/direct/afrimmlu_direct_lin.yaml | 3 - .../afrimmlu/direct/afrimmlu_direct_lug.yaml | 3 - .../afrimmlu/direct/afrimmlu_direct_orm.yaml | 3 - .../afrimmlu/direct/afrimmlu_direct_sna.yaml | 3 - .../afrimmlu/direct/afrimmlu_direct_sot.yaml | 3 - .../afrimmlu/direct/afrimmlu_direct_swa.yaml | 3 - .../afrimmlu/direct/afrimmlu_direct_twi.yaml | 3 - .../afrimmlu/direct/afrimmlu_direct_wol.yaml | 3 - .../afrimmlu/direct/afrimmlu_direct_xho.yaml | 3 - .../afrimmlu/direct/afrimmlu_direct_yor.yaml | 3 - .../afrimmlu/direct/afrimmlu_direct_zul.yaml | 3 - .../afrimmlu/direct/prompt_1/afrimmlu_direct | 37 +++ .../direct/prompt_1/afrimmlu_direct_amh.yaml | 4 + .../direct/prompt_1/afrimmlu_direct_eng.yaml | 4 + .../direct/prompt_1/afrimmlu_direct_ewe.yaml | 4 + .../direct/prompt_1/afrimmlu_direct_fra.yaml | 4 + .../direct/prompt_1/afrimmlu_direct_hau.yaml | 4 + .../direct/prompt_1/afrimmlu_direct_ibo.yaml | 4 + .../direct/prompt_1/afrimmlu_direct_kin.yaml | 4 + .../direct/prompt_1/afrimmlu_direct_lin.yaml | 4 + .../direct/prompt_1/afrimmlu_direct_lug.yaml | 4 + .../direct/prompt_1/afrimmlu_direct_orm.yaml | 4 + .../direct/prompt_1/afrimmlu_direct_sna.yaml | 4 + .../direct/prompt_1/afrimmlu_direct_sot.yaml | 4 + .../direct/prompt_1/afrimmlu_direct_swa.yaml | 4 + .../direct/prompt_1/afrimmlu_direct_twi.yaml | 4 + .../direct/prompt_1/afrimmlu_direct_wol.yaml | 4 + .../direct/prompt_1/afrimmlu_direct_xho.yaml | 4 + .../direct/prompt_1/afrimmlu_direct_yor.yaml | 4 + .../direct/prompt_1/afrimmlu_direct_zul.yaml | 4 + .../afrimmlu/direct/{ => prompt_1}/utils.py | 0 .../afrimmlu/direct/prompt_2/afrimmlu_direct | 37 +++ .../direct/prompt_2/afrimmlu_direct_amh.yaml | 4 + .../direct/prompt_2/afrimmlu_direct_eng.yaml | 4 + .../direct/prompt_2/afrimmlu_direct_ewe.yaml | 4 + .../direct/prompt_2/afrimmlu_direct_fra.yaml | 4 + .../direct/prompt_2/afrimmlu_direct_hau.yaml | 4 + .../direct/prompt_2/afrimmlu_direct_ibo.yaml | 4 + .../direct/prompt_2/afrimmlu_direct_kin.yaml | 4 + .../direct/prompt_2/afrimmlu_direct_lin.yaml | 4 + .../direct/prompt_2/afrimmlu_direct_lug.yaml | 4 + .../direct/prompt_2/afrimmlu_direct_orm.yaml | 4 + .../direct/prompt_2/afrimmlu_direct_sna.yaml | 4 + .../direct/prompt_2/afrimmlu_direct_sot.yaml | 4 + .../direct/prompt_2/afrimmlu_direct_swa.yaml | 4 + .../direct/prompt_2/afrimmlu_direct_twi.yaml | 4 + .../direct/prompt_2/afrimmlu_direct_wol.yaml | 4 + .../direct/prompt_2/afrimmlu_direct_xho.yaml | 4 + .../direct/prompt_2/afrimmlu_direct_yor.yaml | 4 + .../direct/prompt_2/afrimmlu_direct_zul.yaml | 4 + .../tasks/afrimmlu/direct/prompt_2/utils.py | 30 +++ .../afrimmlu/direct/prompt_3/afrimmlu_direct | 37 +++ .../direct/prompt_3/afrimmlu_direct_amh.yaml | 4 + .../direct/prompt_3/afrimmlu_direct_eng.yaml | 4 + .../direct/prompt_3/afrimmlu_direct_ewe.yaml | 4 + .../direct/prompt_3/afrimmlu_direct_fra.yaml | 4 + .../direct/prompt_3/afrimmlu_direct_hau.yaml | 4 + .../direct/prompt_3/afrimmlu_direct_ibo.yaml | 4 + .../direct/prompt_3/afrimmlu_direct_kin.yaml | 4 + .../direct/prompt_3/afrimmlu_direct_lin.yaml | 4 + .../direct/prompt_3/afrimmlu_direct_lug.yaml | 4 + .../direct/prompt_3/afrimmlu_direct_orm.yaml | 4 + .../direct/prompt_3/afrimmlu_direct_sna.yaml | 4 + .../direct/prompt_3/afrimmlu_direct_sot.yaml | 4 + .../direct/prompt_3/afrimmlu_direct_swa.yaml | 4 + .../direct/prompt_3/afrimmlu_direct_twi.yaml | 4 + .../direct/prompt_3/afrimmlu_direct_wol.yaml | 4 + .../direct/prompt_3/afrimmlu_direct_xho.yaml | 4 + .../direct/prompt_3/afrimmlu_direct_yor.yaml | 4 + .../direct/prompt_3/afrimmlu_direct_zul.yaml | 4 + .../tasks/afrimmlu/direct/prompt_3/utils.py | 32 +++ .../afrimmlu/direct/prompt_4/afrimmlu_direct | 37 +++ .../direct/prompt_4/afrimmlu_direct_amh.yaml | 4 + .../direct/prompt_4/afrimmlu_direct_eng.yaml | 4 + .../direct/prompt_4/afrimmlu_direct_ewe.yaml | 4 + .../direct/prompt_4/afrimmlu_direct_fra.yaml | 4 + .../direct/prompt_4/afrimmlu_direct_hau.yaml | 4 + .../direct/prompt_4/afrimmlu_direct_ibo.yaml | 4 + .../direct/prompt_4/afrimmlu_direct_kin.yaml | 4 + .../direct/prompt_4/afrimmlu_direct_lin.yaml | 4 + .../direct/prompt_4/afrimmlu_direct_lug.yaml | 4 + .../direct/prompt_4/afrimmlu_direct_orm.yaml | 4 + .../direct/prompt_4/afrimmlu_direct_sna.yaml | 4 + .../direct/prompt_4/afrimmlu_direct_sot.yaml | 4 + .../direct/prompt_4/afrimmlu_direct_swa.yaml | 4 + .../direct/prompt_4/afrimmlu_direct_twi.yaml | 4 + .../direct/prompt_4/afrimmlu_direct_wol.yaml | 4 + .../direct/prompt_4/afrimmlu_direct_xho.yaml | 4 + .../direct/prompt_4/afrimmlu_direct_yor.yaml | 4 + .../direct/prompt_4/afrimmlu_direct_zul.yaml | 4 + .../tasks/afrimmlu/direct/prompt_4/utils.py | 28 +++ .../afrimmlu/direct/prompt_5/afrimmlu_direct | 37 +++ .../direct/prompt_5/afrimmlu_direct_amh.yaml | 4 + .../direct/prompt_5/afrimmlu_direct_eng.yaml | 4 + .../direct/prompt_5/afrimmlu_direct_ewe.yaml | 4 + .../direct/prompt_5/afrimmlu_direct_fra.yaml | 4 + .../direct/prompt_5/afrimmlu_direct_hau.yaml | 4 + .../direct/prompt_5/afrimmlu_direct_ibo.yaml | 4 + .../direct/prompt_5/afrimmlu_direct_kin.yaml | 4 + .../direct/prompt_5/afrimmlu_direct_lin.yaml | 4 + .../direct/prompt_5/afrimmlu_direct_lug.yaml | 4 + .../direct/prompt_5/afrimmlu_direct_orm.yaml | 4 + .../direct/prompt_5/afrimmlu_direct_sna.yaml | 4 + .../direct/prompt_5/afrimmlu_direct_sot.yaml | 4 + .../direct/prompt_5/afrimmlu_direct_swa.yaml | 4 + .../direct/prompt_5/afrimmlu_direct_twi.yaml | 4 + .../direct/prompt_5/afrimmlu_direct_wol.yaml | 4 + .../direct/prompt_5/afrimmlu_direct_xho.yaml | 4 + .../direct/prompt_5/afrimmlu_direct_yor.yaml | 4 + .../direct/prompt_5/afrimmlu_direct_zul.yaml | 4 + .../tasks/afrimmlu/direct/prompt_5/utils.py | 29 +++ lm_eval/tasks/afrimmlu/gen_utils.py | 103 ++++++++ .../translate/afrimmlu_common_translate_yaml | 34 --- .../translate/afrimmlu_translate_amh.yaml | 3 - .../translate/afrimmlu_translate_eng.yaml | 3 - .../translate/afrimmlu_translate_ewe.yaml | 3 - .../translate/afrimmlu_translate_fra.yaml | 3 - .../translate/afrimmlu_translate_hau.yaml | 3 - .../translate/afrimmlu_translate_ibo.yaml | 3 - .../translate/afrimmlu_translate_kin.yaml | 3 - .../translate/afrimmlu_translate_lin.yaml | 3 - .../translate/afrimmlu_translate_lug.yaml | 3 - .../translate/afrimmlu_translate_orm.yaml | 3 - .../translate/afrimmlu_translate_sna.yaml | 3 - .../translate/afrimmlu_translate_sot.yaml | 3 - .../translate/afrimmlu_translate_swa.yaml | 3 - .../translate/afrimmlu_translate_twi.yaml | 3 - .../translate/afrimmlu_translate_wol.yaml | 3 - .../translate/afrimmlu_translate_xho.yaml | 3 - .../translate/afrimmlu_translate_yor.yaml | 3 - .../translate/afrimmlu_translate_zul.yaml | 3 - .../tasks/afrimmlu/translate/afrimmlu_tt.yaml | 9 + .../translate/prompt_1/afrimmlu_translate | 32 +++ .../prompt_1/afrimmlu_translate_amh.yaml | 4 + .../prompt_1/afrimmlu_translate_ewe.yaml | 4 + .../prompt_1/afrimmlu_translate_fra.yaml | 4 + .../prompt_1/afrimmlu_translate_hau.yaml | 4 + .../prompt_1/afrimmlu_translate_ibo.yaml | 4 + .../prompt_1/afrimmlu_translate_kin.yaml | 4 + .../prompt_1/afrimmlu_translate_lin.yaml | 4 + .../prompt_1/afrimmlu_translate_lug.yaml | 4 + .../prompt_1/afrimmlu_translate_orm.yaml | 4 + .../prompt_1/afrimmlu_translate_sna.yaml | 4 + .../prompt_1/afrimmlu_translate_sot.yaml | 4 + .../prompt_1/afrimmlu_translate_swa.yaml | 4 + .../prompt_1/afrimmlu_translate_twi.yaml | 4 + .../prompt_1/afrimmlu_translate_wol.yaml | 4 + .../prompt_1/afrimmlu_translate_xho.yaml | 4 + .../prompt_1/afrimmlu_translate_yor.yaml | 4 + .../prompt_1/afrimmlu_translate_zul.yaml | 4 + .../afrimmlu/translate/prompt_1/utils.py | 32 +++ .../translate/prompt_2/afrimmlu_translate | 32 +++ .../prompt_2/afrimmlu_translate_amh.yaml | 4 + .../prompt_2/afrimmlu_translate_ewe.yaml | 4 + .../prompt_2/afrimmlu_translate_fra.yaml | 4 + .../prompt_2/afrimmlu_translate_hau.yaml | 4 + .../prompt_2/afrimmlu_translate_ibo.yaml | 4 + .../prompt_2/afrimmlu_translate_kin.yaml | 4 + .../prompt_2/afrimmlu_translate_lin.yaml | 4 + .../prompt_2/afrimmlu_translate_lug.yaml | 4 + .../prompt_2/afrimmlu_translate_orm.yaml | 4 + .../prompt_2/afrimmlu_translate_sna.yaml | 4 + .../prompt_2/afrimmlu_translate_sot.yaml | 4 + .../prompt_2/afrimmlu_translate_swa.yaml | 4 + .../prompt_2/afrimmlu_translate_twi.yaml | 4 + .../prompt_2/afrimmlu_translate_wol.yaml | 4 + .../prompt_2/afrimmlu_translate_xho.yaml | 4 + .../prompt_2/afrimmlu_translate_yor.yaml | 4 + .../prompt_2/afrimmlu_translate_zul.yaml | 4 + .../afrimmlu/translate/prompt_2/utils.py | 30 +++ .../translate/prompt_3/afrimmlu_translate | 32 +++ .../prompt_3/afrimmlu_translate_amh.yaml | 4 + .../prompt_3/afrimmlu_translate_ewe.yaml | 4 + .../prompt_3/afrimmlu_translate_fra.yaml | 4 + .../prompt_3/afrimmlu_translate_hau.yaml | 4 + .../prompt_3/afrimmlu_translate_ibo.yaml | 4 + .../prompt_3/afrimmlu_translate_kin.yaml | 4 + .../prompt_3/afrimmlu_translate_lin.yaml | 4 + .../prompt_3/afrimmlu_translate_lug.yaml | 4 + .../prompt_3/afrimmlu_translate_orm.yaml | 4 + .../prompt_3/afrimmlu_translate_sna.yaml | 4 + .../prompt_3/afrimmlu_translate_sot.yaml | 4 + .../prompt_3/afrimmlu_translate_swa.yaml | 4 + .../prompt_3/afrimmlu_translate_twi.yaml | 4 + .../prompt_3/afrimmlu_translate_wol.yaml | 4 + .../prompt_3/afrimmlu_translate_xho.yaml | 4 + .../prompt_3/afrimmlu_translate_yor.yaml | 4 + .../prompt_3/afrimmlu_translate_zul.yaml | 4 + .../afrimmlu/translate/prompt_3/utils.py | 32 +++ .../translate/prompt_4/afrimmlu_translate | 32 +++ .../prompt_4/afrimmlu_translate_amh.yaml | 4 + .../prompt_4/afrimmlu_translate_ewe.yaml | 4 + .../prompt_4/afrimmlu_translate_fra.yaml | 4 + .../prompt_4/afrimmlu_translate_hau.yaml | 4 + .../prompt_4/afrimmlu_translate_ibo.yaml | 4 + .../prompt_4/afrimmlu_translate_kin.yaml | 4 + .../prompt_4/afrimmlu_translate_lin.yaml | 4 + .../prompt_4/afrimmlu_translate_lug.yaml | 4 + .../prompt_4/afrimmlu_translate_orm.yaml | 4 + .../prompt_4/afrimmlu_translate_sna.yaml | 4 + .../prompt_4/afrimmlu_translate_sot.yaml | 4 + .../prompt_4/afrimmlu_translate_swa.yaml | 4 + .../prompt_4/afrimmlu_translate_twi.yaml | 4 + .../prompt_4/afrimmlu_translate_wol.yaml | 4 + .../prompt_4/afrimmlu_translate_xho.yaml | 4 + .../prompt_4/afrimmlu_translate_yor.yaml | 4 + .../prompt_4/afrimmlu_translate_zul.yaml | 4 + .../afrimmlu/translate/prompt_4/utils.py | 28 +++ .../translate/prompt_5/afrimmlu_translate | 32 +++ .../prompt_5/afrimmlu_translate_amh.yaml | 4 + .../prompt_5/afrimmlu_translate_ewe.yaml | 4 + .../prompt_5/afrimmlu_translate_fra.yaml | 4 + .../prompt_5/afrimmlu_translate_hau.yaml | 4 + .../prompt_5/afrimmlu_translate_ibo.yaml | 4 + .../prompt_5/afrimmlu_translate_kin.yaml | 4 + .../prompt_5/afrimmlu_translate_lin.yaml | 4 + .../prompt_5/afrimmlu_translate_lug.yaml | 4 + .../prompt_5/afrimmlu_translate_orm.yaml | 4 + .../prompt_5/afrimmlu_translate_sna.yaml | 4 + .../prompt_5/afrimmlu_translate_sot.yaml | 4 + .../prompt_5/afrimmlu_translate_swa.yaml | 4 + .../prompt_5/afrimmlu_translate_twi.yaml | 4 + .../prompt_5/afrimmlu_translate_wol.yaml | 4 + .../prompt_5/afrimmlu_translate_xho.yaml | 4 + .../prompt_5/afrimmlu_translate_yor.yaml | 4 + .../prompt_5/afrimmlu_translate_zul.yaml | 4 + .../afrimmlu/translate/prompt_5/utils.py | 28 +++ lm_eval/tasks/afrimmlu/translate/utils.py | 32 --- lm_eval/tasks/afrixnli/direct/afrixnli.yaml | 13 + .../direct/prompt_1/afrixnli_amh.yaml | 15 ++ .../direct/prompt_1/afrixnli_eng.yaml | 15 ++ .../direct/prompt_1/afrixnli_ewe.yaml | 15 ++ .../direct/prompt_1/afrixnli_fra.yaml | 15 ++ .../direct/prompt_1/afrixnli_hau.yaml | 15 ++ .../direct/prompt_1/afrixnli_ibo.yaml | 15 ++ .../direct/prompt_1/afrixnli_kin.yaml | 15 ++ .../direct/prompt_1/afrixnli_lin.yaml | 15 ++ .../direct/prompt_1/afrixnli_lug.yaml | 15 ++ .../direct/prompt_1/afrixnli_orm.yaml | 15 ++ .../direct/prompt_1/afrixnli_sna.yaml | 15 ++ .../direct/prompt_1/afrixnli_sot.yaml | 15 ++ .../direct/prompt_1/afrixnli_swa.yaml | 15 ++ .../direct/prompt_1/afrixnli_twi.yaml | 15 ++ .../direct/prompt_1/afrixnli_wol.yaml | 15 ++ .../direct/prompt_1/afrixnli_xho.yaml | 15 ++ .../afrixnli/direct/prompt_1/afrixnli_yaml | 30 +++ .../direct/prompt_1/afrixnli_yor.yaml | 15 ++ .../direct/prompt_1/afrixnli_zul.yaml | 15 ++ .../tasks/afrixnli/direct/prompt_1/utils.py | 19 ++ .../direct/prompt_2/afrixnli_amh.yaml | 4 + .../direct/prompt_2/afrixnli_eng.yaml | 4 + .../direct/prompt_2/afrixnli_ewe.yaml | 4 + .../direct/prompt_2/afrixnli_fra.yaml | 4 + .../direct/prompt_2/afrixnli_hau.yaml | 4 + .../direct/prompt_2/afrixnli_ibo.yaml | 4 + .../direct/prompt_2/afrixnli_kin.yaml | 4 + .../direct/prompt_2/afrixnli_lin.yaml | 4 + .../direct/prompt_2/afrixnli_lug.yaml | 4 + .../direct/prompt_2/afrixnli_orm.yaml | 4 + .../direct/prompt_2/afrixnli_sna.yaml | 4 + .../direct/prompt_2/afrixnli_sot.yaml | 4 + .../direct/prompt_2/afrixnli_swa.yaml | 4 + .../direct/prompt_2/afrixnli_twi.yaml | 4 + .../direct/prompt_2/afrixnli_wol.yaml | 4 + .../direct/prompt_2/afrixnli_xho.yaml | 4 + .../afrixnli/direct/prompt_2/afrixnli_yaml | 34 +++ .../direct/prompt_2/afrixnli_yor.yaml | 4 + .../direct/prompt_2/afrixnli_zul.yaml | 4 + .../tasks/afrixnli/direct/prompt_2/utils.py | 6 + .../direct/prompt_3/afrixnli_amh.yaml | 8 + .../direct/prompt_3/afrixnli_eng.yaml | 8 + .../direct/prompt_3/afrixnli_ewe.yaml | 8 + .../direct/prompt_3/afrixnli_fra.yaml | 8 + .../direct/prompt_3/afrixnli_hau.yaml | 8 + .../direct/prompt_3/afrixnli_ibo.yaml | 8 + .../direct/prompt_3/afrixnli_kin.yaml | 8 + .../direct/prompt_3/afrixnli_lin.yaml | 8 + .../direct/prompt_3/afrixnli_lug.yaml | 8 + .../direct/prompt_3/afrixnli_orm.yaml | 8 + .../direct/prompt_3/afrixnli_sna.yaml | 8 + .../direct/prompt_3/afrixnli_sot.yaml | 8 + .../direct/prompt_3/afrixnli_swa.yaml | 8 + .../direct/prompt_3/afrixnli_twi.yaml | 8 + .../direct/prompt_3/afrixnli_wol.yaml | 8 + .../direct/prompt_3/afrixnli_xho.yaml | 8 + .../afrixnli/direct/prompt_3/afrixnli_yaml | 30 +++ .../direct/prompt_3/afrixnli_yor.yaml | 8 + .../direct/prompt_3/afrixnli_zul.yaml | 8 + .../tasks/afrixnli/direct/prompt_3/utils.py | 6 + .../direct/prompt_4/afrixnli_amh.yaml | 9 + .../direct/prompt_4/afrixnli_eng.yaml | 9 + .../direct/prompt_4/afrixnli_ewe.yaml | 8 + .../direct/prompt_4/afrixnli_fra.yaml | 9 + .../direct/prompt_4/afrixnli_hau.yaml | 8 + .../direct/prompt_4/afrixnli_ibo.yaml | 8 + .../direct/prompt_4/afrixnli_kin.yaml | 9 + .../direct/prompt_4/afrixnli_lin.yaml | 9 + .../direct/prompt_4/afrixnli_lug.yaml | 9 + .../direct/prompt_4/afrixnli_orm.yaml | 8 + .../direct/prompt_4/afrixnli_sna.yaml | 9 + .../direct/prompt_4/afrixnli_sot.yaml | 9 + .../direct/prompt_4/afrixnli_swa.yaml | 9 + .../direct/prompt_4/afrixnli_twi.yaml | 8 + .../direct/prompt_4/afrixnli_wol.yaml | 8 + .../direct/prompt_4/afrixnli_xho.yaml | 9 + .../afrixnli/direct/prompt_4/afrixnli_yaml | 30 +++ .../direct/prompt_4/afrixnli_yor.yaml | 9 + .../direct/prompt_4/afrixnli_zul.yaml | 8 + .../tasks/afrixnli/direct/prompt_4/utils.py | 19 ++ .../direct/prompt_5/afrixnli_amh.yaml | 6 + .../direct/prompt_5/afrixnli_eng.yaml | 6 + .../direct/prompt_5/afrixnli_ewe.yaml | 6 + .../direct/prompt_5/afrixnli_fra.yaml | 6 + .../direct/prompt_5/afrixnli_hau.yaml | 6 + .../direct/prompt_5/afrixnli_ibo.yaml | 6 + .../direct/prompt_5/afrixnli_kin.yaml | 6 + .../direct/prompt_5/afrixnli_lin.yaml | 6 + .../direct/prompt_5/afrixnli_lug.yaml | 6 + .../direct/prompt_5/afrixnli_orm.yaml | 6 + .../direct/prompt_5/afrixnli_sna.yaml | 6 + .../direct/prompt_5/afrixnli_sot.yaml | 6 + .../direct/prompt_5/afrixnli_swa.yaml | 6 + .../direct/prompt_5/afrixnli_twi.yaml | 6 + .../direct/prompt_5/afrixnli_wol.yaml | 6 + .../direct/prompt_5/afrixnli_xho.yaml | 6 + .../afrixnli/direct/prompt_5/afrixnli_yaml | 30 +++ .../direct/prompt_5/afrixnli_yor.yaml | 6 + .../direct/prompt_5/afrixnli_zul.yaml | 6 + .../tasks/afrixnli/direct/prompt_5/utils.py | 6 + lm_eval/tasks/afrixnli/gen_utils.py | 129 ++++++++++ .../tasks/afrixnli/translate/afrixnli_tt.yaml | 9 + .../prompt_1/afrixnli_translate_amh.yaml | 15 ++ .../prompt_1/afrixnli_translate_ewe.yaml | 15 ++ .../prompt_1/afrixnli_translate_fra.yaml | 15 ++ .../prompt_1/afrixnli_translate_hau.yaml | 15 ++ .../prompt_1/afrixnli_translate_ibo.yaml | 15 ++ .../prompt_1/afrixnli_translate_kin.yaml | 15 ++ .../prompt_1/afrixnli_translate_lin.yaml | 15 ++ .../prompt_1/afrixnli_translate_lug.yaml | 15 ++ .../prompt_1/afrixnli_translate_orm.yaml | 15 ++ .../prompt_1/afrixnli_translate_sna.yaml | 15 ++ .../prompt_1/afrixnli_translate_sot.yaml | 15 ++ .../prompt_1/afrixnli_translate_swa.yaml | 15 ++ .../prompt_1/afrixnli_translate_twi.yaml | 15 ++ .../prompt_1/afrixnli_translate_wol.yaml | 15 ++ .../prompt_1/afrixnli_translate_xho.yaml | 15 ++ .../prompt_1/afrixnli_translate_yaml | 27 +++ .../prompt_1/afrixnli_translate_yor.yaml | 15 ++ .../prompt_1/afrixnli_translate_zul.yaml | 15 ++ .../afrixnli/translate/prompt_1/utils.py | 19 ++ .../prompt_2/afrixnli_translate_amh.yaml | 4 + .../prompt_2/afrixnli_translate_ewe.yaml | 4 + .../prompt_2/afrixnli_translate_fra.yaml | 4 + .../prompt_2/afrixnli_translate_hau.yaml | 4 + .../prompt_2/afrixnli_translate_ibo.yaml | 4 + .../prompt_2/afrixnli_translate_kin.yaml | 4 + .../prompt_2/afrixnli_translate_lin.yaml | 4 + .../prompt_2/afrixnli_translate_lug.yaml | 4 + .../prompt_2/afrixnli_translate_orm.yaml | 4 + .../prompt_2/afrixnli_translate_sna.yaml | 4 + .../prompt_2/afrixnli_translate_sot.yaml | 4 + .../prompt_2/afrixnli_translate_swa.yaml | 4 + .../prompt_2/afrixnli_translate_twi.yaml | 4 + .../prompt_2/afrixnli_translate_wol.yaml | 4 + .../prompt_2/afrixnli_translate_xho.yaml | 4 + .../prompt_2/afrixnli_translate_yaml | 31 +++ .../prompt_2/afrixnli_translate_yor.yaml | 4 + .../prompt_2/afrixnli_translate_zul.yaml | 4 + .../afrixnli/translate/prompt_2/utils.py | 6 + .../prompt_3/afrixnli_translate_amh.yaml | 8 + .../prompt_3/afrixnli_translate_ewe.yaml | 8 + .../prompt_3/afrixnli_translate_fra.yaml | 8 + .../prompt_3/afrixnli_translate_hau.yaml | 8 + .../prompt_3/afrixnli_translate_ibo.yaml | 8 + .../prompt_3/afrixnli_translate_kin.yaml | 8 + .../prompt_3/afrixnli_translate_lin.yaml | 8 + .../prompt_3/afrixnli_translate_lug.yaml | 8 + .../prompt_3/afrixnli_translate_orm.yaml | 8 + .../prompt_3/afrixnli_translate_sna.yaml | 8 + .../prompt_3/afrixnli_translate_sot.yaml | 8 + .../prompt_3/afrixnli_translate_swa.yaml | 8 + .../prompt_3/afrixnli_translate_twi.yaml | 8 + .../prompt_3/afrixnli_translate_wol.yaml | 8 + .../prompt_3/afrixnli_translate_xho.yaml | 8 + .../prompt_3/afrixnli_translate_yaml | 27 +++ .../prompt_3/afrixnli_translate_yor.yaml | 8 + .../prompt_3/afrixnli_translate_zul.yaml | 8 + .../afrixnli/translate/prompt_3/utils.py | 21 ++ .../prompt_4/afrixnli_translate_amh.yaml | 9 + .../prompt_4/afrixnli_translate_ewe.yaml | 8 + .../prompt_4/afrixnli_translate_fra.yaml | 9 + .../prompt_4/afrixnli_translate_hau.yaml | 8 + .../prompt_4/afrixnli_translate_ibo.yaml | 8 + .../prompt_4/afrixnli_translate_kin.yaml | 9 + .../prompt_4/afrixnli_translate_lin.yaml | 9 + .../prompt_4/afrixnli_translate_lug.yaml | 9 + .../prompt_4/afrixnli_translate_orm.yaml | 8 + .../prompt_4/afrixnli_translate_sna.yaml | 9 + .../prompt_4/afrixnli_translate_sot.yaml | 9 + .../prompt_4/afrixnli_translate_swa.yaml | 9 + .../prompt_4/afrixnli_translate_twi.yaml | 8 + .../prompt_4/afrixnli_translate_wol.yaml | 8 + .../prompt_4/afrixnli_translate_xho.yaml | 9 + .../prompt_4/afrixnli_translate_yaml | 27 +++ .../prompt_4/afrixnli_translate_yor.yaml | 9 + .../prompt_4/afrixnli_translate_zul.yaml | 8 + .../afrixnli/translate/prompt_4/utils.py | 19 ++ .../prompt_5/afrixnli_translate_amh.yaml | 6 + .../prompt_5/afrixnli_translate_ewe.yaml | 6 + .../prompt_5/afrixnli_translate_fra.yaml | 6 + .../prompt_5/afrixnli_translate_hau.yaml | 6 + .../prompt_5/afrixnli_translate_ibo.yaml | 6 + .../prompt_5/afrixnli_translate_kin.yaml | 6 + .../prompt_5/afrixnli_translate_lin.yaml | 6 + .../prompt_5/afrixnli_translate_lug.yaml | 6 + .../prompt_5/afrixnli_translate_orm.yaml | 6 + .../prompt_5/afrixnli_translate_sna.yaml | 6 + .../prompt_5/afrixnli_translate_sot.yaml | 6 + .../prompt_5/afrixnli_translate_swa.yaml | 6 + .../prompt_5/afrixnli_translate_twi.yaml | 6 + .../prompt_5/afrixnli_translate_wol.yaml | 6 + .../prompt_5/afrixnli_translate_xho.yaml | 6 + .../prompt_5/afrixnli_translate_yaml | 27 +++ .../prompt_5/afrixnli_translate_yor.yaml | 6 + .../prompt_5/afrixnli_translate_zul.yaml | 6 + .../afrixnli/translate/prompt_5/utils.py | 6 + lm_eval/tasks/afrobench/README.md | 72 ++++++ lm_eval/tasks/afrobench/adr/README.md | 7 + .../tasks/afrobench/adr/afridiacritics.yaml | 13 + lm_eval/tasks/afrobench/adr/gen_utils.py | 105 ++++++++ .../adr/prompt_1/afridiacritics_bbj.yaml | 6 + .../adr/prompt_1/afridiacritics_fon.yaml | 6 + .../adr/prompt_1/afridiacritics_ibo.yaml | 6 + .../adr/prompt_1/afridiacritics_wol.yaml | 6 + .../adr/prompt_1/afridiacritics_yaml | 25 ++ .../adr/prompt_1/afridiacritics_yor.yaml | 6 + .../adr/prompt_2/afridiacritics_bbj.yaml | 7 + .../adr/prompt_2/afridiacritics_fon.yaml | 7 + .../adr/prompt_2/afridiacritics_ibo.yaml | 7 + .../adr/prompt_2/afridiacritics_wol.yaml | 7 + .../adr/prompt_2/afridiacritics_yaml | 25 ++ .../adr/prompt_2/afridiacritics_yor.yaml | 7 + .../adr/prompt_3/afridiacritics_bbj.yaml | 6 + .../adr/prompt_3/afridiacritics_fon.yaml | 6 + .../adr/prompt_3/afridiacritics_ibo.yaml | 6 + .../adr/prompt_3/afridiacritics_wol.yaml | 6 + .../adr/prompt_3/afridiacritics_yaml | 25 ++ .../adr/prompt_3/afridiacritics_yor.yaml | 6 + .../adr/prompt_4/afridiacritics_bbj.yaml | 7 + .../adr/prompt_4/afridiacritics_fon.yaml | 6 + .../adr/prompt_4/afridiacritics_ibo.yaml | 6 + .../adr/prompt_4/afridiacritics_wol.yaml | 7 + .../adr/prompt_4/afridiacritics_yaml | 25 ++ .../adr/prompt_4/afridiacritics_yor.yaml | 7 + .../adr/prompt_5/afridiacritics_bbj.yaml | 8 + .../adr/prompt_5/afridiacritics_fon.yaml | 8 + .../adr/prompt_5/afridiacritics_ibo.yaml | 8 + .../adr/prompt_5/afridiacritics_wol.yaml | 8 + .../adr/prompt_5/afridiacritics_yaml | 25 ++ .../adr/prompt_5/afridiacritics_yor.yaml | 8 + lm_eval/tasks/afrobench/afriqa/README.md | 24 ++ lm_eval/tasks/afrobench/afriqa/afriqa.yaml | 13 + .../tasks/afrobench/afriqa/prompt_1/afriqa | 42 ++++ .../afrobench/afriqa/prompt_1/afriqa_bem.yaml | 12 + .../afrobench/afriqa/prompt_1/afriqa_fon.yaml | 12 + .../afrobench/afriqa/prompt_1/afriqa_hau.yaml | 12 + .../afrobench/afriqa/prompt_1/afriqa_ibo.yaml | 12 + .../afrobench/afriqa/prompt_1/afriqa_kin.yaml | 12 + .../afrobench/afriqa/prompt_1/afriqa_swa.yaml | 15 ++ .../afrobench/afriqa/prompt_1/afriqa_twi.yaml | 12 + .../afrobench/afriqa/prompt_1/afriqa_yor.yaml | 12 + .../afrobench/afriqa/prompt_1/afriqa_zul.yaml | 12 + .../tasks/afrobench/afriqa/prompt_1/utils.py | 53 ++++ .../tasks/afrobench/afriqa/prompt_2/afriqa | 42 ++++ .../afrobench/afriqa/prompt_2/afriqa_bem.yaml | 13 + .../afrobench/afriqa/prompt_2/afriqa_fon.yaml | 13 + .../afrobench/afriqa/prompt_2/afriqa_hau.yaml | 13 + .../afrobench/afriqa/prompt_2/afriqa_ibo.yaml | 13 + .../afrobench/afriqa/prompt_2/afriqa_kin.yaml | 13 + .../afrobench/afriqa/prompt_2/afriqa_swa.yaml | 16 ++ .../afrobench/afriqa/prompt_2/afriqa_twi.yaml | 13 + .../afrobench/afriqa/prompt_2/afriqa_yor.yaml | 13 + .../afrobench/afriqa/prompt_2/afriqa_zul.yaml | 13 + .../tasks/afrobench/afriqa/prompt_2/utils.py | 53 ++++ .../tasks/afrobench/afriqa/prompt_3/afriqa | 42 ++++ .../afrobench/afriqa/prompt_3/afriqa_bem.yaml | 12 + .../afrobench/afriqa/prompt_3/afriqa_fon.yaml | 12 + .../afrobench/afriqa/prompt_3/afriqa_hau.yaml | 12 + .../afrobench/afriqa/prompt_3/afriqa_ibo.yaml | 12 + .../afrobench/afriqa/prompt_3/afriqa_kin.yaml | 12 + .../afrobench/afriqa/prompt_3/afriqa_swa.yaml | 15 ++ .../afrobench/afriqa/prompt_3/afriqa_twi.yaml | 12 + .../afrobench/afriqa/prompt_3/afriqa_yor.yaml | 12 + .../afrobench/afriqa/prompt_3/afriqa_zul.yaml | 12 + .../tasks/afrobench/afriqa/prompt_3/utils.py | 53 ++++ .../tasks/afrobench/afriqa/prompt_4/afriqa | 42 ++++ .../afrobench/afriqa/prompt_4/afriqa_bem.yaml | 13 + .../afrobench/afriqa/prompt_4/afriqa_fon.yaml | 13 + .../afrobench/afriqa/prompt_4/afriqa_hau.yaml | 13 + .../afrobench/afriqa/prompt_4/afriqa_ibo.yaml | 13 + .../afrobench/afriqa/prompt_4/afriqa_kin.yaml | 13 + .../afrobench/afriqa/prompt_4/afriqa_swa.yaml | 16 ++ .../afrobench/afriqa/prompt_4/afriqa_twi.yaml | 13 + .../afrobench/afriqa/prompt_4/afriqa_yor.yaml | 13 + .../afrobench/afriqa/prompt_4/afriqa_zul.yaml | 13 + .../tasks/afrobench/afriqa/prompt_4/utils.py | 53 ++++ .../tasks/afrobench/afriqa/prompt_5/afriqa | 42 ++++ .../afrobench/afriqa/prompt_5/afriqa_bem.yaml | 12 + .../afrobench/afriqa/prompt_5/afriqa_fon.yaml | 12 + .../afrobench/afriqa/prompt_5/afriqa_hau.yaml | 12 + .../afrobench/afriqa/prompt_5/afriqa_ibo.yaml | 12 + .../afrobench/afriqa/prompt_5/afriqa_kin.yaml | 12 + .../afrobench/afriqa/prompt_5/afriqa_swa.yaml | 15 ++ .../afrobench/afriqa/prompt_5/afriqa_twi.yaml | 12 + .../afrobench/afriqa/prompt_5/afriqa_yor.yaml | 12 + .../afrobench/afriqa/prompt_5/afriqa_zul.yaml | 12 + .../tasks/afrobench/afriqa/prompt_5/utils.py | 53 ++++ lm_eval/tasks/afrobench/afriqa/utils.py | 125 ++++++++++ lm_eval/tasks/afrobench/afrisenti/README.md | 58 +++++ .../tasks/afrobench/afrisenti/afrisenti.yaml | 13 + lm_eval/tasks/afrobench/afrisenti/fewshot.sh | 109 +++++++++ .../afrobench/afrisenti/prompt_1/afrisenti | 41 ++++ .../afrisenti/prompt_1/afrisenti_amh.yaml | 4 + .../afrisenti/prompt_1/afrisenti_arq.yaml | 4 + .../afrisenti/prompt_1/afrisenti_ary.yaml | 4 + .../afrisenti/prompt_1/afrisenti_hau.yaml | 4 + .../afrisenti/prompt_1/afrisenti_ibo.yaml | 4 + .../afrisenti/prompt_1/afrisenti_kin.yaml | 4 + .../afrisenti/prompt_1/afrisenti_orm.yaml | 4 + .../afrisenti/prompt_1/afrisenti_pcm.yaml | 4 + .../afrisenti/prompt_1/afrisenti_por.yaml | 4 + .../afrisenti/prompt_1/afrisenti_swa.yaml | 4 + .../afrisenti/prompt_1/afrisenti_tir.yaml | 4 + .../afrisenti/prompt_1/afrisenti_tso.yaml | 4 + .../afrisenti/prompt_1/afrisenti_twi.yaml | 4 + .../afrisenti/prompt_1/afrisenti_yor.yaml | 4 + .../tasks/afrobench/afrisenti/prompt_1/run.sh | 34 +++ .../afrobench/afrisenti/prompt_1/utils.py | 1 + .../tasks/afrobench/afrisenti/prompt_1/xx.py | 13 + .../afrobench/afrisenti/prompt_2/afrisenti | 39 +++ .../afrisenti/prompt_2/afrisenti_amh.yaml | 6 + .../afrisenti/prompt_2/afrisenti_arq.yaml | 6 + .../afrisenti/prompt_2/afrisenti_ary.yaml | 6 + .../afrisenti/prompt_2/afrisenti_hau.yaml | 6 + .../afrisenti/prompt_2/afrisenti_ibo.yaml | 6 + .../afrisenti/prompt_2/afrisenti_kin.yaml | 6 + .../afrisenti/prompt_2/afrisenti_orm.yaml | 6 + .../afrisenti/prompt_2/afrisenti_pcm.yaml | 6 + .../afrisenti/prompt_2/afrisenti_por.yaml | 6 + .../afrisenti/prompt_2/afrisenti_swa.yaml | 6 + .../afrisenti/prompt_2/afrisenti_tir.yaml | 6 + .../afrisenti/prompt_2/afrisenti_tso.yaml | 6 + .../afrisenti/prompt_2/afrisenti_twi.yaml | 6 + .../afrisenti/prompt_2/afrisenti_yor.yaml | 6 + .../tasks/afrobench/afrisenti/prompt_2/run.sh | 33 +++ .../afrobench/afrisenti/prompt_2/utils.py | 1 + .../tasks/afrobench/afrisenti/prompt_2/xx.py | 5 + .../afrobench/afrisenti/prompt_3/afrisenti | 39 +++ .../afrisenti/prompt_3/afrisenti_amh.yaml | 7 + .../afrisenti/prompt_3/afrisenti_arq.yaml | 8 + .../afrisenti/prompt_3/afrisenti_ary.yaml | 8 + .../afrisenti/prompt_3/afrisenti_hau.yaml | 7 + .../afrisenti/prompt_3/afrisenti_ibo.yaml | 7 + .../afrisenti/prompt_3/afrisenti_kin.yaml | 7 + .../afrisenti/prompt_3/afrisenti_orm.yaml | 7 + .../afrisenti/prompt_3/afrisenti_pcm.yaml | 8 + .../afrisenti/prompt_3/afrisenti_por.yaml | 8 + .../afrisenti/prompt_3/afrisenti_swa.yaml | 7 + .../afrisenti/prompt_3/afrisenti_tir.yaml | 7 + .../afrisenti/prompt_3/afrisenti_tso.yaml | 7 + .../afrisenti/prompt_3/afrisenti_twi.yaml | 7 + .../afrisenti/prompt_3/afrisenti_yor.yaml | 7 + .../afrobench/afrisenti/prompt_3/utils.py | 1 + .../tasks/afrobench/afrisenti/prompt_3/xx.py | 5 + .../afrobench/afrisenti/prompt_4/afrisenti | 39 +++ .../afrisenti/prompt_4/afrisenti_amh.yaml | 6 + .../afrisenti/prompt_4/afrisenti_arq.yaml | 6 + .../afrisenti/prompt_4/afrisenti_ary.yaml | 6 + .../afrisenti/prompt_4/afrisenti_hau.yaml | 6 + .../afrisenti/prompt_4/afrisenti_ibo.yaml | 6 + .../afrisenti/prompt_4/afrisenti_kin.yaml | 6 + .../afrisenti/prompt_4/afrisenti_orm.yaml | 6 + .../afrisenti/prompt_4/afrisenti_pcm.yaml | 6 + .../afrisenti/prompt_4/afrisenti_por.yaml | 6 + .../afrisenti/prompt_4/afrisenti_swa.yaml | 6 + .../afrisenti/prompt_4/afrisenti_tir.yaml | 6 + .../afrisenti/prompt_4/afrisenti_tso.yaml | 6 + .../afrisenti/prompt_4/afrisenti_twi.yaml | 6 + .../afrisenti/prompt_4/afrisenti_yor.yaml | 6 + .../afrobench/afrisenti/prompt_4/utils.py | 1 + .../tasks/afrobench/afrisenti/prompt_4/xx.py | 5 + .../afrobench/afrisenti/prompt_5/afrisenti | 39 +++ .../afrisenti/prompt_5/afrisenti_amh.yaml | 13 + .../afrisenti/prompt_5/afrisenti_arq.yaml | 13 + .../afrisenti/prompt_5/afrisenti_ary.yaml | 13 + .../afrisenti/prompt_5/afrisenti_hau.yaml | 13 + .../afrisenti/prompt_5/afrisenti_ibo.yaml | 13 + .../afrisenti/prompt_5/afrisenti_kin.yaml | 13 + .../afrisenti/prompt_5/afrisenti_orm.yaml | 13 + .../afrisenti/prompt_5/afrisenti_pcm.yaml | 13 + .../afrisenti/prompt_5/afrisenti_por.yaml | 13 + .../afrisenti/prompt_5/afrisenti_swa.yaml | 13 + .../afrisenti/prompt_5/afrisenti_tir.yaml | 13 + .../afrisenti/prompt_5/afrisenti_tso.yaml | 13 + .../afrisenti/prompt_5/afrisenti_twi.yaml | 13 + .../afrisenti/prompt_5/afrisenti_yor.yaml | 13 + .../afrobench/afrisenti/prompt_5/utils.py | 1 + .../tasks/afrobench/afrisenti/prompt_5/xx.py | 8 + lm_eval/tasks/afrobench/afrisenti/utils.py | 124 ++++++++++ lm_eval/tasks/afrobench/afrobench-lite.yaml | 15 ++ lm_eval/tasks/afrobench/afrobench.yaml | 23 ++ lm_eval/tasks/afrobench/belebele/README.md | 41 ++++ .../tasks/afrobench/belebele/belebele.yaml | 13 + .../afrobench/belebele/prompt_1/belebele | 23 ++ .../belebele/prompt_1/belebele_afr.yaml | 17 ++ .../belebele/prompt_1/belebele_amh.yaml | 17 ++ .../belebele/prompt_1/belebele_ary.yaml | 17 ++ .../belebele/prompt_1/belebele_arz.yaml | 17 ++ .../belebele/prompt_1/belebele_bam.yaml | 17 ++ .../belebele/prompt_1/belebele_eng.yaml | 17 ++ .../belebele/prompt_1/belebele_fra.yaml | 17 ++ .../belebele/prompt_1/belebele_fuv.yaml | 17 ++ .../belebele/prompt_1/belebele_gaz.yaml | 17 ++ .../belebele/prompt_1/belebele_hau.yaml | 17 ++ .../belebele/prompt_1/belebele_ibo.yaml | 17 ++ .../belebele/prompt_1/belebele_kea.yaml | 17 ++ .../belebele/prompt_1/belebele_kin.yaml | 17 ++ .../belebele/prompt_1/belebele_lin.yaml | 17 ++ .../belebele/prompt_1/belebele_lug.yaml | 17 ++ .../belebele/prompt_1/belebele_luo.yaml | 17 ++ .../belebele/prompt_1/belebele_nya.yaml | 17 ++ .../belebele/prompt_1/belebele_plt.yaml | 17 ++ .../belebele/prompt_1/belebele_por.yaml | 17 ++ .../belebele/prompt_1/belebele_sna.yaml | 17 ++ .../belebele/prompt_1/belebele_som.yaml | 17 ++ .../belebele/prompt_1/belebele_sot.yaml | 17 ++ .../belebele/prompt_1/belebele_ssw.yaml | 17 ++ .../belebele/prompt_1/belebele_swa.yaml | 17 ++ .../belebele/prompt_1/belebele_tir.yaml | 17 ++ .../belebele/prompt_1/belebele_tsn.yaml | 17 ++ .../belebele/prompt_1/belebele_tso.yaml | 17 ++ .../belebele/prompt_1/belebele_wol.yaml | 17 ++ .../belebele/prompt_1/belebele_xho.yaml | 17 ++ .../belebele/prompt_1/belebele_yor.yaml | 17 ++ .../belebele/prompt_1/belebele_zul.yaml | 17 ++ .../afrobench/belebele/prompt_2/belebele | 23 ++ .../belebele/prompt_2/belebele_afr.yaml | 17 ++ .../belebele/prompt_2/belebele_amh.yaml | 17 ++ .../belebele/prompt_2/belebele_ary.yaml | 17 ++ .../belebele/prompt_2/belebele_arz.yaml | 17 ++ .../belebele/prompt_2/belebele_bam.yaml | 17 ++ .../belebele/prompt_2/belebele_eng.yaml | 17 ++ .../belebele/prompt_2/belebele_fra.yaml | 17 ++ .../belebele/prompt_2/belebele_fuv.yaml | 17 ++ .../belebele/prompt_2/belebele_gaz.yaml | 17 ++ .../belebele/prompt_2/belebele_hau.yaml | 17 ++ .../belebele/prompt_2/belebele_ibo.yaml | 17 ++ .../belebele/prompt_2/belebele_kea.yaml | 17 ++ .../belebele/prompt_2/belebele_kin.yaml | 17 ++ .../belebele/prompt_2/belebele_lin.yaml | 17 ++ .../belebele/prompt_2/belebele_lug.yaml | 17 ++ .../belebele/prompt_2/belebele_luo.yaml | 17 ++ .../belebele/prompt_2/belebele_nya.yaml | 17 ++ .../belebele/prompt_2/belebele_plt.yaml | 17 ++ .../belebele/prompt_2/belebele_por.yaml | 17 ++ .../belebele/prompt_2/belebele_sna.yaml | 17 ++ .../belebele/prompt_2/belebele_som.yaml | 17 ++ .../belebele/prompt_2/belebele_sot.yaml | 17 ++ .../belebele/prompt_2/belebele_ssw.yaml | 17 ++ .../belebele/prompt_2/belebele_swa.yaml | 17 ++ .../belebele/prompt_2/belebele_tir.yaml | 17 ++ .../belebele/prompt_2/belebele_tsn.yaml | 17 ++ .../belebele/prompt_2/belebele_tso.yaml | 17 ++ .../belebele/prompt_2/belebele_wol.yaml | 17 ++ .../belebele/prompt_2/belebele_xho.yaml | 17 ++ .../belebele/prompt_2/belebele_yor.yaml | 17 ++ .../belebele/prompt_2/belebele_zul.yaml | 17 ++ .../afrobench/belebele/prompt_3/belebele | 23 ++ .../belebele/prompt_3/belebele_afr.yaml | 17 ++ .../belebele/prompt_3/belebele_amh.yaml | 17 ++ .../belebele/prompt_3/belebele_ary.yaml | 17 ++ .../belebele/prompt_3/belebele_arz.yaml | 17 ++ .../belebele/prompt_3/belebele_bam.yaml | 17 ++ .../belebele/prompt_3/belebele_eng.yaml | 17 ++ .../belebele/prompt_3/belebele_fra.yaml | 17 ++ .../belebele/prompt_3/belebele_fuv.yaml | 17 ++ .../belebele/prompt_3/belebele_gaz.yaml | 17 ++ .../belebele/prompt_3/belebele_hau.yaml | 17 ++ .../belebele/prompt_3/belebele_ibo.yaml | 17 ++ .../belebele/prompt_3/belebele_kea.yaml | 17 ++ .../belebele/prompt_3/belebele_kin.yaml | 17 ++ .../belebele/prompt_3/belebele_lin.yaml | 17 ++ .../belebele/prompt_3/belebele_lug.yaml | 17 ++ .../belebele/prompt_3/belebele_luo.yaml | 17 ++ .../belebele/prompt_3/belebele_nya.yaml | 17 ++ .../belebele/prompt_3/belebele_plt.yaml | 17 ++ .../belebele/prompt_3/belebele_por.yaml | 17 ++ .../belebele/prompt_3/belebele_sna.yaml | 17 ++ .../belebele/prompt_3/belebele_som.yaml | 17 ++ .../belebele/prompt_3/belebele_sot.yaml | 17 ++ .../belebele/prompt_3/belebele_ssw.yaml | 17 ++ .../belebele/prompt_3/belebele_swa.yaml | 17 ++ .../belebele/prompt_3/belebele_tir.yaml | 17 ++ .../belebele/prompt_3/belebele_tsn.yaml | 17 ++ .../belebele/prompt_3/belebele_tso.yaml | 17 ++ .../belebele/prompt_3/belebele_wol.yaml | 17 ++ .../belebele/prompt_3/belebele_xho.yaml | 17 ++ .../belebele/prompt_3/belebele_yor.yaml | 17 ++ .../belebele/prompt_3/belebele_zul.yaml | 17 ++ .../afrobench/belebele/prompt_4/belebele | 23 ++ .../belebele/prompt_4/belebele_afr.yaml | 21 ++ .../belebele/prompt_4/belebele_amh.yaml | 21 ++ .../belebele/prompt_4/belebele_ary.yaml | 21 ++ .../belebele/prompt_4/belebele_arz.yaml | 21 ++ .../belebele/prompt_4/belebele_bam.yaml | 21 ++ .../belebele/prompt_4/belebele_eng.yaml | 21 ++ .../belebele/prompt_4/belebele_fra.yaml | 21 ++ .../belebele/prompt_4/belebele_fuv.yaml | 21 ++ .../belebele/prompt_4/belebele_gaz.yaml | 21 ++ .../belebele/prompt_4/belebele_hau.yaml | 21 ++ .../belebele/prompt_4/belebele_ibo.yaml | 21 ++ .../belebele/prompt_4/belebele_kea.yaml | 21 ++ .../belebele/prompt_4/belebele_kin.yaml | 21 ++ .../belebele/prompt_4/belebele_lin.yaml | 21 ++ .../belebele/prompt_4/belebele_lug.yaml | 21 ++ .../belebele/prompt_4/belebele_luo.yaml | 21 ++ .../belebele/prompt_4/belebele_nya.yaml | 21 ++ .../belebele/prompt_4/belebele_plt.yaml | 21 ++ .../belebele/prompt_4/belebele_por.yaml | 21 ++ .../belebele/prompt_4/belebele_sna.yaml | 21 ++ .../belebele/prompt_4/belebele_som.yaml | 21 ++ .../belebele/prompt_4/belebele_sot.yaml | 21 ++ .../belebele/prompt_4/belebele_ssw.yaml | 21 ++ .../belebele/prompt_4/belebele_swa.yaml | 21 ++ .../belebele/prompt_4/belebele_tir.yaml | 21 ++ .../belebele/prompt_4/belebele_tsn.yaml | 21 ++ .../belebele/prompt_4/belebele_tso.yaml | 21 ++ .../belebele/prompt_4/belebele_wol.yaml | 21 ++ .../belebele/prompt_4/belebele_xho.yaml | 21 ++ .../belebele/prompt_4/belebele_yor.yaml | 21 ++ .../belebele/prompt_4/belebele_zul.yaml | 21 ++ .../afrobench/belebele/prompt_5/belebele | 23 ++ .../belebele/prompt_5/belebele_afr.yaml | 19 ++ .../belebele/prompt_5/belebele_amh.yaml | 19 ++ .../belebele/prompt_5/belebele_ary.yaml | 19 ++ .../belebele/prompt_5/belebele_arz.yaml | 19 ++ .../belebele/prompt_5/belebele_bam.yaml | 19 ++ .../belebele/prompt_5/belebele_eng.yaml | 19 ++ .../belebele/prompt_5/belebele_fra.yaml | 19 ++ .../belebele/prompt_5/belebele_fuv.yaml | 19 ++ .../belebele/prompt_5/belebele_gaz.yaml | 19 ++ .../belebele/prompt_5/belebele_hau.yaml | 19 ++ .../belebele/prompt_5/belebele_ibo.yaml | 19 ++ .../belebele/prompt_5/belebele_kea.yaml | 19 ++ .../belebele/prompt_5/belebele_kin.yaml | 19 ++ .../belebele/prompt_5/belebele_lin.yaml | 19 ++ .../belebele/prompt_5/belebele_lug.yaml | 19 ++ .../belebele/prompt_5/belebele_luo.yaml | 19 ++ .../belebele/prompt_5/belebele_nya.yaml | 19 ++ .../belebele/prompt_5/belebele_plt.yaml | 19 ++ .../belebele/prompt_5/belebele_por.yaml | 19 ++ .../belebele/prompt_5/belebele_sna.yaml | 19 ++ .../belebele/prompt_5/belebele_som.yaml | 19 ++ .../belebele/prompt_5/belebele_sot.yaml | 19 ++ .../belebele/prompt_5/belebele_ssw.yaml | 19 ++ .../belebele/prompt_5/belebele_swa.yaml | 19 ++ .../belebele/prompt_5/belebele_tir.yaml | 19 ++ .../belebele/prompt_5/belebele_tsn.yaml | 19 ++ .../belebele/prompt_5/belebele_tso.yaml | 19 ++ .../belebele/prompt_5/belebele_wol.yaml | 19 ++ .../belebele/prompt_5/belebele_xho.yaml | 19 ++ .../belebele/prompt_5/belebele_yor.yaml | 19 ++ .../belebele/prompt_5/belebele_zul.yaml | 19 ++ lm_eval/tasks/afrobench/belebele/utils.py | 155 ++++++++++++ lm_eval/tasks/afrobench/flores/README.md | 31 +++ lm_eval/tasks/afrobench/flores/flores.yaml | 14 ++ lm_eval/tasks/afrobench/flores/gen_utils.py | 202 ++++++++++++++++ .../flores/prompt_1/african-english/flores | 27 +++ .../flores_ace_Arab-eng_Latn.yaml | 6 + .../flores_ace_Latn-eng_Latn.yaml | 6 + .../flores_acq_Arab-eng_Latn.yaml | 6 + .../flores_aeb_Arab-eng_Latn.yaml | 6 + .../flores_afr_Latn-eng_Latn.yaml | 6 + .../flores_aka_Latn-eng_Latn.yaml | 6 + .../flores_amh_Ethi-eng_Latn.yaml | 6 + .../flores_ary_Arab-eng_Latn.yaml | 6 + .../flores_arz_Arab-eng_Latn.yaml | 6 + .../flores_bam_Latn-eng_Latn.yaml | 6 + .../flores_ban_Latn-eng_Latn.yaml | 6 + .../flores_bem_Latn-eng_Latn.yaml | 6 + .../flores_cjk_Latn-eng_Latn.yaml | 6 + .../flores_dik_Latn-eng_Latn.yaml | 6 + .../flores_dyu_Latn-eng_Latn.yaml | 6 + .../flores_ewe_Latn-eng_Latn.yaml | 6 + .../flores_fon_Latn-eng_Latn.yaml | 6 + .../flores_fra_Latn-eng_Latn.yaml | 6 + .../flores_fuv_Latn-eng_Latn.yaml | 6 + .../flores_gaz_Latn-eng_Latn.yaml | 6 + .../flores_hau_Latn-eng_Latn.yaml | 6 + .../flores_ibo_Latn-eng_Latn.yaml | 6 + .../flores_kab_Latn-eng_Latn.yaml | 6 + .../flores_kam_Latn-eng_Latn.yaml | 6 + .../flores_kbp_Latn-eng_Latn.yaml | 6 + .../flores_kea_Latn-eng_Latn.yaml | 6 + .../flores_kik_Latn-eng_Latn.yaml | 6 + .../flores_kin_Latn-eng_Latn.yaml | 6 + .../flores_kmb_Latn-eng_Latn.yaml | 6 + .../flores_knc_Arab-eng_Latn.yaml | 6 + .../flores_knc_Latn-eng_Latn.yaml | 6 + .../flores_kon_Latn-eng_Latn.yaml | 6 + .../flores_lin_Latn-eng_Latn.yaml | 6 + .../flores_lua_Latn-eng_Latn.yaml | 6 + .../flores_lug_Latn-eng_Latn.yaml | 6 + .../flores_luo_Latn-eng_Latn.yaml | 6 + .../flores_mos_Latn-eng_Latn.yaml | 6 + .../flores_nso_Latn-eng_Latn.yaml | 6 + .../flores_nus_Latn-eng_Latn.yaml | 6 + .../flores_nya_Latn-eng_Latn.yaml | 6 + .../flores_plt_Latn-eng_Latn.yaml | 6 + .../flores_run_Latn-eng_Latn.yaml | 6 + .../flores_sag_Latn-eng_Latn.yaml | 6 + .../flores_sna_Latn-eng_Latn.yaml | 6 + .../flores_som_Latn-eng_Latn.yaml | 6 + .../flores_sot_Latn-eng_Latn.yaml | 6 + .../flores_ssw_Latn-eng_Latn.yaml | 6 + .../flores_sun_Latn-eng_Latn.yaml | 6 + .../flores_swh_Latn-eng_Latn.yaml | 6 + .../flores_taq_Latn-eng_Latn.yaml | 6 + .../flores_taq_Tfng-eng_Latn.yaml | 6 + .../flores_tir_Ethi-eng_Latn.yaml | 6 + .../flores_tsn_Latn-eng_Latn.yaml | 6 + .../flores_tso_Latn-eng_Latn.yaml | 6 + .../flores_tum_Latn-eng_Latn.yaml | 6 + .../flores_twi_Latn-eng_Latn.yaml | 6 + .../flores_tzm_Tfng-eng_Latn.yaml | 6 + .../flores_umb_Latn-eng_Latn.yaml | 6 + .../flores_wol_Latn-eng_Latn.yaml | 6 + .../flores_xho_Latn-eng_Latn.yaml | 6 + .../flores_yor_Latn-eng_Latn.yaml | 6 + .../flores_zul_Latn-eng_Latn.yaml | 6 + .../flores/prompt_1/english-african/flores | 27 +++ .../flores_eng_Latn-ace_Arab.yaml | 6 + .../flores_eng_Latn-ace_Latn.yaml | 6 + .../flores_eng_Latn-acq_Arab.yaml | 6 + .../flores_eng_Latn-aeb_Arab.yaml | 6 + .../flores_eng_Latn-afr_Latn.yaml | 6 + .../flores_eng_Latn-aka_Latn.yaml | 6 + .../flores_eng_Latn-amh_Ethi.yaml | 6 + .../flores_eng_Latn-ary_Arab.yaml | 6 + .../flores_eng_Latn-arz_Arab.yaml | 6 + .../flores_eng_Latn-bam_Latn.yaml | 6 + .../flores_eng_Latn-ban_Latn.yaml | 6 + .../flores_eng_Latn-bem_Latn.yaml | 6 + .../flores_eng_Latn-cjk_Latn.yaml | 6 + .../flores_eng_Latn-dik_Latn.yaml | 6 + .../flores_eng_Latn-dyu_Latn.yaml | 6 + .../flores_eng_Latn-ewe_Latn.yaml | 6 + .../flores_eng_Latn-fon_Latn.yaml | 6 + .../flores_eng_Latn-fra_Latn.yaml | 6 + .../flores_eng_Latn-fuv_Latn.yaml | 6 + .../flores_eng_Latn-gaz_Latn.yaml | 6 + .../flores_eng_Latn-hau_Latn.yaml | 6 + .../flores_eng_Latn-ibo_Latn.yaml | 6 + .../flores_eng_Latn-kab_Latn.yaml | 6 + .../flores_eng_Latn-kam_Latn.yaml | 6 + .../flores_eng_Latn-kbp_Latn.yaml | 6 + .../flores_eng_Latn-kea_Latn.yaml | 6 + .../flores_eng_Latn-kik_Latn.yaml | 6 + .../flores_eng_Latn-kin_Latn.yaml | 6 + .../flores_eng_Latn-kmb_Latn.yaml | 6 + .../flores_eng_Latn-knc_Arab.yaml | 6 + .../flores_eng_Latn-knc_Latn.yaml | 6 + .../flores_eng_Latn-kon_Latn.yaml | 6 + .../flores_eng_Latn-lin_Latn.yaml | 6 + .../flores_eng_Latn-lua_Latn.yaml | 6 + .../flores_eng_Latn-lug_Latn.yaml | 6 + .../flores_eng_Latn-luo_Latn.yaml | 6 + .../flores_eng_Latn-mos_Latn.yaml | 6 + .../flores_eng_Latn-nso_Latn.yaml | 6 + .../flores_eng_Latn-nus_Latn.yaml | 6 + .../flores_eng_Latn-nya_Latn.yaml | 6 + .../flores_eng_Latn-plt_Latn.yaml | 6 + .../flores_eng_Latn-run_Latn.yaml | 6 + .../flores_eng_Latn-sag_Latn.yaml | 6 + .../flores_eng_Latn-sna_Latn.yaml | 6 + .../flores_eng_Latn-som_Latn.yaml | 6 + .../flores_eng_Latn-sot_Latn.yaml | 6 + .../flores_eng_Latn-ssw_Latn.yaml | 6 + .../flores_eng_Latn-sun_Latn.yaml | 6 + .../flores_eng_Latn-swh_Latn.yaml | 6 + .../flores_eng_Latn-taq_Latn.yaml | 6 + .../flores_eng_Latn-taq_Tfng.yaml | 6 + .../flores_eng_Latn-tir_Ethi.yaml | 6 + .../flores_eng_Latn-tsn_Latn.yaml | 6 + .../flores_eng_Latn-tso_Latn.yaml | 6 + .../flores_eng_Latn-tum_Latn.yaml | 6 + .../flores_eng_Latn-twi_Latn.yaml | 6 + .../flores_eng_Latn-tzm_Tfng.yaml | 6 + .../flores_eng_Latn-umb_Latn.yaml | 6 + .../flores_eng_Latn-wol_Latn.yaml | 6 + .../flores_eng_Latn-xho_Latn.yaml | 6 + .../flores_eng_Latn-yor_Latn.yaml | 6 + .../flores_eng_Latn-zul_Latn.yaml | 6 + .../tasks/afrobench/flores/prompt_1/flores | 24 ++ .../flores/prompt_2/african-english/flores | 27 +++ .../flores_ace_Arab-eng_Latn.yaml | 8 + .../flores_ace_Latn-eng_Latn.yaml | 8 + .../flores_acq_Arab-eng_Latn.yaml | 7 + .../flores_aeb_Arab-eng_Latn.yaml | 7 + .../flores_afr_Latn-eng_Latn.yaml | 7 + .../flores_aka_Latn-eng_Latn.yaml | 7 + .../flores_amh_Ethi-eng_Latn.yaml | 7 + .../flores_ary_Arab-eng_Latn.yaml | 7 + .../flores_arz_Arab-eng_Latn.yaml | 7 + .../flores_bam_Latn-eng_Latn.yaml | 7 + .../flores_ban_Latn-eng_Latn.yaml | 7 + .../flores_bem_Latn-eng_Latn.yaml | 7 + .../flores_cjk_Latn-eng_Latn.yaml | 7 + .../flores_dik_Latn-eng_Latn.yaml | 7 + .../flores_dyu_Latn-eng_Latn.yaml | 7 + .../flores_ewe_Latn-eng_Latn.yaml | 7 + .../flores_fon_Latn-eng_Latn.yaml | 7 + .../flores_fra_Latn-eng_Latn.yaml | 7 + .../flores_fuv_Latn-eng_Latn.yaml | 7 + .../flores_gaz_Latn-eng_Latn.yaml | 7 + .../flores_hau_Latn-eng_Latn.yaml | 7 + .../flores_ibo_Latn-eng_Latn.yaml | 7 + .../flores_kab_Latn-eng_Latn.yaml | 7 + .../flores_kam_Latn-eng_Latn.yaml | 7 + .../flores_kbp_Latn-eng_Latn.yaml | 7 + .../flores_kea_Latn-eng_Latn.yaml | 7 + .../flores_kik_Latn-eng_Latn.yaml | 7 + .../flores_kin_Latn-eng_Latn.yaml | 7 + .../flores_kmb_Latn-eng_Latn.yaml | 7 + .../flores_knc_Arab-eng_Latn.yaml | 8 + .../flores_knc_Latn-eng_Latn.yaml | 8 + .../flores_kon_Latn-eng_Latn.yaml | 7 + .../flores_lin_Latn-eng_Latn.yaml | 7 + .../flores_lua_Latn-eng_Latn.yaml | 7 + .../flores_lug_Latn-eng_Latn.yaml | 7 + .../flores_luo_Latn-eng_Latn.yaml | 7 + .../flores_mos_Latn-eng_Latn.yaml | 7 + .../flores_nso_Latn-eng_Latn.yaml | 7 + .../flores_nus_Latn-eng_Latn.yaml | 7 + .../flores_nya_Latn-eng_Latn.yaml | 7 + .../flores_plt_Latn-eng_Latn.yaml | 7 + .../flores_run_Latn-eng_Latn.yaml | 7 + .../flores_sag_Latn-eng_Latn.yaml | 7 + .../flores_sna_Latn-eng_Latn.yaml | 7 + .../flores_som_Latn-eng_Latn.yaml | 7 + .../flores_sot_Latn-eng_Latn.yaml | 7 + .../flores_ssw_Latn-eng_Latn.yaml | 7 + .../flores_sun_Latn-eng_Latn.yaml | 7 + .../flores_swh_Latn-eng_Latn.yaml | 7 + .../flores_taq_Latn-eng_Latn.yaml | 7 + .../flores_taq_Tfng-eng_Latn.yaml | 8 + .../flores_tir_Ethi-eng_Latn.yaml | 7 + .../flores_tsn_Latn-eng_Latn.yaml | 7 + .../flores_tso_Latn-eng_Latn.yaml | 7 + .../flores_tum_Latn-eng_Latn.yaml | 7 + .../flores_twi_Latn-eng_Latn.yaml | 7 + .../flores_tzm_Tfng-eng_Latn.yaml | 8 + .../flores_umb_Latn-eng_Latn.yaml | 7 + .../flores_wol_Latn-eng_Latn.yaml | 7 + .../flores_xho_Latn-eng_Latn.yaml | 7 + .../flores_yor_Latn-eng_Latn.yaml | 7 + .../flores_zul_Latn-eng_Latn.yaml | 7 + .../flores/prompt_2/english-african/flores | 27 +++ .../flores_eng_Latn-ace_Arab.yaml | 8 + .../flores_eng_Latn-ace_Latn.yaml | 8 + .../flores_eng_Latn-acq_Arab.yaml | 7 + .../flores_eng_Latn-aeb_Arab.yaml | 7 + .../flores_eng_Latn-afr_Latn.yaml | 7 + .../flores_eng_Latn-aka_Latn.yaml | 7 + .../flores_eng_Latn-amh_Ethi.yaml | 7 + .../flores_eng_Latn-ary_Arab.yaml | 7 + .../flores_eng_Latn-arz_Arab.yaml | 7 + .../flores_eng_Latn-bam_Latn.yaml | 7 + .../flores_eng_Latn-ban_Latn.yaml | 7 + .../flores_eng_Latn-bem_Latn.yaml | 7 + .../flores_eng_Latn-cjk_Latn.yaml | 7 + .../flores_eng_Latn-dik_Latn.yaml | 7 + .../flores_eng_Latn-dyu_Latn.yaml | 7 + .../flores_eng_Latn-ewe_Latn.yaml | 7 + .../flores_eng_Latn-fon_Latn.yaml | 7 + .../flores_eng_Latn-fra_Latn.yaml | 7 + .../flores_eng_Latn-fuv_Latn.yaml | 7 + .../flores_eng_Latn-gaz_Latn.yaml | 7 + .../flores_eng_Latn-hau_Latn.yaml | 7 + .../flores_eng_Latn-ibo_Latn.yaml | 7 + .../flores_eng_Latn-kab_Latn.yaml | 7 + .../flores_eng_Latn-kam_Latn.yaml | 7 + .../flores_eng_Latn-kbp_Latn.yaml | 7 + .../flores_eng_Latn-kea_Latn.yaml | 7 + .../flores_eng_Latn-kik_Latn.yaml | 7 + .../flores_eng_Latn-kin_Latn.yaml | 7 + .../flores_eng_Latn-kmb_Latn.yaml | 7 + .../flores_eng_Latn-knc_Arab.yaml | 8 + .../flores_eng_Latn-knc_Latn.yaml | 8 + .../flores_eng_Latn-kon_Latn.yaml | 7 + .../flores_eng_Latn-lin_Latn.yaml | 7 + .../flores_eng_Latn-lua_Latn.yaml | 7 + .../flores_eng_Latn-lug_Latn.yaml | 7 + .../flores_eng_Latn-luo_Latn.yaml | 7 + .../flores_eng_Latn-mos_Latn.yaml | 7 + .../flores_eng_Latn-nso_Latn.yaml | 7 + .../flores_eng_Latn-nus_Latn.yaml | 7 + .../flores_eng_Latn-nya_Latn.yaml | 7 + .../flores_eng_Latn-plt_Latn.yaml | 7 + .../flores_eng_Latn-run_Latn.yaml | 7 + .../flores_eng_Latn-sag_Latn.yaml | 7 + .../flores_eng_Latn-sna_Latn.yaml | 7 + .../flores_eng_Latn-som_Latn.yaml | 7 + .../flores_eng_Latn-sot_Latn.yaml | 7 + .../flores_eng_Latn-ssw_Latn.yaml | 7 + .../flores_eng_Latn-sun_Latn.yaml | 7 + .../flores_eng_Latn-swh_Latn.yaml | 7 + .../flores_eng_Latn-taq_Latn.yaml | 7 + .../flores_eng_Latn-taq_Tfng.yaml | 8 + .../flores_eng_Latn-tir_Ethi.yaml | 7 + .../flores_eng_Latn-tsn_Latn.yaml | 7 + .../flores_eng_Latn-tso_Latn.yaml | 7 + .../flores_eng_Latn-tum_Latn.yaml | 7 + .../flores_eng_Latn-twi_Latn.yaml | 7 + .../flores_eng_Latn-tzm_Tfng.yaml | 7 + .../flores_eng_Latn-umb_Latn.yaml | 7 + .../flores_eng_Latn-wol_Latn.yaml | 7 + .../flores_eng_Latn-xho_Latn.yaml | 7 + .../flores_eng_Latn-yor_Latn.yaml | 7 + .../flores_eng_Latn-zul_Latn.yaml | 7 + .../tasks/afrobench/flores/prompt_2/flores | 24 ++ .../flores/prompt_3/african-english/flores | 27 +++ .../flores_ace_Arab-eng_Latn.yaml | 8 + .../flores_ace_Latn-eng_Latn.yaml | 8 + .../flores_acq_Arab-eng_Latn.yaml | 8 + .../flores_aeb_Arab-eng_Latn.yaml | 7 + .../flores_afr_Latn-eng_Latn.yaml | 7 + .../flores_aka_Latn-eng_Latn.yaml | 7 + .../flores_amh_Ethi-eng_Latn.yaml | 7 + .../flores_ary_Arab-eng_Latn.yaml | 7 + .../flores_arz_Arab-eng_Latn.yaml | 7 + .../flores_bam_Latn-eng_Latn.yaml | 7 + .../flores_ban_Latn-eng_Latn.yaml | 7 + .../flores_bem_Latn-eng_Latn.yaml | 7 + .../flores_cjk_Latn-eng_Latn.yaml | 7 + .../flores_dik_Latn-eng_Latn.yaml | 8 + .../flores_dyu_Latn-eng_Latn.yaml | 7 + .../flores_ewe_Latn-eng_Latn.yaml | 7 + .../flores_fon_Latn-eng_Latn.yaml | 7 + .../flores_fra_Latn-eng_Latn.yaml | 7 + .../flores_fuv_Latn-eng_Latn.yaml | 8 + .../flores_gaz_Latn-eng_Latn.yaml | 7 + .../flores_hau_Latn-eng_Latn.yaml | 7 + .../flores_ibo_Latn-eng_Latn.yaml | 7 + .../flores_kab_Latn-eng_Latn.yaml | 7 + .../flores_kam_Latn-eng_Latn.yaml | 7 + .../flores_kbp_Latn-eng_Latn.yaml | 7 + .../flores_kea_Latn-eng_Latn.yaml | 7 + .../flores_kik_Latn-eng_Latn.yaml | 7 + .../flores_kin_Latn-eng_Latn.yaml | 7 + .../flores_kmb_Latn-eng_Latn.yaml | 7 + .../flores_knc_Arab-eng_Latn.yaml | 8 + .../flores_knc_Latn-eng_Latn.yaml | 8 + .../flores_kon_Latn-eng_Latn.yaml | 7 + .../flores_lin_Latn-eng_Latn.yaml | 7 + .../flores_lua_Latn-eng_Latn.yaml | 7 + .../flores_lug_Latn-eng_Latn.yaml | 7 + .../flores_luo_Latn-eng_Latn.yaml | 7 + .../flores_mos_Latn-eng_Latn.yaml | 7 + .../flores_nso_Latn-eng_Latn.yaml | 7 + .../flores_nus_Latn-eng_Latn.yaml | 7 + .../flores_nya_Latn-eng_Latn.yaml | 7 + .../flores_plt_Latn-eng_Latn.yaml | 8 + .../flores_run_Latn-eng_Latn.yaml | 7 + .../flores_sag_Latn-eng_Latn.yaml | 7 + .../flores_sna_Latn-eng_Latn.yaml | 7 + .../flores_som_Latn-eng_Latn.yaml | 7 + .../flores_sot_Latn-eng_Latn.yaml | 7 + .../flores_ssw_Latn-eng_Latn.yaml | 7 + .../flores_sun_Latn-eng_Latn.yaml | 7 + .../flores_swh_Latn-eng_Latn.yaml | 7 + .../flores_taq_Latn-eng_Latn.yaml | 7 + .../flores_taq_Tfng-eng_Latn.yaml | 8 + .../flores_tir_Ethi-eng_Latn.yaml | 7 + .../flores_tsn_Latn-eng_Latn.yaml | 7 + .../flores_tso_Latn-eng_Latn.yaml | 7 + .../flores_tum_Latn-eng_Latn.yaml | 7 + .../flores_twi_Latn-eng_Latn.yaml | 7 + .../flores_tzm_Tfng-eng_Latn.yaml | 8 + .../flores_umb_Latn-eng_Latn.yaml | 7 + .../flores_wol_Latn-eng_Latn.yaml | 7 + .../flores_xho_Latn-eng_Latn.yaml | 7 + .../flores_yor_Latn-eng_Latn.yaml | 7 + .../flores_zul_Latn-eng_Latn.yaml | 7 + .../flores/prompt_3/english-african/flores | 27 +++ .../flores_eng_Latn-ace_Arab.yaml | 8 + .../flores_eng_Latn-ace_Latn.yaml | 8 + .../flores_eng_Latn-acq_Arab.yaml | 8 + .../flores_eng_Latn-aeb_Arab.yaml | 7 + .../flores_eng_Latn-afr_Latn.yaml | 7 + .../flores_eng_Latn-aka_Latn.yaml | 7 + .../flores_eng_Latn-amh_Ethi.yaml | 7 + .../flores_eng_Latn-ary_Arab.yaml | 7 + .../flores_eng_Latn-arz_Arab.yaml | 7 + .../flores_eng_Latn-bam_Latn.yaml | 7 + .../flores_eng_Latn-ban_Latn.yaml | 7 + .../flores_eng_Latn-bem_Latn.yaml | 7 + .../flores_eng_Latn-cjk_Latn.yaml | 7 + .../flores_eng_Latn-dik_Latn.yaml | 8 + .../flores_eng_Latn-dyu_Latn.yaml | 7 + .../flores_eng_Latn-ewe_Latn.yaml | 7 + .../flores_eng_Latn-fon_Latn.yaml | 7 + .../flores_eng_Latn-fra_Latn.yaml | 7 + .../flores_eng_Latn-fuv_Latn.yaml | 8 + .../flores_eng_Latn-gaz_Latn.yaml | 7 + .../flores_eng_Latn-hau_Latn.yaml | 7 + .../flores_eng_Latn-ibo_Latn.yaml | 7 + .../flores_eng_Latn-kab_Latn.yaml | 7 + .../flores_eng_Latn-kam_Latn.yaml | 7 + .../flores_eng_Latn-kbp_Latn.yaml | 7 + .../flores_eng_Latn-kea_Latn.yaml | 7 + .../flores_eng_Latn-kik_Latn.yaml | 7 + .../flores_eng_Latn-kin_Latn.yaml | 7 + .../flores_eng_Latn-kmb_Latn.yaml | 7 + .../flores_eng_Latn-knc_Arab.yaml | 8 + .../flores_eng_Latn-knc_Latn.yaml | 8 + .../flores_eng_Latn-kon_Latn.yaml | 7 + .../flores_eng_Latn-lin_Latn.yaml | 7 + .../flores_eng_Latn-lua_Latn.yaml | 7 + .../flores_eng_Latn-lug_Latn.yaml | 7 + .../flores_eng_Latn-luo_Latn.yaml | 7 + .../flores_eng_Latn-mos_Latn.yaml | 7 + .../flores_eng_Latn-nso_Latn.yaml | 7 + .../flores_eng_Latn-nus_Latn.yaml | 7 + .../flores_eng_Latn-nya_Latn.yaml | 7 + .../flores_eng_Latn-plt_Latn.yaml | 8 + .../flores_eng_Latn-run_Latn.yaml | 7 + .../flores_eng_Latn-sag_Latn.yaml | 7 + .../flores_eng_Latn-sna_Latn.yaml | 7 + .../flores_eng_Latn-som_Latn.yaml | 7 + .../flores_eng_Latn-sot_Latn.yaml | 7 + .../flores_eng_Latn-ssw_Latn.yaml | 7 + .../flores_eng_Latn-sun_Latn.yaml | 7 + .../flores_eng_Latn-swh_Latn.yaml | 7 + .../flores_eng_Latn-taq_Latn.yaml | 7 + .../flores_eng_Latn-taq_Tfng.yaml | 8 + .../flores_eng_Latn-tir_Ethi.yaml | 7 + .../flores_eng_Latn-tsn_Latn.yaml | 7 + .../flores_eng_Latn-tso_Latn.yaml | 7 + .../flores_eng_Latn-tum_Latn.yaml | 7 + .../flores_eng_Latn-twi_Latn.yaml | 7 + .../flores_eng_Latn-tzm_Tfng.yaml | 8 + .../flores_eng_Latn-umb_Latn.yaml | 7 + .../flores_eng_Latn-wol_Latn.yaml | 7 + .../flores_eng_Latn-xho_Latn.yaml | 7 + .../flores_eng_Latn-yor_Latn.yaml | 7 + .../flores_eng_Latn-zul_Latn.yaml | 7 + .../tasks/afrobench/flores/prompt_3/flores | 24 ++ .../tasks/afrobench/injongointent/README.md | 23 ++ .../afrobench/injongointent/gen_utils.py | 159 ++++++++++++ .../injongointent/injongointent.yaml | 13 + .../injongointent/prompt_1/injongointent | 75 ++++++ .../prompt_1/injongointent_amh.yaml | 13 + .../prompt_1/injongointent_eng.yaml | 16 ++ .../prompt_1/injongointent_ewe.yaml | 13 + .../prompt_1/injongointent_hau.yaml | 13 + .../prompt_1/injongointent_ibo.yaml | 13 + .../prompt_1/injongointent_kin.yaml | 13 + .../prompt_1/injongointent_lin.yaml | 13 + .../prompt_1/injongointent_lug.yaml | 13 + .../prompt_1/injongointent_orm.yaml | 13 + .../prompt_1/injongointent_sna.yaml | 13 + .../prompt_1/injongointent_sot.yaml | 13 + .../prompt_1/injongointent_swa.yaml | 13 + .../prompt_1/injongointent_twi.yaml | 13 + .../prompt_1/injongointent_wol.yaml | 13 + .../prompt_1/injongointent_xho.yaml | 13 + .../prompt_1/injongointent_yor.yaml | 13 + .../prompt_1/injongointent_zul.yaml | 13 + .../afrobench/injongointent/prompt_1/utils.py | 1 + .../injongointent/prompt_2/injongointent | 75 ++++++ .../prompt_2/injongointent_amh.yaml | 13 + .../prompt_2/injongointent_eng.yaml | 16 ++ .../prompt_2/injongointent_ewe.yaml | 13 + .../prompt_2/injongointent_hau.yaml | 13 + .../prompt_2/injongointent_ibo.yaml | 13 + .../prompt_2/injongointent_kin.yaml | 13 + .../prompt_2/injongointent_lin.yaml | 13 + .../prompt_2/injongointent_lug.yaml | 13 + .../prompt_2/injongointent_orm.yaml | 13 + .../prompt_2/injongointent_sna.yaml | 13 + .../prompt_2/injongointent_sot.yaml | 13 + .../prompt_2/injongointent_swa.yaml | 13 + .../prompt_2/injongointent_twi.yaml | 13 + .../prompt_2/injongointent_wol.yaml | 13 + .../prompt_2/injongointent_xho.yaml | 13 + .../prompt_2/injongointent_yor.yaml | 13 + .../prompt_2/injongointent_zul.yaml | 13 + .../afrobench/injongointent/prompt_2/utils.py | 1 + .../injongointent/prompt_3/injongointent | 75 ++++++ .../prompt_3/injongointent_amh.yaml | 13 + .../prompt_3/injongointent_eng.yaml | 16 ++ .../prompt_3/injongointent_ewe.yaml | 13 + .../prompt_3/injongointent_hau.yaml | 13 + .../prompt_3/injongointent_ibo.yaml | 13 + .../prompt_3/injongointent_kin.yaml | 13 + .../prompt_3/injongointent_lin.yaml | 13 + .../prompt_3/injongointent_lug.yaml | 13 + .../prompt_3/injongointent_orm.yaml | 13 + .../prompt_3/injongointent_sna.yaml | 13 + .../prompt_3/injongointent_sot.yaml | 13 + .../prompt_3/injongointent_swa.yaml | 13 + .../prompt_3/injongointent_twi.yaml | 13 + .../prompt_3/injongointent_wol.yaml | 13 + .../prompt_3/injongointent_xho.yaml | 13 + .../prompt_3/injongointent_yor.yaml | 13 + .../prompt_3/injongointent_zul.yaml | 13 + .../afrobench/injongointent/prompt_3/utils.py | 1 + .../injongointent/prompt_4/injongointent | 75 ++++++ .../prompt_4/injongointent_amh.yaml | 14 ++ .../prompt_4/injongointent_eng.yaml | 17 ++ .../prompt_4/injongointent_ewe.yaml | 13 + .../prompt_4/injongointent_hau.yaml | 14 ++ .../prompt_4/injongointent_ibo.yaml | 13 + .../prompt_4/injongointent_kin.yaml | 14 ++ .../prompt_4/injongointent_lin.yaml | 14 ++ .../prompt_4/injongointent_lug.yaml | 14 ++ .../prompt_4/injongointent_orm.yaml | 14 ++ .../prompt_4/injongointent_sna.yaml | 14 ++ .../prompt_4/injongointent_sot.yaml | 14 ++ .../prompt_4/injongointent_swa.yaml | 14 ++ .../prompt_4/injongointent_twi.yaml | 13 + .../prompt_4/injongointent_wol.yaml | 14 ++ .../prompt_4/injongointent_xho.yaml | 14 ++ .../prompt_4/injongointent_yor.yaml | 14 ++ .../prompt_4/injongointent_zul.yaml | 13 + .../afrobench/injongointent/prompt_4/utils.py | 1 + .../injongointent/prompt_5/injongointent | 75 ++++++ .../prompt_5/injongointent_amh.yaml | 13 + .../prompt_5/injongointent_eng.yaml | 16 ++ .../prompt_5/injongointent_ewe.yaml | 13 + .../prompt_5/injongointent_hau.yaml | 13 + .../prompt_5/injongointent_ibo.yaml | 13 + .../prompt_5/injongointent_kin.yaml | 13 + .../prompt_5/injongointent_lin.yaml | 13 + .../prompt_5/injongointent_lug.yaml | 13 + .../prompt_5/injongointent_orm.yaml | 13 + .../prompt_5/injongointent_sna.yaml | 13 + .../prompt_5/injongointent_sot.yaml | 13 + .../prompt_5/injongointent_swa.yaml | 13 + .../prompt_5/injongointent_twi.yaml | 13 + .../prompt_5/injongointent_wol.yaml | 13 + .../prompt_5/injongointent_xho.yaml | 13 + .../prompt_5/injongointent_yor.yaml | 13 + .../prompt_5/injongointent_zul.yaml | 13 + .../afrobench/injongointent/prompt_5/utils.py | 1 + lm_eval/tasks/afrobench/mafand/README.md | 73 ++++++ lm_eval/tasks/afrobench/mafand/gen_utils.py | 147 ++++++++++++ lm_eval/tasks/afrobench/mafand/mafand.yaml | 14 ++ .../mafand/prompt_1/african-english/mafand | 28 +++ .../african-english/mafand_amh-en.yaml | 4 + .../african-english/mafand_bam-fr.yaml | 4 + .../african-english/mafand_bbj-fr.yaml | 4 + .../african-english/mafand_ewe-fr.yaml | 4 + .../african-english/mafand_fon-fr.yaml | 4 + .../african-english/mafand_hau-en.yaml | 4 + .../african-english/mafand_ibo-en.yaml | 4 + .../african-english/mafand_kin-en.yaml | 4 + .../african-english/mafand_lug-en.yaml | 4 + .../african-english/mafand_luo-en.yaml | 4 + .../african-english/mafand_mos-fr.yaml | 4 + .../african-english/mafand_nya-en.yaml | 4 + .../african-english/mafand_pcm-en.yaml | 4 + .../african-english/mafand_sna-en.yaml | 4 + .../african-english/mafand_swa-en.yaml | 4 + .../african-english/mafand_tsn-en.yaml | 4 + .../african-english/mafand_twi-en.yaml | 4 + .../african-english/mafand_wol-fr.yaml | 4 + .../african-english/mafand_xho-en.yaml | 4 + .../african-english/mafand_yor-en.yaml | 4 + .../african-english/mafand_zul-en.yaml | 4 + .../mafand/prompt_1/african-english/utils.py | 121 ++++++++++ .../mafand/prompt_1/english-african/mafand | 28 +++ .../english-african/mafand_en-amh.yaml | 4 + .../english-african/mafand_en-hau.yaml | 4 + .../english-african/mafand_en-ibo.yaml | 4 + .../english-african/mafand_en-kin.yaml | 4 + .../english-african/mafand_en-lug.yaml | 4 + .../english-african/mafand_en-luo.yaml | 4 + .../english-african/mafand_en-nya.yaml | 4 + .../english-african/mafand_en-pcm.yaml | 4 + .../english-african/mafand_en-sna.yaml | 4 + .../english-african/mafand_en-swa.yaml | 4 + .../english-african/mafand_en-tsn.yaml | 4 + .../english-african/mafand_en-twi.yaml | 4 + .../english-african/mafand_en-xho.yaml | 4 + .../english-african/mafand_en-yor.yaml | 4 + .../english-african/mafand_en-zul.yaml | 4 + .../english-african/mafand_fr-bam.yaml | 4 + .../english-african/mafand_fr-bbj.yaml | 4 + .../english-african/mafand_fr-ewe.yaml | 4 + .../english-african/mafand_fr-fon.yaml | 4 + .../english-african/mafand_fr-mos.yaml | 4 + .../english-african/mafand_fr-wol.yaml | 4 + .../mafand/prompt_1/english-african/utils.py | 121 ++++++++++ .../mafand/prompt_2/african-english/mafand | 28 +++ .../african-english/mafand_amh-en.yaml | 4 + .../african-english/mafand_bam-fr.yaml | 4 + .../african-english/mafand_bbj-fr.yaml | 4 + .../african-english/mafand_ewe-fr.yaml | 4 + .../african-english/mafand_fon-fr.yaml | 4 + .../african-english/mafand_hau-en.yaml | 4 + .../african-english/mafand_ibo-en.yaml | 4 + .../african-english/mafand_kin-en.yaml | 4 + .../african-english/mafand_lug-en.yaml | 4 + .../african-english/mafand_luo-en.yaml | 4 + .../african-english/mafand_mos-fr.yaml | 4 + .../african-english/mafand_nya-en.yaml | 4 + .../african-english/mafand_pcm-en.yaml | 4 + .../african-english/mafand_sna-en.yaml | 4 + .../african-english/mafand_swa-en.yaml | 4 + .../african-english/mafand_tsn-en.yaml | 4 + .../african-english/mafand_twi-en.yaml | 4 + .../african-english/mafand_wol-fr.yaml | 4 + .../african-english/mafand_xho-en.yaml | 4 + .../african-english/mafand_yor-en.yaml | 4 + .../african-english/mafand_zul-en.yaml | 4 + .../mafand/prompt_2/african-english/utils.py | 121 ++++++++++ .../mafand/prompt_2/english-african/mafand | 28 +++ .../english-african/mafand_en-amh.yaml | 4 + .../english-african/mafand_en-hau.yaml | 4 + .../english-african/mafand_en-ibo.yaml | 4 + .../english-african/mafand_en-kin.yaml | 4 + .../english-african/mafand_en-lug.yaml | 4 + .../english-african/mafand_en-luo.yaml | 4 + .../english-african/mafand_en-nya.yaml | 4 + .../english-african/mafand_en-pcm.yaml | 4 + .../english-african/mafand_en-sna.yaml | 4 + .../english-african/mafand_en-swa.yaml | 4 + .../english-african/mafand_en-tsn.yaml | 4 + .../english-african/mafand_en-twi.yaml | 4 + .../english-african/mafand_en-xho.yaml | 4 + .../english-african/mafand_en-yor.yaml | 4 + .../english-african/mafand_en-zul.yaml | 4 + .../english-african/mafand_fr-bam.yaml | 4 + .../english-african/mafand_fr-bbj.yaml | 4 + .../english-african/mafand_fr-ewe.yaml | 4 + .../english-african/mafand_fr-fon.yaml | 4 + .../english-african/mafand_fr-mos.yaml | 4 + .../english-african/mafand_fr-wol.yaml | 4 + .../mafand/prompt_2/english-african/utils.py | 121 ++++++++++ .../mafand/prompt_3/african-english/mafand | 28 +++ .../african-english/mafand_amh-en.yaml | 4 + .../african-english/mafand_bam-fr.yaml | 4 + .../african-english/mafand_bbj-fr.yaml | 4 + .../african-english/mafand_ewe-fr.yaml | 4 + .../african-english/mafand_fon-fr.yaml | 4 + .../african-english/mafand_hau-en.yaml | 4 + .../african-english/mafand_ibo-en.yaml | 4 + .../african-english/mafand_kin-en.yaml | 4 + .../african-english/mafand_lug-en.yaml | 4 + .../african-english/mafand_luo-en.yaml | 4 + .../african-english/mafand_mos-fr.yaml | 4 + .../african-english/mafand_nya-en.yaml | 4 + .../african-english/mafand_pcm-en.yaml | 4 + .../african-english/mafand_sna-en.yaml | 4 + .../african-english/mafand_swa-en.yaml | 4 + .../african-english/mafand_tsn-en.yaml | 4 + .../african-english/mafand_twi-en.yaml | 4 + .../african-english/mafand_wol-fr.yaml | 4 + .../african-english/mafand_xho-en.yaml | 4 + .../african-english/mafand_yor-en.yaml | 4 + .../african-english/mafand_zul-en.yaml | 4 + .../mafand/prompt_3/african-english/utils.py | 121 ++++++++++ .../mafand/prompt_3/english-african/mafand | 28 +++ .../english-african/mafand_en-amh.yaml | 4 + .../english-african/mafand_en-hau.yaml | 4 + .../english-african/mafand_en-ibo.yaml | 4 + .../english-african/mafand_en-kin.yaml | 4 + .../english-african/mafand_en-lug.yaml | 4 + .../english-african/mafand_en-luo.yaml | 4 + .../english-african/mafand_en-nya.yaml | 4 + .../english-african/mafand_en-pcm.yaml | 4 + .../english-african/mafand_en-sna.yaml | 4 + .../english-african/mafand_en-swa.yaml | 4 + .../english-african/mafand_en-tsn.yaml | 4 + .../english-african/mafand_en-twi.yaml | 4 + .../english-african/mafand_en-xho.yaml | 4 + .../english-african/mafand_en-yor.yaml | 4 + .../english-african/mafand_en-zul.yaml | 4 + .../english-african/mafand_fr-bam.yaml | 4 + .../english-african/mafand_fr-bbj.yaml | 4 + .../english-african/mafand_fr-ewe.yaml | 4 + .../english-african/mafand_fr-fon.yaml | 4 + .../english-african/mafand_fr-mos.yaml | 4 + .../english-african/mafand_fr-wol.yaml | 4 + .../mafand/prompt_3/english-african/utils.py | 121 ++++++++++ lm_eval/tasks/afrobench/masakhaner/README.md | 76 ++++++ .../tasks/afrobench/masakhaner/gen_utils.py | 138 +++++++++++ .../afrobench/masakhaner/masakhaner.yaml | 13 + .../afrobench/masakhaner/prompt_1/masakhaner | 26 ++ .../masakhaner/prompt_1/masakhaner_am.yaml | 11 + .../masakhaner/prompt_1/masakhaner_bbj.yaml | 11 + .../masakhaner/prompt_1/masakhaner_bm.yaml | 11 + .../masakhaner/prompt_1/masakhaner_ee.yaml | 11 + .../masakhaner/prompt_1/masakhaner_ha.yaml | 11 + .../masakhaner/prompt_1/masakhaner_ig.yaml | 11 + .../masakhaner/prompt_1/masakhaner_lg.yaml | 11 + .../masakhaner/prompt_1/masakhaner_luo.yaml | 11 + .../masakhaner/prompt_1/masakhaner_mos.yaml | 11 + .../masakhaner/prompt_1/masakhaner_ny.yaml | 11 + .../masakhaner/prompt_1/masakhaner_pcm.yaml | 11 + .../masakhaner/prompt_1/masakhaner_rw.yaml | 11 + .../masakhaner/prompt_1/masakhaner_sn.yaml | 11 + .../masakhaner/prompt_1/masakhaner_sw.yaml | 11 + .../masakhaner/prompt_1/masakhaner_tn.yaml | 11 + .../masakhaner/prompt_1/masakhaner_tw.yaml | 11 + .../masakhaner/prompt_1/masakhaner_wo.yaml | 11 + .../masakhaner/prompt_1/masakhaner_xh.yaml | 11 + .../masakhaner/prompt_1/masakhaner_yo.yaml | 11 + .../masakhaner/prompt_1/masakhaner_zu.yaml | 11 + .../afrobench/masakhaner/prompt_1/utils.py | 146 +++++++++++ .../afrobench/masakhaner/prompt_2/masakhaner | 26 ++ .../masakhaner/prompt_2/masakhaner_am.yaml | 13 + .../masakhaner/prompt_2/masakhaner_bbj.yaml | 13 + .../masakhaner/prompt_2/masakhaner_bm.yaml | 13 + .../masakhaner/prompt_2/masakhaner_ee.yaml | 13 + .../masakhaner/prompt_2/masakhaner_ha.yaml | 13 + .../masakhaner/prompt_2/masakhaner_ig.yaml | 13 + .../masakhaner/prompt_2/masakhaner_lg.yaml | 13 + .../masakhaner/prompt_2/masakhaner_luo.yaml | 13 + .../masakhaner/prompt_2/masakhaner_mos.yaml | 13 + .../masakhaner/prompt_2/masakhaner_ny.yaml | 13 + .../masakhaner/prompt_2/masakhaner_pcm.yaml | 13 + .../masakhaner/prompt_2/masakhaner_rw.yaml | 13 + .../masakhaner/prompt_2/masakhaner_sn.yaml | 13 + .../masakhaner/prompt_2/masakhaner_sw.yaml | 13 + .../masakhaner/prompt_2/masakhaner_tn.yaml | 13 + .../masakhaner/prompt_2/masakhaner_tw.yaml | 13 + .../masakhaner/prompt_2/masakhaner_wo.yaml | 13 + .../masakhaner/prompt_2/masakhaner_xh.yaml | 13 + .../masakhaner/prompt_2/masakhaner_yo.yaml | 13 + .../masakhaner/prompt_2/masakhaner_zu.yaml | 13 + .../afrobench/masakhaner/prompt_2/utils.py | 146 +++++++++++ .../afrobench/masakhaner/prompt_3/masakhaner | 26 ++ .../masakhaner/prompt_3/masakhaner_am.yaml | 10 + .../masakhaner/prompt_3/masakhaner_bbj.yaml | 10 + .../masakhaner/prompt_3/masakhaner_bm.yaml | 10 + .../masakhaner/prompt_3/masakhaner_ee.yaml | 10 + .../masakhaner/prompt_3/masakhaner_ha.yaml | 10 + .../masakhaner/prompt_3/masakhaner_ig.yaml | 10 + .../masakhaner/prompt_3/masakhaner_lg.yaml | 10 + .../masakhaner/prompt_3/masakhaner_luo.yaml | 10 + .../masakhaner/prompt_3/masakhaner_mos.yaml | 10 + .../masakhaner/prompt_3/masakhaner_ny.yaml | 10 + .../masakhaner/prompt_3/masakhaner_pcm.yaml | 10 + .../masakhaner/prompt_3/masakhaner_rw.yaml | 10 + .../masakhaner/prompt_3/masakhaner_sn.yaml | 10 + .../masakhaner/prompt_3/masakhaner_sw.yaml | 10 + .../masakhaner/prompt_3/masakhaner_tn.yaml | 10 + .../masakhaner/prompt_3/masakhaner_tw.yaml | 10 + .../masakhaner/prompt_3/masakhaner_wo.yaml | 10 + .../masakhaner/prompt_3/masakhaner_xh.yaml | 10 + .../masakhaner/prompt_3/masakhaner_yo.yaml | 10 + .../masakhaner/prompt_3/masakhaner_zu.yaml | 10 + .../afrobench/masakhaner/prompt_3/utils.py | 146 +++++++++++ .../afrobench/masakhaner/prompt_4/masakhaner | 26 ++ .../masakhaner/prompt_4/masakhaner_am.yaml | 10 + .../masakhaner/prompt_4/masakhaner_bbj.yaml | 10 + .../masakhaner/prompt_4/masakhaner_bm.yaml | 10 + .../masakhaner/prompt_4/masakhaner_ee.yaml | 9 + .../masakhaner/prompt_4/masakhaner_ha.yaml | 9 + .../masakhaner/prompt_4/masakhaner_ig.yaml | 9 + .../masakhaner/prompt_4/masakhaner_lg.yaml | 10 + .../masakhaner/prompt_4/masakhaner_luo.yaml | 9 + .../masakhaner/prompt_4/masakhaner_mos.yaml | 9 + .../masakhaner/prompt_4/masakhaner_ny.yaml | 10 + .../masakhaner/prompt_4/masakhaner_pcm.yaml | 10 + .../masakhaner/prompt_4/masakhaner_rw.yaml | 10 + .../masakhaner/prompt_4/masakhaner_sn.yaml | 10 + .../masakhaner/prompt_4/masakhaner_sw.yaml | 10 + .../masakhaner/prompt_4/masakhaner_tn.yaml | 10 + .../masakhaner/prompt_4/masakhaner_tw.yaml | 9 + .../masakhaner/prompt_4/masakhaner_wo.yaml | 9 + .../masakhaner/prompt_4/masakhaner_xh.yaml | 10 + .../masakhaner/prompt_4/masakhaner_yo.yaml | 9 + .../masakhaner/prompt_4/masakhaner_zu.yaml | 10 + .../afrobench/masakhaner/prompt_4/utils.py | 146 +++++++++++ .../afrobench/masakhaner/prompt_5/masakhaner | 26 ++ .../masakhaner/prompt_5/masakhaner_am.yaml | 9 + .../masakhaner/prompt_5/masakhaner_bbj.yaml | 9 + .../masakhaner/prompt_5/masakhaner_bm.yaml | 9 + .../masakhaner/prompt_5/masakhaner_ee.yaml | 9 + .../masakhaner/prompt_5/masakhaner_ha.yaml | 9 + .../masakhaner/prompt_5/masakhaner_ig.yaml | 9 + .../masakhaner/prompt_5/masakhaner_lg.yaml | 9 + .../masakhaner/prompt_5/masakhaner_luo.yaml | 9 + .../masakhaner/prompt_5/masakhaner_mos.yaml | 9 + .../masakhaner/prompt_5/masakhaner_ny.yaml | 9 + .../masakhaner/prompt_5/masakhaner_pcm.yaml | 9 + .../masakhaner/prompt_5/masakhaner_rw.yaml | 9 + .../masakhaner/prompt_5/masakhaner_sn.yaml | 9 + .../masakhaner/prompt_5/masakhaner_sw.yaml | 9 + .../masakhaner/prompt_5/masakhaner_tn.yaml | 9 + .../masakhaner/prompt_5/masakhaner_tw.yaml | 9 + .../masakhaner/prompt_5/masakhaner_wo.yaml | 9 + .../masakhaner/prompt_5/masakhaner_xh.yaml | 9 + .../masakhaner/prompt_5/masakhaner_yo.yaml | 9 + .../masakhaner/prompt_5/masakhaner_zu.yaml | 9 + .../afrobench/masakhaner/prompt_5/utils.py | 146 +++++++++++ lm_eval/tasks/afrobench/masakhanews/README.md | 99 ++++++++ .../afrobench/masakhanews/masakhanews.yaml | 13 + .../masakhanews/prompt_1/masakhanews | 43 ++++ .../masakhanews/prompt_1/masakhanews_amh.yaml | 7 + .../masakhanews/prompt_1/masakhanews_eng.yaml | 7 + .../masakhanews/prompt_1/masakhanews_fra.yaml | 7 + .../masakhanews/prompt_1/masakhanews_hau.yaml | 7 + .../masakhanews/prompt_1/masakhanews_ibo.yaml | 7 + .../masakhanews/prompt_1/masakhanews_lin.yaml | 7 + .../masakhanews/prompt_1/masakhanews_lug.yaml | 7 + .../masakhanews/prompt_1/masakhanews_orm.yaml | 7 + .../masakhanews/prompt_1/masakhanews_pcm.yaml | 7 + .../masakhanews/prompt_1/masakhanews_run.yaml | 7 + .../masakhanews/prompt_1/masakhanews_sna.yaml | 7 + .../masakhanews/prompt_1/masakhanews_som.yaml | 7 + .../masakhanews/prompt_1/masakhanews_swa.yaml | 7 + .../masakhanews/prompt_1/masakhanews_tir.yaml | 7 + .../masakhanews/prompt_1/masakhanews_xho.yaml | 7 + .../masakhanews/prompt_1/masakhanews_yor.yaml | 7 + .../afrobench/masakhanews/prompt_1/utils.py | 1 + .../masakhanews/prompt_2/masakhanews | 43 ++++ .../masakhanews/prompt_2/masakhanews_amh.yaml | 10 + .../masakhanews/prompt_2/masakhanews_eng.yaml | 10 + .../masakhanews/prompt_2/masakhanews_fra.yaml | 10 + .../masakhanews/prompt_2/masakhanews_hau.yaml | 10 + .../masakhanews/prompt_2/masakhanews_ibo.yaml | 10 + .../masakhanews/prompt_2/masakhanews_lin.yaml | 10 + .../masakhanews/prompt_2/masakhanews_lug.yaml | 10 + .../masakhanews/prompt_2/masakhanews_orm.yaml | 10 + .../masakhanews/prompt_2/masakhanews_pcm.yaml | 10 + .../masakhanews/prompt_2/masakhanews_run.yaml | 10 + .../masakhanews/prompt_2/masakhanews_sna.yaml | 10 + .../masakhanews/prompt_2/masakhanews_som.yaml | 10 + .../masakhanews/prompt_2/masakhanews_swa.yaml | 10 + .../masakhanews/prompt_2/masakhanews_tir.yaml | 10 + .../masakhanews/prompt_2/masakhanews_xho.yaml | 10 + .../masakhanews/prompt_2/masakhanews_yor.yaml | 10 + .../afrobench/masakhanews/prompt_2/utils.py | 1 + .../masakhanews/prompt_3/masakhanews | 43 ++++ .../masakhanews/prompt_3/masakhanews_amh.yaml | 8 + .../masakhanews/prompt_3/masakhanews_eng.yaml | 8 + .../masakhanews/prompt_3/masakhanews_fra.yaml | 8 + .../masakhanews/prompt_3/masakhanews_hau.yaml | 8 + .../masakhanews/prompt_3/masakhanews_ibo.yaml | 8 + .../masakhanews/prompt_3/masakhanews_lin.yaml | 8 + .../masakhanews/prompt_3/masakhanews_lug.yaml | 8 + .../masakhanews/prompt_3/masakhanews_orm.yaml | 8 + .../masakhanews/prompt_3/masakhanews_pcm.yaml | 8 + .../masakhanews/prompt_3/masakhanews_run.yaml | 8 + .../masakhanews/prompt_3/masakhanews_sna.yaml | 8 + .../masakhanews/prompt_3/masakhanews_som.yaml | 8 + .../masakhanews/prompt_3/masakhanews_swa.yaml | 8 + .../masakhanews/prompt_3/masakhanews_tir.yaml | 8 + .../masakhanews/prompt_3/masakhanews_xho.yaml | 8 + .../masakhanews/prompt_3/masakhanews_yor.yaml | 8 + .../afrobench/masakhanews/prompt_3/utils.py | 1 + .../masakhanews/prompt_4/masakhanews | 43 ++++ .../masakhanews/prompt_4/masakhanews_amh.yaml | 7 + .../masakhanews/prompt_4/masakhanews_eng.yaml | 7 + .../masakhanews/prompt_4/masakhanews_fra.yaml | 7 + .../masakhanews/prompt_4/masakhanews_hau.yaml | 7 + .../masakhanews/prompt_4/masakhanews_ibo.yaml | 7 + .../masakhanews/prompt_4/masakhanews_lin.yaml | 7 + .../masakhanews/prompt_4/masakhanews_lug.yaml | 7 + .../masakhanews/prompt_4/masakhanews_orm.yaml | 7 + .../masakhanews/prompt_4/masakhanews_pcm.yaml | 7 + .../masakhanews/prompt_4/masakhanews_run.yaml | 7 + .../masakhanews/prompt_4/masakhanews_sna.yaml | 7 + .../masakhanews/prompt_4/masakhanews_som.yaml | 7 + .../masakhanews/prompt_4/masakhanews_swa.yaml | 7 + .../masakhanews/prompt_4/masakhanews_tir.yaml | 7 + .../masakhanews/prompt_4/masakhanews_xho.yaml | 7 + .../masakhanews/prompt_4/masakhanews_yor.yaml | 7 + .../afrobench/masakhanews/prompt_4/utils.py | 1 + .../masakhanews/prompt_5/masakhanews | 43 ++++ .../masakhanews/prompt_5/masakhanews_amh.yaml | 18 ++ .../masakhanews/prompt_5/masakhanews_eng.yaml | 18 ++ .../masakhanews/prompt_5/masakhanews_fra.yaml | 18 ++ .../masakhanews/prompt_5/masakhanews_hau.yaml | 18 ++ .../masakhanews/prompt_5/masakhanews_ibo.yaml | 18 ++ .../masakhanews/prompt_5/masakhanews_lin.yaml | 18 ++ .../masakhanews/prompt_5/masakhanews_lug.yaml | 18 ++ .../masakhanews/prompt_5/masakhanews_orm.yaml | 18 ++ .../masakhanews/prompt_5/masakhanews_pcm.yaml | 18 ++ .../masakhanews/prompt_5/masakhanews_run.yaml | 18 ++ .../masakhanews/prompt_5/masakhanews_sna.yaml | 18 ++ .../masakhanews/prompt_5/masakhanews_som.yaml | 18 ++ .../masakhanews/prompt_5/masakhanews_swa.yaml | 18 ++ .../masakhanews/prompt_5/masakhanews_tir.yaml | 18 ++ .../masakhanews/prompt_5/masakhanews_xho.yaml | 18 ++ .../masakhanews/prompt_5/masakhanews_yor.yaml | 18 ++ .../afrobench/masakhanews/prompt_5/utils.py | 1 + lm_eval/tasks/afrobench/masakhanews/utils.py | 127 ++++++++++ lm_eval/tasks/afrobench/masakhapos/README.md | 75 ++++++ .../tasks/afrobench/masakhapos/gen_utils.py | 151 ++++++++++++ .../afrobench/masakhapos/masakhapos.yaml | 13 + .../masakhapos/prompt_1/masakhapos_bam.yaml | 13 + .../masakhapos/prompt_1/masakhapos_bbj.yaml | 13 + .../masakhapos/prompt_1/masakhapos_ewe.yaml | 13 + .../masakhapos/prompt_1/masakhapos_fon.yaml | 13 + .../masakhapos/prompt_1/masakhapos_hau.yaml | 13 + .../masakhapos/prompt_1/masakhapos_ibo.yaml | 13 + .../masakhapos/prompt_1/masakhapos_kin.yaml | 13 + .../masakhapos/prompt_1/masakhapos_lug.yaml | 13 + .../masakhapos/prompt_1/masakhapos_luo.yaml | 13 + .../masakhapos/prompt_1/masakhapos_mos.yaml | 13 + .../masakhapos/prompt_1/masakhapos_nya.yaml | 13 + .../masakhapos/prompt_1/masakhapos_pcm.yaml | 13 + .../masakhapos/prompt_1/masakhapos_sna.yaml | 13 + .../masakhapos/prompt_1/masakhapos_swa.yaml | 13 + .../masakhapos/prompt_1/masakhapos_tsn.yaml | 13 + .../masakhapos/prompt_1/masakhapos_twi.yaml | 13 + .../masakhapos/prompt_1/masakhapos_wol.yaml | 13 + .../masakhapos/prompt_1/masakhapos_xho.yaml | 13 + .../masakhapos/prompt_1/masakhapos_yaml | 32 +++ .../masakhapos/prompt_1/masakhapos_yor.yaml | 13 + .../masakhapos/prompt_1/masakhapos_zul.yaml | 13 + .../afrobench/masakhapos/prompt_1/utils.py | 55 +++++ .../masakhapos/prompt_2/masakhapos_bam.yaml | 14 ++ .../masakhapos/prompt_2/masakhapos_bbj.yaml | 14 ++ .../masakhapos/prompt_2/masakhapos_ewe.yaml | 14 ++ .../masakhapos/prompt_2/masakhapos_fon.yaml | 14 ++ .../masakhapos/prompt_2/masakhapos_hau.yaml | 14 ++ .../masakhapos/prompt_2/masakhapos_ibo.yaml | 14 ++ .../masakhapos/prompt_2/masakhapos_kin.yaml | 14 ++ .../masakhapos/prompt_2/masakhapos_lug.yaml | 14 ++ .../masakhapos/prompt_2/masakhapos_luo.yaml | 14 ++ .../masakhapos/prompt_2/masakhapos_mos.yaml | 14 ++ .../masakhapos/prompt_2/masakhapos_nya.yaml | 14 ++ .../masakhapos/prompt_2/masakhapos_pcm.yaml | 14 ++ .../masakhapos/prompt_2/masakhapos_sna.yaml | 14 ++ .../masakhapos/prompt_2/masakhapos_swa.yaml | 14 ++ .../masakhapos/prompt_2/masakhapos_tsn.yaml | 14 ++ .../masakhapos/prompt_2/masakhapos_twi.yaml | 14 ++ .../masakhapos/prompt_2/masakhapos_wol.yaml | 14 ++ .../masakhapos/prompt_2/masakhapos_xho.yaml | 14 ++ .../masakhapos/prompt_2/masakhapos_yaml | 32 +++ .../masakhapos/prompt_2/masakhapos_yor.yaml | 14 ++ .../masakhapos/prompt_2/masakhapos_zul.yaml | 14 ++ .../afrobench/masakhapos/prompt_2/utils.py | 55 +++++ .../masakhapos/prompt_3/masakhapos_bam.yaml | 14 ++ .../masakhapos/prompt_3/masakhapos_bbj.yaml | 14 ++ .../masakhapos/prompt_3/masakhapos_ewe.yaml | 14 ++ .../masakhapos/prompt_3/masakhapos_fon.yaml | 14 ++ .../masakhapos/prompt_3/masakhapos_hau.yaml | 14 ++ .../masakhapos/prompt_3/masakhapos_ibo.yaml | 14 ++ .../masakhapos/prompt_3/masakhapos_kin.yaml | 14 ++ .../masakhapos/prompt_3/masakhapos_lug.yaml | 14 ++ .../masakhapos/prompt_3/masakhapos_luo.yaml | 14 ++ .../masakhapos/prompt_3/masakhapos_mos.yaml | 14 ++ .../masakhapos/prompt_3/masakhapos_nya.yaml | 14 ++ .../masakhapos/prompt_3/masakhapos_pcm.yaml | 14 ++ .../masakhapos/prompt_3/masakhapos_sna.yaml | 14 ++ .../masakhapos/prompt_3/masakhapos_swa.yaml | 14 ++ .../masakhapos/prompt_3/masakhapos_tsn.yaml | 14 ++ .../masakhapos/prompt_3/masakhapos_twi.yaml | 14 ++ .../masakhapos/prompt_3/masakhapos_wol.yaml | 14 ++ .../masakhapos/prompt_3/masakhapos_xho.yaml | 14 ++ .../masakhapos/prompt_3/masakhapos_yaml | 32 +++ .../masakhapos/prompt_3/masakhapos_yor.yaml | 14 ++ .../masakhapos/prompt_3/masakhapos_zul.yaml | 14 ++ .../afrobench/masakhapos/prompt_3/utils.py | 55 +++++ .../masakhapos/prompt_4/masakhapos_bam.yaml | 13 + .../masakhapos/prompt_4/masakhapos_bbj.yaml | 13 + .../masakhapos/prompt_4/masakhapos_ewe.yaml | 13 + .../masakhapos/prompt_4/masakhapos_fon.yaml | 13 + .../masakhapos/prompt_4/masakhapos_hau.yaml | 13 + .../masakhapos/prompt_4/masakhapos_ibo.yaml | 13 + .../masakhapos/prompt_4/masakhapos_kin.yaml | 13 + .../masakhapos/prompt_4/masakhapos_lug.yaml | 13 + .../masakhapos/prompt_4/masakhapos_luo.yaml | 13 + .../masakhapos/prompt_4/masakhapos_mos.yaml | 13 + .../masakhapos/prompt_4/masakhapos_nya.yaml | 13 + .../masakhapos/prompt_4/masakhapos_pcm.yaml | 13 + .../masakhapos/prompt_4/masakhapos_sna.yaml | 13 + .../masakhapos/prompt_4/masakhapos_swa.yaml | 13 + .../masakhapos/prompt_4/masakhapos_tsn.yaml | 13 + .../masakhapos/prompt_4/masakhapos_twi.yaml | 13 + .../masakhapos/prompt_4/masakhapos_wol.yaml | 13 + .../masakhapos/prompt_4/masakhapos_xho.yaml | 13 + .../masakhapos/prompt_4/masakhapos_yaml | 32 +++ .../masakhapos/prompt_4/masakhapos_yor.yaml | 13 + .../masakhapos/prompt_4/masakhapos_zul.yaml | 13 + .../afrobench/masakhapos/prompt_4/utils.py | 55 +++++ .../masakhapos/prompt_5/masakhapos_bam.yaml | 18 ++ .../masakhapos/prompt_5/masakhapos_bbj.yaml | 18 ++ .../masakhapos/prompt_5/masakhapos_ewe.yaml | 18 ++ .../masakhapos/prompt_5/masakhapos_fon.yaml | 18 ++ .../masakhapos/prompt_5/masakhapos_hau.yaml | 18 ++ .../masakhapos/prompt_5/masakhapos_ibo.yaml | 18 ++ .../masakhapos/prompt_5/masakhapos_kin.yaml | 18 ++ .../masakhapos/prompt_5/masakhapos_lug.yaml | 18 ++ .../masakhapos/prompt_5/masakhapos_luo.yaml | 18 ++ .../masakhapos/prompt_5/masakhapos_mos.yaml | 18 ++ .../masakhapos/prompt_5/masakhapos_nya.yaml | 18 ++ .../masakhapos/prompt_5/masakhapos_pcm.yaml | 18 ++ .../masakhapos/prompt_5/masakhapos_sna.yaml | 18 ++ .../masakhapos/prompt_5/masakhapos_swa.yaml | 18 ++ .../masakhapos/prompt_5/masakhapos_tsn.yaml | 18 ++ .../masakhapos/prompt_5/masakhapos_twi.yaml | 18 ++ .../masakhapos/prompt_5/masakhapos_wol.yaml | 18 ++ .../masakhapos/prompt_5/masakhapos_xho.yaml | 18 ++ .../masakhapos/prompt_5/masakhapos_yaml | 32 +++ .../masakhapos/prompt_5/masakhapos_yor.yaml | 18 ++ .../masakhapos/prompt_5/masakhapos_zul.yaml | 18 ++ .../afrobench/masakhapos/prompt_5/utils.py | 55 +++++ lm_eval/tasks/afrobench/masakhapos/utils.py | 40 +++ lm_eval/tasks/afrobench/naijarc/README.md | 25 ++ lm_eval/tasks/afrobench/naijarc/naijarc.yaml | 13 + .../tasks/afrobench/naijarc/prompt_1/naijarc | 24 ++ .../naijarc/prompt_1/naijarc_hau.yaml | 17 ++ .../naijarc/prompt_1/naijarc_ibo.yaml | 17 ++ .../naijarc/prompt_1/naijarc_yor.yaml | 17 ++ .../tasks/afrobench/naijarc/prompt_2/naijarc | 23 ++ .../naijarc/prompt_2/naijarc_hau.yaml | 17 ++ .../naijarc/prompt_2/naijarc_ibo.yaml | 17 ++ .../naijarc/prompt_2/naijarc_yor.yaml | 17 ++ .../tasks/afrobench/naijarc/prompt_3/naijarc | 23 ++ .../naijarc/prompt_3/naijarc_hau.yaml | 17 ++ .../naijarc/prompt_3/naijarc_ibo.yaml | 17 ++ .../naijarc/prompt_3/naijarc_yor.yaml | 17 ++ .../tasks/afrobench/naijarc/prompt_4/naijarc | 23 ++ .../naijarc/prompt_4/naijarc_hau.yaml | 21 ++ .../naijarc/prompt_4/naijarc_ibo.yaml | 21 ++ .../naijarc/prompt_4/naijarc_yor.yaml | 21 ++ .../tasks/afrobench/naijarc/prompt_5/naijarc | 23 ++ .../naijarc/prompt_5/naijarc_hau.yaml | 19 ++ .../naijarc/prompt_5/naijarc_ibo.yaml | 19 ++ .../naijarc/prompt_5/naijarc_yor.yaml | 19 ++ lm_eval/tasks/afrobench/naijarc/utils.py | 93 +++++++ lm_eval/tasks/afrobench/nollysenti/README.md | 35 +++ .../afrobench/nollysenti/nollysenti.yaml | 13 + .../afrobench/nollysenti/prompt_1/nollysenti | 38 +++ .../nollysenti/prompt_1/nollysenti_eng.yaml | 3 + .../nollysenti/prompt_1/nollysenti_hau.yaml | 3 + .../nollysenti/prompt_1/nollysenti_ibo.yaml | 3 + .../nollysenti/prompt_1/nollysenti_pcm.yaml | 3 + .../nollysenti/prompt_1/nollysenti_yor.yaml | 3 + .../afrobench/nollysenti/prompt_1/utils.py | 1 + .../afrobench/nollysenti/prompt_2/nollysenti | 37 +++ .../nollysenti/prompt_2/nollysenti_eng.yaml | 4 + .../nollysenti/prompt_2/nollysenti_hau.yaml | 4 + .../nollysenti/prompt_2/nollysenti_ibo.yaml | 4 + .../nollysenti/prompt_2/nollysenti_pcm.yaml | 4 + .../nollysenti/prompt_2/nollysenti_yor.yaml | 4 + .../afrobench/nollysenti/prompt_2/utils.py | 1 + .../afrobench/nollysenti/prompt_3/nollysenti | 37 +++ .../nollysenti/prompt_3/nollysenti_eng.yaml | 7 + .../nollysenti/prompt_3/nollysenti_hau.yaml | 7 + .../nollysenti/prompt_3/nollysenti_ibo.yaml | 7 + .../nollysenti/prompt_3/nollysenti_pcm.yaml | 7 + .../nollysenti/prompt_3/nollysenti_yor.yaml | 7 + .../afrobench/nollysenti/prompt_3/utils.py | 1 + .../afrobench/nollysenti/prompt_4/nollysenti | 37 +++ .../nollysenti/prompt_4/nollysenti_eng.yaml | 6 + .../nollysenti/prompt_4/nollysenti_hau.yaml | 6 + .../nollysenti/prompt_4/nollysenti_ibo.yaml | 6 + .../nollysenti/prompt_4/nollysenti_pcm.yaml | 6 + .../nollysenti/prompt_4/nollysenti_yor.yaml | 6 + .../afrobench/nollysenti/prompt_4/utils.py | 1 + .../afrobench/nollysenti/prompt_5/nollysenti | 37 +++ .../nollysenti/prompt_5/nollysenti_eng.yaml | 12 + .../nollysenti/prompt_5/nollysenti_hau.yaml | 12 + .../nollysenti/prompt_5/nollysenti_ibo.yaml | 12 + .../nollysenti/prompt_5/nollysenti_pcm.yaml | 12 + .../nollysenti/prompt_5/nollysenti_yor.yaml | 12 + .../afrobench/nollysenti/prompt_5/utils.py | 1 + lm_eval/tasks/afrobench/ntrex/README.md | 38 +++ lm_eval/tasks/afrobench/ntrex/gen_utils.py | 171 +++++++++++++ lm_eval/tasks/afrobench/ntrex/ntrex.yaml | 14 ++ .../ntrex/prompt_1/african-english/ntrex | 26 ++ .../ntrex_afr_Latn-eng_Latn.yaml | 6 + .../ntrex_amh_Ethi-eng_Latn.yaml | 6 + .../ntrex_arb_Arab-eng_Latn.yaml | 6 + .../ntrex_bem_Latn-eng_Latn.yaml | 6 + .../ntrex_ewe_Latn-eng_Latn.yaml | 6 + .../ntrex_fra_Latn-eng_Latn.yaml | 6 + .../ntrex_hau_Latn-eng_Latn.yaml | 6 + .../ntrex_ibo_Latn-eng_Latn.yaml | 6 + .../ntrex_kin_Latn-eng_Latn.yaml | 6 + .../ntrex_mey_Arab-eng_Latn.yaml | 6 + .../ntrex_mlg_Latn-eng_Latn.yaml | 6 + .../ntrex_msa_Latn-eng_Latn.yaml | 6 + .../ntrex_nde_Latn-eng_Latn.yaml | 6 + .../ntrex_nso_Latn-eng_Latn.yaml | 6 + .../ntrex_nya_Latn-eng_Latn.yaml | 6 + .../ntrex_orm_Ethi-eng_Latn.yaml | 6 + .../ntrex_shi_Arab-eng_Latn.yaml | 6 + .../ntrex_sna_Latn-eng_Latn.yaml | 6 + .../ntrex_som_Latn-eng_Latn.yaml | 6 + .../ntrex_ssw_Latn-eng_Latn.yaml | 6 + .../ntrex_swa_Latn-eng_Latn.yaml | 6 + .../ntrex_tam_Taml-eng_Latn.yaml | 6 + .../ntrex_tel_Telu-eng_Latn.yaml | 6 + .../ntrex_tir_Ethi-eng_Latn.yaml | 6 + .../ntrex_ton_Latn-eng_Latn.yaml | 6 + .../ntrex_tsn_Latn-eng_Latn.yaml | 6 + .../ntrex_urd_Arab-eng_Latn.yaml | 6 + .../ntrex_ven_Latn-eng_Latn.yaml | 6 + .../ntrex_wol_Latn-eng_Latn.yaml | 6 + .../ntrex_xho_Latn-eng_Latn.yaml | 6 + .../ntrex_yor_Latn-eng_Latn.yaml | 6 + .../ntrex_zul_Latn-eng_Latn.yaml | 6 + .../ntrex/prompt_1/english-african/ntrex | 26 ++ .../ntrex_eng_Latn-afr_Latn.yaml | 6 + .../ntrex_eng_Latn-amh_Ethi.yaml | 6 + .../ntrex_eng_Latn-arb_Arab.yaml | 6 + .../ntrex_eng_Latn-bem_Latn.yaml | 6 + .../ntrex_eng_Latn-ewe_Latn.yaml | 6 + .../ntrex_eng_Latn-fra_Latn.yaml | 6 + .../ntrex_eng_Latn-hau_Latn.yaml | 6 + .../ntrex_eng_Latn-ibo_Latn.yaml | 6 + .../ntrex_eng_Latn-kin_Latn.yaml | 6 + .../ntrex_eng_Latn-mey_Arab.yaml | 6 + .../ntrex_eng_Latn-mlg_Latn.yaml | 6 + .../ntrex_eng_Latn-msa_Latn.yaml | 6 + .../ntrex_eng_Latn-nde_Latn.yaml | 6 + .../ntrex_eng_Latn-nso_Latn.yaml | 6 + .../ntrex_eng_Latn-nya_Latn.yaml | 6 + .../ntrex_eng_Latn-orm_Ethi.yaml | 6 + .../ntrex_eng_Latn-shi_Arab.yaml | 6 + .../ntrex_eng_Latn-sna_Latn.yaml | 6 + .../ntrex_eng_Latn-som_Latn.yaml | 6 + .../ntrex_eng_Latn-ssw_Latn.yaml | 6 + .../ntrex_eng_Latn-swa_Latn.yaml | 6 + .../ntrex_eng_Latn-tam_Taml.yaml | 6 + .../ntrex_eng_Latn-tel_Telu.yaml | 6 + .../ntrex_eng_Latn-tir_Ethi.yaml | 6 + .../ntrex_eng_Latn-ton_Latn.yaml | 6 + .../ntrex_eng_Latn-tsn_Latn.yaml | 6 + .../ntrex_eng_Latn-urd_Arab.yaml | 6 + .../ntrex_eng_Latn-ven_Latn.yaml | 6 + .../ntrex_eng_Latn-wol_Latn.yaml | 6 + .../ntrex_eng_Latn-xho_Latn.yaml | 6 + .../ntrex_eng_Latn-yor_Latn.yaml | 6 + .../ntrex_eng_Latn-zul_Latn.yaml | 6 + .../ntrex/prompt_2/african-english/ntrex | 25 ++ .../ntrex_afr_Latn-eng_Latn.yaml | 7 + .../ntrex_amh_Ethi-eng_Latn.yaml | 7 + .../ntrex_arb_Arab-eng_Latn.yaml | 7 + .../ntrex_bem_Latn-eng_Latn.yaml | 7 + .../ntrex_ewe_Latn-eng_Latn.yaml | 7 + .../ntrex_fra_Latn-eng_Latn.yaml | 7 + .../ntrex_hau_Latn-eng_Latn.yaml | 7 + .../ntrex_ibo_Latn-eng_Latn.yaml | 7 + .../ntrex_kin_Latn-eng_Latn.yaml | 7 + .../ntrex_mey_Arab-eng_Latn.yaml | 7 + .../ntrex_mlg_Latn-eng_Latn.yaml | 7 + .../ntrex_msa_Latn-eng_Latn.yaml | 7 + .../ntrex_nde_Latn-eng_Latn.yaml | 7 + .../ntrex_nso_Latn-eng_Latn.yaml | 7 + .../ntrex_nya_Latn-eng_Latn.yaml | 7 + .../ntrex_orm_Ethi-eng_Latn.yaml | 7 + .../ntrex_shi_Arab-eng_Latn.yaml | 7 + .../ntrex_sna_Latn-eng_Latn.yaml | 7 + .../ntrex_som_Latn-eng_Latn.yaml | 7 + .../ntrex_ssw_Latn-eng_Latn.yaml | 7 + .../ntrex_swa_Latn-eng_Latn.yaml | 7 + .../ntrex_tam_Taml-eng_Latn.yaml | 7 + .../ntrex_tel_Telu-eng_Latn.yaml | 7 + .../ntrex_tir_Ethi-eng_Latn.yaml | 7 + .../ntrex_ton_Latn-eng_Latn.yaml | 7 + .../ntrex_tsn_Latn-eng_Latn.yaml | 7 + .../ntrex_urd_Arab-eng_Latn.yaml | 7 + .../ntrex_ven_Latn-eng_Latn.yaml | 7 + .../ntrex_wol_Latn-eng_Latn.yaml | 7 + .../ntrex_xho_Latn-eng_Latn.yaml | 7 + .../ntrex_yor_Latn-eng_Latn.yaml | 7 + .../ntrex_zul_Latn-eng_Latn.yaml | 7 + .../ntrex/prompt_2/english-african/ntrex | 25 ++ .../ntrex_eng_Latn-afr_Latn.yaml | 7 + .../ntrex_eng_Latn-amh_Ethi.yaml | 7 + .../ntrex_eng_Latn-arb_Arab.yaml | 7 + .../ntrex_eng_Latn-bem_Latn.yaml | 7 + .../ntrex_eng_Latn-ewe_Latn.yaml | 7 + .../ntrex_eng_Latn-fra_Latn.yaml | 7 + .../ntrex_eng_Latn-hau_Latn.yaml | 7 + .../ntrex_eng_Latn-ibo_Latn.yaml | 7 + .../ntrex_eng_Latn-kin_Latn.yaml | 7 + .../ntrex_eng_Latn-mey_Arab.yaml | 7 + .../ntrex_eng_Latn-mlg_Latn.yaml | 7 + .../ntrex_eng_Latn-msa_Latn.yaml | 7 + .../ntrex_eng_Latn-nde_Latn.yaml | 7 + .../ntrex_eng_Latn-nso_Latn.yaml | 7 + .../ntrex_eng_Latn-nya_Latn.yaml | 7 + .../ntrex_eng_Latn-orm_Ethi.yaml | 7 + .../ntrex_eng_Latn-shi_Arab.yaml | 7 + .../ntrex_eng_Latn-sna_Latn.yaml | 7 + .../ntrex_eng_Latn-som_Latn.yaml | 7 + .../ntrex_eng_Latn-ssw_Latn.yaml | 7 + .../ntrex_eng_Latn-swa_Latn.yaml | 7 + .../ntrex_eng_Latn-tam_Taml.yaml | 7 + .../ntrex_eng_Latn-tel_Telu.yaml | 7 + .../ntrex_eng_Latn-tir_Ethi.yaml | 7 + .../ntrex_eng_Latn-ton_Latn.yaml | 7 + .../ntrex_eng_Latn-tsn_Latn.yaml | 7 + .../ntrex_eng_Latn-urd_Arab.yaml | 7 + .../ntrex_eng_Latn-ven_Latn.yaml | 7 + .../ntrex_eng_Latn-wol_Latn.yaml | 7 + .../ntrex_eng_Latn-xho_Latn.yaml | 7 + .../ntrex_eng_Latn-yor_Latn.yaml | 7 + .../ntrex_eng_Latn-zul_Latn.yaml | 7 + .../ntrex/prompt_3/african-english/ntrex | 25 ++ .../ntrex_afr_Latn-eng_Latn.yaml | 7 + .../ntrex_amh_Ethi-eng_Latn.yaml | 7 + .../ntrex_arb_Arab-eng_Latn.yaml | 7 + .../ntrex_bem_Latn-eng_Latn.yaml | 7 + .../ntrex_ewe_Latn-eng_Latn.yaml | 7 + .../ntrex_fra_Latn-eng_Latn.yaml | 7 + .../ntrex_hau_Latn-eng_Latn.yaml | 7 + .../ntrex_ibo_Latn-eng_Latn.yaml | 7 + .../ntrex_kin_Latn-eng_Latn.yaml | 7 + .../ntrex_mey_Arab-eng_Latn.yaml | 8 + .../ntrex_mlg_Latn-eng_Latn.yaml | 7 + .../ntrex_msa_Latn-eng_Latn.yaml | 7 + .../ntrex_nde_Latn-eng_Latn.yaml | 7 + .../ntrex_nso_Latn-eng_Latn.yaml | 7 + .../ntrex_nya_Latn-eng_Latn.yaml | 7 + .../ntrex_orm_Ethi-eng_Latn.yaml | 7 + .../ntrex_shi_Arab-eng_Latn.yaml | 7 + .../ntrex_sna_Latn-eng_Latn.yaml | 7 + .../ntrex_som_Latn-eng_Latn.yaml | 7 + .../ntrex_ssw_Latn-eng_Latn.yaml | 7 + .../ntrex_swa_Latn-eng_Latn.yaml | 7 + .../ntrex_tam_Taml-eng_Latn.yaml | 7 + .../ntrex_tel_Telu-eng_Latn.yaml | 7 + .../ntrex_tir_Ethi-eng_Latn.yaml | 7 + .../ntrex_ton_Latn-eng_Latn.yaml | 7 + .../ntrex_tsn_Latn-eng_Latn.yaml | 7 + .../ntrex_urd_Arab-eng_Latn.yaml | 7 + .../ntrex_ven_Latn-eng_Latn.yaml | 7 + .../ntrex_wol_Latn-eng_Latn.yaml | 7 + .../ntrex_xho_Latn-eng_Latn.yaml | 7 + .../ntrex_yor_Latn-eng_Latn.yaml | 7 + .../ntrex_zul_Latn-eng_Latn.yaml | 7 + .../ntrex/prompt_3/english-african/ntrex | 25 ++ .../ntrex_eng_Latn-afr_Latn.yaml | 7 + .../ntrex_eng_Latn-amh_Ethi.yaml | 7 + .../ntrex_eng_Latn-arb_Arab.yaml | 7 + .../ntrex_eng_Latn-bem_Latn.yaml | 7 + .../ntrex_eng_Latn-ewe_Latn.yaml | 7 + .../ntrex_eng_Latn-fra_Latn.yaml | 7 + .../ntrex_eng_Latn-hau_Latn.yaml | 7 + .../ntrex_eng_Latn-ibo_Latn.yaml | 7 + .../ntrex_eng_Latn-kin_Latn.yaml | 7 + .../ntrex_eng_Latn-mey_Arab.yaml | 8 + .../ntrex_eng_Latn-mlg_Latn.yaml | 7 + .../ntrex_eng_Latn-msa_Latn.yaml | 7 + .../ntrex_eng_Latn-nde_Latn.yaml | 7 + .../ntrex_eng_Latn-nso_Latn.yaml | 7 + .../ntrex_eng_Latn-nya_Latn.yaml | 7 + .../ntrex_eng_Latn-orm_Ethi.yaml | 7 + .../ntrex_eng_Latn-shi_Arab.yaml | 7 + .../ntrex_eng_Latn-sna_Latn.yaml | 7 + .../ntrex_eng_Latn-som_Latn.yaml | 7 + .../ntrex_eng_Latn-ssw_Latn.yaml | 7 + .../ntrex_eng_Latn-swa_Latn.yaml | 7 + .../ntrex_eng_Latn-tam_Taml.yaml | 7 + .../ntrex_eng_Latn-tel_Telu.yaml | 7 + .../ntrex_eng_Latn-tir_Ethi.yaml | 7 + .../ntrex_eng_Latn-ton_Latn.yaml | 7 + .../ntrex_eng_Latn-tsn_Latn.yaml | 7 + .../ntrex_eng_Latn-urd_Arab.yaml | 7 + .../ntrex_eng_Latn-ven_Latn.yaml | 7 + .../ntrex_eng_Latn-wol_Latn.yaml | 7 + .../ntrex_eng_Latn-xho_Latn.yaml | 7 + .../ntrex_eng_Latn-yor_Latn.yaml | 7 + .../ntrex_eng_Latn-zul_Latn.yaml | 7 + lm_eval/tasks/afrobench/openai_mmlu/README.md | 25 ++ .../afrobench/openai_mmlu/openai_mmlu.yaml | 13 + .../openai_mmlu/prompt_1/openai_mmlu | 22 ++ .../openai_mmlu/prompt_1/openai_mmlu_ara.yaml | 15 ++ .../openai_mmlu/prompt_1/openai_mmlu_swa.yaml | 15 ++ .../openai_mmlu/prompt_1/openai_mmlu_yor.yaml | 15 ++ .../openai_mmlu/prompt_2/openai_mmlu | 22 ++ .../openai_mmlu/prompt_2/openai_mmlu_ara.yaml | 15 ++ .../openai_mmlu/prompt_2/openai_mmlu_swa.yaml | 15 ++ .../openai_mmlu/prompt_2/openai_mmlu_yor.yaml | 15 ++ .../openai_mmlu/prompt_3/openai_mmlu | 23 ++ .../openai_mmlu/prompt_3/openai_mmlu_ara.yaml | 15 ++ .../openai_mmlu/prompt_3/openai_mmlu_swa.yaml | 15 ++ .../openai_mmlu/prompt_3/openai_mmlu_yor.yaml | 15 ++ .../openai_mmlu/prompt_4/openai_mmlu | 23 ++ .../openai_mmlu/prompt_4/openai_mmlu_ara.yaml | 18 ++ .../openai_mmlu/prompt_4/openai_mmlu_swa.yaml | 18 ++ .../openai_mmlu/prompt_4/openai_mmlu_yor.yaml | 18 ++ .../openai_mmlu/prompt_5/openai_mmlu | 23 ++ .../openai_mmlu/prompt_5/openai_mmlu_ara.yaml | 17 ++ .../openai_mmlu/prompt_5/openai_mmlu_swa.yaml | 17 ++ .../openai_mmlu/prompt_5/openai_mmlu_yor.yaml | 17 ++ lm_eval/tasks/afrobench/openai_mmlu/utils.py | 99 ++++++++ lm_eval/tasks/afrobench/salt/README.md | 17 ++ lm_eval/tasks/afrobench/salt/gen_utils.py | 149 ++++++++++++ lm_eval/tasks/afrobench/salt/prompt_1/salt | 24 ++ .../afrobench/salt/prompt_1/salt_ach-eng.yaml | 6 + .../afrobench/salt/prompt_1/salt_eng-ach.yaml | 6 + .../afrobench/salt/prompt_1/salt_eng-ibo.yaml | 6 + .../afrobench/salt/prompt_1/salt_eng-lgg.yaml | 6 + .../afrobench/salt/prompt_1/salt_eng-lug.yaml | 6 + .../afrobench/salt/prompt_1/salt_eng-nyn.yaml | 6 + .../afrobench/salt/prompt_1/salt_eng-swa.yaml | 6 + .../afrobench/salt/prompt_1/salt_eng-teo.yaml | 6 + .../afrobench/salt/prompt_1/salt_ibo-eng.yaml | 6 + .../afrobench/salt/prompt_1/salt_lgg-eng.yaml | 6 + .../afrobench/salt/prompt_1/salt_lug-eng.yaml | 6 + .../afrobench/salt/prompt_1/salt_nyn-eng.yaml | 6 + .../afrobench/salt/prompt_1/salt_swa-eng.yaml | 6 + .../afrobench/salt/prompt_1/salt_teo-eng.yaml | 6 + lm_eval/tasks/afrobench/salt/prompt_2/salt | 24 ++ .../afrobench/salt/prompt_2/salt_ach-eng.yaml | 7 + .../afrobench/salt/prompt_2/salt_eng-ach.yaml | 7 + .../afrobench/salt/prompt_2/salt_eng-ibo.yaml | 7 + .../afrobench/salt/prompt_2/salt_eng-lgg.yaml | 7 + .../afrobench/salt/prompt_2/salt_eng-lug.yaml | 7 + .../afrobench/salt/prompt_2/salt_eng-nyn.yaml | 7 + .../afrobench/salt/prompt_2/salt_eng-swa.yaml | 7 + .../afrobench/salt/prompt_2/salt_eng-teo.yaml | 7 + .../afrobench/salt/prompt_2/salt_ibo-eng.yaml | 7 + .../afrobench/salt/prompt_2/salt_lgg-eng.yaml | 7 + .../afrobench/salt/prompt_2/salt_lug-eng.yaml | 7 + .../afrobench/salt/prompt_2/salt_nyn-eng.yaml | 7 + .../afrobench/salt/prompt_2/salt_swa-eng.yaml | 7 + .../afrobench/salt/prompt_2/salt_teo-eng.yaml | 7 + lm_eval/tasks/afrobench/salt/prompt_3/salt | 24 ++ .../afrobench/salt/prompt_3/salt_ach-eng.yaml | 7 + .../afrobench/salt/prompt_3/salt_eng-ach.yaml | 7 + .../afrobench/salt/prompt_3/salt_eng-ibo.yaml | 7 + .../afrobench/salt/prompt_3/salt_eng-lgg.yaml | 7 + .../afrobench/salt/prompt_3/salt_eng-lug.yaml | 7 + .../afrobench/salt/prompt_3/salt_eng-nyn.yaml | 8 + .../afrobench/salt/prompt_3/salt_eng-swa.yaml | 7 + .../afrobench/salt/prompt_3/salt_eng-teo.yaml | 7 + .../afrobench/salt/prompt_3/salt_ibo-eng.yaml | 7 + .../afrobench/salt/prompt_3/salt_lgg-eng.yaml | 7 + .../afrobench/salt/prompt_3/salt_lug-eng.yaml | 7 + .../afrobench/salt/prompt_3/salt_nyn-eng.yaml | 7 + .../afrobench/salt/prompt_3/salt_swa-eng.yaml | 7 + .../afrobench/salt/prompt_3/salt_teo-eng.yaml | 7 + lm_eval/tasks/afrobench/salt/salt.yaml | 11 + .../sample_run_scripts/run_afrobench.sh | 32 +++ .../sample_run_scripts/run_afrobench_lite.sh | 32 +++ lm_eval/tasks/afrobench/sib/README.md | 37 +++ lm_eval/tasks/afrobench/sib/prompt_1/sib | 43 ++++ .../tasks/afrobench/sib/prompt_1/sib_aeb.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_afr.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_aka.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_amh.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_ary.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_arz.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_bam.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_bem.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_cjk.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_dik.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_dyu.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_eng.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_ewe.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_fon.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_fra.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_fuv.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_gaz.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_hau.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_ibo.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_kab.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_kam.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_kbp.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_kea.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_kik.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_kin.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_kmb.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_knc.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_kon.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_lin.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_lua.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_lug.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_luo.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_mos.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_nso.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_nus.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_nya.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_plt.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_por.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_run.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_sag.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_sna.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_som.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_sot.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_ssw.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_swa.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_taq.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_tir.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_tso.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_tum.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_twi.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_tzm.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_umb.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_wol.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_xho.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_yor.yaml | 7 + .../tasks/afrobench/sib/prompt_1/sib_zul.yaml | 7 + lm_eval/tasks/afrobench/sib/prompt_1/utils.py | 1 + lm_eval/tasks/afrobench/sib/prompt_2/sib | 43 ++++ .../tasks/afrobench/sib/prompt_2/sib_aeb.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_afr.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_aka.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_amh.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_ary.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_arz.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_bam.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_bem.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_cjk.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_dik.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_dyu.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_eng.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_ewe.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_fon.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_fra.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_fuv.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_gaz.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_hau.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_ibo.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_kab.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_kam.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_kbp.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_kea.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_kik.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_kin.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_kmb.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_knc.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_kon.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_lin.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_lua.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_lug.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_luo.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_mos.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_nso.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_nus.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_nya.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_plt.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_por.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_run.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_sag.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_sna.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_som.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_sot.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_ssw.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_swa.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_taq.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_tir.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_tso.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_tum.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_twi.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_tzm.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_umb.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_wol.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_xho.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_yor.yaml | 10 + .../tasks/afrobench/sib/prompt_2/sib_zul.yaml | 10 + lm_eval/tasks/afrobench/sib/prompt_2/utils.py | 1 + lm_eval/tasks/afrobench/sib/prompt_3/sib | 43 ++++ .../tasks/afrobench/sib/prompt_3/sib_aeb.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_afr.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_aka.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_amh.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_ary.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_arz.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_bam.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_bem.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_cjk.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_dik.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_dyu.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_eng.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_ewe.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_fon.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_fra.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_fuv.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_gaz.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_hau.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_ibo.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_kab.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_kam.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_kbp.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_kea.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_kik.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_kin.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_kmb.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_knc.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_kon.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_lin.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_lua.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_lug.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_luo.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_mos.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_nso.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_nus.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_nya.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_plt.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_por.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_run.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_sag.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_sna.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_som.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_sot.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_ssw.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_swa.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_taq.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_tir.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_tso.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_tum.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_twi.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_tzm.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_umb.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_wol.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_xho.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_yor.yaml | 8 + .../tasks/afrobench/sib/prompt_3/sib_zul.yaml | 8 + lm_eval/tasks/afrobench/sib/prompt_3/utils.py | 1 + lm_eval/tasks/afrobench/sib/prompt_4/sib | 43 ++++ .../tasks/afrobench/sib/prompt_4/sib_aeb.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_afr.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_aka.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_amh.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_ary.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_arz.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_bam.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_bem.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_cjk.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_dik.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_dyu.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_eng.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_ewe.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_fon.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_fra.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_fuv.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_gaz.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_hau.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_ibo.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_kab.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_kam.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_kbp.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_kea.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_kik.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_kin.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_kmb.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_knc.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_kon.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_lin.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_lua.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_lug.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_luo.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_mos.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_nso.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_nus.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_nya.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_plt.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_por.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_run.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_sag.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_sna.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_som.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_sot.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_ssw.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_swa.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_taq.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_tir.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_tso.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_tum.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_twi.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_tzm.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_umb.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_wol.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_xho.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_yor.yaml | 7 + .../tasks/afrobench/sib/prompt_4/sib_zul.yaml | 7 + lm_eval/tasks/afrobench/sib/prompt_4/utils.py | 1 + lm_eval/tasks/afrobench/sib/prompt_5/sib | 43 ++++ .../tasks/afrobench/sib/prompt_5/sib_aeb.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_afr.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_aka.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_amh.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_ary.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_arz.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_bam.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_bem.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_cjk.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_dik.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_dyu.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_eng.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_ewe.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_fon.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_fra.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_fuv.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_gaz.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_hau.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_ibo.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_kab.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_kam.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_kbp.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_kea.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_kik.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_kin.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_kmb.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_knc.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_kon.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_lin.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_lua.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_lug.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_luo.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_mos.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_nso.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_nus.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_nya.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_plt.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_por.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_run.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_sag.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_sna.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_som.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_sot.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_ssw.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_swa.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_taq.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_tir.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_tso.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_tum.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_twi.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_tzm.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_umb.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_wol.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_xho.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_yor.yaml | 18 ++ .../tasks/afrobench/sib/prompt_5/sib_zul.yaml | 18 ++ lm_eval/tasks/afrobench/sib/prompt_5/utils.py | 1 + lm_eval/tasks/afrobench/sib/sib.yaml | 13 + lm_eval/tasks/afrobench/sib/utils.py | 227 ++++++++++++++++++ .../tasks/afrobench/uhura-arc-easy/README.md | 25 ++ .../uhura-arc-easy/prompt_1/uhura-arc-easy | 39 +++ .../prompt_1/uhura-arc-easy_am.yaml | 8 + .../prompt_1/uhura-arc-easy_en.yaml | 8 + .../prompt_1/uhura-arc-easy_ha.yaml | 8 + .../prompt_1/uhura-arc-easy_nso.yaml | 9 + .../prompt_1/uhura-arc-easy_sw.yaml | 8 + .../prompt_1/uhura-arc-easy_yo.yaml | 8 + .../prompt_1/uhura-arc-easy_zu.yaml | 9 + .../uhura-arc-easy/prompt_1/utils.py | 1 + .../uhura-arc-easy/prompt_2/uhura-arc-easy | 38 +++ .../prompt_2/uhura-arc-easy_am.yaml | 7 + .../prompt_2/uhura-arc-easy_en.yaml | 7 + .../prompt_2/uhura-arc-easy_ha.yaml | 7 + .../prompt_2/uhura-arc-easy_nso.yaml | 8 + .../prompt_2/uhura-arc-easy_sw.yaml | 7 + .../prompt_2/uhura-arc-easy_yo.yaml | 7 + .../prompt_2/uhura-arc-easy_zu.yaml | 8 + .../uhura-arc-easy/prompt_2/utils.py | 1 + .../uhura-arc-easy/prompt_3/uhura-arc-easy | 38 +++ .../prompt_3/uhura-arc-easy_am.yaml | 7 + .../prompt_3/uhura-arc-easy_en.yaml | 7 + .../prompt_3/uhura-arc-easy_ha.yaml | 7 + .../prompt_3/uhura-arc-easy_nso.yaml | 8 + .../prompt_3/uhura-arc-easy_sw.yaml | 7 + .../prompt_3/uhura-arc-easy_yo.yaml | 7 + .../prompt_3/uhura-arc-easy_zu.yaml | 8 + .../uhura-arc-easy/prompt_3/utils.py | 1 + .../uhura-arc-easy/prompt_4/uhura-arc-easy | 38 +++ .../prompt_4/uhura-arc-easy_am.yaml | 6 + .../prompt_4/uhura-arc-easy_en.yaml | 6 + .../prompt_4/uhura-arc-easy_ha.yaml | 6 + .../prompt_4/uhura-arc-easy_nso.yaml | 7 + .../prompt_4/uhura-arc-easy_sw.yaml | 6 + .../prompt_4/uhura-arc-easy_yo.yaml | 6 + .../prompt_4/uhura-arc-easy_zu.yaml | 7 + .../uhura-arc-easy/prompt_4/utils.py | 1 + .../uhura-arc-easy/prompt_5/uhura-arc-easy | 38 +++ .../prompt_5/uhura-arc-easy_am.yaml | 7 + .../prompt_5/uhura-arc-easy_en.yaml | 7 + .../prompt_5/uhura-arc-easy_ha.yaml | 7 + .../prompt_5/uhura-arc-easy_nso.yaml | 8 + .../prompt_5/uhura-arc-easy_sw.yaml | 7 + .../prompt_5/uhura-arc-easy_yo.yaml | 7 + .../prompt_5/uhura-arc-easy_zu.yaml | 8 + .../uhura-arc-easy/prompt_5/utils.py | 1 + .../tasks/afrobench/uhura-arc-easy/uhura.yaml | 13 + .../tasks/afrobench/uhura-arc-easy/utils.py | 129 ++++++++++ lm_eval/tasks/afrobench/xlsum/README.md | 34 +++ .../tasks/afrobench/xlsum/prompt_1/utils.py | 18 ++ lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum | 22 ++ .../xlsum/prompt_1/xlsum_amharic.yaml | 9 + .../xlsum/prompt_1/xlsum_arabic.yaml | 9 + .../afrobench/xlsum/prompt_1/xlsum_hausa.yaml | 9 + .../afrobench/xlsum/prompt_1/xlsum_igbo.yaml | 9 + .../xlsum/prompt_1/xlsum_kirundi.yaml | 9 + .../afrobench/xlsum/prompt_1/xlsum_oromo.yaml | 9 + .../xlsum/prompt_1/xlsum_pidgin.yaml | 9 + .../xlsum/prompt_1/xlsum_somali.yaml | 9 + .../xlsum/prompt_1/xlsum_swahili.yaml | 9 + .../xlsum/prompt_1/xlsum_telugu.yaml | 9 + .../xlsum/prompt_1/xlsum_tigrinya.yaml | 9 + .../xlsum/prompt_1/xlsum_yoruba.yaml | 9 + .../tasks/afrobench/xlsum/prompt_2/utils.py | 18 ++ lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum | 22 ++ .../xlsum/prompt_2/xlsum_amharic.yaml | 9 + .../xlsum/prompt_2/xlsum_arabic.yaml | 9 + .../afrobench/xlsum/prompt_2/xlsum_hausa.yaml | 9 + .../afrobench/xlsum/prompt_2/xlsum_igbo.yaml | 9 + .../xlsum/prompt_2/xlsum_kirundi.yaml | 9 + .../afrobench/xlsum/prompt_2/xlsum_oromo.yaml | 9 + .../xlsum/prompt_2/xlsum_pidgin.yaml | 9 + .../xlsum/prompt_2/xlsum_somali.yaml | 9 + .../xlsum/prompt_2/xlsum_swahili.yaml | 9 + .../xlsum/prompt_2/xlsum_telugu.yaml | 9 + .../xlsum/prompt_2/xlsum_tigrinya.yaml | 9 + .../xlsum/prompt_2/xlsum_yoruba.yaml | 9 + .../tasks/afrobench/xlsum/prompt_3/utils.py | 18 ++ lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum | 22 ++ .../xlsum/prompt_3/xlsum_amharic.yaml | 10 + .../xlsum/prompt_3/xlsum_arabic.yaml | 10 + .../afrobench/xlsum/prompt_3/xlsum_hausa.yaml | 10 + .../afrobench/xlsum/prompt_3/xlsum_igbo.yaml | 10 + .../xlsum/prompt_3/xlsum_kirundi.yaml | 10 + .../afrobench/xlsum/prompt_3/xlsum_oromo.yaml | 10 + .../xlsum/prompt_3/xlsum_pidgin.yaml | 10 + .../xlsum/prompt_3/xlsum_somali.yaml | 10 + .../xlsum/prompt_3/xlsum_swahili.yaml | 10 + .../xlsum/prompt_3/xlsum_telugu.yaml | 10 + .../xlsum/prompt_3/xlsum_tigrinya.yaml | 10 + .../xlsum/prompt_3/xlsum_yoruba.yaml | 10 + lm_eval/tasks/afrobench/xlsum/utils.py | 118 +++++++++ lm_eval/tasks/afrobench/xlsum/xlsum.yaml | 11 + 2895 files changed, 32371 insertions(+), 969 deletions(-) create mode 100644 lm_eval/tasks/afrimgsm/direct/afrimgsm.yaml delete mode 100644 lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_amh.yaml delete mode 100644 lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_eng.yaml delete mode 100644 lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_ewe.yaml delete mode 100644 lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_fra.yaml delete mode 100644 lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_hau.yaml delete mode 100644 lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_ibo.yaml delete mode 100644 lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_kin.yaml delete mode 100644 lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_lin.yaml delete mode 100644 lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_lug.yaml delete mode 100644 lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_orm.yaml delete mode 100644 lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_sna.yaml delete mode 100644 lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_sot.yaml delete mode 100644 lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_swa.yaml delete mode 100644 lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_twi.yaml delete mode 100644 lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_wol.yaml delete mode 100644 lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_xho.yaml delete mode 100644 lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_yor.yaml delete mode 100644 lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_zul.yaml delete mode 100644 lm_eval/tasks/afrimgsm/direct/direct_yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_amh.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_eng.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_ewe.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_fra.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_hau.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_ibo.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_kin.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_lin.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_lug.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_orm.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_sna.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_sot.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_swa.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_twi.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_vai.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_wol.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_xho.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_yor.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_zul.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_amh.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_eng.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_ewe.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_fra.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_hau.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_ibo.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_kin.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_lin.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_lug.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_orm.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_sna.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_sot.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_swa.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_twi.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_vai.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_wol.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_xho.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_yor.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_zul.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_amh.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_eng.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_ewe.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_fra.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_hau.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_ibo.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_kin.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_lin.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_lug.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_orm.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_sna.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_sot.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_swa.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_twi.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_vai.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_wol.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_xho.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_yor.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_zul.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_amh.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_eng.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_ewe.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_fra.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_hau.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_ibo.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_kin.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_lin.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_lug.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_orm.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_sna.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_sot.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_swa.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_twi.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_vai.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_wol.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_xho.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_yor.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_zul.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_amh.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_eng.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_ewe.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_fra.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_hau.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_ibo.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_kin.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_lin.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_lug.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_orm.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_sna.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_sot.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_swa.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_twi.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_vai.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_wol.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_xho.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_yor.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_zul.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/afrimgsm_cot.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_amh.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_eng.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_ewe.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_fra.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_hau.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_ibo.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_kin.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_lin.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_lug.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_orm.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_sna.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_sot.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_swa.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_twi.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_vai.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_wol.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_xho.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_yor.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_zul.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_amh.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_eng.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_ewe.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_fra.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_hau.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_ibo.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_kin.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_lin.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_lug.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_orm.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_sna.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_sot.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_swa.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_twi.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_vai.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_wol.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_xho.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_yor.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_zul.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_amh.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_eng.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_ewe.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_fra.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_hau.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_ibo.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_kin.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_lin.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_lug.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_orm.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_sna.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_sot.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_swa.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_twi.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_vai.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_wol.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_xho.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_yor.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_zul.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_amh.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_eng.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_ewe.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_fra.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_hau.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_ibo.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_kin.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_lin.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_lug.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_orm.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_sna.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_sot.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_swa.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_twi.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_vai.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_wol.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_xho.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_yor.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_zul.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_amh.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_eng.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_ewe.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_fra.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_hau.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_ibo.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_kin.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_lin.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_lug.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_orm.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_sna.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_sot.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_swa.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_twi.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_vai.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_wol.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_xho.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_yor.yaml create mode 100644 lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_zul.yaml delete mode 100644 lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_amh.yaml delete mode 100644 lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_eng.yaml delete mode 100644 lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_ewe.yaml delete mode 100644 lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_fra.yaml delete mode 100644 lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_hau.yaml delete mode 100644 lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_ibo.yaml delete mode 100644 lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_kin.yaml delete mode 100644 lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_lin.yaml delete mode 100644 lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_lug.yaml delete mode 100644 lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_orm.yaml delete mode 100644 lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_sna.yaml delete mode 100644 lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_sot.yaml delete mode 100644 lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_swa.yaml delete mode 100644 lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_twi.yaml delete mode 100644 lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_wol.yaml delete mode 100644 lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_xho.yaml delete mode 100644 lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_yor.yaml delete mode 100644 lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_zul.yaml delete mode 100644 lm_eval/tasks/afrimgsm/en_cot/cot_yaml create mode 100644 lm_eval/tasks/afrimgsm/gen_utils.py delete mode 100644 lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_amh.yaml delete mode 100644 lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_eng.yaml delete mode 100644 lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_ewe.yaml delete mode 100644 lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_fra.yaml delete mode 100644 lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_hau.yaml delete mode 100644 lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_ibo.yaml delete mode 100644 lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_kin.yaml delete mode 100644 lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_lin.yaml delete mode 100644 lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_lug.yaml delete mode 100644 lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_orm.yaml delete mode 100644 lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_sna.yaml delete mode 100644 lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_sot.yaml delete mode 100644 lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_swa.yaml delete mode 100644 lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_twi.yaml delete mode 100644 lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_wol.yaml delete mode 100644 lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_xho.yaml delete mode 100644 lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_yor.yaml delete mode 100644 lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_zul.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/afrimgsm_tt.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_amh.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_ewe.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_fra.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_hau.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_ibo.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_kin.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_lin.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_lug.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_orm.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_sna.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_sot.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_swa.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_twi.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_wol.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_xho.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_yor.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_zul.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_amh.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_ewe.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_fra.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_hau.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_ibo.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_kin.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_lin.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_lug.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_orm.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_sna.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_sot.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_swa.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_twi.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_wol.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_xho.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_yor.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_zul.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_amh.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_ewe.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_fra.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_hau.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_ibo.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_kin.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_lin.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_lug.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_orm.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_sna.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_sot.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_swa.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_twi.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_wol.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_xho.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_yor.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_zul.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_amh.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_ewe.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_fra.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_hau.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_ibo.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_kin.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_lin.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_lug.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_orm.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_sna.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_sot.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_swa.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_twi.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_wol.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_xho.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_yor.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_zul.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_amh.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_ewe.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_fra.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_hau.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_ibo.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_kin.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_lin.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_lug.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_orm.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_sna.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_sot.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_swa.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_twi.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_wol.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_xho.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_yor.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_zul.yaml delete mode 100644 lm_eval/tasks/afrimgsm/translate/translate_direct_yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/afrimgsm_tt_cot.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_amh.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_ewe.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_fra.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_hau.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_ibo.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_kin.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_lin.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_lug.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_orm.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_sna.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_sot.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_swa.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_twi.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_vai.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_wol.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_xho.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_yor.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_zul.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_amh.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_ewe.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_fra.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_hau.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_ibo.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_kin.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_lin.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_lug.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_orm.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_sna.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_sot.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_swa.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_twi.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_vai.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_wol.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_xho.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_yor.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_zul.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_amh.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_ewe.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_fra.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_hau.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_ibo.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_kin.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_lin.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_lug.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_orm.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_sna.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_sot.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_swa.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_twi.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_vai.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_wol.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_xho.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_yor.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_zul.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_amh.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_ewe.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_fra.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_hau.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_ibo.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_kin.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_lin.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_lug.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_orm.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_sna.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_sot.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_swa.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_twi.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_vai.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_wol.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_xho.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_yor.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_zul.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_amh.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_ewe.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_fra.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_hau.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_ibo.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_kin.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_lin.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_lug.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_orm.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_sna.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_sot.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_swa.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_twi.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_vai.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_wol.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_xho.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_yor.yaml create mode 100644 lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_zul.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/afrimmlu.yaml delete mode 100644 lm_eval/tasks/afrimmlu/direct/afrimmlu_common_yaml delete mode 100644 lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_amh.yaml delete mode 100644 lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_eng.yaml delete mode 100644 lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_ewe.yaml delete mode 100644 lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_fra.yaml delete mode 100644 lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_hau.yaml delete mode 100644 lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_ibo.yaml delete mode 100644 lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_kin.yaml delete mode 100644 lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_lin.yaml delete mode 100644 lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_lug.yaml delete mode 100644 lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_orm.yaml delete mode 100644 lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_sna.yaml delete mode 100644 lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_sot.yaml delete mode 100644 lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_swa.yaml delete mode 100644 lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_twi.yaml delete mode 100644 lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_wol.yaml delete mode 100644 lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_xho.yaml delete mode 100644 lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_yor.yaml delete mode 100644 lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_zul.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_amh.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_eng.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_ewe.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_fra.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_hau.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_ibo.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_kin.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_lin.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_lug.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_orm.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_sna.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_sot.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_swa.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_twi.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_wol.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_xho.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_yor.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_zul.yaml rename lm_eval/tasks/afrimmlu/direct/{ => prompt_1}/utils.py (100%) create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_amh.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_eng.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_ewe.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_fra.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_hau.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_ibo.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_kin.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_lin.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_lug.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_orm.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_sna.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_sot.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_swa.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_twi.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_wol.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_xho.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_yor.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_zul.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_2/utils.py create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_amh.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_eng.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_ewe.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_fra.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_hau.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_ibo.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_kin.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_lin.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_lug.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_orm.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_sna.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_sot.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_swa.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_twi.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_wol.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_xho.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_yor.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_zul.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_3/utils.py create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_amh.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_eng.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_ewe.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_fra.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_hau.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_ibo.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_kin.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_lin.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_lug.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_orm.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_sna.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_sot.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_swa.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_twi.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_wol.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_xho.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_yor.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_zul.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_4/utils.py create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_amh.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_eng.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_ewe.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_fra.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_hau.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_ibo.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_kin.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_lin.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_lug.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_orm.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_sna.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_sot.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_swa.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_twi.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_wol.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_xho.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_yor.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_zul.yaml create mode 100644 lm_eval/tasks/afrimmlu/direct/prompt_5/utils.py create mode 100644 lm_eval/tasks/afrimmlu/gen_utils.py delete mode 100644 lm_eval/tasks/afrimmlu/translate/afrimmlu_common_translate_yaml delete mode 100644 lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_amh.yaml delete mode 100644 lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_eng.yaml delete mode 100644 lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_ewe.yaml delete mode 100644 lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_fra.yaml delete mode 100644 lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_hau.yaml delete mode 100644 lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_ibo.yaml delete mode 100644 lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_kin.yaml delete mode 100644 lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_lin.yaml delete mode 100644 lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_lug.yaml delete mode 100644 lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_orm.yaml delete mode 100644 lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_sna.yaml delete mode 100644 lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_sot.yaml delete mode 100644 lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_swa.yaml delete mode 100644 lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_twi.yaml delete mode 100644 lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_wol.yaml delete mode 100644 lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_xho.yaml delete mode 100644 lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_yor.yaml delete mode 100644 lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_zul.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/afrimmlu_tt.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_amh.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_ewe.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_fra.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_hau.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_ibo.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_kin.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_lin.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_lug.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_orm.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_sna.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_sot.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_swa.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_twi.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_wol.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_xho.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_yor.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_zul.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_1/utils.py create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_amh.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_ewe.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_fra.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_hau.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_ibo.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_kin.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_lin.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_lug.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_orm.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_sna.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_sot.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_swa.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_twi.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_wol.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_xho.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_yor.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_zul.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_2/utils.py create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_amh.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_ewe.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_fra.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_hau.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_ibo.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_kin.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_lin.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_lug.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_orm.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_sna.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_sot.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_swa.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_twi.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_wol.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_xho.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_yor.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_zul.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_3/utils.py create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_amh.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_ewe.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_fra.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_hau.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_ibo.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_kin.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_lin.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_lug.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_orm.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_sna.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_sot.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_swa.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_twi.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_wol.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_xho.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_yor.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_zul.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_4/utils.py create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_amh.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_ewe.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_fra.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_hau.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_ibo.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_kin.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_lin.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_lug.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_orm.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_sna.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_sot.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_swa.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_twi.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_wol.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_xho.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_yor.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_zul.yaml create mode 100644 lm_eval/tasks/afrimmlu/translate/prompt_5/utils.py delete mode 100644 lm_eval/tasks/afrimmlu/translate/utils.py create mode 100644 lm_eval/tasks/afrixnli/direct/afrixnli.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_amh.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_eng.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_ewe.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_fra.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_hau.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_ibo.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_kin.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_lin.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_lug.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_orm.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_sna.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_sot.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_swa.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_twi.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_wol.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_xho.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_yor.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_zul.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_1/utils.py create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_amh.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_eng.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_ewe.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_fra.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_hau.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_ibo.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_kin.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_lin.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_lug.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_orm.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_sna.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_sot.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_swa.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_twi.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_wol.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_xho.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_yor.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_zul.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_2/utils.py create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_amh.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_eng.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_ewe.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_fra.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_hau.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_ibo.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_kin.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_lin.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_lug.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_orm.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_sna.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_sot.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_swa.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_twi.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_wol.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_xho.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_yor.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_zul.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_3/utils.py create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_amh.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_eng.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_ewe.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_fra.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_hau.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_ibo.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_kin.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_lin.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_lug.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_orm.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_sna.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_sot.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_swa.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_twi.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_wol.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_xho.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_yor.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_zul.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_4/utils.py create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_amh.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_eng.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_ewe.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_fra.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_hau.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_ibo.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_kin.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_lin.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_lug.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_orm.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_sna.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_sot.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_swa.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_twi.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_wol.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_xho.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_yor.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_zul.yaml create mode 100644 lm_eval/tasks/afrixnli/direct/prompt_5/utils.py create mode 100644 lm_eval/tasks/afrixnli/gen_utils.py create mode 100644 lm_eval/tasks/afrixnli/translate/afrixnli_tt.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_amh.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_ewe.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_fra.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_hau.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_ibo.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_kin.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_lin.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_lug.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_orm.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_sna.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_sot.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_swa.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_twi.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_wol.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_xho.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_yor.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_zul.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_1/utils.py create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_amh.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_ewe.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_fra.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_hau.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_ibo.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_kin.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_lin.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_lug.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_orm.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_sna.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_sot.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_swa.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_twi.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_wol.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_xho.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_yor.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_zul.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_2/utils.py create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_amh.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_ewe.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_fra.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_hau.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_ibo.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_kin.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_lin.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_lug.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_orm.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_sna.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_sot.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_swa.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_twi.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_wol.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_xho.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_yor.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_zul.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_3/utils.py create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_amh.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_ewe.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_fra.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_hau.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_ibo.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_kin.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_lin.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_lug.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_orm.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_sna.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_sot.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_swa.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_twi.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_wol.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_xho.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_yor.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_zul.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_4/utils.py create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_amh.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_ewe.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_fra.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_hau.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_ibo.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_kin.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_lin.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_lug.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_orm.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_sna.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_sot.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_swa.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_twi.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_wol.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_xho.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_yor.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_zul.yaml create mode 100644 lm_eval/tasks/afrixnli/translate/prompt_5/utils.py create mode 100644 lm_eval/tasks/afrobench/README.md create mode 100644 lm_eval/tasks/afrobench/adr/README.md create mode 100644 lm_eval/tasks/afrobench/adr/afridiacritics.yaml create mode 100644 lm_eval/tasks/afrobench/adr/gen_utils.py create mode 100644 lm_eval/tasks/afrobench/adr/prompt_1/afridiacritics_bbj.yaml create mode 100644 lm_eval/tasks/afrobench/adr/prompt_1/afridiacritics_fon.yaml create mode 100644 lm_eval/tasks/afrobench/adr/prompt_1/afridiacritics_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/adr/prompt_1/afridiacritics_wol.yaml create mode 100644 lm_eval/tasks/afrobench/adr/prompt_1/afridiacritics_yaml create mode 100644 lm_eval/tasks/afrobench/adr/prompt_1/afridiacritics_yor.yaml create mode 100644 lm_eval/tasks/afrobench/adr/prompt_2/afridiacritics_bbj.yaml create mode 100644 lm_eval/tasks/afrobench/adr/prompt_2/afridiacritics_fon.yaml create mode 100644 lm_eval/tasks/afrobench/adr/prompt_2/afridiacritics_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/adr/prompt_2/afridiacritics_wol.yaml create mode 100644 lm_eval/tasks/afrobench/adr/prompt_2/afridiacritics_yaml create mode 100644 lm_eval/tasks/afrobench/adr/prompt_2/afridiacritics_yor.yaml create mode 100644 lm_eval/tasks/afrobench/adr/prompt_3/afridiacritics_bbj.yaml create mode 100644 lm_eval/tasks/afrobench/adr/prompt_3/afridiacritics_fon.yaml create mode 100644 lm_eval/tasks/afrobench/adr/prompt_3/afridiacritics_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/adr/prompt_3/afridiacritics_wol.yaml create mode 100644 lm_eval/tasks/afrobench/adr/prompt_3/afridiacritics_yaml create mode 100644 lm_eval/tasks/afrobench/adr/prompt_3/afridiacritics_yor.yaml create mode 100644 lm_eval/tasks/afrobench/adr/prompt_4/afridiacritics_bbj.yaml create mode 100644 lm_eval/tasks/afrobench/adr/prompt_4/afridiacritics_fon.yaml create mode 100644 lm_eval/tasks/afrobench/adr/prompt_4/afridiacritics_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/adr/prompt_4/afridiacritics_wol.yaml create mode 100644 lm_eval/tasks/afrobench/adr/prompt_4/afridiacritics_yaml create mode 100644 lm_eval/tasks/afrobench/adr/prompt_4/afridiacritics_yor.yaml create mode 100644 lm_eval/tasks/afrobench/adr/prompt_5/afridiacritics_bbj.yaml create mode 100644 lm_eval/tasks/afrobench/adr/prompt_5/afridiacritics_fon.yaml create mode 100644 lm_eval/tasks/afrobench/adr/prompt_5/afridiacritics_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/adr/prompt_5/afridiacritics_wol.yaml create mode 100644 lm_eval/tasks/afrobench/adr/prompt_5/afridiacritics_yaml create mode 100644 lm_eval/tasks/afrobench/adr/prompt_5/afridiacritics_yor.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/README.md create mode 100644 lm_eval/tasks/afrobench/afriqa/afriqa.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_bem.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_fon.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_hau.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_kin.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_swa.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_twi.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_yor.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_zul.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_1/utils.py create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_bem.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_fon.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_hau.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_kin.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_swa.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_twi.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_yor.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_zul.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_2/utils.py create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_bem.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_fon.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_hau.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_kin.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_swa.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_twi.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_yor.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_zul.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_3/utils.py create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa_bem.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa_fon.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa_hau.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa_kin.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa_swa.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa_twi.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa_yor.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa_zul.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_4/utils.py create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa_bem.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa_fon.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa_hau.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa_kin.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa_swa.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa_twi.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa_yor.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa_zul.yaml create mode 100644 lm_eval/tasks/afrobench/afriqa/prompt_5/utils.py create mode 100644 lm_eval/tasks/afrobench/afriqa/utils.py create mode 100644 lm_eval/tasks/afrobench/afrisenti/README.md create mode 100644 lm_eval/tasks/afrobench/afrisenti/afrisenti.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/fewshot.sh create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_amh.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_arq.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_ary.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_hau.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_kin.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_orm.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_pcm.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_por.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_swa.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_tir.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_tso.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_twi.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_yor.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_1/run.sh create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_1/utils.py create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_1/xx.py create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_amh.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_arq.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_ary.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_hau.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_kin.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_orm.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_pcm.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_por.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_swa.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_tir.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_tso.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_twi.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_yor.yaml create mode 100755 lm_eval/tasks/afrobench/afrisenti/prompt_2/run.sh create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_2/utils.py create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_2/xx.py create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_amh.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_arq.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_ary.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_hau.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_kin.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_orm.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_pcm.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_por.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_swa.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_tir.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_tso.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_twi.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_yor.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_3/utils.py create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_3/xx.py create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_amh.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_arq.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_ary.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_hau.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_kin.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_orm.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_pcm.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_por.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_swa.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_tir.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_tso.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_twi.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_yor.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_4/utils.py create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_4/xx.py create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_amh.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_arq.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_ary.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_hau.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_kin.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_orm.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_pcm.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_por.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_swa.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_tir.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_tso.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_twi.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_yor.yaml create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_5/utils.py create mode 100644 lm_eval/tasks/afrobench/afrisenti/prompt_5/xx.py create mode 100644 lm_eval/tasks/afrobench/afrisenti/utils.py create mode 100644 lm_eval/tasks/afrobench/afrobench-lite.yaml create mode 100644 lm_eval/tasks/afrobench/afrobench.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/README.md create mode 100644 lm_eval/tasks/afrobench/belebele/belebele.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_1/belebele create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_1/belebele_afr.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_1/belebele_amh.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_1/belebele_ary.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_1/belebele_arz.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_1/belebele_bam.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_1/belebele_eng.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_1/belebele_fra.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_1/belebele_fuv.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_1/belebele_gaz.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_1/belebele_hau.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_1/belebele_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_1/belebele_kea.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_1/belebele_kin.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_1/belebele_lin.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_1/belebele_lug.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_1/belebele_luo.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_1/belebele_nya.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_1/belebele_plt.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_1/belebele_por.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_1/belebele_sna.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_1/belebele_som.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_1/belebele_sot.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_1/belebele_ssw.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_1/belebele_swa.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_1/belebele_tir.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_1/belebele_tsn.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_1/belebele_tso.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_1/belebele_wol.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_1/belebele_xho.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_1/belebele_yor.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_1/belebele_zul.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_2/belebele create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_2/belebele_afr.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_2/belebele_amh.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_2/belebele_ary.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_2/belebele_arz.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_2/belebele_bam.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_2/belebele_eng.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_2/belebele_fra.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_2/belebele_fuv.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_2/belebele_gaz.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_2/belebele_hau.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_2/belebele_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_2/belebele_kea.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_2/belebele_kin.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_2/belebele_lin.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_2/belebele_lug.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_2/belebele_luo.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_2/belebele_nya.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_2/belebele_plt.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_2/belebele_por.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_2/belebele_sna.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_2/belebele_som.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_2/belebele_sot.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_2/belebele_ssw.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_2/belebele_swa.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_2/belebele_tir.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_2/belebele_tsn.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_2/belebele_tso.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_2/belebele_wol.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_2/belebele_xho.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_2/belebele_yor.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_2/belebele_zul.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_3/belebele create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_3/belebele_afr.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_3/belebele_amh.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_3/belebele_ary.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_3/belebele_arz.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_3/belebele_bam.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_3/belebele_eng.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_3/belebele_fra.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_3/belebele_fuv.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_3/belebele_gaz.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_3/belebele_hau.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_3/belebele_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_3/belebele_kea.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_3/belebele_kin.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_3/belebele_lin.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_3/belebele_lug.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_3/belebele_luo.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_3/belebele_nya.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_3/belebele_plt.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_3/belebele_por.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_3/belebele_sna.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_3/belebele_som.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_3/belebele_sot.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_3/belebele_ssw.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_3/belebele_swa.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_3/belebele_tir.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_3/belebele_tsn.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_3/belebele_tso.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_3/belebele_wol.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_3/belebele_xho.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_3/belebele_yor.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_3/belebele_zul.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_4/belebele create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_4/belebele_afr.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_4/belebele_amh.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_4/belebele_ary.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_4/belebele_arz.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_4/belebele_bam.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_4/belebele_eng.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_4/belebele_fra.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_4/belebele_fuv.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_4/belebele_gaz.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_4/belebele_hau.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_4/belebele_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_4/belebele_kea.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_4/belebele_kin.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_4/belebele_lin.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_4/belebele_lug.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_4/belebele_luo.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_4/belebele_nya.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_4/belebele_plt.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_4/belebele_por.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_4/belebele_sna.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_4/belebele_som.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_4/belebele_sot.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_4/belebele_ssw.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_4/belebele_swa.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_4/belebele_tir.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_4/belebele_tsn.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_4/belebele_tso.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_4/belebele_wol.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_4/belebele_xho.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_4/belebele_yor.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_4/belebele_zul.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_5/belebele create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_5/belebele_afr.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_5/belebele_amh.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_5/belebele_ary.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_5/belebele_arz.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_5/belebele_bam.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_5/belebele_eng.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_5/belebele_fra.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_5/belebele_fuv.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_5/belebele_gaz.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_5/belebele_hau.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_5/belebele_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_5/belebele_kea.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_5/belebele_kin.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_5/belebele_lin.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_5/belebele_lug.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_5/belebele_luo.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_5/belebele_nya.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_5/belebele_plt.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_5/belebele_por.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_5/belebele_sna.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_5/belebele_som.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_5/belebele_sot.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_5/belebele_ssw.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_5/belebele_swa.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_5/belebele_tir.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_5/belebele_tsn.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_5/belebele_tso.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_5/belebele_wol.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_5/belebele_xho.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_5/belebele_yor.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/prompt_5/belebele_zul.yaml create mode 100644 lm_eval/tasks/afrobench/belebele/utils.py create mode 100644 lm_eval/tasks/afrobench/flores/README.md create mode 100644 lm_eval/tasks/afrobench/flores/flores.yaml create mode 100644 lm_eval/tasks/afrobench/flores/gen_utils.py create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_ace_Arab-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_ace_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_acq_Arab-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_aeb_Arab-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_afr_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_aka_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_amh_Ethi-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_ary_Arab-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_arz_Arab-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_bam_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_ban_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_bem_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_cjk_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_dik_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_dyu_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_ewe_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_fon_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_fra_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_fuv_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_gaz_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_hau_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_ibo_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_kab_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_kam_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_kbp_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_kea_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_kik_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_kin_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_kmb_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_knc_Arab-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_knc_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_kon_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_lin_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_lua_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_lug_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_luo_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_mos_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_nso_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_nus_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_nya_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_plt_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_run_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_sag_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_sna_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_som_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_sot_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_ssw_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_sun_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_swh_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_taq_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_taq_Tfng-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_tir_Ethi-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_tsn_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_tso_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_tum_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_twi_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_tzm_Tfng-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_umb_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_wol_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_xho_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_yor_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_zul_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-ace_Arab.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-ace_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-acq_Arab.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-aeb_Arab.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-afr_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-aka_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-amh_Ethi.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-ary_Arab.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-arz_Arab.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-bam_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-ban_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-bem_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-cjk_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-dik_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-dyu_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-ewe_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-fon_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-fra_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-fuv_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-gaz_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-hau_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-ibo_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kab_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kam_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kbp_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kea_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kik_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kin_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kmb_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-knc_Arab.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-knc_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kon_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-lin_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-lua_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-lug_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-luo_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-mos_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-nso_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-nus_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-nya_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-plt_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-run_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-sag_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-sna_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-som_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-sot_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-ssw_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-sun_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-swh_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-taq_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-taq_Tfng.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-tir_Ethi.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-tsn_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-tso_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-tum_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-twi_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-tzm_Tfng.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-umb_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-wol_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-xho_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-yor_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-zul_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_1/flores create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ace_Arab-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ace_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_acq_Arab-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_aeb_Arab-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_afr_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_aka_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_amh_Ethi-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ary_Arab-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_arz_Arab-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_bam_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ban_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_bem_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_cjk_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_dik_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_dyu_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ewe_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_fon_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_fra_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_fuv_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_gaz_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_hau_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ibo_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kab_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kam_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kbp_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kea_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kik_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kin_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kmb_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_knc_Arab-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_knc_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kon_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_lin_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_lua_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_lug_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_luo_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_mos_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_nso_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_nus_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_nya_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_plt_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_run_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_sag_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_sna_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_som_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_sot_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ssw_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_sun_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_swh_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_taq_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_taq_Tfng-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_tir_Ethi-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_tsn_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_tso_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_tum_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_twi_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_tzm_Tfng-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_umb_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_wol_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_xho_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_yor_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_zul_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ace_Arab.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ace_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-acq_Arab.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-aeb_Arab.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-afr_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-aka_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-amh_Ethi.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ary_Arab.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-arz_Arab.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-bam_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ban_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-bem_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-cjk_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-dik_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-dyu_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ewe_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-fon_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-fra_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-fuv_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-gaz_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-hau_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ibo_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kab_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kam_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kbp_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kea_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kik_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kin_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kmb_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-knc_Arab.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-knc_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kon_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-lin_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-lua_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-lug_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-luo_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-mos_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-nso_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-nus_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-nya_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-plt_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-run_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-sag_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-sna_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-som_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-sot_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ssw_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-sun_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-swh_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-taq_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-taq_Tfng.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-tir_Ethi.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-tsn_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-tso_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-tum_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-twi_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-tzm_Tfng.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-umb_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-wol_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-xho_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-yor_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-zul_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_2/flores create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ace_Arab-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ace_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_acq_Arab-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_aeb_Arab-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_afr_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_aka_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_amh_Ethi-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ary_Arab-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_arz_Arab-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_bam_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ban_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_bem_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_cjk_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_dik_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_dyu_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ewe_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_fon_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_fra_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_fuv_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_gaz_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_hau_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ibo_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kab_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kam_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kbp_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kea_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kik_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kin_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kmb_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_knc_Arab-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_knc_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kon_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_lin_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_lua_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_lug_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_luo_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_mos_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_nso_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_nus_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_nya_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_plt_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_run_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_sag_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_sna_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_som_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_sot_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ssw_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_sun_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_swh_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_taq_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_taq_Tfng-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_tir_Ethi-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_tsn_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_tso_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_tum_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_twi_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_tzm_Tfng-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_umb_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_wol_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_xho_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_yor_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_zul_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-ace_Arab.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-ace_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-acq_Arab.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-aeb_Arab.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-afr_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-aka_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-amh_Ethi.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-ary_Arab.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-arz_Arab.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-bam_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-ban_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-bem_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-cjk_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-dik_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-dyu_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-ewe_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-fon_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-fra_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-fuv_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-gaz_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-hau_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-ibo_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-kab_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-kam_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-kbp_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-kea_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-kik_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-kin_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-kmb_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-knc_Arab.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-knc_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-kon_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-lin_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-lua_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-lug_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-luo_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-mos_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-nso_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-nus_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-nya_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-plt_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-run_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-sag_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-sna_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-som_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-sot_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-ssw_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-sun_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-swh_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-taq_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-taq_Tfng.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-tir_Ethi.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-tsn_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-tso_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-tum_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-twi_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-tzm_Tfng.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-umb_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-wol_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-xho_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-yor_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-zul_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/flores/prompt_3/flores create mode 100644 lm_eval/tasks/afrobench/injongointent/README.md create mode 100644 lm_eval/tasks/afrobench/injongointent/gen_utils.py create mode 100644 lm_eval/tasks/afrobench/injongointent/injongointent.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_amh.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_eng.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_ewe.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_hau.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_kin.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_lin.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_lug.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_orm.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_sna.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_sot.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_swa.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_twi.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_wol.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_xho.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_yor.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_zul.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_1/utils.py create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_amh.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_eng.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_ewe.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_hau.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_kin.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_lin.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_lug.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_orm.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_sna.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_sot.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_swa.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_twi.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_wol.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_xho.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_yor.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_zul.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_2/utils.py create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_amh.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_eng.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_ewe.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_hau.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_kin.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_lin.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_lug.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_orm.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_sna.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_sot.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_swa.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_twi.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_wol.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_xho.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_yor.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_zul.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_3/utils.py create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_amh.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_eng.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_ewe.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_hau.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_kin.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_lin.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_lug.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_orm.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_sna.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_sot.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_swa.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_twi.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_wol.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_xho.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_yor.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_zul.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_4/utils.py create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_amh.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_eng.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_ewe.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_hau.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_kin.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_lin.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_lug.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_orm.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_sna.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_sot.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_swa.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_twi.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_wol.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_xho.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_yor.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_zul.yaml create mode 100644 lm_eval/tasks/afrobench/injongointent/prompt_5/utils.py create mode 100644 lm_eval/tasks/afrobench/mafand/README.md create mode 100644 lm_eval/tasks/afrobench/mafand/gen_utils.py create mode 100644 lm_eval/tasks/afrobench/mafand/mafand.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_amh-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_bam-fr.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_bbj-fr.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_ewe-fr.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_fon-fr.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_hau-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_ibo-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_kin-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_lug-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_luo-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_mos-fr.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_nya-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_pcm-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_sna-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_swa-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_tsn-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_twi-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_wol-fr.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_xho-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_yor-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_zul-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/african-english/utils.py create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-amh.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-hau.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-ibo.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-kin.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-lug.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-luo.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-nya.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-pcm.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-sna.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-swa.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-tsn.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-twi.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-xho.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-yor.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-zul.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-bam.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-bbj.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-ewe.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-fon.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-mos.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-wol.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_1/english-african/utils.py create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_amh-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_bam-fr.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_bbj-fr.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_ewe-fr.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_fon-fr.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_hau-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_ibo-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_kin-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_lug-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_luo-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_mos-fr.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_nya-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_pcm-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_sna-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_swa-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_tsn-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_twi-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_wol-fr.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_xho-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_yor-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_zul-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/african-english/utils.py create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-amh.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-hau.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-ibo.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-kin.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-lug.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-luo.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-nya.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-pcm.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-sna.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-swa.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-tsn.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-twi.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-xho.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-yor.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-zul.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_fr-bam.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_fr-bbj.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_fr-ewe.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_fr-fon.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_fr-mos.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_fr-wol.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_2/english-african/utils.py create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_amh-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_bam-fr.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_bbj-fr.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_ewe-fr.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_fon-fr.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_hau-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_ibo-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_kin-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_lug-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_luo-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_mos-fr.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_nya-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_pcm-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_sna-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_swa-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_tsn-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_twi-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_wol-fr.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_xho-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_yor-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_zul-en.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/african-english/utils.py create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-amh.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-hau.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-ibo.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-kin.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-lug.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-luo.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-nya.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-pcm.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-sna.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-swa.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-tsn.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-twi.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-xho.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-yor.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-zul.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_fr-bam.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_fr-bbj.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_fr-ewe.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_fr-fon.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_fr-mos.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_fr-wol.yaml create mode 100644 lm_eval/tasks/afrobench/mafand/prompt_3/english-african/utils.py create mode 100644 lm_eval/tasks/afrobench/masakhaner/README.md create mode 100644 lm_eval/tasks/afrobench/masakhaner/gen_utils.py create mode 100644 lm_eval/tasks/afrobench/masakhaner/masakhaner.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_am.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_bbj.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_bm.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_ee.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_ha.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_ig.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_lg.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_luo.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_mos.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_ny.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_pcm.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_rw.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_sn.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_sw.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_tn.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_tw.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_wo.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_xh.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_yo.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_zu.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_1/utils.py create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_am.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_bbj.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_bm.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_ee.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_ha.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_ig.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_lg.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_luo.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_mos.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_ny.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_pcm.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_rw.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_sn.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_sw.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_tn.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_tw.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_wo.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_xh.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_yo.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_zu.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_2/utils.py create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_am.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_bbj.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_bm.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_ee.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_ha.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_ig.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_lg.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_luo.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_mos.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_ny.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_pcm.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_rw.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_sn.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_sw.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_tn.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_tw.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_wo.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_xh.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_yo.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_zu.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_3/utils.py create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_am.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_bbj.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_bm.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_ee.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_ha.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_ig.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_lg.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_luo.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_mos.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_ny.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_pcm.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_rw.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_sn.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_sw.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_tn.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_tw.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_wo.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_xh.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_yo.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_zu.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_4/utils.py create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_am.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_bbj.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_bm.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_ee.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_ha.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_ig.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_lg.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_luo.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_mos.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_ny.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_pcm.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_rw.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_sn.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_sw.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_tn.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_tw.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_wo.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_xh.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_yo.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_zu.yaml create mode 100644 lm_eval/tasks/afrobench/masakhaner/prompt_5/utils.py create mode 100644 lm_eval/tasks/afrobench/masakhanews/README.md create mode 100644 lm_eval/tasks/afrobench/masakhanews/masakhanews.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_amh.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_eng.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_fra.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_hau.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_lin.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_lug.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_orm.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_pcm.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_run.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_sna.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_som.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_swa.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_tir.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_xho.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_yor.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_1/utils.py create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_amh.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_eng.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_fra.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_hau.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_lin.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_lug.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_orm.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_pcm.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_run.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_sna.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_som.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_swa.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_tir.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_xho.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_yor.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_2/utils.py create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_amh.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_eng.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_fra.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_hau.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_lin.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_lug.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_orm.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_pcm.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_run.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_sna.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_som.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_swa.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_tir.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_xho.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_yor.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_3/utils.py create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_amh.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_eng.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_fra.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_hau.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_lin.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_lug.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_orm.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_pcm.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_run.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_sna.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_som.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_swa.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_tir.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_xho.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_yor.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_4/utils.py create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_amh.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_eng.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_fra.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_hau.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_lin.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_lug.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_orm.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_pcm.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_run.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_sna.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_som.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_swa.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_tir.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_xho.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_yor.yaml create mode 100644 lm_eval/tasks/afrobench/masakhanews/prompt_5/utils.py create mode 100644 lm_eval/tasks/afrobench/masakhanews/utils.py create mode 100644 lm_eval/tasks/afrobench/masakhapos/README.md create mode 100644 lm_eval/tasks/afrobench/masakhapos/gen_utils.py create mode 100644 lm_eval/tasks/afrobench/masakhapos/masakhapos.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_bam.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_bbj.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_ewe.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_fon.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_hau.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_kin.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_lug.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_luo.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_mos.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_nya.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_pcm.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_sna.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_swa.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_tsn.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_twi.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_wol.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_xho.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_yor.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_zul.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_1/utils.py create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_bam.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_bbj.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_ewe.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_fon.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_hau.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_kin.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_lug.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_luo.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_mos.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_nya.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_pcm.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_sna.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_swa.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_tsn.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_twi.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_wol.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_xho.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_yor.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_zul.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_2/utils.py create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_bam.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_bbj.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_ewe.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_fon.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_hau.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_kin.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_lug.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_luo.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_mos.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_nya.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_pcm.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_sna.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_swa.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_tsn.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_twi.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_wol.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_xho.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_yor.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_zul.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_3/utils.py create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_bam.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_bbj.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_ewe.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_fon.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_hau.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_kin.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_lug.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_luo.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_mos.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_nya.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_pcm.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_sna.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_swa.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_tsn.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_twi.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_wol.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_xho.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_yor.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_zul.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_4/utils.py create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_bam.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_bbj.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_ewe.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_fon.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_hau.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_kin.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_lug.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_luo.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_mos.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_nya.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_pcm.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_sna.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_swa.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_tsn.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_twi.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_wol.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_xho.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_yor.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_zul.yaml create mode 100644 lm_eval/tasks/afrobench/masakhapos/prompt_5/utils.py create mode 100644 lm_eval/tasks/afrobench/masakhapos/utils.py create mode 100644 lm_eval/tasks/afrobench/naijarc/README.md create mode 100644 lm_eval/tasks/afrobench/naijarc/naijarc.yaml create mode 100644 lm_eval/tasks/afrobench/naijarc/prompt_1/naijarc create mode 100644 lm_eval/tasks/afrobench/naijarc/prompt_1/naijarc_hau.yaml create mode 100644 lm_eval/tasks/afrobench/naijarc/prompt_1/naijarc_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/naijarc/prompt_1/naijarc_yor.yaml create mode 100644 lm_eval/tasks/afrobench/naijarc/prompt_2/naijarc create mode 100644 lm_eval/tasks/afrobench/naijarc/prompt_2/naijarc_hau.yaml create mode 100644 lm_eval/tasks/afrobench/naijarc/prompt_2/naijarc_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/naijarc/prompt_2/naijarc_yor.yaml create mode 100644 lm_eval/tasks/afrobench/naijarc/prompt_3/naijarc create mode 100644 lm_eval/tasks/afrobench/naijarc/prompt_3/naijarc_hau.yaml create mode 100644 lm_eval/tasks/afrobench/naijarc/prompt_3/naijarc_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/naijarc/prompt_3/naijarc_yor.yaml create mode 100644 lm_eval/tasks/afrobench/naijarc/prompt_4/naijarc create mode 100644 lm_eval/tasks/afrobench/naijarc/prompt_4/naijarc_hau.yaml create mode 100644 lm_eval/tasks/afrobench/naijarc/prompt_4/naijarc_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/naijarc/prompt_4/naijarc_yor.yaml create mode 100644 lm_eval/tasks/afrobench/naijarc/prompt_5/naijarc create mode 100644 lm_eval/tasks/afrobench/naijarc/prompt_5/naijarc_hau.yaml create mode 100644 lm_eval/tasks/afrobench/naijarc/prompt_5/naijarc_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/naijarc/prompt_5/naijarc_yor.yaml create mode 100644 lm_eval/tasks/afrobench/naijarc/utils.py create mode 100644 lm_eval/tasks/afrobench/nollysenti/README.md create mode 100644 lm_eval/tasks/afrobench/nollysenti/nollysenti.yaml create mode 100644 lm_eval/tasks/afrobench/nollysenti/prompt_1/nollysenti create mode 100644 lm_eval/tasks/afrobench/nollysenti/prompt_1/nollysenti_eng.yaml create mode 100644 lm_eval/tasks/afrobench/nollysenti/prompt_1/nollysenti_hau.yaml create mode 100644 lm_eval/tasks/afrobench/nollysenti/prompt_1/nollysenti_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/nollysenti/prompt_1/nollysenti_pcm.yaml create mode 100644 lm_eval/tasks/afrobench/nollysenti/prompt_1/nollysenti_yor.yaml create mode 100644 lm_eval/tasks/afrobench/nollysenti/prompt_1/utils.py create mode 100644 lm_eval/tasks/afrobench/nollysenti/prompt_2/nollysenti create mode 100644 lm_eval/tasks/afrobench/nollysenti/prompt_2/nollysenti_eng.yaml create mode 100644 lm_eval/tasks/afrobench/nollysenti/prompt_2/nollysenti_hau.yaml create mode 100644 lm_eval/tasks/afrobench/nollysenti/prompt_2/nollysenti_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/nollysenti/prompt_2/nollysenti_pcm.yaml create mode 100644 lm_eval/tasks/afrobench/nollysenti/prompt_2/nollysenti_yor.yaml create mode 100644 lm_eval/tasks/afrobench/nollysenti/prompt_2/utils.py create mode 100644 lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti create mode 100644 lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti_eng.yaml create mode 100644 lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti_hau.yaml create mode 100644 lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti_pcm.yaml create mode 100644 lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti_yor.yaml create mode 100644 lm_eval/tasks/afrobench/nollysenti/prompt_3/utils.py create mode 100644 lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti create mode 100644 lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti_eng.yaml create mode 100644 lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti_hau.yaml create mode 100644 lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti_pcm.yaml create mode 100644 lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti_yor.yaml create mode 100644 lm_eval/tasks/afrobench/nollysenti/prompt_4/utils.py create mode 100644 lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti create mode 100644 lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti_eng.yaml create mode 100644 lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti_hau.yaml create mode 100644 lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti_pcm.yaml create mode 100644 lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti_yor.yaml create mode 100644 lm_eval/tasks/afrobench/nollysenti/prompt_5/utils.py create mode 100644 lm_eval/tasks/afrobench/ntrex/README.md create mode 100644 lm_eval/tasks/afrobench/ntrex/gen_utils.py create mode 100644 lm_eval/tasks/afrobench/ntrex/ntrex.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_afr_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_amh_Ethi-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_arb_Arab-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_bem_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_ewe_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_fra_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_hau_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_ibo_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_kin_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_mey_Arab-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_mlg_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_msa_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_nde_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_nso_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_nya_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_orm_Ethi-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_shi_Arab-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_sna_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_som_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_ssw_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_swa_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_tam_Taml-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_tel_Telu-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_tir_Ethi-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_ton_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_tsn_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_urd_Arab-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_ven_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_wol_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_xho_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_yor_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_zul_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-afr_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-amh_Ethi.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-arb_Arab.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-bem_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-ewe_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-fra_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-hau_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-ibo_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-kin_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-mey_Arab.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-mlg_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-msa_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-nde_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-nso_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-nya_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-orm_Ethi.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-shi_Arab.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-sna_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-som_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-ssw_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-swa_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-tam_Taml.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-tel_Telu.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-tir_Ethi.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-ton_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-tsn_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-urd_Arab.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-ven_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-wol_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-xho_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-yor_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-zul_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_afr_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_amh_Ethi-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_arb_Arab-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_bem_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_ewe_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_fra_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_hau_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_ibo_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_kin_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_mey_Arab-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_mlg_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_msa_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_nde_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_nso_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_nya_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_orm_Ethi-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_shi_Arab-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_sna_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_som_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_ssw_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_swa_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_tam_Taml-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_tel_Telu-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_tir_Ethi-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_ton_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_tsn_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_urd_Arab-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_ven_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_wol_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_xho_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_yor_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_zul_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-afr_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-amh_Ethi.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-arb_Arab.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-bem_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-ewe_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-fra_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-hau_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-ibo_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-kin_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-mey_Arab.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-mlg_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-msa_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-nde_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-nso_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-nya_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-orm_Ethi.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-shi_Arab.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-sna_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-som_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-ssw_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-swa_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-tam_Taml.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-tel_Telu.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-tir_Ethi.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-ton_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-tsn_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-urd_Arab.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-ven_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-wol_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-xho_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-yor_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-zul_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_afr_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_amh_Ethi-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_arb_Arab-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_bem_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_ewe_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_fra_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_hau_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_ibo_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_kin_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_mey_Arab-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_mlg_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_msa_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_nde_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_nso_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_nya_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_orm_Ethi-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_shi_Arab-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_sna_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_som_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_ssw_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_swa_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_tam_Taml-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_tel_Telu-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_tir_Ethi-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_ton_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_tsn_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_urd_Arab-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_ven_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_wol_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_xho_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_yor_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_zul_Latn-eng_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-afr_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-amh_Ethi.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-arb_Arab.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-bem_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-ewe_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-fra_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-hau_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-ibo_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-kin_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-mey_Arab.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-mlg_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-msa_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-nde_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-nso_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-nya_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-orm_Ethi.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-shi_Arab.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-sna_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-som_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-ssw_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-swa_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-tam_Taml.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-tel_Telu.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-tir_Ethi.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-ton_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-tsn_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-urd_Arab.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-ven_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-wol_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-xho_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-yor_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-zul_Latn.yaml create mode 100644 lm_eval/tasks/afrobench/openai_mmlu/README.md create mode 100644 lm_eval/tasks/afrobench/openai_mmlu/openai_mmlu.yaml create mode 100644 lm_eval/tasks/afrobench/openai_mmlu/prompt_1/openai_mmlu create mode 100644 lm_eval/tasks/afrobench/openai_mmlu/prompt_1/openai_mmlu_ara.yaml create mode 100644 lm_eval/tasks/afrobench/openai_mmlu/prompt_1/openai_mmlu_swa.yaml create mode 100644 lm_eval/tasks/afrobench/openai_mmlu/prompt_1/openai_mmlu_yor.yaml create mode 100644 lm_eval/tasks/afrobench/openai_mmlu/prompt_2/openai_mmlu create mode 100644 lm_eval/tasks/afrobench/openai_mmlu/prompt_2/openai_mmlu_ara.yaml create mode 100644 lm_eval/tasks/afrobench/openai_mmlu/prompt_2/openai_mmlu_swa.yaml create mode 100644 lm_eval/tasks/afrobench/openai_mmlu/prompt_2/openai_mmlu_yor.yaml create mode 100644 lm_eval/tasks/afrobench/openai_mmlu/prompt_3/openai_mmlu create mode 100644 lm_eval/tasks/afrobench/openai_mmlu/prompt_3/openai_mmlu_ara.yaml create mode 100644 lm_eval/tasks/afrobench/openai_mmlu/prompt_3/openai_mmlu_swa.yaml create mode 100644 lm_eval/tasks/afrobench/openai_mmlu/prompt_3/openai_mmlu_yor.yaml create mode 100644 lm_eval/tasks/afrobench/openai_mmlu/prompt_4/openai_mmlu create mode 100644 lm_eval/tasks/afrobench/openai_mmlu/prompt_4/openai_mmlu_ara.yaml create mode 100644 lm_eval/tasks/afrobench/openai_mmlu/prompt_4/openai_mmlu_swa.yaml create mode 100644 lm_eval/tasks/afrobench/openai_mmlu/prompt_4/openai_mmlu_yor.yaml create mode 100644 lm_eval/tasks/afrobench/openai_mmlu/prompt_5/openai_mmlu create mode 100644 lm_eval/tasks/afrobench/openai_mmlu/prompt_5/openai_mmlu_ara.yaml create mode 100644 lm_eval/tasks/afrobench/openai_mmlu/prompt_5/openai_mmlu_swa.yaml create mode 100644 lm_eval/tasks/afrobench/openai_mmlu/prompt_5/openai_mmlu_yor.yaml create mode 100644 lm_eval/tasks/afrobench/openai_mmlu/utils.py create mode 100644 lm_eval/tasks/afrobench/salt/README.md create mode 100644 lm_eval/tasks/afrobench/salt/gen_utils.py create mode 100644 lm_eval/tasks/afrobench/salt/prompt_1/salt create mode 100644 lm_eval/tasks/afrobench/salt/prompt_1/salt_ach-eng.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-ach.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-ibo.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-lgg.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-lug.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-nyn.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-swa.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-teo.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_1/salt_ibo-eng.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_1/salt_lgg-eng.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_1/salt_lug-eng.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_1/salt_nyn-eng.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_1/salt_swa-eng.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_1/salt_teo-eng.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_2/salt create mode 100644 lm_eval/tasks/afrobench/salt/prompt_2/salt_ach-eng.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-ach.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-ibo.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-lgg.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-lug.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-nyn.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-swa.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-teo.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_2/salt_ibo-eng.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_2/salt_lgg-eng.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_2/salt_lug-eng.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_2/salt_nyn-eng.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_2/salt_swa-eng.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_2/salt_teo-eng.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_3/salt create mode 100644 lm_eval/tasks/afrobench/salt/prompt_3/salt_ach-eng.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-ach.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-ibo.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-lgg.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-lug.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-nyn.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-swa.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-teo.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_3/salt_ibo-eng.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_3/salt_lgg-eng.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_3/salt_lug-eng.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_3/salt_nyn-eng.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_3/salt_swa-eng.yaml create mode 100644 lm_eval/tasks/afrobench/salt/prompt_3/salt_teo-eng.yaml create mode 100644 lm_eval/tasks/afrobench/salt/salt.yaml create mode 100644 lm_eval/tasks/afrobench/sample_run_scripts/run_afrobench.sh create mode 100644 lm_eval/tasks/afrobench/sample_run_scripts/run_afrobench_lite.sh create mode 100644 lm_eval/tasks/afrobench/sib/README.md create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_aeb.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_afr.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_aka.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_amh.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_ary.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_arz.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_bam.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_bem.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_cjk.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_dik.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_dyu.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_eng.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_ewe.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_fon.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_fra.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_fuv.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_gaz.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_hau.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_kab.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_kam.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_kbp.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_kea.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_kik.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_kin.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_kmb.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_knc.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_kon.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_lin.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_lua.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_lug.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_luo.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_mos.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_nso.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_nus.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_nya.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_plt.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_por.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_run.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_sag.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_sna.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_som.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_sot.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_ssw.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_swa.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_taq.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_tir.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_tso.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_tum.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_twi.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_tzm.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_umb.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_wol.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_xho.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_yor.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/sib_zul.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_1/utils.py create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_aeb.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_afr.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_aka.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_amh.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_ary.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_arz.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_bam.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_bem.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_cjk.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_dik.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_dyu.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_eng.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_ewe.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_fon.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_fra.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_fuv.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_gaz.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_hau.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_kab.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_kam.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_kbp.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_kea.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_kik.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_kin.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_kmb.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_knc.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_kon.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_lin.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_lua.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_lug.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_luo.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_mos.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_nso.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_nus.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_nya.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_plt.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_por.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_run.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_sag.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_sna.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_som.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_sot.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_ssw.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_swa.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_taq.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_tir.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_tso.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_tum.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_twi.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_tzm.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_umb.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_wol.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_xho.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_yor.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/sib_zul.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_2/utils.py create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_aeb.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_afr.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_aka.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_amh.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_ary.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_arz.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_bam.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_bem.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_cjk.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_dik.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_dyu.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_eng.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_ewe.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_fon.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_fra.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_fuv.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_gaz.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_hau.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_kab.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_kam.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_kbp.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_kea.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_kik.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_kin.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_kmb.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_knc.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_kon.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_lin.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_lua.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_lug.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_luo.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_mos.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_nso.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_nus.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_nya.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_plt.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_por.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_run.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_sag.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_sna.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_som.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_sot.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_ssw.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_swa.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_taq.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_tir.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_tso.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_tum.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_twi.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_tzm.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_umb.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_wol.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_xho.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_yor.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/sib_zul.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_3/utils.py create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_aeb.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_afr.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_aka.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_amh.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_ary.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_arz.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_bam.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_bem.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_cjk.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_dik.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_dyu.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_eng.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_ewe.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_fon.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_fra.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_fuv.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_gaz.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_hau.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_kab.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_kam.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_kbp.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_kea.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_kik.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_kin.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_kmb.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_knc.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_kon.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_lin.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_lua.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_lug.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_luo.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_mos.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_nso.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_nus.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_nya.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_plt.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_por.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_run.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_sag.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_sna.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_som.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_sot.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_ssw.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_swa.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_taq.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_tir.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_tso.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_tum.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_twi.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_tzm.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_umb.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_wol.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_xho.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_yor.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/sib_zul.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_4/utils.py create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_aeb.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_afr.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_aka.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_amh.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_ary.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_arz.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_bam.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_bem.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_cjk.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_dik.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_dyu.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_eng.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_ewe.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_fon.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_fra.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_fuv.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_gaz.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_hau.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_ibo.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_kab.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_kam.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_kbp.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_kea.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_kik.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_kin.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_kmb.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_knc.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_kon.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_lin.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_lua.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_lug.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_luo.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_mos.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_nso.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_nus.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_nya.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_plt.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_por.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_run.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_sag.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_sna.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_som.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_sot.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_ssw.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_swa.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_taq.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_tir.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_tso.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_tum.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_twi.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_tzm.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_umb.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_wol.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_xho.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_yor.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/sib_zul.yaml create mode 100644 lm_eval/tasks/afrobench/sib/prompt_5/utils.py create mode 100644 lm_eval/tasks/afrobench/sib/sib.yaml create mode 100644 lm_eval/tasks/afrobench/sib/utils.py create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/README.md create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_am.yaml create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_en.yaml create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_ha.yaml create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_nso.yaml create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_sw.yaml create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_yo.yaml create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_zu.yaml create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/utils.py create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_am.yaml create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_en.yaml create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_ha.yaml create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_nso.yaml create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_sw.yaml create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_yo.yaml create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_zu.yaml create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/utils.py create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_am.yaml create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_en.yaml create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_ha.yaml create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_nso.yaml create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_sw.yaml create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_yo.yaml create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_zu.yaml create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/utils.py create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_am.yaml create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_en.yaml create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_ha.yaml create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_nso.yaml create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_sw.yaml create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_yo.yaml create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_zu.yaml create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/utils.py create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_am.yaml create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_en.yaml create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_ha.yaml create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_nso.yaml create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_sw.yaml create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_yo.yaml create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_zu.yaml create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/utils.py create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/uhura.yaml create mode 100644 lm_eval/tasks/afrobench/uhura-arc-easy/utils.py create mode 100644 lm_eval/tasks/afrobench/xlsum/README.md create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_1/utils.py create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_amharic.yaml create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_arabic.yaml create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_hausa.yaml create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_igbo.yaml create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_kirundi.yaml create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_oromo.yaml create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_pidgin.yaml create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_somali.yaml create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_swahili.yaml create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_telugu.yaml create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_tigrinya.yaml create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_yoruba.yaml create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_2/utils.py create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_amharic.yaml create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_arabic.yaml create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_hausa.yaml create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_igbo.yaml create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_kirundi.yaml create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_oromo.yaml create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_pidgin.yaml create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_somali.yaml create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_swahili.yaml create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_telugu.yaml create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_tigrinya.yaml create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_yoruba.yaml create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_3/utils.py create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_amharic.yaml create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_arabic.yaml create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_hausa.yaml create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_igbo.yaml create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_kirundi.yaml create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_oromo.yaml create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_pidgin.yaml create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_somali.yaml create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_swahili.yaml create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_telugu.yaml create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_tigrinya.yaml create mode 100644 lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_yoruba.yaml create mode 100644 lm_eval/tasks/afrobench/xlsum/utils.py create mode 100644 lm_eval/tasks/afrobench/xlsum/xlsum.yaml diff --git a/lm_eval/filters/extraction.py b/lm_eval/filters/extraction.py index 9c8d796b..22ca883a 100644 --- a/lm_eval/filters/extraction.py +++ b/lm_eval/filters/extraction.py @@ -54,6 +54,51 @@ class RegexFilter(Filter): return filtered filtered_resps = list(map(lambda x: filter_set(x), resps)) + return filtered_resps + + +@register_filter("regex_pos") +class POSFilter(Filter): + """ """ + + def __init__( + self, + regex_pattern: str = r"\['(.*?)'\]", + group_select=0, + fallback=None, + ) -> None: + """ + pass a string `regex` to run `re.compile(r"regex")` on. + `fallback` defines the output returned if no matches for the regex are located. + """ + if fallback is None: + fallback = ["invalid"] + self.regex_pattern = regex_pattern + self.regex = re.compile(regex_pattern) + self.group_select = group_select + self.fallback = fallback + + def apply(self, resps, docs): + def extract_tagged_tokens(text): + # Extract tagged tokens list from text input using regex + tokens = re.findall(r"\('([^']*)', '([^']*)'\)", text) + return [(token, pos) for token, pos in tokens] + + def extract_pos_tags(result): + pos_tags = [] + if isinstance(result, str): + result = extract_tagged_tokens(result) + pos_tags.extend(pos for _, pos in result) + return pos_tags if pos_tags else self.fallback + + def filter_set(inst): + filtered = [] + for resp in inst: + match = extract_pos_tags(resp) + filtered.append(match) + return filtered + + filtered_resps = map(lambda x: filter_set(x), resps) return filtered_resps diff --git a/lm_eval/filters/transformation.py b/lm_eval/filters/transformation.py index 1a3592b6..722c6740 100644 --- a/lm_eval/filters/transformation.py +++ b/lm_eval/filters/transformation.py @@ -1,3 +1,5 @@ +import re + from lm_eval.api.filter import Filter from lm_eval.api.registry import register_filter @@ -54,3 +56,67 @@ class MapFilter(Filter): return [self.mapping_dict.get(resp, self.default_value) for resp in inst] return [filter_set(resp) for resp in resps] + + +@register_filter("format_span") +class SPANFilter(Filter): + def __init__(self) -> None: + pass + + def apply(self, resps, docs): + def format_ner_text(text): + label_dict = { + "person": "PER", + "location": "LOC", + "organization": "ORG", + "counties": "LOC", + "places": "LOC", + "people": "PER", + "persons": "PER", + "company": "ORG", + "country": "LOC", + "continent": "LOC", + "time": "DATE", + "date": "DATE", + "per": "PER", + "loc": "LOC", + "org": "ORG", + } + text = text.lower() + for key, value in label_dict.items(): + text = text.replace(key, value) + + text = "$".join(i for i in text.split("$$")) + return text.rstrip("$$") + + def format_named_entities(text): + """ + Extract named entities from text and format them as 'label: value $$ label: value'. + Handles grouped entities (e.g., LOC: kenya, uganda) and excludes 'none' values. + """ + # Regular expression to match label: entities pattern + pattern = r"\b(PER|LOC|ORG|DATE):\s*([^$]+)" + # Normalize newline characters + text = text.replace("\n", "$").strip() + matches = re.findall(pattern, text) + + formatted_entities = [] + + for label, values in matches: + # Split multiple entities separated by commas and strip whitespace + entities = [value.strip() for value in values.split(",")] + + # Exclude 'none' entities + for entity in entities: + if entity.lower() != "none": + formatted_entities.append(f"{label.lower()}: {entity}") + + # Join entities with the desired separator + return " $ ".join(formatted_entities) + + def filter_set(inst): + return [ + format_named_entities(format_ner_text(resp.lower())) for resp in inst + ] + + return [filter_set(resp) for resp in resps] diff --git a/lm_eval/tasks/afrimgsm/direct/afrimgsm.yaml b/lm_eval/tasks/afrimgsm/direct/afrimgsm.yaml new file mode 100644 index 00000000..2316a748 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/afrimgsm.yaml @@ -0,0 +1,13 @@ +group: afrimgsm-irokobench +task: + - afrimgsm_tasks_prompt_1 + - afrimgsm_tasks_prompt_2 + - afrimgsm_tasks_prompt_3 + - afrimgsm_tasks_prompt_4 + - afrimgsm_tasks_prompt_5 +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 2 diff --git a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_amh.yaml b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_amh.yaml deleted file mode 100644 index 04d0bdd6..00000000 --- a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_amh.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: amh -doc_to_target: '{% if answer is not none %}{{answer[15:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: direct_yaml -task: afrimgsm_direct_amh diff --git a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_eng.yaml b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_eng.yaml deleted file mode 100644 index 5804270d..00000000 --- a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_eng.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: eng -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: direct_yaml -task: afrimgsm_direct_eng diff --git a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_ewe.yaml b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_ewe.yaml deleted file mode 100644 index 4eae6fc4..00000000 --- a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_ewe.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: ewe -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: direct_yaml -task: afrimgsm_direct_ewe diff --git a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_fra.yaml b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_fra.yaml deleted file mode 100644 index 16aeacf2..00000000 --- a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_fra.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: fra -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: direct_yaml -task: afrimgsm_direct_fra diff --git a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_hau.yaml b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_hau.yaml deleted file mode 100644 index 3a6668e9..00000000 --- a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_hau.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: hau -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: direct_yaml -task: afrimgsm_direct_hau diff --git a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_ibo.yaml b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_ibo.yaml deleted file mode 100644 index ab79986a..00000000 --- a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_ibo.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: ibo -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: direct_yaml -task: afrimgsm_direct_ibo diff --git a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_kin.yaml b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_kin.yaml deleted file mode 100644 index d4c9c75a..00000000 --- a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_kin.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: kin -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: direct_yaml -task: afrimgsm_direct_kin diff --git a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_lin.yaml b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_lin.yaml deleted file mode 100644 index 7136d737..00000000 --- a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_lin.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: lin -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: direct_yaml -task: afrimgsm_direct_lin diff --git a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_lug.yaml b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_lug.yaml deleted file mode 100644 index 03fc0c28..00000000 --- a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_lug.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: lug -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: direct_yaml -task: afrimgsm_direct_lug diff --git a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_orm.yaml b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_orm.yaml deleted file mode 100644 index 49d7e933..00000000 --- a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_orm.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: orm -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: direct_yaml -task: afrimgsm_direct_orm diff --git a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_sna.yaml b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_sna.yaml deleted file mode 100644 index a61de85a..00000000 --- a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_sna.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: sna -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: direct_yaml -task: afrimgsm_direct_sna diff --git a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_sot.yaml b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_sot.yaml deleted file mode 100644 index 455c1adc..00000000 --- a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_sot.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: sot -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: direct_yaml -task: afrimgsm_direct_sot diff --git a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_swa.yaml b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_swa.yaml deleted file mode 100644 index 462ddfd3..00000000 --- a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_swa.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: swa -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: direct_yaml -task: afrimgsm_direct_swa diff --git a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_twi.yaml b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_twi.yaml deleted file mode 100644 index 8c4673b7..00000000 --- a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_twi.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: twi -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: direct_yaml -task: afrimgsm_direct_twi diff --git a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_wol.yaml b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_wol.yaml deleted file mode 100644 index 08a8e030..00000000 --- a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_wol.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: wol -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: direct_yaml -task: afrimgsm_direct_wol diff --git a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_xho.yaml b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_xho.yaml deleted file mode 100644 index 2103d182..00000000 --- a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_xho.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: xho -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: direct_yaml -task: afrimgsm_direct_xho diff --git a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_yor.yaml b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_yor.yaml deleted file mode 100644 index aa084c32..00000000 --- a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_yor.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: yor -doc_to_target: '{% if answer is not none %}{{answer[16:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: direct_yaml -task: afrimgsm_direct_yor diff --git a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_zul.yaml b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_zul.yaml deleted file mode 100644 index dcffb694..00000000 --- a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_zul.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: zul -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: direct_yaml -task: afrimgsm_direct_zul diff --git a/lm_eval/tasks/afrimgsm/direct/direct_yaml b/lm_eval/tasks/afrimgsm/direct/direct_yaml deleted file mode 100644 index f9819fe6..00000000 --- a/lm_eval/tasks/afrimgsm/direct/direct_yaml +++ /dev/null @@ -1,37 +0,0 @@ -# This file will be included in the generated language-specific task configs. -# It doesn't have a yaml file extension as it is not meant to be imported directly -# by the harness. -tag: - - afrimgsm - - afrimgsm_direct -dataset_path: masakhane/afrimgsm -dataset_name: null # Overridden by language-specific config. -output_type: generate_until -# training_split: train -test_split: test -target_delimiter: "" -generation_kwargs: - until: - - "\n\n" - - "\n" - do_sample: false - temperature: 0.0 -filter_list: - - name: remove_whitespace - filter: - - function: remove_whitespace - - function: take_first - - filter: - - function: regex - group_select: -1 - regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) - - function: take_first - name: flexible-extract -metric_list: - - metric: exact_match - aggregation: mean - higher_is_better: true - ignore_case: true - ignore_punctuation: true -metadata: - version: 2.0 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_amh.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_amh.yaml new file mode 100644 index 00000000..23007e36 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: amh +include: afrimgsm_yaml +task: afrimgsm_amh_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_eng.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_eng.yaml new file mode 100644 index 00000000..1d569422 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_eng.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: eng +include: afrimgsm_yaml +task: afrimgsm_eng_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_ewe.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_ewe.yaml new file mode 100644 index 00000000..68980400 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ewe +include: afrimgsm_yaml +task: afrimgsm_ewe_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_fra.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_fra.yaml new file mode 100644 index 00000000..04d57dbd --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_fra.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fra +include: afrimgsm_yaml +task: afrimgsm_fra_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_hau.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_hau.yaml new file mode 100644 index 00000000..aef377d2 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: hau +include: afrimgsm_yaml +task: afrimgsm_hau_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_ibo.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_ibo.yaml new file mode 100644 index 00000000..bbbb7ef8 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ibo +include: afrimgsm_yaml +task: afrimgsm_ibo_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_kin.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_kin.yaml new file mode 100644 index 00000000..dcfc7160 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: kin +include: afrimgsm_yaml +task: afrimgsm_kin_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_lin.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_lin.yaml new file mode 100644 index 00000000..be6a24ce --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_lin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lin +include: afrimgsm_yaml +task: afrimgsm_lin_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_lug.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_lug.yaml new file mode 100644 index 00000000..4bffe69f --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lug +include: afrimgsm_yaml +task: afrimgsm_lug_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_orm.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_orm.yaml new file mode 100644 index 00000000..b916281c --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_orm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: orm +include: afrimgsm_yaml +task: afrimgsm_orm_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_sna.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_sna.yaml new file mode 100644 index 00000000..1afa6bb3 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sna +include: afrimgsm_yaml +task: afrimgsm_sna_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_sot.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_sot.yaml new file mode 100644 index 00000000..f6cd4cdc --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_sot.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sot +include: afrimgsm_yaml +task: afrimgsm_sot_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_swa.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_swa.yaml new file mode 100644 index 00000000..be6dba71 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrimgsm_yaml +task: afrimgsm_swa_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_twi.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_twi.yaml new file mode 100644 index 00000000..a82235d7 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: twi +include: afrimgsm_yaml +task: afrimgsm_twi_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_vai.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_vai.yaml new file mode 100644 index 00000000..e04d28f0 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_vai.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: vai +include: afrimgsm_yaml +task: afrimgsm_vai_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_wol.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_wol.yaml new file mode 100644 index 00000000..34b77376 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: wol +include: afrimgsm_yaml +task: afrimgsm_wol_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_xho.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_xho.yaml new file mode 100644 index 00000000..d17530bd --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: xho +include: afrimgsm_yaml +task: afrimgsm_xho_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_yaml b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_yaml new file mode 100644 index 00000000..19d4f7d1 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_yaml @@ -0,0 +1,35 @@ +tag: + - afrimgsm_tasks + - afrimgsm_tasks_prompt_1 +dataset_path: masakhane/afrimgsm +dataset_name: null # Overridden by language-specific config. +output_type: generate_until +test_split: test +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' +target_delimiter: "" +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +filter_list: + - name: remove_whitespace + filter: + - function: remove_whitespace + - function: take_first + - filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 2.0 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_yor.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_yor.yaml new file mode 100644 index 00000000..ba89f9ac --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_yor.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: yor +include: afrimgsm_yaml +task: afrimgsm_yor_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_zul.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_zul.yaml new file mode 100644 index 00000000..07b89135 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_1/afrimgsm_zul.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: zul +include: afrimgsm_yaml +task: afrimgsm_zul_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_amh.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_amh.yaml new file mode 100644 index 00000000..ac0812c1 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: amh +include: afrimgsm_yaml +task: afrimgsm_amh_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_eng.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_eng.yaml new file mode 100644 index 00000000..94000037 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_eng.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: eng +include: afrimgsm_yaml +task: afrimgsm_eng_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_ewe.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_ewe.yaml new file mode 100644 index 00000000..d0cd4926 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ewe +include: afrimgsm_yaml +task: afrimgsm_ewe_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_fra.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_fra.yaml new file mode 100644 index 00000000..2d709e9c --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_fra.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fra +include: afrimgsm_yaml +task: afrimgsm_fra_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_hau.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_hau.yaml new file mode 100644 index 00000000..3e7d62ab --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: hau +include: afrimgsm_yaml +task: afrimgsm_hau_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_ibo.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_ibo.yaml new file mode 100644 index 00000000..fce1d518 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ibo +include: afrimgsm_yaml +task: afrimgsm_ibo_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_kin.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_kin.yaml new file mode 100644 index 00000000..9c7b6525 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: kin +include: afrimgsm_yaml +task: afrimgsm_kin_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_lin.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_lin.yaml new file mode 100644 index 00000000..71594885 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_lin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lin +include: afrimgsm_yaml +task: afrimgsm_lin_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_lug.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_lug.yaml new file mode 100644 index 00000000..cf463374 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lug +include: afrimgsm_yaml +task: afrimgsm_lug_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_orm.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_orm.yaml new file mode 100644 index 00000000..e3d9d96e --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_orm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: orm +include: afrimgsm_yaml +task: afrimgsm_orm_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_sna.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_sna.yaml new file mode 100644 index 00000000..8e27f832 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sna +include: afrimgsm_yaml +task: afrimgsm_sna_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_sot.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_sot.yaml new file mode 100644 index 00000000..5ad77562 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_sot.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sot +include: afrimgsm_yaml +task: afrimgsm_sot_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_swa.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_swa.yaml new file mode 100644 index 00000000..fea74a3d --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrimgsm_yaml +task: afrimgsm_swa_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_twi.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_twi.yaml new file mode 100644 index 00000000..311639a1 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: twi +include: afrimgsm_yaml +task: afrimgsm_twi_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_vai.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_vai.yaml new file mode 100644 index 00000000..655b23de --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_vai.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: vai +include: afrimgsm_yaml +task: afrimgsm_vai_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_wol.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_wol.yaml new file mode 100644 index 00000000..49355162 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: wol +include: afrimgsm_yaml +task: afrimgsm_wol_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_xho.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_xho.yaml new file mode 100644 index 00000000..1c076be5 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: xho +include: afrimgsm_yaml +task: afrimgsm_xho_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_yaml b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_yaml new file mode 100644 index 00000000..2eaceade --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_yaml @@ -0,0 +1,34 @@ +tag: + - afrimgsm_tasks + - afrimgsm_tasks_prompt_1 +dataset_path: masakhane/afrimgsm +output_type: generate_until +test_split: test +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: "Give direct numerical answers for the question provided. \n\nQuestion: {{question}} \nAnswer: " +target_delimiter: "" +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +filter_list: + - name: remove_whitespace + filter: + - function: remove_whitespace + - function: take_first + - filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 2.0 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_yor.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_yor.yaml new file mode 100644 index 00000000..6faf98bd --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_yor.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: yor +include: afrimgsm_yaml +task: afrimgsm_yor_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_zul.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_zul.yaml new file mode 100644 index 00000000..0b340608 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_2/afrimgsm_zul.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: zul +include: afrimgsm_yaml +task: afrimgsm_zul_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_amh.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_amh.yaml new file mode 100644 index 00000000..eb3ed830 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: amh +include: afrimgsm_yaml +task: afrimgsm_amh_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_eng.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_eng.yaml new file mode 100644 index 00000000..87efd748 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_eng.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: eng +include: afrimgsm_yaml +task: afrimgsm_eng_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_ewe.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_ewe.yaml new file mode 100644 index 00000000..4636069a --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ewe +include: afrimgsm_yaml +task: afrimgsm_ewe_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_fra.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_fra.yaml new file mode 100644 index 00000000..b1faf85c --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_fra.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fra +include: afrimgsm_yaml +task: afrimgsm_fra_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_hau.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_hau.yaml new file mode 100644 index 00000000..c0cfcbe4 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: hau +include: afrimgsm_yaml +task: afrimgsm_hau_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_ibo.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_ibo.yaml new file mode 100644 index 00000000..fc451b94 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ibo +include: afrimgsm_yaml +task: afrimgsm_ibo_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_kin.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_kin.yaml new file mode 100644 index 00000000..f01edaae --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: kin +include: afrimgsm_yaml +task: afrimgsm_kin_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_lin.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_lin.yaml new file mode 100644 index 00000000..86ccf1e4 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_lin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lin +include: afrimgsm_yaml +task: afrimgsm_lin_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_lug.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_lug.yaml new file mode 100644 index 00000000..ff6b6de3 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lug +include: afrimgsm_yaml +task: afrimgsm_lug_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_orm.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_orm.yaml new file mode 100644 index 00000000..8a5ff414 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_orm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: orm +include: afrimgsm_yaml +task: afrimgsm_orm_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_sna.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_sna.yaml new file mode 100644 index 00000000..4000b9dc --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sna +include: afrimgsm_yaml +task: afrimgsm_sna_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_sot.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_sot.yaml new file mode 100644 index 00000000..a9941fce --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_sot.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sot +include: afrimgsm_yaml +task: afrimgsm_sot_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_swa.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_swa.yaml new file mode 100644 index 00000000..abd83580 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrimgsm_yaml +task: afrimgsm_swa_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_twi.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_twi.yaml new file mode 100644 index 00000000..12652498 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: twi +include: afrimgsm_yaml +task: afrimgsm_twi_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_vai.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_vai.yaml new file mode 100644 index 00000000..aae0b5f5 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_vai.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: vai +include: afrimgsm_yaml +task: afrimgsm_vai_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_wol.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_wol.yaml new file mode 100644 index 00000000..2a8fd58b --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: wol +include: afrimgsm_yaml +task: afrimgsm_wol_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_xho.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_xho.yaml new file mode 100644 index 00000000..067e88d7 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: xho +include: afrimgsm_yaml +task: afrimgsm_xho_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_yaml b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_yaml new file mode 100644 index 00000000..8dd3f5ca --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_yaml @@ -0,0 +1,34 @@ +tag: + - afrimgsm_tasks + - afrimgsm_tasks_prompt_3 +dataset_path: masakhane/afrimgsm +output_type: generate_until +test_split: test +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: "Solve the following math question \n\nQuestion: {{question}} \nAnswer: " +target_delimiter: "" +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +filter_list: + - name: remove_whitespace + filter: + - function: remove_whitespace + - function: take_first + - filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 2.0 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_yor.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_yor.yaml new file mode 100644 index 00000000..2d644810 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_yor.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: yor +include: afrimgsm_yaml +task: afrimgsm_yor_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_zul.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_zul.yaml new file mode 100644 index 00000000..4b94a2c1 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_3/afrimgsm_zul.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: zul +include: afrimgsm_yaml +task: afrimgsm_zul_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_amh.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_amh.yaml new file mode 100644 index 00000000..00e45eb4 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_amh.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\ + \ that the response is clear and without any supplementary information. \n\nQuestion:\ + \ {{question}} \nAnswer: " +include: afrimgsm_yaml +task: afrimgsm_amh_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_eng.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_eng.yaml new file mode 100644 index 00000000..0e3abef5 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng +doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\ + \ that the response is clear and without any supplementary information. \n\nQuestion:\ + \ {{question}} \nAnswer: " +include: afrimgsm_yaml +task: afrimgsm_eng_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_ewe.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_ewe.yaml new file mode 100644 index 00000000..cda1994b --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_ewe.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ewe +doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\ + \ that the response is clear and without any supplementary information. \n\nQuestion:\ + \ {{question}} \nAnswer: " +include: afrimgsm_yaml +task: afrimgsm_ewe_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_fra.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_fra.yaml new file mode 100644 index 00000000..b7e5c155 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_fra.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fra +doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\ + \ that the response is clear and without any supplementary information. \n\nQuestion:\ + \ {{question}} \nAnswer: " +include: afrimgsm_yaml +task: afrimgsm_fra_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_hau.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_hau.yaml new file mode 100644 index 00000000..e6ad1a00 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_hau.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\ + \ that the response is clear and without any supplementary information. \n\nQuestion:\ + \ {{question}} \nAnswer: " +include: afrimgsm_yaml +task: afrimgsm_hau_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_ibo.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_ibo.yaml new file mode 100644 index 00000000..7d214574 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_ibo.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\ + \ that the response is clear and without any supplementary information. \n\nQuestion:\ + \ {{question}} \nAnswer: " +include: afrimgsm_yaml +task: afrimgsm_ibo_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_kin.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_kin.yaml new file mode 100644 index 00000000..a2c7cb9f --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_kin.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\ + \ that the response is clear and without any supplementary information. \n\nQuestion:\ + \ {{question}} \nAnswer: " +include: afrimgsm_yaml +task: afrimgsm_kin_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_lin.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_lin.yaml new file mode 100644 index 00000000..44d2d2f3 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_lin.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\ + \ that the response is clear and without any supplementary information. \n\nQuestion:\ + \ {{question}} \nAnswer: " +include: afrimgsm_yaml +task: afrimgsm_lin_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_lug.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_lug.yaml new file mode 100644 index 00000000..d8a12945 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_lug.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\ + \ that the response is clear and without any supplementary information. \n\nQuestion:\ + \ {{question}} \nAnswer: " +include: afrimgsm_yaml +task: afrimgsm_lug_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_orm.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_orm.yaml new file mode 100644 index 00000000..dcf7123f --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_orm.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\ + \ that the response is clear and without any supplementary information. \n\nQuestion:\ + \ {{question}} \nAnswer: " +include: afrimgsm_yaml +task: afrimgsm_orm_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_sna.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_sna.yaml new file mode 100644 index 00000000..3d3414e3 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_sna.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\ + \ that the response is clear and without any supplementary information. \n\nQuestion:\ + \ {{question}} \nAnswer: " +include: afrimgsm_yaml +task: afrimgsm_sna_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_sot.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_sot.yaml new file mode 100644 index 00000000..ac0017a3 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_sot.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sot +doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\ + \ that the response is clear and without any supplementary information. \n\nQuestion:\ + \ {{question}} \nAnswer: " +include: afrimgsm_yaml +task: afrimgsm_sot_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_swa.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_swa.yaml new file mode 100644 index 00000000..043311d3 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_swa.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\ + \ that the response is clear and without any supplementary information. \n\nQuestion:\ + \ {{question}} \nAnswer: " +include: afrimgsm_yaml +task: afrimgsm_swa_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_twi.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_twi.yaml new file mode 100644 index 00000000..f3e030d0 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_twi.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\ + \ that the response is clear and without any supplementary information. \n\nQuestion:\ + \ {{question}} \nAnswer: " +include: afrimgsm_yaml +task: afrimgsm_twi_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_vai.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_vai.yaml new file mode 100644 index 00000000..2dbe04c1 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_vai.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: vai +doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\ + \ that the response is clear and without any supplementary information. \n\nQuestion:\ + \ {{question}} \nAnswer: " +include: afrimgsm_yaml +task: afrimgsm_vai_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_wol.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_wol.yaml new file mode 100644 index 00000000..55f546ab --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_wol.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\ + \ that the response is clear and without any supplementary information. \n\nQuestion:\ + \ {{question}} \nAnswer: " +include: afrimgsm_yaml +task: afrimgsm_wol_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_xho.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_xho.yaml new file mode 100644 index 00000000..a7595498 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_xho.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\ + \ that the response is clear and without any supplementary information. \n\nQuestion:\ + \ {{question}} \nAnswer: " +include: afrimgsm_yaml +task: afrimgsm_xho_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_yaml b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_yaml new file mode 100644 index 00000000..5f34f774 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_yaml @@ -0,0 +1,33 @@ +tag: + - afrimgsm_tasks + - afrimgsm_tasks_prompt_4 +dataset_path: masakhane/afrimgsm +output_type: generate_until +test_split: test +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +target_delimiter: "" +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +filter_list: + - name: remove_whitespace + filter: + - function: remove_whitespace + - function: take_first + - filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 2.0 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_yor.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_yor.yaml new file mode 100644 index 00000000..fd4d01bb --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_yor.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\ + \ that the response is clear and without any supplementary information. \n\nQuestion:\ + \ {{question}} \nAnswer: " +include: afrimgsm_yaml +task: afrimgsm_yor_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_zul.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_zul.yaml new file mode 100644 index 00000000..5151f266 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_4/afrimgsm_zul.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\ + \ that the response is clear and without any supplementary information. \n\nQuestion:\ + \ {{question}} \nAnswer: " +include: afrimgsm_yaml +task: afrimgsm_zul_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_amh.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_amh.yaml new file mode 100644 index 00000000..e7ad215f --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_amh.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: "For mathematical questions provided in Amharic language. Supply the\ + \ accurate numeric answer to the provided question. \n\nQuestion: {{question}} \n\ + Answer: " +include: afrimgsm_yaml +task: afrimgsm_amh_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_eng.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_eng.yaml new file mode 100644 index 00000000..a4de5e95 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng +doc_to_text: "For mathematical questions provided in English language. Supply the\ + \ accurate numeric answer to the provided question. \n\nQuestion: {{question}} \n\ + Answer: " +include: afrimgsm_yaml +task: afrimgsm_eng_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_ewe.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_ewe.yaml new file mode 100644 index 00000000..bdb3c4f2 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_ewe.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ewe +doc_to_text: "For mathematical questions provided in Ewe language. Supply the accurate\ + \ numeric answer to the provided question. \n\nQuestion: {{question}} \nAnswer: " +include: afrimgsm_yaml +task: afrimgsm_ewe_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_fra.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_fra.yaml new file mode 100644 index 00000000..a93e79ec --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_fra.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: fra +doc_to_text: "For mathematical questions provided in French language. Supply the accurate\ + \ numeric answer to the provided question. \n\nQuestion: {{question}} \nAnswer: " +include: afrimgsm_yaml +task: afrimgsm_fra_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_hau.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_hau.yaml new file mode 100644 index 00000000..74fabdee --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_hau.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: "For mathematical questions provided in Hausa language. Supply the accurate\ + \ numeric answer to the provided question. \n\nQuestion: {{question}} \nAnswer: " +include: afrimgsm_yaml +task: afrimgsm_hau_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_ibo.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_ibo.yaml new file mode 100644 index 00000000..c2dd77f2 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_ibo.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "For mathematical questions provided in Igbo language. Supply the accurate\ + \ numeric answer to the provided question. \n\nQuestion: {{question}} \nAnswer: " +include: afrimgsm_yaml +task: afrimgsm_ibo_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_kin.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_kin.yaml new file mode 100644 index 00000000..ba3f3c2f --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_kin.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: "For mathematical questions provided in Kinyarwanda language. Supply\ + \ the accurate numeric answer to the provided question. \n\nQuestion: {{question}}\ + \ \nAnswer: " +include: afrimgsm_yaml +task: afrimgsm_kin_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_lin.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_lin.yaml new file mode 100644 index 00000000..74131916 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_lin.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: "For mathematical questions provided in Lingala language. Supply the\ + \ accurate numeric answer to the provided question. \n\nQuestion: {{question}} \n\ + Answer: " +include: afrimgsm_yaml +task: afrimgsm_lin_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_lug.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_lug.yaml new file mode 100644 index 00000000..b92bc4e6 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_lug.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: "For mathematical questions provided in Luganda language. Supply the\ + \ accurate numeric answer to the provided question. \n\nQuestion: {{question}} \n\ + Answer: " +include: afrimgsm_yaml +task: afrimgsm_lug_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_orm.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_orm.yaml new file mode 100644 index 00000000..0c33dd44 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_orm.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: "For mathematical questions provided in Oromo language. Supply the accurate\ + \ numeric answer to the provided question. \n\nQuestion: {{question}} \nAnswer: " +include: afrimgsm_yaml +task: afrimgsm_orm_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_sna.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_sna.yaml new file mode 100644 index 00000000..a2518c36 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_sna.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: "For mathematical questions provided in chiShona language. Supply the\ + \ accurate numeric answer to the provided question. \n\nQuestion: {{question}} \n\ + Answer: " +include: afrimgsm_yaml +task: afrimgsm_sna_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_sot.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_sot.yaml new file mode 100644 index 00000000..06cb1b05 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_sot.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sot +doc_to_text: "For mathematical questions provided in Sesotho language. Supply the\ + \ accurate numeric answer to the provided question. \n\nQuestion: {{question}} \n\ + Answer: " +include: afrimgsm_yaml +task: afrimgsm_sot_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_swa.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_swa.yaml new file mode 100644 index 00000000..0a08c8e3 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_swa.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: "For mathematical questions provided in Swahili language. Supply the\ + \ accurate numeric answer to the provided question. \n\nQuestion: {{question}} \n\ + Answer: " +include: afrimgsm_yaml +task: afrimgsm_swa_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_twi.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_twi.yaml new file mode 100644 index 00000000..54de3ce5 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_twi.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: "For mathematical questions provided in Twi language. Supply the accurate\ + \ numeric answer to the provided question. \n\nQuestion: {{question}} \nAnswer: " +include: afrimgsm_yaml +task: afrimgsm_twi_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_vai.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_vai.yaml new file mode 100644 index 00000000..ab3337a7 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_vai.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: vai +doc_to_text: "For mathematical questions provided in Vai language. Supply the accurate\ + \ numeric answer to the provided question. \n\nQuestion: {{question}} \nAnswer: " +include: afrimgsm_yaml +task: afrimgsm_vai_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_wol.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_wol.yaml new file mode 100644 index 00000000..4e711e8e --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_wol.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: "For mathematical questions provided in Wolof language. Supply the accurate\ + \ numeric answer to the provided question. \n\nQuestion: {{question}} \nAnswer: " +include: afrimgsm_yaml +task: afrimgsm_wol_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_xho.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_xho.yaml new file mode 100644 index 00000000..728cacf8 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_xho.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: "For mathematical questions provided in isiXhosa language. Supply the\ + \ accurate numeric answer to the provided question. \n\nQuestion: {{question}} \n\ + Answer: " +include: afrimgsm_yaml +task: afrimgsm_xho_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_yaml b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_yaml new file mode 100644 index 00000000..ca8bb03f --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_yaml @@ -0,0 +1,33 @@ +tag: + - afrimgsm_tasks + - afrimgsm_tasks_prompt_5 +dataset_path: masakhane/afrimgsm +output_type: generate_until +test_split: test +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +target_delimiter: "" +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +filter_list: + - name: remove_whitespace + filter: + - function: remove_whitespace + - function: take_first + - filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 2.0 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_yor.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_yor.yaml new file mode 100644 index 00000000..cd0bea64 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_yor.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: "For mathematical questions provided in Yoruba language. Supply the accurate\ + \ numeric answer to the provided question. \n\nQuestion: {{question}} \nAnswer: " +include: afrimgsm_yaml +task: afrimgsm_yor_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_zul.yaml b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_zul.yaml new file mode 100644 index 00000000..fb8474e3 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/prompt_5/afrimgsm_zul.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: "For mathematical questions provided in Zulu language. Supply the accurate\ + \ numeric answer to the provided question. \n\nQuestion: {{question}} \nAnswer: " +include: afrimgsm_yaml +task: afrimgsm_zul_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/afrimgsm_cot.yaml b/lm_eval/tasks/afrimgsm/direct_cot/afrimgsm_cot.yaml new file mode 100644 index 00000000..d07832b4 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/afrimgsm_cot.yaml @@ -0,0 +1,9 @@ +group: afrimgsm_cot-irokobench +task: + - afrimgsm_cot_tasks +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 2 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_amh.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_amh.yaml new file mode 100644 index 00000000..c9f0d931 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: amh +include: afrimgsm_cot_yaml +task: afrimgsm_cot_amh_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_eng.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_eng.yaml new file mode 100644 index 00000000..57c0e564 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_eng.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: eng +include: afrimgsm_cot_yaml +task: afrimgsm_cot_eng_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_ewe.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_ewe.yaml new file mode 100644 index 00000000..55fdff7c --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ewe +include: afrimgsm_cot_yaml +task: afrimgsm_cot_ewe_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_fra.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_fra.yaml new file mode 100644 index 00000000..717a45d9 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_fra.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fra +include: afrimgsm_cot_yaml +task: afrimgsm_cot_fra_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_hau.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_hau.yaml new file mode 100644 index 00000000..f42e0ee5 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: hau +include: afrimgsm_cot_yaml +task: afrimgsm_cot_hau_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_ibo.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_ibo.yaml new file mode 100644 index 00000000..dfabc3e3 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ibo +include: afrimgsm_cot_yaml +task: afrimgsm_cot_ibo_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_kin.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_kin.yaml new file mode 100644 index 00000000..55d20e01 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: kin +include: afrimgsm_cot_yaml +task: afrimgsm_cot_kin_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_lin.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_lin.yaml new file mode 100644 index 00000000..ecbf38d8 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_lin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lin +include: afrimgsm_cot_yaml +task: afrimgsm_cot_lin_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_lug.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_lug.yaml new file mode 100644 index 00000000..033cbce0 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lug +include: afrimgsm_cot_yaml +task: afrimgsm_cot_lug_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_orm.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_orm.yaml new file mode 100644 index 00000000..7ce25aee --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_orm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: orm +include: afrimgsm_cot_yaml +task: afrimgsm_cot_orm_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_sna.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_sna.yaml new file mode 100644 index 00000000..fae029f1 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sna +include: afrimgsm_cot_yaml +task: afrimgsm_cot_sna_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_sot.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_sot.yaml new file mode 100644 index 00000000..d0d12077 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_sot.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sot +include: afrimgsm_cot_yaml +task: afrimgsm_cot_sot_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_swa.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_swa.yaml new file mode 100644 index 00000000..da4e39cf --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrimgsm_cot_yaml +task: afrimgsm_cot_swa_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_twi.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_twi.yaml new file mode 100644 index 00000000..24f55349 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: twi +include: afrimgsm_cot_yaml +task: afrimgsm_cot_twi_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_vai.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_vai.yaml new file mode 100644 index 00000000..cc637170 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_vai.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: vai +include: afrimgsm_cot_yaml +task: afrimgsm_cot_vai_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_wol.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_wol.yaml new file mode 100644 index 00000000..c86b09d6 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: wol +include: afrimgsm_cot_yaml +task: afrimgsm_cot_wol_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_xho.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_xho.yaml new file mode 100644 index 00000000..6f03080c --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: xho +include: afrimgsm_cot_yaml +task: afrimgsm_cot_xho_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_yaml new file mode 100644 index 00000000..6ab733bf --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_yaml @@ -0,0 +1,37 @@ +tag: + - afrimgsm_cot_tasks + - afrimgsm_cot_tasks_prompt_1 +dataset_path: masakhane/afrimgsm +dataset_name: null # Overridden by language-specific config. +output_type: generate_until +training_split: train +test_split: test +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> + - <|eot_id|> +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +filter_list: + - name: "strict-match" + filter: + - function: "regex" + regex_pattern: "The answer is (\\-?[0-9\\.\\,]+)" + - function: "take_first" + - filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +metadata: + version: 2.0 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_yor.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_yor.yaml new file mode 100644 index 00000000..c6858ab2 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_yor.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: yor +include: afrimgsm_cot_yaml +task: afrimgsm_cot_yor_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_zul.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_zul.yaml new file mode 100644 index 00000000..5cacc2be --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_1/afrimgsm_cot_zul.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: zul +include: afrimgsm_cot_yaml +task: afrimgsm_cot_zul_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_amh.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_amh.yaml new file mode 100644 index 00000000..6d5d43fb --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: amh +include: afrimgsm_cot_yaml +task: afrimgsm_cot_amh_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_eng.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_eng.yaml new file mode 100644 index 00000000..84a6b26d --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_eng.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: eng +include: afrimgsm_cot_yaml +task: afrimgsm_cot_eng_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_ewe.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_ewe.yaml new file mode 100644 index 00000000..e7ef2907 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ewe +include: afrimgsm_cot_yaml +task: afrimgsm_cot_ewe_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_fra.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_fra.yaml new file mode 100644 index 00000000..987ac630 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_fra.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fra +include: afrimgsm_cot_yaml +task: afrimgsm_cot_fra_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_hau.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_hau.yaml new file mode 100644 index 00000000..488f693a --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: hau +include: afrimgsm_cot_yaml +task: afrimgsm_cot_hau_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_ibo.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_ibo.yaml new file mode 100644 index 00000000..aefa0aa2 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ibo +include: afrimgsm_cot_yaml +task: afrimgsm_cot_ibo_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_kin.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_kin.yaml new file mode 100644 index 00000000..e183dcd4 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: kin +include: afrimgsm_cot_yaml +task: afrimgsm_cot_kin_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_lin.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_lin.yaml new file mode 100644 index 00000000..840a99ac --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_lin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lin +include: afrimgsm_cot_yaml +task: afrimgsm_cot_lin_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_lug.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_lug.yaml new file mode 100644 index 00000000..75e8b892 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lug +include: afrimgsm_cot_yaml +task: afrimgsm_cot_lug_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_orm.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_orm.yaml new file mode 100644 index 00000000..a36d8935 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_orm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: orm +include: afrimgsm_cot_yaml +task: afrimgsm_cot_orm_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_sna.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_sna.yaml new file mode 100644 index 00000000..25187cec --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sna +include: afrimgsm_cot_yaml +task: afrimgsm_cot_sna_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_sot.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_sot.yaml new file mode 100644 index 00000000..22fc718b --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_sot.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sot +include: afrimgsm_cot_yaml +task: afrimgsm_cot_sot_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_swa.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_swa.yaml new file mode 100644 index 00000000..b0d91d0c --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrimgsm_cot_yaml +task: afrimgsm_cot_swa_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_twi.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_twi.yaml new file mode 100644 index 00000000..03b59a39 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: twi +include: afrimgsm_cot_yaml +task: afrimgsm_cot_twi_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_vai.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_vai.yaml new file mode 100644 index 00000000..8fa4cf5e --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_vai.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: vai +include: afrimgsm_cot_yaml +task: afrimgsm_cot_vai_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_wol.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_wol.yaml new file mode 100644 index 00000000..2611de84 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: wol +include: afrimgsm_cot_yaml +task: afrimgsm_cot_wol_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_xho.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_xho.yaml new file mode 100644 index 00000000..33059776 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: xho +include: afrimgsm_cot_yaml +task: afrimgsm_cot_xho_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_yaml new file mode 100644 index 00000000..505336ba --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_yaml @@ -0,0 +1,37 @@ +tag: + - afrimgsm_cot_tasks + - afrimgsm_cot_tasks_prompt_2 +dataset_path: masakhane/afrimgsm +dataset_name: null # Overridden by language-specific config. +output_type: generate_until +training_split: train +test_split: test +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: 'Give direct numerical answers for the question provided. \n\nQuestion: {{question}} \Step-by-Step Answer: ' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> + - <|eot_id|> +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +filter_list: + - name: "strict-match" + filter: + - function: "regex" + regex_pattern: "The answer is (\\-?[0-9\\.\\,]+)" + - function: "take_first" + - filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +metadata: + version: 2.0 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_yor.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_yor.yaml new file mode 100644 index 00000000..991297c4 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_yor.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: yor +include: afrimgsm_cot_yaml +task: afrimgsm_cot_yor_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_zul.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_zul.yaml new file mode 100644 index 00000000..833edbb1 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_2/afrimgsm_cot_zul.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: zul +include: afrimgsm_cot_yaml +task: afrimgsm_cot_zul_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_amh.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_amh.yaml new file mode 100644 index 00000000..00f830a2 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: amh +include: afrimgsm_cot_yaml +task: afrimgsm_cot_amh_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_eng.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_eng.yaml new file mode 100644 index 00000000..ea0937f2 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_eng.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: eng +include: afrimgsm_cot_yaml +task: afrimgsm_cot_eng_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_ewe.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_ewe.yaml new file mode 100644 index 00000000..dfe111d7 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ewe +include: afrimgsm_cot_yaml +task: afrimgsm_cot_ewe_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_fra.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_fra.yaml new file mode 100644 index 00000000..eb82d3a4 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_fra.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fra +include: afrimgsm_cot_yaml +task: afrimgsm_cot_fra_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_hau.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_hau.yaml new file mode 100644 index 00000000..3162114b --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: hau +include: afrimgsm_cot_yaml +task: afrimgsm_cot_hau_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_ibo.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_ibo.yaml new file mode 100644 index 00000000..f46191a3 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ibo +include: afrimgsm_cot_yaml +task: afrimgsm_cot_ibo_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_kin.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_kin.yaml new file mode 100644 index 00000000..8ddc82ee --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: kin +include: afrimgsm_cot_yaml +task: afrimgsm_cot_kin_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_lin.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_lin.yaml new file mode 100644 index 00000000..769ae73a --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_lin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lin +include: afrimgsm_cot_yaml +task: afrimgsm_cot_lin_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_lug.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_lug.yaml new file mode 100644 index 00000000..e04769a6 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lug +include: afrimgsm_cot_yaml +task: afrimgsm_cot_lug_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_orm.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_orm.yaml new file mode 100644 index 00000000..79a69658 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_orm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: orm +include: afrimgsm_cot_yaml +task: afrimgsm_cot_orm_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_sna.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_sna.yaml new file mode 100644 index 00000000..f08d4425 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sna +include: afrimgsm_cot_yaml +task: afrimgsm_cot_sna_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_sot.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_sot.yaml new file mode 100644 index 00000000..76501f53 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_sot.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sot +include: afrimgsm_cot_yaml +task: afrimgsm_cot_sot_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_swa.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_swa.yaml new file mode 100644 index 00000000..76ea5f96 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrimgsm_cot_yaml +task: afrimgsm_cot_swa_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_twi.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_twi.yaml new file mode 100644 index 00000000..c45b3f0f --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: twi +include: afrimgsm_cot_yaml +task: afrimgsm_cot_twi_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_vai.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_vai.yaml new file mode 100644 index 00000000..ca50c481 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_vai.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: vai +include: afrimgsm_cot_yaml +task: afrimgsm_cot_vai_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_wol.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_wol.yaml new file mode 100644 index 00000000..16dbc506 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: wol +include: afrimgsm_cot_yaml +task: afrimgsm_cot_wol_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_xho.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_xho.yaml new file mode 100644 index 00000000..a329b8eb --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: xho +include: afrimgsm_cot_yaml +task: afrimgsm_cot_xho_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_yaml new file mode 100644 index 00000000..d4d3657d --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_yaml @@ -0,0 +1,37 @@ +tag: + - afrimgsm_cot_tasks + - afrimgsm_cot_tasks_prompt_3 +dataset_path: masakhane/afrimgsm +dataset_name: null # Overridden by language-specific config. +output_type: generate_until +training_split: train +test_split: test +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: 'Solve the following math question \n\nQuestion: {{question}} \nStep-by-Step Answer: ' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> + - <|eot_id|> +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +filter_list: + - name: "strict-match" + filter: + - function: "regex" + regex_pattern: "The answer is (\\-?[0-9\\.\\,]+)" + - function: "take_first" + - filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +metadata: + version: 2.0 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_yor.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_yor.yaml new file mode 100644 index 00000000..003fb634 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_yor.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: yor +include: afrimgsm_cot_yaml +task: afrimgsm_cot_yor_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_zul.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_zul.yaml new file mode 100644 index 00000000..c01468ec --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_3/afrimgsm_cot_zul.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: zul +include: afrimgsm_cot_yaml +task: afrimgsm_cot_zul_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_amh.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_amh.yaml new file mode 100644 index 00000000..6624ddfe --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_amh.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_yaml +task: afrimgsm_cot_amh_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_eng.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_eng.yaml new file mode 100644 index 00000000..fa82cb87 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_yaml +task: afrimgsm_cot_eng_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_ewe.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_ewe.yaml new file mode 100644 index 00000000..135bd975 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_ewe.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ewe +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_yaml +task: afrimgsm_cot_ewe_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_fra.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_fra.yaml new file mode 100644 index 00000000..81a060b2 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_fra.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fra +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_yaml +task: afrimgsm_cot_fra_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_hau.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_hau.yaml new file mode 100644 index 00000000..b53dba58 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_hau.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_yaml +task: afrimgsm_cot_hau_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_ibo.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_ibo.yaml new file mode 100644 index 00000000..2a4236e1 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_ibo.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_yaml +task: afrimgsm_cot_ibo_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_kin.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_kin.yaml new file mode 100644 index 00000000..51407a66 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_kin.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_yaml +task: afrimgsm_cot_kin_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_lin.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_lin.yaml new file mode 100644 index 00000000..248ffeee --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_lin.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_yaml +task: afrimgsm_cot_lin_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_lug.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_lug.yaml new file mode 100644 index 00000000..fbf7c8cd --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_lug.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_yaml +task: afrimgsm_cot_lug_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_orm.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_orm.yaml new file mode 100644 index 00000000..218c3f90 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_orm.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_yaml +task: afrimgsm_cot_orm_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_sna.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_sna.yaml new file mode 100644 index 00000000..81e4840a --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_sna.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_yaml +task: afrimgsm_cot_sna_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_sot.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_sot.yaml new file mode 100644 index 00000000..47bcd414 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_sot.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sot +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_yaml +task: afrimgsm_cot_sot_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_swa.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_swa.yaml new file mode 100644 index 00000000..e0b57a14 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_swa.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_yaml +task: afrimgsm_cot_swa_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_twi.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_twi.yaml new file mode 100644 index 00000000..abdbdec7 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_twi.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_yaml +task: afrimgsm_cot_twi_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_vai.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_vai.yaml new file mode 100644 index 00000000..a0b7913b --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_vai.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: vai +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_yaml +task: afrimgsm_cot_vai_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_wol.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_wol.yaml new file mode 100644 index 00000000..aa75a3f5 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_wol.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_yaml +task: afrimgsm_cot_wol_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_xho.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_xho.yaml new file mode 100644 index 00000000..8c125ebe --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_xho.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_yaml +task: afrimgsm_cot_xho_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_yaml new file mode 100644 index 00000000..59013d84 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_yaml @@ -0,0 +1,36 @@ +tag: + - afrimgsm_cot_tasks + - afrimgsm_cot_tasks_prompt_4 +dataset_path: masakhane/afrimgsm +dataset_name: null # Overridden by language-specific config. +output_type: generate_until +training_split: train +test_split: test +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> + - <|eot_id|> +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +filter_list: + - name: "strict-match" + filter: + - function: "regex" + regex_pattern: "The answer is (\\-?[0-9\\.\\,]+)" + - function: "take_first" + - filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +metadata: + version: 2.0 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_yor.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_yor.yaml new file mode 100644 index 00000000..2c960b75 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_yor.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_yaml +task: afrimgsm_cot_yor_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_zul.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_zul.yaml new file mode 100644 index 00000000..2641b2e5 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_4/afrimgsm_cot_zul.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_yaml +task: afrimgsm_cot_zul_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_amh.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_amh.yaml new file mode 100644 index 00000000..ea512485 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_amh.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: "For mathematical questions provided in Amharic language. Supply the\ + \ accurate step by step answer to the provided question. \n\nQuestion: {{question}}\ + \ \nStep by step answer: " +include: afrimgsm_cot_yaml +task: afrimgsm_cot_amh_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_eng.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_eng.yaml new file mode 100644 index 00000000..9b485061 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng +doc_to_text: "For mathematical questions provided in English language. Supply the\ + \ accurate step by step answer to the provided question. \n\nQuestion: {{question}}\ + \ \nStep by step answer: " +include: afrimgsm_cot_yaml +task: afrimgsm_cot_eng_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_ewe.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_ewe.yaml new file mode 100644 index 00000000..e52f4327 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_ewe.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ewe +doc_to_text: "For mathematical questions provided in Ewe language. Supply the accurate\ + \ step by step answer to the provided question. \n\nQuestion: {{question}} \nStep\ + \ by step answer: " +include: afrimgsm_cot_yaml +task: afrimgsm_cot_ewe_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_fra.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_fra.yaml new file mode 100644 index 00000000..f311e12a --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_fra.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fra +doc_to_text: "For mathematical questions provided in French language. Supply the accurate\ + \ step by step answer to the provided question. \n\nQuestion: {{question}} \nStep\ + \ by step answer: " +include: afrimgsm_cot_yaml +task: afrimgsm_cot_fra_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_hau.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_hau.yaml new file mode 100644 index 00000000..91cc7ace --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_hau.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: "For mathematical questions provided in Hausa language. Supply the accurate\ + \ step by step answer to the provided question. \n\nQuestion: {{question}} \nStep\ + \ by step answer: " +include: afrimgsm_cot_yaml +task: afrimgsm_cot_hau_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_ibo.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_ibo.yaml new file mode 100644 index 00000000..e2c6a5cc --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_ibo.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "For mathematical questions provided in Igbo language. Supply the accurate\ + \ step by step answer to the provided question. \n\nQuestion: {{question}} \nStep\ + \ by step answer: " +include: afrimgsm_cot_yaml +task: afrimgsm_cot_ibo_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_kin.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_kin.yaml new file mode 100644 index 00000000..36c19a99 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_kin.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: "For mathematical questions provided in Kinyarwanda language. Supply\ + \ the accurate step by step answer to the provided question. \n\nQuestion: {{question}}\ + \ \nStep by step answer: " +include: afrimgsm_cot_yaml +task: afrimgsm_cot_kin_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_lin.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_lin.yaml new file mode 100644 index 00000000..419da8ab --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_lin.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: "For mathematical questions provided in Lingala language. Supply the\ + \ accurate step by step answer to the provided question. \n\nQuestion: {{question}}\ + \ \nStep by step answer: " +include: afrimgsm_cot_yaml +task: afrimgsm_cot_lin_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_lug.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_lug.yaml new file mode 100644 index 00000000..918a3e31 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_lug.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: "For mathematical questions provided in Luganda language. Supply the\ + \ accurate step by step answer to the provided question. \n\nQuestion: {{question}}\ + \ \nStep by step answer: " +include: afrimgsm_cot_yaml +task: afrimgsm_cot_lug_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_orm.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_orm.yaml new file mode 100644 index 00000000..a9a448f2 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_orm.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: "For mathematical questions provided in Oromo language. Supply the accurate\ + \ step by step answer to the provided question. \n\nQuestion: {{question}} \nStep\ + \ by step answer: " +include: afrimgsm_cot_yaml +task: afrimgsm_cot_orm_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_sna.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_sna.yaml new file mode 100644 index 00000000..645b2898 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_sna.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: "For mathematical questions provided in chiShona language. Supply the\ + \ accurate step by step answer to the provided question. \n\nQuestion: {{question}}\ + \ \nStep by step answer: " +include: afrimgsm_cot_yaml +task: afrimgsm_cot_sna_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_sot.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_sot.yaml new file mode 100644 index 00000000..a0b940d9 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_sot.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sot +doc_to_text: "For mathematical questions provided in Sesotho language. Supply the\ + \ accurate step by step answer to the provided question. \n\nQuestion: {{question}}\ + \ \nStep by step answer: " +include: afrimgsm_cot_yaml +task: afrimgsm_cot_sot_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_swa.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_swa.yaml new file mode 100644 index 00000000..093ccfa2 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_swa.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: "For mathematical questions provided in Swahili language. Supply the\ + \ accurate step by step answer to the provided question. \n\nQuestion: {{question}}\ + \ \nStep by step answer: " +include: afrimgsm_cot_yaml +task: afrimgsm_cot_swa_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_twi.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_twi.yaml new file mode 100644 index 00000000..dd0436e7 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_twi.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: "For mathematical questions provided in Twi language. Supply the accurate\ + \ step by step answer to the provided question. \n\nQuestion: {{question}} \nStep\ + \ by step answer: " +include: afrimgsm_cot_yaml +task: afrimgsm_cot_twi_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_vai.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_vai.yaml new file mode 100644 index 00000000..0b348be3 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_vai.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: vai +doc_to_text: "For mathematical questions provided in Vai language. Supply the accurate\ + \ step by step answer to the provided question. \n\nQuestion: {{question}} \nStep\ + \ by step answer: " +include: afrimgsm_cot_yaml +task: afrimgsm_cot_vai_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_wol.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_wol.yaml new file mode 100644 index 00000000..b73863ad --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_wol.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: "For mathematical questions provided in Wolof language. Supply the accurate\ + \ step by step answer to the provided question. \n\nQuestion: {{question}} \nStep\ + \ by step answer: " +include: afrimgsm_cot_yaml +task: afrimgsm_cot_wol_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_xho.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_xho.yaml new file mode 100644 index 00000000..1b77d56f --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_xho.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: "For mathematical questions provided in isiXhosa language. Supply the\ + \ accurate step by step answer to the provided question. \n\nQuestion: {{question}}\ + \ \nStep by step answer: " +include: afrimgsm_cot_yaml +task: afrimgsm_cot_xho_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_yaml new file mode 100644 index 00000000..de150891 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_yaml @@ -0,0 +1,36 @@ +tag: + - afrimgsm_cot_tasks + - afrimgsm_cot_tasks_prompt_5 +dataset_path: masakhane/afrimgsm +dataset_name: null # Overridden by language-specific config. +output_type: generate_until +training_split: train +test_split: test +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> + - <|eot_id|> +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +filter_list: + - name: "strict-match" + filter: + - function: "regex" + regex_pattern: "The answer is (\\-?[0-9\\.\\,]+)" + - function: "take_first" + - filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +metadata: + version: 2.0 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_yor.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_yor.yaml new file mode 100644 index 00000000..9032313a --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_yor.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: "For mathematical questions provided in Yoruba language. Supply the accurate\ + \ step by step answer to the provided question. \n\nQuestion: {{question}} \nStep\ + \ by step answer: " +include: afrimgsm_cot_yaml +task: afrimgsm_cot_yor_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_zul.yaml b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_zul.yaml new file mode 100644 index 00000000..0b6ef003 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct_cot/prompt_5/afrimgsm_cot_zul.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: "For mathematical questions provided in Zulu language. Supply the accurate\ + \ step by step answer to the provided question. \n\nQuestion: {{question}} \nStep\ + \ by step answer: " +include: afrimgsm_cot_yaml +task: afrimgsm_cot_zul_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_amh.yaml b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_amh.yaml deleted file mode 100644 index f00400d9..00000000 --- a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_amh.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: amh -doc_to_target: '{% if answer is not none %}{{answer[15:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: cot_yaml -task: afrimgsm_en_cot_amh diff --git a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_eng.yaml b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_eng.yaml deleted file mode 100644 index c62bf206..00000000 --- a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_eng.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: eng -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: cot_yaml -task: afrimgsm_en_cot_eng diff --git a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_ewe.yaml b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_ewe.yaml deleted file mode 100644 index ea246f7c..00000000 --- a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_ewe.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: ewe -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: cot_yaml -task: afrimgsm_en_cot_ewe diff --git a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_fra.yaml b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_fra.yaml deleted file mode 100644 index 16bf57b7..00000000 --- a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_fra.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: fra -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: cot_yaml -task: afrimgsm_en_cot_fra diff --git a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_hau.yaml b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_hau.yaml deleted file mode 100644 index 2a397baf..00000000 --- a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_hau.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: hau -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: cot_yaml -task: afrimgsm_en_cot_hau diff --git a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_ibo.yaml b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_ibo.yaml deleted file mode 100644 index 9bd7bf62..00000000 --- a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_ibo.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: ibo -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: cot_yaml -task: afrimgsm_en_cot_ibo diff --git a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_kin.yaml b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_kin.yaml deleted file mode 100644 index 841913b7..00000000 --- a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_kin.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: kin -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: cot_yaml -task: afrimgsm_en_cot_kin diff --git a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_lin.yaml b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_lin.yaml deleted file mode 100644 index 76d7fdb9..00000000 --- a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_lin.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: lin -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: cot_yaml -task: afrimgsm_en_cot_lin diff --git a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_lug.yaml b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_lug.yaml deleted file mode 100644 index 84c05bb2..00000000 --- a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_lug.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: lug -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: cot_yaml -task: afrimgsm_en_cot_lug diff --git a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_orm.yaml b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_orm.yaml deleted file mode 100644 index e9e5600e..00000000 --- a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_orm.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: orm -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: cot_yaml -task: afrimgsm_en_cot_orm diff --git a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_sna.yaml b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_sna.yaml deleted file mode 100644 index 05868962..00000000 --- a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_sna.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: sna -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: cot_yaml -task: afrimgsm_en_cot_sna diff --git a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_sot.yaml b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_sot.yaml deleted file mode 100644 index ae443f18..00000000 --- a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_sot.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: sot -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: cot_yaml -task: afrimgsm_en_cot_sot diff --git a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_swa.yaml b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_swa.yaml deleted file mode 100644 index 1aa2d07d..00000000 --- a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_swa.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: swa -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: cot_yaml -task: afrimgsm_en_cot_swa diff --git a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_twi.yaml b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_twi.yaml deleted file mode 100644 index 2957cb37..00000000 --- a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_twi.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: twi -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: cot_yaml -task: afrimgsm_en_cot_twi diff --git a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_wol.yaml b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_wol.yaml deleted file mode 100644 index 6ecf4c44..00000000 --- a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_wol.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: wol -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: cot_yaml -task: afrimgsm_en_cot_wol diff --git a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_xho.yaml b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_xho.yaml deleted file mode 100644 index 9dc6691b..00000000 --- a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_xho.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: xho -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: cot_yaml -task: afrimgsm_en_cot_xho diff --git a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_yor.yaml b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_yor.yaml deleted file mode 100644 index 8ef29830..00000000 --- a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_yor.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: yor -doc_to_target: '{% if answer is not none %}{{answer[16:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: cot_yaml -task: afrimgsm_en_cot_yor diff --git a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_zul.yaml b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_zul.yaml deleted file mode 100644 index 24f486e0..00000000 --- a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_zul.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: zul -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: cot_yaml -task: afrimgsm_en_cot_zul diff --git a/lm_eval/tasks/afrimgsm/en_cot/cot_yaml b/lm_eval/tasks/afrimgsm/en_cot/cot_yaml deleted file mode 100644 index b4a0071d..00000000 --- a/lm_eval/tasks/afrimgsm/en_cot/cot_yaml +++ /dev/null @@ -1,37 +0,0 @@ -# This file will be included in the generated language-specific task configs. -# It doesn't have a yaml file extension as it is not meant to be imported directly by the harness. -tag: - - afrimgsm - - afrimgsm_en_cot -dataset_path: masakhane/afrimgsm -dataset_name: null # Overridden by language-specific config. -output_type: generate_until -training_split: train -test_split: test -generation_kwargs: - until: - - "\n\n" - - "\n" - do_sample: false - temperature: 0.0 -target_delimiter: " " -metric_list: - - metric: exact_match - aggregation: mean - higher_is_better: true - ignore_case: true - ignore_punctuation: true -filter_list: - - name: "strict-match" - filter: - - function: "regex" - regex_pattern: "The answer is (\\-?[0-9\\.\\,]+)" - - function: "take_first" - - filter: - - function: regex - group_select: -1 - regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) - - function: take_first - name: flexible-extract -metadata: - version: 2.0 diff --git a/lm_eval/tasks/afrimgsm/gen_utils.py b/lm_eval/tasks/afrimgsm/gen_utils.py new file mode 100644 index 00000000..ecef389f --- /dev/null +++ b/lm_eval/tasks/afrimgsm/gen_utils.py @@ -0,0 +1,122 @@ +import argparse +import os + +import yaml + + +class FunctionTag: + def __init__(self, value): + self.value = value + + +def prompt_func(mode, lang): + prompt_map = { + "prompt_4": "Answer the given question with the step by step solution appropriate numerical value, ensuring that the response is " + "clear and without any supplementary information. \n\nQuestion: {{question}} \nStep by step answer: ", + "prompt_5": f"For mathematical questions provided in {lang} language. Supply the accurate step by step answer to the " + "provided question. \n\nQuestion: {{question}} \nStep by step answer: ", + } + return prompt_map[mode] + + +def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None: + """ + Generate a yaml file for each language. + + :param output_dir: The directory to output the files to. + :param overwrite: Whether to overwrite files if they already exist. + """ + err = [] + languages = { + "eng": "English", + "amh": "Amharic", + "ibo": "Igbo", + "fra": "French", + "sna": "chiShona", + "wol": "Wolof", + "ewe": "Ewe", + "lin": "Lingala", + "lug": "Luganda", + "xho": "isiXhosa", + "kin": "Kinyarwanda", + "twi": "Twi", + "zul": "Zulu", + "orm": "Oromo", + "yor": "Yoruba", + "hau": "Hausa", + "sot": "Sesotho", + "swa": "Swahili", + "vai": "Vai", + } + + for lang in languages.keys(): + try: + file_name = f"afrimgsm_cot_{lang}.yaml" + task_name = f"afrimgsm_cot_{lang}_{mode}" + yaml_template = "afrimgsm_cot_yaml" + if "translate" in output_dir.split("/")[-1]: + file_name = f"afrimgsm_cot_translate_{lang}.yaml" + task_name = f"afrimgsm_cot_translate_{lang}_{mode}" + yaml_template = "afrimgsm_cot_translate_yaml" + if int(mode.split("_")[-1]) > 3: + yaml_details = { + "include": yaml_template, + "task": task_name, + "dataset_name": lang, + "doc_to_text": prompt_func(mode, languages[lang]), + } + else: + yaml_details = { + "include": yaml_template, + "task": task_name, + "dataset_name": lang, + } + os.makedirs(f"{output_dir}/{mode}", exist_ok=True) + with open( + f"{output_dir}/{mode}/{file_name}", + "w" if overwrite else "x", + encoding="utf8", + ) as f: + f.write("# Generated by utils.py\n") + yaml.dump( + yaml_details, + f, + allow_unicode=True, + ) + except FileExistsError: + err.append(file_name) + + if len(err) > 0: + raise FileExistsError( + "Files were not created because they already exist (use --overwrite flag):" + f" {', '.join(err)}" + ) + + +def main() -> None: + """Parse CLI args and generate language-specific yaml files.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + default=True, + action="store_true", + help="Overwrite files if they already exist", + ) + parser.add_argument( + "--output-dir", + default="./translate_cot", + help="Directory to write yaml files to", + ) + parser.add_argument( + "--mode", + default="prompt_5", + choices=["prompt_1", "prompt_2", "prompt_3", "prompt_4", "prompt_5"], + help="Prompt number", + ) + args = parser.parse_args() + + gen_lang_yamls(output_dir=args.output_dir, overwrite=args.overwrite, mode=args.mode) + + +if __name__ == "__main__": + main() diff --git a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_amh.yaml b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_amh.yaml deleted file mode 100644 index 55fbe4bf..00000000 --- a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_amh.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: amh -doc_to_target: '{% if answer is not none %}{{answer[15:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: translate_direct_yaml -task: afrimgsm_translate_direct_amh diff --git a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_eng.yaml b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_eng.yaml deleted file mode 100644 index 1d729a5c..00000000 --- a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_eng.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: eng -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: translate_direct_yaml -task: afrimgsm_translate_direct_eng diff --git a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_ewe.yaml b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_ewe.yaml deleted file mode 100644 index 26191dc8..00000000 --- a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_ewe.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: ewe -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: translate_direct_yaml -task: afrimgsm_translate_direct_ewe diff --git a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_fra.yaml b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_fra.yaml deleted file mode 100644 index 9f0331ee..00000000 --- a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_fra.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: fra -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: translate_direct_yaml -task: afrimgsm_translate_direct_fra diff --git a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_hau.yaml b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_hau.yaml deleted file mode 100644 index 850dad63..00000000 --- a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_hau.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: hau -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: translate_direct_yaml -task: afrimgsm_translate_direct_hau diff --git a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_ibo.yaml b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_ibo.yaml deleted file mode 100644 index 8b81178c..00000000 --- a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_ibo.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: ibo -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: translate_direct_yaml -task: afrimgsm_translate_direct_ibo diff --git a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_kin.yaml b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_kin.yaml deleted file mode 100644 index 5a8f53e2..00000000 --- a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_kin.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: kin -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: translate_direct_yaml -task: afrimgsm_translate_direct_kin diff --git a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_lin.yaml b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_lin.yaml deleted file mode 100644 index 58044ee2..00000000 --- a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_lin.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: lin -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: translate_direct_yaml -task: afrimgsm_translate_direct_lin diff --git a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_lug.yaml b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_lug.yaml deleted file mode 100644 index 87013c14..00000000 --- a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_lug.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: lug -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: translate_direct_yaml -task: afrimgsm_translate_direct_lug diff --git a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_orm.yaml b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_orm.yaml deleted file mode 100644 index 1dd19325..00000000 --- a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_orm.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: orm -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: translate_direct_yaml -task: afrimgsm_translate_direct_orm diff --git a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_sna.yaml b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_sna.yaml deleted file mode 100644 index d710b1da..00000000 --- a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_sna.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: sna -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: translate_direct_yaml -task: afrimgsm_translate_direct_sna diff --git a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_sot.yaml b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_sot.yaml deleted file mode 100644 index 643eaaee..00000000 --- a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_sot.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: sot -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: translate_direct_yaml -task: afrimgsm_translate_direct_sot diff --git a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_swa.yaml b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_swa.yaml deleted file mode 100644 index b882e89c..00000000 --- a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_swa.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: swa -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: translate_direct_yaml -task: afrimgsm_translate_direct_swa diff --git a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_twi.yaml b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_twi.yaml deleted file mode 100644 index ac946eb7..00000000 --- a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_twi.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: twi -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: translate_direct_yaml -task: afrimgsm_translate_direct_twi diff --git a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_wol.yaml b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_wol.yaml deleted file mode 100644 index dbcc6b2e..00000000 --- a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_wol.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: wol -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: translate_direct_yaml -task: afrimgsm_translate_direct_wol diff --git a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_xho.yaml b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_xho.yaml deleted file mode 100644 index dfb3d74f..00000000 --- a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_xho.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: xho -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: translate_direct_yaml -task: afrimgsm_translate_direct_xho diff --git a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_yor.yaml b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_yor.yaml deleted file mode 100644 index 6b4c346f..00000000 --- a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_yor.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: yor -doc_to_target: '{% if answer is not none %}{{answer[16:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: translate_direct_yaml -task: afrimgsm_translate_direct_yor diff --git a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_zul.yaml b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_zul.yaml deleted file mode 100644 index 5e79edff..00000000 --- a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_zul.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by utils.py -dataset_name: zul -doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' -doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' -generation_kwargs: - do_sample: false - until: - - 'Question:' - - - - <|im_end|> -include: translate_direct_yaml -task: afrimgsm_translate_direct_zul diff --git a/lm_eval/tasks/afrimgsm/translate/afrimgsm_tt.yaml b/lm_eval/tasks/afrimgsm/translate/afrimgsm_tt.yaml new file mode 100644 index 00000000..e1cc68ab --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/afrimgsm_tt.yaml @@ -0,0 +1,9 @@ +group: afrimgsm_tt-irokobench +task: + - afrimgsm_tt_tasks +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 2 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_amh.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_amh.yaml new file mode 100644 index 00000000..0f067e53 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: amh +include: afrimgsm_translate_yaml +task: afrimgsm_translate_amh_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_ewe.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_ewe.yaml new file mode 100644 index 00000000..1420deed --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ewe +include: afrimgsm_translate_yaml +task: afrimgsm_translate_ewe_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_fra.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_fra.yaml new file mode 100644 index 00000000..b38e82f2 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_fra.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fra +include: afrimgsm_translate_yaml +task: afrimgsm_translate_fra_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_hau.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_hau.yaml new file mode 100644 index 00000000..768bcab9 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: hau +include: afrimgsm_translate_yaml +task: afrimgsm_translate_hau_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_ibo.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_ibo.yaml new file mode 100644 index 00000000..5333b163 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ibo +include: afrimgsm_translate_yaml +task: afrimgsm_translate_ibo_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_kin.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_kin.yaml new file mode 100644 index 00000000..ae231d6d --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: kin +include: afrimgsm_translate_yaml +task: afrimgsm_translate_kin_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_lin.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_lin.yaml new file mode 100644 index 00000000..65349c7e --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_lin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lin +include: afrimgsm_translate_yaml +task: afrimgsm_translate_lin_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_lug.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_lug.yaml new file mode 100644 index 00000000..7643fc12 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lug +include: afrimgsm_translate_yaml +task: afrimgsm_translate_lug_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_orm.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_orm.yaml new file mode 100644 index 00000000..55e19927 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_orm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: orm +include: afrimgsm_translate_yaml +task: afrimgsm_translate_orm_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_sna.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_sna.yaml new file mode 100644 index 00000000..2f8826ab --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sna +include: afrimgsm_translate_yaml +task: afrimgsm_translate_sna_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_sot.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_sot.yaml new file mode 100644 index 00000000..2b206e3f --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_sot.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sot +include: afrimgsm_translate_yaml +task: afrimgsm_translate_sot_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_swa.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_swa.yaml new file mode 100644 index 00000000..3aede319 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrimgsm_translate_yaml +task: afrimgsm_translate_swa_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_twi.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_twi.yaml new file mode 100644 index 00000000..c8e23103 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: twi +include: afrimgsm_translate_yaml +task: afrimgsm_translate_twi_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_wol.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_wol.yaml new file mode 100644 index 00000000..4b97922f --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: wol +include: afrimgsm_translate_yaml +task: afrimgsm_translate_wol_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_xho.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_xho.yaml new file mode 100644 index 00000000..1abdd50b --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: xho +include: afrimgsm_translate_yaml +task: afrimgsm_translate_xho_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_yaml b/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_yaml new file mode 100644 index 00000000..61451089 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_yaml @@ -0,0 +1,32 @@ +tag: afrimgsm_tt_tasks +dataset_path: masakhane/afrimgsm-translate-test +output_type: generate_until +test_split: test +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' +target_delimiter: "" +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +filter_list: + - name: remove_whitespace + filter: + - function: remove_whitespace + - function: take_first + - filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 2.0 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_yor.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_yor.yaml new file mode 100644 index 00000000..d3927ba8 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_yor.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: yor +include: afrimgsm_translate_yaml +task: afrimgsm_translate_yor_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_zul.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_zul.yaml new file mode 100644 index 00000000..a57260d9 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_1/afrimgsm_translate_zul.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: zul +include: afrimgsm_translate_yaml +task: afrimgsm_translate_zul_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_amh.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_amh.yaml new file mode 100644 index 00000000..49b559be --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: amh +include: afrimgsm_translate_yaml +task: afrimgsm_translate_amh_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_ewe.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_ewe.yaml new file mode 100644 index 00000000..cf82f862 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ewe +include: afrimgsm_translate_yaml +task: afrimgsm_translate_ewe_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_fra.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_fra.yaml new file mode 100644 index 00000000..924ac026 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_fra.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fra +include: afrimgsm_translate_yaml +task: afrimgsm_translate_fra_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_hau.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_hau.yaml new file mode 100644 index 00000000..86d8dbbc --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: hau +include: afrimgsm_translate_yaml +task: afrimgsm_translate_hau_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_ibo.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_ibo.yaml new file mode 100644 index 00000000..466ced5c --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ibo +include: afrimgsm_translate_yaml +task: afrimgsm_translate_ibo_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_kin.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_kin.yaml new file mode 100644 index 00000000..53078341 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: kin +include: afrimgsm_translate_yaml +task: afrimgsm_translate_kin_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_lin.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_lin.yaml new file mode 100644 index 00000000..72aa73d2 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_lin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lin +include: afrimgsm_translate_yaml +task: afrimgsm_translate_lin_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_lug.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_lug.yaml new file mode 100644 index 00000000..88ae24a2 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lug +include: afrimgsm_translate_yaml +task: afrimgsm_translate_lug_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_orm.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_orm.yaml new file mode 100644 index 00000000..7e2ffcc3 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_orm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: orm +include: afrimgsm_translate_yaml +task: afrimgsm_translate_orm_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_sna.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_sna.yaml new file mode 100644 index 00000000..137ccbcd --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sna +include: afrimgsm_translate_yaml +task: afrimgsm_translate_sna_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_sot.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_sot.yaml new file mode 100644 index 00000000..5bd7e53c --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_sot.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sot +include: afrimgsm_translate_yaml +task: afrimgsm_translate_sot_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_swa.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_swa.yaml new file mode 100644 index 00000000..5134b3c4 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrimgsm_translate_yaml +task: afrimgsm_translate_swa_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_twi.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_twi.yaml new file mode 100644 index 00000000..f6135d99 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: twi +include: afrimgsm_translate_yaml +task: afrimgsm_translate_twi_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_wol.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_wol.yaml new file mode 100644 index 00000000..db00be88 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: wol +include: afrimgsm_translate_yaml +task: afrimgsm_translate_wol_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_xho.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_xho.yaml new file mode 100644 index 00000000..3be8dd64 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: xho +include: afrimgsm_translate_yaml +task: afrimgsm_translate_xho_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_yaml b/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_yaml new file mode 100644 index 00000000..63766339 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_yaml @@ -0,0 +1,34 @@ +tag: afrimgsm_tt_tasks +dataset_path: masakhane/afrimgsm-translate-test +output_type: generate_until +test_split: test +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: "Give direct numerical answers for the question provided. \n\nQuestion: {{question}} \nAnswer: " +target_delimiter: "" +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +should_decontaminate: true +doc_to_decontamination_query: "Answer: " +filter_list: + - name: remove_whitespace + filter: + - function: remove_whitespace + - function: take_first + - filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 2.0 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_yor.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_yor.yaml new file mode 100644 index 00000000..01a54e15 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_yor.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: yor +include: afrimgsm_translate_yaml +task: afrimgsm_translate_yor_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_zul.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_zul.yaml new file mode 100644 index 00000000..5f7e74df --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_2/afrimgsm_translate_zul.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: zul +include: afrimgsm_translate_yaml +task: afrimgsm_translate_zul_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_amh.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_amh.yaml new file mode 100644 index 00000000..04a14a1b --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: amh +include: afrimgsm_translate_yaml +task: afrimgsm_translate_amh_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_ewe.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_ewe.yaml new file mode 100644 index 00000000..3cda09e4 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ewe +include: afrimgsm_translate_yaml +task: afrimgsm_translate_ewe_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_fra.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_fra.yaml new file mode 100644 index 00000000..49c95be2 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_fra.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fra +include: afrimgsm_translate_yaml +task: afrimgsm_translate_fra_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_hau.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_hau.yaml new file mode 100644 index 00000000..9d16ac8f --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: hau +include: afrimgsm_translate_yaml +task: afrimgsm_translate_hau_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_ibo.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_ibo.yaml new file mode 100644 index 00000000..2bbb66ff --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ibo +include: afrimgsm_translate_yaml +task: afrimgsm_translate_ibo_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_kin.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_kin.yaml new file mode 100644 index 00000000..488061a3 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: kin +include: afrimgsm_translate_yaml +task: afrimgsm_translate_kin_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_lin.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_lin.yaml new file mode 100644 index 00000000..928ba457 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_lin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lin +include: afrimgsm_translate_yaml +task: afrimgsm_translate_lin_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_lug.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_lug.yaml new file mode 100644 index 00000000..bdc0c807 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lug +include: afrimgsm_translate_yaml +task: afrimgsm_translate_lug_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_orm.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_orm.yaml new file mode 100644 index 00000000..04ec7565 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_orm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: orm +include: afrimgsm_translate_yaml +task: afrimgsm_translate_orm_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_sna.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_sna.yaml new file mode 100644 index 00000000..22ab7bde --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sna +include: afrimgsm_translate_yaml +task: afrimgsm_translate_sna_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_sot.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_sot.yaml new file mode 100644 index 00000000..617340d0 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_sot.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sot +include: afrimgsm_translate_yaml +task: afrimgsm_translate_sot_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_swa.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_swa.yaml new file mode 100644 index 00000000..337ad6e4 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrimgsm_translate_yaml +task: afrimgsm_translate_swa_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_twi.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_twi.yaml new file mode 100644 index 00000000..eb13aba5 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: twi +include: afrimgsm_translate_yaml +task: afrimgsm_translate_twi_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_wol.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_wol.yaml new file mode 100644 index 00000000..f759e6aa --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: wol +include: afrimgsm_translate_yaml +task: afrimgsm_translate_wol_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_xho.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_xho.yaml new file mode 100644 index 00000000..50ab80df --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: xho +include: afrimgsm_translate_yaml +task: afrimgsm_translate_xho_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_yaml b/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_yaml new file mode 100644 index 00000000..544fa0cc --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_yaml @@ -0,0 +1,32 @@ +tag: afrimgsm_tt_tasks +dataset_path: masakhane/afrimgsm-translate-test +output_type: generate_until +test_split: test +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: "Solve the following math question \n\nQuestion: {{question}} \nAnswer: " +target_delimiter: "" +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +filter_list: + - name: remove_whitespace + filter: + - function: remove_whitespace + - function: take_first + - filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 2.0 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_yor.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_yor.yaml new file mode 100644 index 00000000..ee1c7917 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_yor.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: yor +include: afrimgsm_translate_yaml +task: afrimgsm_translate_yor_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_zul.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_zul.yaml new file mode 100644 index 00000000..f3e21704 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_3/afrimgsm_translate_zul.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: zul +include: afrimgsm_translate_yaml +task: afrimgsm_translate_zul_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_amh.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_amh.yaml new file mode 100644 index 00000000..60387afc --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_amh.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\ + \ that the response is clear and without any supplementary information. \n\nQuestion:\ + \ {{question}} \nAnswer: " +include: afrimgsm_translate_yaml +task: afrimgsm_translate_amh_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_ewe.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_ewe.yaml new file mode 100644 index 00000000..7633bc3e --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_ewe.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ewe +doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\ + \ that the response is clear and without any supplementary information. \n\nQuestion:\ + \ {{question}} \nAnswer: " +include: afrimgsm_translate_yaml +task: afrimgsm_translate_ewe_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_fra.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_fra.yaml new file mode 100644 index 00000000..a8e16ea9 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_fra.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fra +doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\ + \ that the response is clear and without any supplementary information. \n\nQuestion:\ + \ {{question}} \nAnswer: " +include: afrimgsm_translate_yaml +task: afrimgsm_translate_fra_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_hau.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_hau.yaml new file mode 100644 index 00000000..98282050 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_hau.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\ + \ that the response is clear and without any supplementary information. \n\nQuestion:\ + \ {{question}} \nAnswer: " +include: afrimgsm_translate_yaml +task: afrimgsm_translate_hau_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_ibo.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_ibo.yaml new file mode 100644 index 00000000..b8acf8d0 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_ibo.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\ + \ that the response is clear and without any supplementary information. \n\nQuestion:\ + \ {{question}} \nAnswer: " +include: afrimgsm_translate_yaml +task: afrimgsm_translate_ibo_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_kin.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_kin.yaml new file mode 100644 index 00000000..74ac1173 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_kin.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\ + \ that the response is clear and without any supplementary information. \n\nQuestion:\ + \ {{question}} \nAnswer: " +include: afrimgsm_translate_yaml +task: afrimgsm_translate_kin_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_lin.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_lin.yaml new file mode 100644 index 00000000..6cf11361 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_lin.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\ + \ that the response is clear and without any supplementary information. \n\nQuestion:\ + \ {{question}} \nAnswer: " +include: afrimgsm_translate_yaml +task: afrimgsm_translate_lin_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_lug.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_lug.yaml new file mode 100644 index 00000000..5dffbdb8 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_lug.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\ + \ that the response is clear and without any supplementary information. \n\nQuestion:\ + \ {{question}} \nAnswer: " +include: afrimgsm_translate_yaml +task: afrimgsm_translate_lug_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_orm.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_orm.yaml new file mode 100644 index 00000000..30f776f4 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_orm.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\ + \ that the response is clear and without any supplementary information. \n\nQuestion:\ + \ {{question}} \nAnswer: " +include: afrimgsm_translate_yaml +task: afrimgsm_translate_orm_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_sna.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_sna.yaml new file mode 100644 index 00000000..63efa250 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_sna.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\ + \ that the response is clear and without any supplementary information. \n\nQuestion:\ + \ {{question}} \nAnswer: " +include: afrimgsm_translate_yaml +task: afrimgsm_translate_sna_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_sot.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_sot.yaml new file mode 100644 index 00000000..19b86220 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_sot.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sot +doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\ + \ that the response is clear and without any supplementary information. \n\nQuestion:\ + \ {{question}} \nAnswer: " +include: afrimgsm_translate_yaml +task: afrimgsm_translate_sot_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_swa.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_swa.yaml new file mode 100644 index 00000000..20236ae8 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_swa.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\ + \ that the response is clear and without any supplementary information. \n\nQuestion:\ + \ {{question}} \nAnswer: " +include: afrimgsm_translate_yaml +task: afrimgsm_translate_swa_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_twi.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_twi.yaml new file mode 100644 index 00000000..5fe7e747 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_twi.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\ + \ that the response is clear and without any supplementary information. \n\nQuestion:\ + \ {{question}} \nAnswer: " +include: afrimgsm_translate_yaml +task: afrimgsm_translate_twi_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_wol.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_wol.yaml new file mode 100644 index 00000000..a8fb5640 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_wol.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\ + \ that the response is clear and without any supplementary information. \n\nQuestion:\ + \ {{question}} \nAnswer: " +include: afrimgsm_translate_yaml +task: afrimgsm_translate_wol_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_xho.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_xho.yaml new file mode 100644 index 00000000..fdb63749 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_xho.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\ + \ that the response is clear and without any supplementary information. \n\nQuestion:\ + \ {{question}} \nAnswer: " +include: afrimgsm_translate_yaml +task: afrimgsm_translate_xho_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_yaml b/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_yaml new file mode 100644 index 00000000..2d390394 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_yaml @@ -0,0 +1,31 @@ +tag: afrimgsm_tt_tasks +dataset_path: masakhane/afrimgsm-translate-test +output_type: generate_until +test_split: test +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +target_delimiter: "" +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +filter_list: + - name: remove_whitespace + filter: + - function: remove_whitespace + - function: take_first + - filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 2.0 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_yor.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_yor.yaml new file mode 100644 index 00000000..f5cb74d4 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_yor.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\ + \ that the response is clear and without any supplementary information. \n\nQuestion:\ + \ {{question}} \nAnswer: " +include: afrimgsm_translate_yaml +task: afrimgsm_translate_yor_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_zul.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_zul.yaml new file mode 100644 index 00000000..6f0a068e --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_4/afrimgsm_translate_zul.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: "Answer the given question with the appropriate numerical value, ensuring\ + \ that the response is clear and without any supplementary information. \n\nQuestion:\ + \ {{question}} \nAnswer: " +include: afrimgsm_translate_yaml +task: afrimgsm_translate_zul_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_amh.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_amh.yaml new file mode 100644 index 00000000..48ca09aa --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_amh.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: "For mathematical questions provided in Amharic language. Supply the\ + \ accurate numeric answer to the provided question. \n\nQuestion: {{question}} \n\ + Answer: " +include: afrimgsm_translate_yaml +task: afrimgsm_translate_amh_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_ewe.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_ewe.yaml new file mode 100644 index 00000000..a4a254f0 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_ewe.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ewe +doc_to_text: "For mathematical questions provided in Ewe language. Supply the accurate\ + \ numeric answer to the provided question. \n\nQuestion: {{question}} \nAnswer: " +include: afrimgsm_translate_yaml +task: afrimgsm_translate_ewe_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_fra.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_fra.yaml new file mode 100644 index 00000000..ac623045 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_fra.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: fra +doc_to_text: "For mathematical questions provided in French language. Supply the accurate\ + \ numeric answer to the provided question. \n\nQuestion: {{question}} \nAnswer: " +include: afrimgsm_translate_yaml +task: afrimgsm_translate_fra_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_hau.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_hau.yaml new file mode 100644 index 00000000..695f1f37 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_hau.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: "For mathematical questions provided in Hausa language. Supply the accurate\ + \ numeric answer to the provided question. \n\nQuestion: {{question}} \nAnswer: " +include: afrimgsm_translate_yaml +task: afrimgsm_translate_hau_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_ibo.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_ibo.yaml new file mode 100644 index 00000000..7fd530e7 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_ibo.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "For mathematical questions provided in Igbo language. Supply the accurate\ + \ numeric answer to the provided question. \n\nQuestion: {{question}} \nAnswer: " +include: afrimgsm_translate_yaml +task: afrimgsm_translate_ibo_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_kin.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_kin.yaml new file mode 100644 index 00000000..52ea0a78 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_kin.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: "For mathematical questions provided in Kinyarwanda language. Supply\ + \ the accurate numeric answer to the provided question. \n\nQuestion: {{question}}\ + \ \nAnswer: " +include: afrimgsm_translate_yaml +task: afrimgsm_translate_kin_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_lin.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_lin.yaml new file mode 100644 index 00000000..07cf6a6b --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_lin.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: "For mathematical questions provided in Lingala language. Supply the\ + \ accurate numeric answer to the provided question. \n\nQuestion: {{question}} \n\ + Answer: " +include: afrimgsm_translate_yaml +task: afrimgsm_translate_lin_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_lug.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_lug.yaml new file mode 100644 index 00000000..fa3461be --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_lug.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: "For mathematical questions provided in Luganda language. Supply the\ + \ accurate numeric answer to the provided question. \n\nQuestion: {{question}} \n\ + Answer: " +include: afrimgsm_translate_yaml +task: afrimgsm_translate_lug_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_orm.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_orm.yaml new file mode 100644 index 00000000..c1a00385 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_orm.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: "For mathematical questions provided in Oromo language. Supply the accurate\ + \ numeric answer to the provided question. \n\nQuestion: {{question}} \nAnswer: " +include: afrimgsm_translate_yaml +task: afrimgsm_translate_orm_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_sna.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_sna.yaml new file mode 100644 index 00000000..c7f08a78 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_sna.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: "For mathematical questions provided in chiShona language. Supply the\ + \ accurate numeric answer to the provided question. \n\nQuestion: {{question}} \n\ + Answer: " +include: afrimgsm_translate_yaml +task: afrimgsm_translate_sna_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_sot.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_sot.yaml new file mode 100644 index 00000000..b258204f --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_sot.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sot +doc_to_text: "For mathematical questions provided in Sesotho language. Supply the\ + \ accurate numeric answer to the provided question. \n\nQuestion: {{question}} \n\ + Answer: " +include: afrimgsm_translate_yaml +task: afrimgsm_translate_sot_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_swa.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_swa.yaml new file mode 100644 index 00000000..a950c84d --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_swa.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: "For mathematical questions provided in Swahili language. Supply the\ + \ accurate numeric answer to the provided question. \n\nQuestion: {{question}} \n\ + Answer: " +include: afrimgsm_translate_yaml +task: afrimgsm_translate_swa_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_twi.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_twi.yaml new file mode 100644 index 00000000..0a048829 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_twi.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: "For mathematical questions provided in Twi language. Supply the accurate\ + \ numeric answer to the provided question. \n\nQuestion: {{question}} \nAnswer: " +include: afrimgsm_translate_yaml +task: afrimgsm_translate_twi_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_wol.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_wol.yaml new file mode 100644 index 00000000..61ffc3f9 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_wol.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: "For mathematical questions provided in Wolof language. Supply the accurate\ + \ numeric answer to the provided question. \n\nQuestion: {{question}} \nAnswer: " +include: afrimgsm_translate_yaml +task: afrimgsm_translate_wol_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_xho.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_xho.yaml new file mode 100644 index 00000000..c308cc7f --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_xho.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: "For mathematical questions provided in isiXhosa language. Supply the\ + \ accurate numeric answer to the provided question. \n\nQuestion: {{question}} \n\ + Answer: " +include: afrimgsm_translate_yaml +task: afrimgsm_translate_xho_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_yaml b/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_yaml new file mode 100644 index 00000000..2d390394 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_yaml @@ -0,0 +1,31 @@ +tag: afrimgsm_tt_tasks +dataset_path: masakhane/afrimgsm-translate-test +output_type: generate_until +test_split: test +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +target_delimiter: "" +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +filter_list: + - name: remove_whitespace + filter: + - function: remove_whitespace + - function: take_first + - filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 2.0 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_yor.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_yor.yaml new file mode 100644 index 00000000..f2a0a0fd --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_yor.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: "For mathematical questions provided in Yoruba language. Supply the accurate\ + \ numeric answer to the provided question. \n\nQuestion: {{question}} \nAnswer: " +include: afrimgsm_translate_yaml +task: afrimgsm_translate_yor_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_zul.yaml b/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_zul.yaml new file mode 100644 index 00000000..b52cfb72 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/prompt_5/afrimgsm_translate_zul.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: "For mathematical questions provided in Zulu language. Supply the accurate\ + \ numeric answer to the provided question. \n\nQuestion: {{question}} \nAnswer: " +include: afrimgsm_translate_yaml +task: afrimgsm_translate_zul_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/translate/translate_direct_yaml b/lm_eval/tasks/afrimgsm/translate/translate_direct_yaml deleted file mode 100644 index f9f1c866..00000000 --- a/lm_eval/tasks/afrimgsm/translate/translate_direct_yaml +++ /dev/null @@ -1,36 +0,0 @@ -# This file will be included in the generated language-specific task configs. -# It doesn't have a yaml file extension as it is not meant to be imported directly -# by the harness. -tag: - - afrimgsm - - afrimgsm_translate -dataset_path: masakhane/afrimgsm-translate-test -dataset_name: null # Overridden by language-specific config. -output_type: generate_until -test_split: test -generation_kwargs: - until: - - "\n\n" - - "\n" - do_sample: false - temperature: 0.0 -target_delimiter: " " -filter_list: - - name: remove_whitespace - filter: - - function: remove_whitespace - - function: take_first - - filter: - - function: regex - group_select: -1 - regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) - - function: take_first - name: flexible-extract -metric_list: - - metric: exact_match - aggregation: mean - higher_is_better: true - ignore_case: true - ignore_punctuation: true -metadata: - version: 2.0 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/afrimgsm_tt_cot.yaml b/lm_eval/tasks/afrimgsm/translate_cot/afrimgsm_tt_cot.yaml new file mode 100644 index 00000000..d43ddd23 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/afrimgsm_tt_cot.yaml @@ -0,0 +1,9 @@ +group: afrimgsm_tt_cot-irokobench +task: + - afrimgsm_tt_cot_tasks +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 2 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_amh.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_amh.yaml new file mode 100644 index 00000000..da7764e8 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: amh +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_amh_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_ewe.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_ewe.yaml new file mode 100644 index 00000000..e65e9298 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ewe +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_ewe_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_fra.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_fra.yaml new file mode 100644 index 00000000..a16b91ff --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_fra.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fra +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_fra_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_hau.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_hau.yaml new file mode 100644 index 00000000..3bee8575 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: hau +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_hau_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_ibo.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_ibo.yaml new file mode 100644 index 00000000..e6f495ea --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ibo +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_ibo_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_kin.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_kin.yaml new file mode 100644 index 00000000..400bf888 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: kin +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_kin_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_lin.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_lin.yaml new file mode 100644 index 00000000..22599e98 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_lin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lin +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_lin_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_lug.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_lug.yaml new file mode 100644 index 00000000..83c9565d --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lug +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_lug_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_orm.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_orm.yaml new file mode 100644 index 00000000..ca19eb14 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_orm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: orm +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_orm_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_sna.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_sna.yaml new file mode 100644 index 00000000..e99d3aa7 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sna +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_sna_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_sot.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_sot.yaml new file mode 100644 index 00000000..9f8fc2ef --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_sot.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sot +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_sot_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_swa.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_swa.yaml new file mode 100644 index 00000000..d0545ccc --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_swa_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_twi.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_twi.yaml new file mode 100644 index 00000000..a0b4f971 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: twi +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_twi_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_vai.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_vai.yaml new file mode 100644 index 00000000..76c18a3f --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_vai.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: vai +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_vai_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_wol.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_wol.yaml new file mode 100644 index 00000000..ee0d6fc9 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: wol +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_wol_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_xho.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_xho.yaml new file mode 100644 index 00000000..6f340a46 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: xho +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_xho_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_yaml new file mode 100644 index 00000000..8ad7f006 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_yaml @@ -0,0 +1,33 @@ +tag: afrimgsm_tt_cot_tasks +dataset_path: masakhane/afrimgsm-translate-test +dataset_name: null # Overridden by language-specific config. +output_type: generate_until +test_split: test +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +filter_list: + - name: "strict-match" + filter: + - function: "regex" + regex_pattern: "The answer is (\\-?[0-9\\.\\,]+)" + - function: "take_first" + - filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +metadata: + version: 2.0 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_yor.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_yor.yaml new file mode 100644 index 00000000..cf093766 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_yor.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: yor +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_yor_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_zul.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_zul.yaml new file mode 100644 index 00000000..1bb302a4 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_1/afrimgsm_cot_translate_zul.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: zul +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_zul_prompt_1 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_amh.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_amh.yaml new file mode 100644 index 00000000..e9f97735 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: amh +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_amh_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_ewe.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_ewe.yaml new file mode 100644 index 00000000..1a837647 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ewe +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_ewe_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_fra.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_fra.yaml new file mode 100644 index 00000000..b496c775 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_fra.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fra +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_fra_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_hau.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_hau.yaml new file mode 100644 index 00000000..1022ae89 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: hau +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_hau_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_ibo.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_ibo.yaml new file mode 100644 index 00000000..dd2a2528 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ibo +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_ibo_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_kin.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_kin.yaml new file mode 100644 index 00000000..4d8a986a --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: kin +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_kin_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_lin.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_lin.yaml new file mode 100644 index 00000000..70d40323 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_lin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lin +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_lin_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_lug.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_lug.yaml new file mode 100644 index 00000000..a774c189 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lug +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_lug_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_orm.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_orm.yaml new file mode 100644 index 00000000..b325b2ce --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_orm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: orm +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_orm_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_sna.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_sna.yaml new file mode 100644 index 00000000..3e852558 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sna +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_sna_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_sot.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_sot.yaml new file mode 100644 index 00000000..a298b504 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_sot.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sot +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_sot_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_swa.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_swa.yaml new file mode 100644 index 00000000..e3de9a4c --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_swa_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_twi.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_twi.yaml new file mode 100644 index 00000000..c2e1ab61 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: twi +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_twi_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_vai.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_vai.yaml new file mode 100644 index 00000000..9186f1e0 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_vai.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: vai +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_vai_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_wol.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_wol.yaml new file mode 100644 index 00000000..185b406b --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: wol +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_wol_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_xho.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_xho.yaml new file mode 100644 index 00000000..52a0e1ca --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: xho +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_xho_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_yaml new file mode 100644 index 00000000..ad059aea --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_yaml @@ -0,0 +1,33 @@ +tag: afrimgsm_tt_cot_tasks +dataset_path: masakhane/afrimgsm-translate-test +dataset_name: null # Overridden by language-specific config. +output_type: generate_until +test_split: test +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: 'Give direct numerical answers for the question provided. \n\nQuestion: {{question}} \Step-by-Step Answer: ' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +filter_list: + - name: "strict-match" + filter: + - function: "regex" + regex_pattern: "The answer is (\\-?[0-9\\.\\,]+)" + - function: "take_first" + - filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +metadata: + version: 2.0 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_yor.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_yor.yaml new file mode 100644 index 00000000..2452b0fa --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_yor.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: yor +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_yor_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_zul.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_zul.yaml new file mode 100644 index 00000000..2ce8151b --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_2/afrimgsm_cot_translate_zul.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: zul +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_zul_prompt_2 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_amh.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_amh.yaml new file mode 100644 index 00000000..b627e575 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: amh +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_amh_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_ewe.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_ewe.yaml new file mode 100644 index 00000000..52dc345f --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ewe +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_ewe_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_fra.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_fra.yaml new file mode 100644 index 00000000..d2b7582c --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_fra.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fra +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_fra_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_hau.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_hau.yaml new file mode 100644 index 00000000..d57be8c8 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: hau +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_hau_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_ibo.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_ibo.yaml new file mode 100644 index 00000000..296ea98f --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ibo +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_ibo_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_kin.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_kin.yaml new file mode 100644 index 00000000..2b555b3e --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: kin +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_kin_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_lin.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_lin.yaml new file mode 100644 index 00000000..0ace69b2 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_lin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lin +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_lin_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_lug.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_lug.yaml new file mode 100644 index 00000000..bd25a166 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lug +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_lug_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_orm.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_orm.yaml new file mode 100644 index 00000000..698c1474 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_orm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: orm +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_orm_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_sna.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_sna.yaml new file mode 100644 index 00000000..354df6bf --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sna +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_sna_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_sot.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_sot.yaml new file mode 100644 index 00000000..5990be74 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_sot.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sot +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_sot_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_swa.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_swa.yaml new file mode 100644 index 00000000..d8666298 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_swa_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_twi.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_twi.yaml new file mode 100644 index 00000000..78ef85fc --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: twi +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_twi_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_vai.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_vai.yaml new file mode 100644 index 00000000..25ec4e8f --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_vai.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: vai +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_vai_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_wol.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_wol.yaml new file mode 100644 index 00000000..7815a0a5 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: wol +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_wol_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_xho.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_xho.yaml new file mode 100644 index 00000000..e45afd3a --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: xho +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_xho_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_yaml new file mode 100644 index 00000000..c0bb7d66 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_yaml @@ -0,0 +1,33 @@ +tag: afrimgsm_tt_cot_tasks +dataset_path: masakhane/afrimgsm-translate-test +dataset_name: null # Overridden by language-specific config. +output_type: generate_until +test_split: test +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: 'Solve the following math question \n\nQuestion: {{question}} \nStep-by-Step Answer: ' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +filter_list: + - name: "strict-match" + filter: + - function: "regex" + regex_pattern: "The answer is (\\-?[0-9\\.\\,]+)" + - function: "take_first" + - filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +metadata: + version: 2.0 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_yor.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_yor.yaml new file mode 100644 index 00000000..39e18cb4 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_yor.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: yor +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_yor_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_zul.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_zul.yaml new file mode 100644 index 00000000..08fbc9e1 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_3/afrimgsm_cot_translate_zul.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: zul +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_zul_prompt_3 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_amh.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_amh.yaml new file mode 100644 index 00000000..4f73f15f --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_amh.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_amh_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_ewe.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_ewe.yaml new file mode 100644 index 00000000..0d57247b --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_ewe.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ewe +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_ewe_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_fra.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_fra.yaml new file mode 100644 index 00000000..6a2f70ca --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_fra.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fra +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_fra_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_hau.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_hau.yaml new file mode 100644 index 00000000..c5e7903c --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_hau.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_hau_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_ibo.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_ibo.yaml new file mode 100644 index 00000000..cf15ed07 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_ibo.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_ibo_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_kin.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_kin.yaml new file mode 100644 index 00000000..c1b57c13 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_kin.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_kin_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_lin.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_lin.yaml new file mode 100644 index 00000000..81cdb1ff --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_lin.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_lin_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_lug.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_lug.yaml new file mode 100644 index 00000000..a949b028 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_lug.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_lug_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_orm.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_orm.yaml new file mode 100644 index 00000000..d4deb092 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_orm.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_orm_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_sna.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_sna.yaml new file mode 100644 index 00000000..5ebac199 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_sna.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_sna_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_sot.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_sot.yaml new file mode 100644 index 00000000..bf83c8f0 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_sot.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sot +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_sot_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_swa.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_swa.yaml new file mode 100644 index 00000000..87b581b8 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_swa.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_swa_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_twi.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_twi.yaml new file mode 100644 index 00000000..223901eb --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_twi.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_twi_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_vai.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_vai.yaml new file mode 100644 index 00000000..92ce3892 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_vai.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: vai +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_vai_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_wol.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_wol.yaml new file mode 100644 index 00000000..c626fde4 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_wol.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_wol_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_xho.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_xho.yaml new file mode 100644 index 00000000..285b679c --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_xho.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_xho_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_yaml new file mode 100644 index 00000000..241787c7 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_yaml @@ -0,0 +1,32 @@ +tag: afrimgsm_tt_cot_tasks +dataset_path: masakhane/afrimgsm-translate-test +dataset_name: null # Overridden by language-specific config. +output_type: generate_until +test_split: test +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +filter_list: + - name: "strict-match" + filter: + - function: "regex" + regex_pattern: "The answer is (\\-?[0-9\\.\\,]+)" + - function: "take_first" + - filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +metadata: + version: 2.0 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_yor.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_yor.yaml new file mode 100644 index 00000000..f76f4cd1 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_yor.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_yor_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_zul.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_zul.yaml new file mode 100644 index 00000000..7023a554 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_4/afrimgsm_cot_translate_zul.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: "Answer the given question with the step by step solution appropriate\ + \ numerical value, ensuring that the response is clear and without any supplementary\ + \ information. \n\nQuestion: {{question}} \nStep by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_zul_prompt_4 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_amh.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_amh.yaml new file mode 100644 index 00000000..d64f088f --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_amh.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: "For mathematical questions provided in Amharic language. Supply the\ + \ accurate step by step answer to the provided question. \n\nQuestion: {{question}}\ + \ \nStep by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_amh_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_ewe.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_ewe.yaml new file mode 100644 index 00000000..de4aa48d --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_ewe.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ewe +doc_to_text: "For mathematical questions provided in Ewe language. Supply the accurate\ + \ step by step answer to the provided question. \n\nQuestion: {{question}} \nStep\ + \ by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_ewe_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_fra.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_fra.yaml new file mode 100644 index 00000000..5cf15ea1 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_fra.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fra +doc_to_text: "For mathematical questions provided in French language. Supply the accurate\ + \ step by step answer to the provided question. \n\nQuestion: {{question}} \nStep\ + \ by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_fra_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_hau.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_hau.yaml new file mode 100644 index 00000000..0dfa643c --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_hau.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: "For mathematical questions provided in Hausa language. Supply the accurate\ + \ step by step answer to the provided question. \n\nQuestion: {{question}} \nStep\ + \ by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_hau_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_ibo.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_ibo.yaml new file mode 100644 index 00000000..959f3890 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_ibo.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "For mathematical questions provided in Igbo language. Supply the accurate\ + \ step by step answer to the provided question. \n\nQuestion: {{question}} \nStep\ + \ by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_ibo_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_kin.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_kin.yaml new file mode 100644 index 00000000..85ff4196 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_kin.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: "For mathematical questions provided in Kinyarwanda language. Supply\ + \ the accurate step by step answer to the provided question. \n\nQuestion: {{question}}\ + \ \nStep by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_kin_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_lin.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_lin.yaml new file mode 100644 index 00000000..87db46c9 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_lin.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: "For mathematical questions provided in Lingala language. Supply the\ + \ accurate step by step answer to the provided question. \n\nQuestion: {{question}}\ + \ \nStep by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_lin_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_lug.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_lug.yaml new file mode 100644 index 00000000..ac0fde85 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_lug.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: "For mathematical questions provided in Luganda language. Supply the\ + \ accurate step by step answer to the provided question. \n\nQuestion: {{question}}\ + \ \nStep by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_lug_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_orm.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_orm.yaml new file mode 100644 index 00000000..bcf34106 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_orm.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: "For mathematical questions provided in Oromo language. Supply the accurate\ + \ step by step answer to the provided question. \n\nQuestion: {{question}} \nStep\ + \ by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_orm_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_sna.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_sna.yaml new file mode 100644 index 00000000..d5eac98d --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_sna.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: "For mathematical questions provided in chiShona language. Supply the\ + \ accurate step by step answer to the provided question. \n\nQuestion: {{question}}\ + \ \nStep by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_sna_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_sot.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_sot.yaml new file mode 100644 index 00000000..9fc015cd --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_sot.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sot +doc_to_text: "For mathematical questions provided in Sesotho language. Supply the\ + \ accurate step by step answer to the provided question. \n\nQuestion: {{question}}\ + \ \nStep by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_sot_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_swa.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_swa.yaml new file mode 100644 index 00000000..179af867 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_swa.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: "For mathematical questions provided in Swahili language. Supply the\ + \ accurate step by step answer to the provided question. \n\nQuestion: {{question}}\ + \ \nStep by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_swa_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_twi.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_twi.yaml new file mode 100644 index 00000000..ebb680a6 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_twi.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: "For mathematical questions provided in Twi language. Supply the accurate\ + \ step by step answer to the provided question. \n\nQuestion: {{question}} \nStep\ + \ by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_twi_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_vai.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_vai.yaml new file mode 100644 index 00000000..3d284864 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_vai.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: vai +doc_to_text: "For mathematical questions provided in Vai language. Supply the accurate\ + \ step by step answer to the provided question. \n\nQuestion: {{question}} \nStep\ + \ by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_vai_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_wol.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_wol.yaml new file mode 100644 index 00000000..799cc29f --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_wol.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: "For mathematical questions provided in Wolof language. Supply the accurate\ + \ step by step answer to the provided question. \n\nQuestion: {{question}} \nStep\ + \ by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_wol_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_xho.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_xho.yaml new file mode 100644 index 00000000..7969fdba --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_xho.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: "For mathematical questions provided in isiXhosa language. Supply the\ + \ accurate step by step answer to the provided question. \n\nQuestion: {{question}}\ + \ \nStep by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_xho_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_yaml new file mode 100644 index 00000000..241787c7 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_yaml @@ -0,0 +1,32 @@ +tag: afrimgsm_tt_cot_tasks +dataset_path: masakhane/afrimgsm-translate-test +dataset_name: null # Overridden by language-specific config. +output_type: generate_until +test_split: test +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +filter_list: + - name: "strict-match" + filter: + - function: "regex" + regex_pattern: "The answer is (\\-?[0-9\\.\\,]+)" + - function: "take_first" + - filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +metadata: + version: 2.0 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_yor.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_yor.yaml new file mode 100644 index 00000000..0d05de22 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_yor.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: "For mathematical questions provided in Yoruba language. Supply the accurate\ + \ step by step answer to the provided question. \n\nQuestion: {{question}} \nStep\ + \ by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_yor_prompt_5 diff --git a/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_zul.yaml b/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_zul.yaml new file mode 100644 index 00000000..68329068 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate_cot/prompt_5/afrimgsm_cot_translate_zul.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: "For mathematical questions provided in Zulu language. Supply the accurate\ + \ step by step answer to the provided question. \n\nQuestion: {{question}} \nStep\ + \ by step answer: " +include: afrimgsm_cot_translate_yaml +task: afrimgsm_cot_translate_zul_prompt_5 diff --git a/lm_eval/tasks/afrimmlu/direct/afrimmlu.yaml b/lm_eval/tasks/afrimmlu/direct/afrimmlu.yaml new file mode 100644 index 00000000..202c3182 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/afrimmlu.yaml @@ -0,0 +1,13 @@ +group: afrimmlu-irokobench +task: + - afrimmlu_tasks_prompt_1 + - afrimmlu_tasks_prompt_2 + - afrimmlu_tasks_prompt_3 + - afrimmlu_tasks_prompt_4 + - afrimmlu_tasks_prompt_5 +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 2 diff --git a/lm_eval/tasks/afrimmlu/direct/afrimmlu_common_yaml b/lm_eval/tasks/afrimmlu/direct/afrimmlu_common_yaml deleted file mode 100644 index 53acc4c8..00000000 --- a/lm_eval/tasks/afrimmlu/direct/afrimmlu_common_yaml +++ /dev/null @@ -1,37 +0,0 @@ -tag: - - afrimmlu - - afrimmlu_direct -task: null -dataset_path: masakhane/afrimmlu -dataset_name: null -output_type: multiple_choice -validation_split: validation -test_split: test -fewshot_split: validation -doc_to_text: !function utils.doc_to_text -doc_to_target: "{{['A', 'B', 'C', 'D'].index(answer)}}" -doc_to_choice: !function utils.doc_to_choice -should_decontaminate: true -doc_to_decontamination_query: "Question: {{question}}\nAnswer:" -metric_list: - - metric: f1 - aggregation: !function utils.weighted_f1_score - # aggregation: mean - average: weighted - hf_evaluate: true - higher_is_better: True - ignore_case: true - ignore_punctuation: true - regexes_to_ignore: - - "," - - "\\$" - - metric: acc - aggregation: mean - higher_is_better: true - ignore_case: true - ignore_punctuation: true - regexes_to_ignore: - - "," - - "\\$" -metadata: - version: 1.0 diff --git a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_amh.yaml b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_amh.yaml deleted file mode 100644 index aa60c668..00000000 --- a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_amh.yaml +++ /dev/null @@ -1,3 +0,0 @@ -dataset_name: amh -include: afrimmlu_common_yaml -task: afrimmlu_direct_amh diff --git a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_eng.yaml b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_eng.yaml deleted file mode 100644 index a1e647cd..00000000 --- a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_eng.yaml +++ /dev/null @@ -1,3 +0,0 @@ -dataset_name: eng -include: afrimmlu_common_yaml -task: afrimmlu_direct_eng diff --git a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_ewe.yaml b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_ewe.yaml deleted file mode 100644 index 1cc45ddc..00000000 --- a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_ewe.yaml +++ /dev/null @@ -1,3 +0,0 @@ -dataset_name: ewe -include: afrimmlu_common_yaml -task: afrimmlu_direct_ewe diff --git a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_fra.yaml b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_fra.yaml deleted file mode 100644 index e6adb6c8..00000000 --- a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_fra.yaml +++ /dev/null @@ -1,3 +0,0 @@ -dataset_name: fra -include: afrimmlu_common_yaml -task: afrimmlu_direct_fra diff --git a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_hau.yaml b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_hau.yaml deleted file mode 100644 index 9cc9a1ae..00000000 --- a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_hau.yaml +++ /dev/null @@ -1,3 +0,0 @@ -dataset_name: hau -include: afrimmlu_common_yaml -task: afrimmlu_direct_hau diff --git a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_ibo.yaml b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_ibo.yaml deleted file mode 100644 index 6abb2c4a..00000000 --- a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_ibo.yaml +++ /dev/null @@ -1,3 +0,0 @@ -dataset_name: ibo -include: afrimmlu_common_yaml -task: afrimmlu_direct_ibo diff --git a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_kin.yaml b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_kin.yaml deleted file mode 100644 index 2f81f709..00000000 --- a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_kin.yaml +++ /dev/null @@ -1,3 +0,0 @@ -dataset_name: kin -include: afrimmlu_common_yaml -task: afrimmlu_direct_kin diff --git a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_lin.yaml b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_lin.yaml deleted file mode 100644 index 55363ed9..00000000 --- a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_lin.yaml +++ /dev/null @@ -1,3 +0,0 @@ -dataset_name: lin -include: afrimmlu_common_yaml -task: afrimmlu_direct_lin diff --git a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_lug.yaml b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_lug.yaml deleted file mode 100644 index 0d484427..00000000 --- a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_lug.yaml +++ /dev/null @@ -1,3 +0,0 @@ -dataset_name: lug -include: afrimmlu_common_yaml -task: afrimmlu_direct_lug diff --git a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_orm.yaml b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_orm.yaml deleted file mode 100644 index 763eb8a7..00000000 --- a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_orm.yaml +++ /dev/null @@ -1,3 +0,0 @@ -dataset_name: orm -include: afrimmlu_common_yaml -task: afrimmlu_direct_orm diff --git a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_sna.yaml b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_sna.yaml deleted file mode 100644 index ed9e69af..00000000 --- a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_sna.yaml +++ /dev/null @@ -1,3 +0,0 @@ -dataset_name: sna -include: afrimmlu_common_yaml -task: afrimmlu_direct_sna diff --git a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_sot.yaml b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_sot.yaml deleted file mode 100644 index acdba0fd..00000000 --- a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_sot.yaml +++ /dev/null @@ -1,3 +0,0 @@ -dataset_name: sot -include: afrimmlu_common_yaml -task: afrimmlu_direct_sot diff --git a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_swa.yaml b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_swa.yaml deleted file mode 100644 index c1aa82b0..00000000 --- a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_swa.yaml +++ /dev/null @@ -1,3 +0,0 @@ -dataset_name: swa -include: afrimmlu_common_yaml -task: afrimmlu_direct_swa diff --git a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_twi.yaml b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_twi.yaml deleted file mode 100644 index 2695d4a1..00000000 --- a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_twi.yaml +++ /dev/null @@ -1,3 +0,0 @@ -dataset_name: twi -include: afrimmlu_common_yaml -task: afrimmlu_direct_twi diff --git a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_wol.yaml b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_wol.yaml deleted file mode 100644 index 027f8376..00000000 --- a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_wol.yaml +++ /dev/null @@ -1,3 +0,0 @@ -dataset_name: wol -include: afrimmlu_common_yaml -task: afrimmlu_direct_wol diff --git a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_xho.yaml b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_xho.yaml deleted file mode 100644 index 8e0c1297..00000000 --- a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_xho.yaml +++ /dev/null @@ -1,3 +0,0 @@ -dataset_name: xho -include: afrimmlu_common_yaml -task: afrimmlu_direct_xho diff --git a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_yor.yaml b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_yor.yaml deleted file mode 100644 index 2a9f7645..00000000 --- a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_yor.yaml +++ /dev/null @@ -1,3 +0,0 @@ -dataset_name: yor -include: afrimmlu_common_yaml -task: afrimmlu_direct_yor diff --git a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_zul.yaml b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_zul.yaml deleted file mode 100644 index 9d8d3b41..00000000 --- a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_zul.yaml +++ /dev/null @@ -1,3 +0,0 @@ -dataset_name: zul -include: afrimmlu_common_yaml -task: afrimmlu_direct_zul diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct b/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct new file mode 100644 index 00000000..a3e17f71 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct @@ -0,0 +1,37 @@ +tag: + - afrimmlu_tasks + - afrimmlu_tasks_prompt_1 + - afrobench_mmlu_tasks +dataset_path: masakhane/afrimmlu +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_text: !function utils.doc_to_text +doc_to_target: "{{['A', 'B', 'C', 'D'].index(answer)}}" +doc_to_choice: !function utils.doc_to_choice +should_decontaminate: true +doc_to_decontamination_query: "Question: {{question}}\nAnswer:" +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_amh.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_amh.yaml new file mode 100644 index 00000000..8a26369b --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: amh +include: afrimmlu_direct +task: afrimmlu_direct_amh_prompt_1 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_eng.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_eng.yaml new file mode 100644 index 00000000..18a34c7b --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_eng.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: eng +include: afrimmlu_direct +task: afrimmlu_direct_eng_prompt_1 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_ewe.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_ewe.yaml new file mode 100644 index 00000000..e85bd7dc --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ewe +include: afrimmlu_direct +task: afrimmlu_direct_ewe_prompt_1 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_fra.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_fra.yaml new file mode 100644 index 00000000..4e8a2875 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_fra.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fra +include: afrimmlu_direct +task: afrimmlu_direct_fra_prompt_1 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_hau.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_hau.yaml new file mode 100644 index 00000000..b438ea31 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: hau +include: afrimmlu_direct +task: afrimmlu_direct_hau_prompt_1 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_ibo.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_ibo.yaml new file mode 100644 index 00000000..8b08d48e --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ibo +include: afrimmlu_direct +task: afrimmlu_direct_ibo_prompt_1 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_kin.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_kin.yaml new file mode 100644 index 00000000..00d82dfa --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: kin +include: afrimmlu_direct +task: afrimmlu_direct_kin_prompt_1 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_lin.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_lin.yaml new file mode 100644 index 00000000..7059c941 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_lin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lin +include: afrimmlu_direct +task: afrimmlu_direct_lin_prompt_1 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_lug.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_lug.yaml new file mode 100644 index 00000000..33016472 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lug +include: afrimmlu_direct +task: afrimmlu_direct_lug_prompt_1 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_orm.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_orm.yaml new file mode 100644 index 00000000..5047ae98 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_orm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: orm +include: afrimmlu_direct +task: afrimmlu_direct_orm_prompt_1 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_sna.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_sna.yaml new file mode 100644 index 00000000..17222f95 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sna +include: afrimmlu_direct +task: afrimmlu_direct_sna_prompt_1 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_sot.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_sot.yaml new file mode 100644 index 00000000..c62ce9bf --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_sot.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sot +include: afrimmlu_direct +task: afrimmlu_direct_sot_prompt_1 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_swa.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_swa.yaml new file mode 100644 index 00000000..c5ebed9f --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrimmlu_direct +task: afrimmlu_direct_swa_prompt_1 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_twi.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_twi.yaml new file mode 100644 index 00000000..fb270c94 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: twi +include: afrimmlu_direct +task: afrimmlu_direct_twi_prompt_1 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_wol.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_wol.yaml new file mode 100644 index 00000000..4ccbc47c --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: wol +include: afrimmlu_direct +task: afrimmlu_direct_wol_prompt_1 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_xho.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_xho.yaml new file mode 100644 index 00000000..3e30d201 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: xho +include: afrimmlu_direct +task: afrimmlu_direct_xho_prompt_1 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_yor.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_yor.yaml new file mode 100644 index 00000000..e3de56f8 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_yor.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: yor +include: afrimmlu_direct +task: afrimmlu_direct_yor_prompt_1 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_zul.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_zul.yaml new file mode 100644 index 00000000..86c56fec --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_1/afrimmlu_direct_zul.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: zul +include: afrimmlu_direct +task: afrimmlu_direct_zul_prompt_1 diff --git a/lm_eval/tasks/afrimmlu/direct/utils.py b/lm_eval/tasks/afrimmlu/direct/prompt_1/utils.py similarity index 100% rename from lm_eval/tasks/afrimmlu/direct/utils.py rename to lm_eval/tasks/afrimmlu/direct/prompt_1/utils.py diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct b/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct new file mode 100644 index 00000000..fefabf7e --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct @@ -0,0 +1,37 @@ +tag: + - afrimmlu_tasks + - afrimmlu_tasks_prompt_2 + - afrobench_mmlu_tasks +dataset_path: masakhane/afrimmlu +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_text: !function utils.doc_to_text +doc_to_target: "{{['A', 'B', 'C', 'D'].index(answer)}}" +doc_to_choice: !function utils.doc_to_choice +should_decontaminate: true +doc_to_decontamination_query: "Question: {{question}}\nAnswer:" +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_amh.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_amh.yaml new file mode 100644 index 00000000..85d85171 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: amh +include: afrimmlu_direct +task: afrimmlu_direct_amh_prompt_2 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_eng.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_eng.yaml new file mode 100644 index 00000000..c46eca5e --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_eng.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: eng +include: afrimmlu_direct +task: afrimmlu_direct_eng_prompt_2 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_ewe.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_ewe.yaml new file mode 100644 index 00000000..26acfcfa --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ewe +include: afrimmlu_direct +task: afrimmlu_direct_ewe_prompt_2 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_fra.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_fra.yaml new file mode 100644 index 00000000..47f0bfb1 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_fra.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fra +include: afrimmlu_direct +task: afrimmlu_direct_fra_prompt_2 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_hau.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_hau.yaml new file mode 100644 index 00000000..29b4a4d2 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: hau +include: afrimmlu_direct +task: afrimmlu_direct_hau_prompt_2 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_ibo.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_ibo.yaml new file mode 100644 index 00000000..0cf7db0e --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ibo +include: afrimmlu_direct +task: afrimmlu_direct_ibo_prompt_2 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_kin.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_kin.yaml new file mode 100644 index 00000000..ce7c2e89 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: kin +include: afrimmlu_direct +task: afrimmlu_direct_kin_prompt_2 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_lin.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_lin.yaml new file mode 100644 index 00000000..51fcea62 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_lin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lin +include: afrimmlu_direct +task: afrimmlu_direct_lin_prompt_2 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_lug.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_lug.yaml new file mode 100644 index 00000000..f4c57ae3 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lug +include: afrimmlu_direct +task: afrimmlu_direct_lug_prompt_2 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_orm.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_orm.yaml new file mode 100644 index 00000000..494d4240 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_orm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: orm +include: afrimmlu_direct +task: afrimmlu_direct_orm_prompt_2 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_sna.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_sna.yaml new file mode 100644 index 00000000..7706ad64 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sna +include: afrimmlu_direct +task: afrimmlu_direct_sna_prompt_2 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_sot.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_sot.yaml new file mode 100644 index 00000000..353bd257 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_sot.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sot +include: afrimmlu_direct +task: afrimmlu_direct_sot_prompt_2 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_swa.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_swa.yaml new file mode 100644 index 00000000..54a16c6c --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrimmlu_direct +task: afrimmlu_direct_swa_prompt_2 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_twi.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_twi.yaml new file mode 100644 index 00000000..8bb35bd5 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: twi +include: afrimmlu_direct +task: afrimmlu_direct_twi_prompt_2 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_wol.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_wol.yaml new file mode 100644 index 00000000..963f7cd2 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: wol +include: afrimmlu_direct +task: afrimmlu_direct_wol_prompt_2 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_xho.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_xho.yaml new file mode 100644 index 00000000..9da0589a --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: xho +include: afrimmlu_direct +task: afrimmlu_direct_xho_prompt_2 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_yor.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_yor.yaml new file mode 100644 index 00000000..39b36541 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_yor.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: yor +include: afrimmlu_direct +task: afrimmlu_direct_yor_prompt_2 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_zul.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_zul.yaml new file mode 100644 index 00000000..8766392a --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_2/afrimmlu_direct_zul.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: zul +include: afrimmlu_direct +task: afrimmlu_direct_zul_prompt_2 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_2/utils.py b/lm_eval/tasks/afrimmlu/direct/prompt_2/utils.py new file mode 100644 index 00000000..e0cfb334 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_2/utils.py @@ -0,0 +1,30 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_choice(doc): + choices = eval(doc["choices"]) + return choices + + +def doc_to_text(doc): + output = """As an expert in {subject}, choose the most accurate answer to the question below. +Your goal is to select the correct option 'A', 'B', 'C', or 'D' by understanding the nuances of the topic. + +Question: {question} +Choices: + A: {choice1} + B: {choice2} + C: {choice3} + D: {choice4} +Answer: """ + + choices = eval(doc["choices"]) + text = output.format( + subject=doc["subject"], + question=doc["question"], + choice1=choices[0], + choice2=choices[1], + choice3=choices[2], + choice4=choices[3], + ) + return text diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct b/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct new file mode 100644 index 00000000..fb2fd165 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct @@ -0,0 +1,37 @@ +tag: + - afrimmlu_tasks + - afrimmlu_tasks_prompt_3 + - afrobench_mmlu_tasks +dataset_path: masakhane/afrimmlu +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_text: !function utils.doc_to_text +doc_to_target: "{{['A', 'B', 'C', 'D'].index(answer)}}" +doc_to_choice: !function utils.doc_to_choice +should_decontaminate: true +doc_to_decontamination_query: "Question: {{question}}\nAnswer:" +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_amh.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_amh.yaml new file mode 100644 index 00000000..c7c28f20 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: amh +include: afrimmlu_direct +task: afrimmlu_direct_amh_prompt_3 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_eng.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_eng.yaml new file mode 100644 index 00000000..83f7cfcb --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_eng.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: eng +include: afrimmlu_direct +task: afrimmlu_direct_eng_prompt_3 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_ewe.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_ewe.yaml new file mode 100644 index 00000000..351bdf33 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ewe +include: afrimmlu_direct +task: afrimmlu_direct_ewe_prompt_3 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_fra.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_fra.yaml new file mode 100644 index 00000000..69197818 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_fra.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fra +include: afrimmlu_direct +task: afrimmlu_direct_fra_prompt_3 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_hau.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_hau.yaml new file mode 100644 index 00000000..90521523 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: hau +include: afrimmlu_direct +task: afrimmlu_direct_hau_prompt_3 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_ibo.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_ibo.yaml new file mode 100644 index 00000000..43a88fe6 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ibo +include: afrimmlu_direct +task: afrimmlu_direct_ibo_prompt_3 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_kin.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_kin.yaml new file mode 100644 index 00000000..977f3ab2 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: kin +include: afrimmlu_direct +task: afrimmlu_direct_kin_prompt_3 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_lin.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_lin.yaml new file mode 100644 index 00000000..2d25584a --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_lin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lin +include: afrimmlu_direct +task: afrimmlu_direct_lin_prompt_3 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_lug.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_lug.yaml new file mode 100644 index 00000000..2b4da1a7 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lug +include: afrimmlu_direct +task: afrimmlu_direct_lug_prompt_3 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_orm.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_orm.yaml new file mode 100644 index 00000000..2738f980 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_orm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: orm +include: afrimmlu_direct +task: afrimmlu_direct_orm_prompt_3 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_sna.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_sna.yaml new file mode 100644 index 00000000..063d111a --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sna +include: afrimmlu_direct +task: afrimmlu_direct_sna_prompt_3 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_sot.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_sot.yaml new file mode 100644 index 00000000..6cf6e66d --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_sot.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sot +include: afrimmlu_direct +task: afrimmlu_direct_sot_prompt_3 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_swa.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_swa.yaml new file mode 100644 index 00000000..e90204d4 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrimmlu_direct +task: afrimmlu_direct_swa_prompt_3 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_twi.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_twi.yaml new file mode 100644 index 00000000..719ebe90 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: twi +include: afrimmlu_direct +task: afrimmlu_direct_twi_prompt_3 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_wol.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_wol.yaml new file mode 100644 index 00000000..8f0f1d0d --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: wol +include: afrimmlu_direct +task: afrimmlu_direct_wol_prompt_3 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_xho.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_xho.yaml new file mode 100644 index 00000000..8fc1af4d --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: xho +include: afrimmlu_direct +task: afrimmlu_direct_xho_prompt_3 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_yor.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_yor.yaml new file mode 100644 index 00000000..a641b03a --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_yor.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: yor +include: afrimmlu_direct +task: afrimmlu_direct_yor_prompt_3 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_zul.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_zul.yaml new file mode 100644 index 00000000..8c6b493d --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_3/afrimmlu_direct_zul.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: zul +include: afrimmlu_direct +task: afrimmlu_direct_zul_prompt_3 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_3/utils.py b/lm_eval/tasks/afrimmlu/direct/prompt_3/utils.py new file mode 100644 index 00000000..bc3da2e2 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_3/utils.py @@ -0,0 +1,32 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_choice(doc): + choices = eval(doc["choices"]) + return choices + + +def doc_to_text(doc): + output = """You are a subject matter expert in {subject}. + + Utilizing your expertise in {subject}, answer the following multiple-choice question + by picking 'A', 'B', 'C', or 'D'. + +Question: {question} +Choices: + A: {choice1} + B: {choice2} + C: {choice3} + D: {choice4} +Answer: """ + + choices = eval(doc["choices"]) + text = output.format( + subject=doc["subject"], + question=doc["question"], + choice1=choices[0], + choice2=choices[1], + choice3=choices[2], + choice4=choices[3], + ) + return text diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct b/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct new file mode 100644 index 00000000..c15b7b2f --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct @@ -0,0 +1,37 @@ +tag: + - afrimmlu_tasks + - afrimmlu_tasks_prompt_4 + - afrobench_mmlu_tasks +dataset_path: masakhane/afrimmlu +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_text: !function utils.doc_to_text +doc_to_target: "{{['A', 'B', 'C', 'D'].index(answer)}}" +doc_to_choice: !function utils.doc_to_choice +should_decontaminate: true +doc_to_decontamination_query: "Question: {{question}}\nAnswer:" +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_amh.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_amh.yaml new file mode 100644 index 00000000..cc862dc2 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: amh +include: afrimmlu_direct +task: afrimmlu_direct_amh_prompt_4 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_eng.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_eng.yaml new file mode 100644 index 00000000..69baef50 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_eng.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: eng +include: afrimmlu_direct +task: afrimmlu_direct_eng_prompt_4 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_ewe.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_ewe.yaml new file mode 100644 index 00000000..f5af1074 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ewe +include: afrimmlu_direct +task: afrimmlu_direct_ewe_prompt_4 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_fra.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_fra.yaml new file mode 100644 index 00000000..d1f94eea --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_fra.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fra +include: afrimmlu_direct +task: afrimmlu_direct_fra_prompt_4 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_hau.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_hau.yaml new file mode 100644 index 00000000..ca8f7c5e --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: hau +include: afrimmlu_direct +task: afrimmlu_direct_hau_prompt_4 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_ibo.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_ibo.yaml new file mode 100644 index 00000000..8a181d07 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ibo +include: afrimmlu_direct +task: afrimmlu_direct_ibo_prompt_4 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_kin.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_kin.yaml new file mode 100644 index 00000000..8f861224 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: kin +include: afrimmlu_direct +task: afrimmlu_direct_kin_prompt_4 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_lin.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_lin.yaml new file mode 100644 index 00000000..3c7d3ecf --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_lin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lin +include: afrimmlu_direct +task: afrimmlu_direct_lin_prompt_4 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_lug.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_lug.yaml new file mode 100644 index 00000000..46720131 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lug +include: afrimmlu_direct +task: afrimmlu_direct_lug_prompt_4 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_orm.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_orm.yaml new file mode 100644 index 00000000..e5266825 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_orm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: orm +include: afrimmlu_direct +task: afrimmlu_direct_orm_prompt_4 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_sna.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_sna.yaml new file mode 100644 index 00000000..af29225a --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sna +include: afrimmlu_direct +task: afrimmlu_direct_sna_prompt_4 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_sot.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_sot.yaml new file mode 100644 index 00000000..0342dc10 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_sot.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sot +include: afrimmlu_direct +task: afrimmlu_direct_sot_prompt_4 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_swa.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_swa.yaml new file mode 100644 index 00000000..ec9a3525 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrimmlu_direct +task: afrimmlu_direct_swa_prompt_4 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_twi.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_twi.yaml new file mode 100644 index 00000000..83dc916c --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: twi +include: afrimmlu_direct +task: afrimmlu_direct_twi_prompt_4 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_wol.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_wol.yaml new file mode 100644 index 00000000..e656af2c --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: wol +include: afrimmlu_direct +task: afrimmlu_direct_wol_prompt_4 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_xho.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_xho.yaml new file mode 100644 index 00000000..ab23d934 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: xho +include: afrimmlu_direct +task: afrimmlu_direct_xho_prompt_4 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_yor.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_yor.yaml new file mode 100644 index 00000000..0dd02548 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_yor.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: yor +include: afrimmlu_direct +task: afrimmlu_direct_yor_prompt_4 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_zul.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_zul.yaml new file mode 100644 index 00000000..98a0937f --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_4/afrimmlu_direct_zul.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: zul +include: afrimmlu_direct +task: afrimmlu_direct_zul_prompt_4 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_4/utils.py b/lm_eval/tasks/afrimmlu/direct/prompt_4/utils.py new file mode 100644 index 00000000..29c23b7f --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_4/utils.py @@ -0,0 +1,28 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_choice(doc): + choices = eval(doc["choices"]) + return choices + + +def doc_to_text(doc): + output = """Analyze each question critically and determine the most correct option based on your understanding of the subject matter + +Question: {question} +Choices: + A: {choice1} + B: {choice2} + C: {choice3} + D: {choice4} +Answer: """ + + choices = eval(doc["choices"]) + text = output.format( + question=doc["question"], + choice1=choices[0], + choice2=choices[1], + choice3=choices[2], + choice4=choices[3], + ) + return text diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct b/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct new file mode 100644 index 00000000..3da1eb82 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct @@ -0,0 +1,37 @@ +tag: + - afrimmlu_tasks + - afrimmlu_tasks_prompt_5 + - afrobench_mmlu_tasks +dataset_path: masakhane/afrimmlu +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_text: !function utils.doc_to_text +doc_to_target: "{{['A', 'B', 'C', 'D'].index(answer)}}" +doc_to_choice: !function utils.doc_to_choice +should_decontaminate: true +doc_to_decontamination_query: "Question: {{question}}\nAnswer:" +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_amh.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_amh.yaml new file mode 100644 index 00000000..cff031d7 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: amh +include: afrimmlu_direct +task: afrimmlu_direct_amh_prompt_5 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_eng.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_eng.yaml new file mode 100644 index 00000000..52f31798 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_eng.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: eng +include: afrimmlu_direct +task: afrimmlu_direct_eng_prompt_5 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_ewe.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_ewe.yaml new file mode 100644 index 00000000..cef2f865 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ewe +include: afrimmlu_direct +task: afrimmlu_direct_ewe_prompt_5 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_fra.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_fra.yaml new file mode 100644 index 00000000..042c0bbb --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_fra.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fra +include: afrimmlu_direct +task: afrimmlu_direct_fra_prompt_5 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_hau.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_hau.yaml new file mode 100644 index 00000000..cd507182 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: hau +include: afrimmlu_direct +task: afrimmlu_direct_hau_prompt_5 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_ibo.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_ibo.yaml new file mode 100644 index 00000000..9e983900 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ibo +include: afrimmlu_direct +task: afrimmlu_direct_ibo_prompt_5 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_kin.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_kin.yaml new file mode 100644 index 00000000..1d157559 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: kin +include: afrimmlu_direct +task: afrimmlu_direct_kin_prompt_5 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_lin.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_lin.yaml new file mode 100644 index 00000000..6eca1f8e --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_lin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lin +include: afrimmlu_direct +task: afrimmlu_direct_lin_prompt_5 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_lug.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_lug.yaml new file mode 100644 index 00000000..854b160d --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lug +include: afrimmlu_direct +task: afrimmlu_direct_lug_prompt_5 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_orm.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_orm.yaml new file mode 100644 index 00000000..9592e585 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_orm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: orm +include: afrimmlu_direct +task: afrimmlu_direct_orm_prompt_5 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_sna.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_sna.yaml new file mode 100644 index 00000000..51d05c68 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sna +include: afrimmlu_direct +task: afrimmlu_direct_sna_prompt_5 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_sot.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_sot.yaml new file mode 100644 index 00000000..cce0e460 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_sot.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sot +include: afrimmlu_direct +task: afrimmlu_direct_sot_prompt_5 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_swa.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_swa.yaml new file mode 100644 index 00000000..c1cd2672 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrimmlu_direct +task: afrimmlu_direct_swa_prompt_5 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_twi.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_twi.yaml new file mode 100644 index 00000000..1e2e258c --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: twi +include: afrimmlu_direct +task: afrimmlu_direct_twi_prompt_5 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_wol.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_wol.yaml new file mode 100644 index 00000000..d721871b --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: wol +include: afrimmlu_direct +task: afrimmlu_direct_wol_prompt_5 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_xho.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_xho.yaml new file mode 100644 index 00000000..1c150726 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: xho +include: afrimmlu_direct +task: afrimmlu_direct_xho_prompt_5 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_yor.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_yor.yaml new file mode 100644 index 00000000..8f528abb --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_yor.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: yor +include: afrimmlu_direct +task: afrimmlu_direct_yor_prompt_5 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_zul.yaml b/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_zul.yaml new file mode 100644 index 00000000..ec83abeb --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_5/afrimmlu_direct_zul.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: zul +include: afrimmlu_direct +task: afrimmlu_direct_zul_prompt_5 diff --git a/lm_eval/tasks/afrimmlu/direct/prompt_5/utils.py b/lm_eval/tasks/afrimmlu/direct/prompt_5/utils.py new file mode 100644 index 00000000..a47ceca9 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/prompt_5/utils.py @@ -0,0 +1,29 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_choice(doc): + choices = eval(doc["choices"]) + return choices + + +def doc_to_text(doc): + output = """Given your proficiency in {subject}, please answer the subsequent multiple-choice question with 'A', 'B', 'C', or 'D'. + +Question: {question} +Choices: + A: {choice1} + B: {choice2} + C: {choice3} + D: {choice4} +Answer: """ + + choices = eval(doc["choices"]) + text = output.format( + subject=doc["subject"], + question=doc["question"], + choice1=choices[0], + choice2=choices[1], + choice3=choices[2], + choice4=choices[3], + ) + return text diff --git a/lm_eval/tasks/afrimmlu/gen_utils.py b/lm_eval/tasks/afrimmlu/gen_utils.py new file mode 100644 index 00000000..a195b6b5 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/gen_utils.py @@ -0,0 +1,103 @@ +import argparse +import os + +import yaml + + +class FunctionTag: + def __init__(self, value): + self.value = value + + +def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None: + """ + Generate a yaml file for each language. + + :param output_dir: The directory to output the files to. + :param overwrite: Whether to overwrite files if they already exist. + """ + err = [] + languages = { + "eng": "English", + "amh": "Amharic", + "ibo": "Igbo", + "fra": "French", + "sna": "chiShona", + "wol": "Wolof", + "ewe": "Ewe", + "lin": "Lingala", + "lug": "Luganda", + "xho": "isiXhosa", + "kin": "Kinyarwanda", + "twi": "Twi", + "zul": "Zulu", + "orm": "Oromo", + "yor": "Yoruba", + "hau": "Hausa", + "sot": "Sesotho", + "swa": "Swahili", + } + + for lang in languages.keys(): + try: + file_name = f"afrimmlu_direct_{lang}.yaml" + task_name = f"afrimmlu_direct_{lang}_{mode}" + yaml_template = "afrimmlu_direct" + if output_dir.split("/")[-1] == "translate": + file_name = f"afrimmlu_translate_{lang}.yaml" + task_name = f"afrimmlu_translate_{lang}_{mode}" + yaml_template = "afrimmlu_translate" + yaml_details = { + "include": yaml_template, + "task": task_name, + "dataset_name": lang, + } + os.makedirs(f"{output_dir}/{mode}", exist_ok=True) + with open( + f"{output_dir}/{mode}/{file_name}", + "w" if overwrite else "x", + encoding="utf8", + ) as f: + f.write("# Generated by utils.py\n") + yaml.dump( + yaml_details, + f, + allow_unicode=True, + ) + except FileExistsError: + err.append(file_name) + + if len(err) > 0: + raise FileExistsError( + "Files were not created because they already exist (use --overwrite flag):" + f" {', '.join(err)}" + ) + + +def main() -> None: + """Parse CLI args and generate language-specific yaml files.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + default=True, + action="store_true", + help="Overwrite files if they already exist", + ) + parser.add_argument( + "--output-dir", + default="./direct", + help="Directory to write yaml files to", + ) + parser.add_argument( + "--mode", + default="prompt_4", + choices=["prompt_1", "prompt_2", "prompt_3", "prompt_4", "prompt_5"], + help="Prompt number", + ) + args = parser.parse_args() + + gen_lang_yamls(output_dir=args.output_dir, overwrite=args.overwrite, mode=args.mode) + + +if __name__ == "__main__": + main() diff --git a/lm_eval/tasks/afrimmlu/translate/afrimmlu_common_translate_yaml b/lm_eval/tasks/afrimmlu/translate/afrimmlu_common_translate_yaml deleted file mode 100644 index fad94678..00000000 --- a/lm_eval/tasks/afrimmlu/translate/afrimmlu_common_translate_yaml +++ /dev/null @@ -1,34 +0,0 @@ -tag: - - afrimmlu_translate -task: null -dataset_path: masakhane/afrimmlu-translate-test -dataset_name: null -output_type: multiple_choice -test_split: test -doc_to_text: !function utils.doc_to_text -doc_to_target: "{{['A', 'B', 'C', 'D'].index(answer)}}" -doc_to_choice: !function utils.doc_to_choice -should_decontaminate: true -doc_to_decontamination_query: "Question: {{question}}\nAnswer:" -metric_list: - - metric: f1 - aggregation: !function utils.weighted_f1_score - # aggregation: mean - average: weighted - hf_evaluate: true - higher_is_better: True - ignore_case: true - ignore_punctuation: true - regexes_to_ignore: - - "," - - "\\$" - - metric: acc - aggregation: mean - higher_is_better: true - ignore_case: true - ignore_punctuation: true - regexes_to_ignore: - - "," - - "\\$" -metadata: - version: 1.0 diff --git a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_amh.yaml b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_amh.yaml deleted file mode 100644 index ac88ffa9..00000000 --- a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_amh.yaml +++ /dev/null @@ -1,3 +0,0 @@ -dataset_name: amh -include: afrimmlu_common_translate_yaml -task: afrimmlu_translate_amh diff --git a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_eng.yaml b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_eng.yaml deleted file mode 100644 index 0be98bee..00000000 --- a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_eng.yaml +++ /dev/null @@ -1,3 +0,0 @@ -dataset_name: eng -include: afrimmlu_common_translate_yaml -task: afrimmlu_translate_eng diff --git a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_ewe.yaml b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_ewe.yaml deleted file mode 100644 index 624342b9..00000000 --- a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_ewe.yaml +++ /dev/null @@ -1,3 +0,0 @@ -dataset_name: ewe -include: afrimmlu_common_translate_yaml -task: afrimmlu_translate_ewe diff --git a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_fra.yaml b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_fra.yaml deleted file mode 100644 index c4fd7e1f..00000000 --- a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_fra.yaml +++ /dev/null @@ -1,3 +0,0 @@ -dataset_name: fra -include: afrimmlu_common_translate_yaml -task: afrimmlu_translate_fra diff --git a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_hau.yaml b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_hau.yaml deleted file mode 100644 index aaeb415f..00000000 --- a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_hau.yaml +++ /dev/null @@ -1,3 +0,0 @@ -dataset_name: hau -include: afrimmlu_common_translate_yaml -task: afrimmlu_translate_hau diff --git a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_ibo.yaml b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_ibo.yaml deleted file mode 100644 index 93fb24e8..00000000 --- a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_ibo.yaml +++ /dev/null @@ -1,3 +0,0 @@ -dataset_name: ibo -include: afrimmlu_common_translate_yaml -task: afrimmlu_translate_ibo diff --git a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_kin.yaml b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_kin.yaml deleted file mode 100644 index f39f6668..00000000 --- a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_kin.yaml +++ /dev/null @@ -1,3 +0,0 @@ -dataset_name: kin -include: afrimmlu_common_translate_yaml -task: afrimmlu_translate_kin diff --git a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_lin.yaml b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_lin.yaml deleted file mode 100644 index c935ee47..00000000 --- a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_lin.yaml +++ /dev/null @@ -1,3 +0,0 @@ -dataset_name: lin -include: afrimmlu_common_translate_yaml -task: afrimmlu_translate_lin diff --git a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_lug.yaml b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_lug.yaml deleted file mode 100644 index 72e4bce0..00000000 --- a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_lug.yaml +++ /dev/null @@ -1,3 +0,0 @@ -dataset_name: lug -include: afrimmlu_common_translate_yaml -task: afrimmlu_translate_lug diff --git a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_orm.yaml b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_orm.yaml deleted file mode 100644 index 3ff90249..00000000 --- a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_orm.yaml +++ /dev/null @@ -1,3 +0,0 @@ -dataset_name: orm -include: afrimmlu_common_translate_yaml -task: afrimmlu_translate_orm diff --git a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_sna.yaml b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_sna.yaml deleted file mode 100644 index 9979740a..00000000 --- a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_sna.yaml +++ /dev/null @@ -1,3 +0,0 @@ -dataset_name: sna -include: afrimmlu_common_translate_yaml -task: afrimmlu_translate_sna diff --git a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_sot.yaml b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_sot.yaml deleted file mode 100644 index deb2b9b8..00000000 --- a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_sot.yaml +++ /dev/null @@ -1,3 +0,0 @@ -dataset_name: sot -include: afrimmlu_common_translate_yaml -task: afrimmlu_translate_sot diff --git a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_swa.yaml b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_swa.yaml deleted file mode 100644 index e58d90bc..00000000 --- a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_swa.yaml +++ /dev/null @@ -1,3 +0,0 @@ -dataset_name: swa -include: afrimmlu_common_translate_yaml -task: afrimmlu_translate_swa diff --git a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_twi.yaml b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_twi.yaml deleted file mode 100644 index 51a2d26a..00000000 --- a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_twi.yaml +++ /dev/null @@ -1,3 +0,0 @@ -dataset_name: twi -include: afrimmlu_common_translate_yaml -task: afrimmlu_translate_twi diff --git a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_wol.yaml b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_wol.yaml deleted file mode 100644 index 006b6847..00000000 --- a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_wol.yaml +++ /dev/null @@ -1,3 +0,0 @@ -dataset_name: wol -include: afrimmlu_common_translate_yaml -task: afrimmlu_translate_wol diff --git a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_xho.yaml b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_xho.yaml deleted file mode 100644 index c0bdf447..00000000 --- a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_xho.yaml +++ /dev/null @@ -1,3 +0,0 @@ -dataset_name: xho -include: afrimmlu_common_translate_yaml -task: afrimmlu_translate_xho diff --git a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_yor.yaml b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_yor.yaml deleted file mode 100644 index 0e7ba600..00000000 --- a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_yor.yaml +++ /dev/null @@ -1,3 +0,0 @@ -dataset_name: yor -include: afrimmlu_common_translate_yaml -task: afrimmlu_translate_yor diff --git a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_zul.yaml b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_zul.yaml deleted file mode 100644 index a18d251c..00000000 --- a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_zul.yaml +++ /dev/null @@ -1,3 +0,0 @@ -dataset_name: zul -include: afrimmlu_common_translate_yaml -task: afrimmlu_translate_zul diff --git a/lm_eval/tasks/afrimmlu/translate/afrimmlu_tt.yaml b/lm_eval/tasks/afrimmlu/translate/afrimmlu_tt.yaml new file mode 100644 index 00000000..bbbf9387 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/afrimmlu_tt.yaml @@ -0,0 +1,9 @@ +group: afrimmlu_tt-irokobench +task: + - afrimmlu_tt_tasks +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 2 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate b/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate new file mode 100644 index 00000000..7a974279 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate @@ -0,0 +1,32 @@ +tag: afrimmlu_tt_tasks +dataset_path: masakhane/afrimmlu-translate-test +dataset_name: null +output_type: multiple_choice +test_split: test +doc_to_text: !function utils.doc_to_text +doc_to_target: "{{['A', 'B', 'C', 'D'].index(answer)}}" +doc_to_choice: !function utils.doc_to_choice +should_decontaminate: true +doc_to_decontamination_query: "Question: {{question}}\nAnswer:" +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_amh.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_amh.yaml new file mode 100644 index 00000000..aaaaa6b8 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: amh +include: afrimmlu_translate +task: afrimmlu_translate_amh_prompt_1 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_ewe.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_ewe.yaml new file mode 100644 index 00000000..45298a1f --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ewe +include: afrimmlu_translate +task: afrimmlu_translate_ewe_prompt_1 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_fra.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_fra.yaml new file mode 100644 index 00000000..6ac43a80 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_fra.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fra +include: afrimmlu_translate +task: afrimmlu_translate_fra_prompt_1 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_hau.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_hau.yaml new file mode 100644 index 00000000..c09424d3 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: hau +include: afrimmlu_translate +task: afrimmlu_translate_hau_prompt_1 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_ibo.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_ibo.yaml new file mode 100644 index 00000000..6fe13991 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ibo +include: afrimmlu_translate +task: afrimmlu_translate_ibo_prompt_1 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_kin.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_kin.yaml new file mode 100644 index 00000000..c6899523 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: kin +include: afrimmlu_translate +task: afrimmlu_translate_kin_prompt_1 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_lin.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_lin.yaml new file mode 100644 index 00000000..e245d7bd --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_lin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lin +include: afrimmlu_translate +task: afrimmlu_translate_lin_prompt_1 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_lug.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_lug.yaml new file mode 100644 index 00000000..bcbac5f6 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lug +include: afrimmlu_translate +task: afrimmlu_translate_lug_prompt_1 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_orm.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_orm.yaml new file mode 100644 index 00000000..84b3d2c3 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_orm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: orm +include: afrimmlu_translate +task: afrimmlu_translate_orm_prompt_1 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_sna.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_sna.yaml new file mode 100644 index 00000000..722ee952 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sna +include: afrimmlu_translate +task: afrimmlu_translate_sna_prompt_1 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_sot.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_sot.yaml new file mode 100644 index 00000000..4e8893aa --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_sot.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sot +include: afrimmlu_translate +task: afrimmlu_translate_sot_prompt_1 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_swa.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_swa.yaml new file mode 100644 index 00000000..eb89697c --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrimmlu_translate +task: afrimmlu_translate_swa_prompt_1 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_twi.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_twi.yaml new file mode 100644 index 00000000..d672f6c7 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: twi +include: afrimmlu_translate +task: afrimmlu_translate_twi_prompt_1 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_wol.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_wol.yaml new file mode 100644 index 00000000..9fb5f370 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: wol +include: afrimmlu_translate +task: afrimmlu_translate_wol_prompt_1 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_xho.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_xho.yaml new file mode 100644 index 00000000..1a06af04 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: xho +include: afrimmlu_translate +task: afrimmlu_translate_xho_prompt_1 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_yor.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_yor.yaml new file mode 100644 index 00000000..0f5eb7de --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_yor.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: yor +include: afrimmlu_translate +task: afrimmlu_translate_yor_prompt_1 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_zul.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_zul.yaml new file mode 100644 index 00000000..ae04b652 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_1/afrimmlu_translate_zul.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: zul +include: afrimmlu_translate +task: afrimmlu_translate_zul_prompt_1 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_1/utils.py b/lm_eval/tasks/afrimmlu/translate/prompt_1/utils.py new file mode 100644 index 00000000..f1bb9162 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_1/utils.py @@ -0,0 +1,32 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_choice(doc): + choices = eval(doc["choices"]) + return choices + + +def doc_to_text(doc): + output = """You are a highly knowledgeable and intelligent artificial intelligence + model answers multiple-choice questions about {subject} + + Question: {question} + + Choices: + A: {choice1} + B: {choice2} + C: {choice3} + D: {choice4} + + Answer: """ + + choices = eval(doc["choices"]) + text = output.format( + subject=doc["subject"], + question=doc["question"], + choice1=choices[0], + choice2=choices[1], + choice3=choices[2], + choice4=choices[3], + ) + return text diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate b/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate new file mode 100644 index 00000000..7a974279 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate @@ -0,0 +1,32 @@ +tag: afrimmlu_tt_tasks +dataset_path: masakhane/afrimmlu-translate-test +dataset_name: null +output_type: multiple_choice +test_split: test +doc_to_text: !function utils.doc_to_text +doc_to_target: "{{['A', 'B', 'C', 'D'].index(answer)}}" +doc_to_choice: !function utils.doc_to_choice +should_decontaminate: true +doc_to_decontamination_query: "Question: {{question}}\nAnswer:" +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_amh.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_amh.yaml new file mode 100644 index 00000000..283b6ff1 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: amh +include: afrimmlu_translate +task: afrimmlu_translate_amh_prompt_2 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_ewe.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_ewe.yaml new file mode 100644 index 00000000..39e9f735 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ewe +include: afrimmlu_translate +task: afrimmlu_translate_ewe_prompt_2 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_fra.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_fra.yaml new file mode 100644 index 00000000..ced80282 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_fra.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fra +include: afrimmlu_translate +task: afrimmlu_translate_fra_prompt_2 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_hau.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_hau.yaml new file mode 100644 index 00000000..0d687cac --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: hau +include: afrimmlu_translate +task: afrimmlu_translate_hau_prompt_2 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_ibo.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_ibo.yaml new file mode 100644 index 00000000..d1edfaa3 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ibo +include: afrimmlu_translate +task: afrimmlu_translate_ibo_prompt_2 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_kin.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_kin.yaml new file mode 100644 index 00000000..48fa1518 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: kin +include: afrimmlu_translate +task: afrimmlu_translate_kin_prompt_2 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_lin.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_lin.yaml new file mode 100644 index 00000000..54163d5c --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_lin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lin +include: afrimmlu_translate +task: afrimmlu_translate_lin_prompt_2 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_lug.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_lug.yaml new file mode 100644 index 00000000..d6b709ec --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lug +include: afrimmlu_translate +task: afrimmlu_translate_lug_prompt_2 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_orm.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_orm.yaml new file mode 100644 index 00000000..2a58330c --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_orm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: orm +include: afrimmlu_translate +task: afrimmlu_translate_orm_prompt_2 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_sna.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_sna.yaml new file mode 100644 index 00000000..0edd502e --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sna +include: afrimmlu_translate +task: afrimmlu_translate_sna_prompt_2 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_sot.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_sot.yaml new file mode 100644 index 00000000..2f499691 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_sot.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sot +include: afrimmlu_translate +task: afrimmlu_translate_sot_prompt_2 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_swa.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_swa.yaml new file mode 100644 index 00000000..f267b6d0 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrimmlu_translate +task: afrimmlu_translate_swa_prompt_2 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_twi.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_twi.yaml new file mode 100644 index 00000000..b145669e --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: twi +include: afrimmlu_translate +task: afrimmlu_translate_twi_prompt_2 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_wol.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_wol.yaml new file mode 100644 index 00000000..c87a8d61 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: wol +include: afrimmlu_translate +task: afrimmlu_translate_wol_prompt_2 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_xho.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_xho.yaml new file mode 100644 index 00000000..7f552712 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: xho +include: afrimmlu_translate +task: afrimmlu_translate_xho_prompt_2 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_yor.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_yor.yaml new file mode 100644 index 00000000..b5cdbc6b --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_yor.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: yor +include: afrimmlu_translate +task: afrimmlu_translate_yor_prompt_2 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_zul.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_zul.yaml new file mode 100644 index 00000000..2ff80402 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_2/afrimmlu_translate_zul.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: zul +include: afrimmlu_translate +task: afrimmlu_translate_zul_prompt_2 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_2/utils.py b/lm_eval/tasks/afrimmlu/translate/prompt_2/utils.py new file mode 100644 index 00000000..e0cfb334 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_2/utils.py @@ -0,0 +1,30 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_choice(doc): + choices = eval(doc["choices"]) + return choices + + +def doc_to_text(doc): + output = """As an expert in {subject}, choose the most accurate answer to the question below. +Your goal is to select the correct option 'A', 'B', 'C', or 'D' by understanding the nuances of the topic. + +Question: {question} +Choices: + A: {choice1} + B: {choice2} + C: {choice3} + D: {choice4} +Answer: """ + + choices = eval(doc["choices"]) + text = output.format( + subject=doc["subject"], + question=doc["question"], + choice1=choices[0], + choice2=choices[1], + choice3=choices[2], + choice4=choices[3], + ) + return text diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate b/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate new file mode 100644 index 00000000..7a974279 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate @@ -0,0 +1,32 @@ +tag: afrimmlu_tt_tasks +dataset_path: masakhane/afrimmlu-translate-test +dataset_name: null +output_type: multiple_choice +test_split: test +doc_to_text: !function utils.doc_to_text +doc_to_target: "{{['A', 'B', 'C', 'D'].index(answer)}}" +doc_to_choice: !function utils.doc_to_choice +should_decontaminate: true +doc_to_decontamination_query: "Question: {{question}}\nAnswer:" +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_amh.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_amh.yaml new file mode 100644 index 00000000..03772573 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: amh +include: afrimmlu_translate +task: afrimmlu_translate_amh_prompt_3 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_ewe.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_ewe.yaml new file mode 100644 index 00000000..aa924284 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ewe +include: afrimmlu_translate +task: afrimmlu_translate_ewe_prompt_3 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_fra.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_fra.yaml new file mode 100644 index 00000000..7bef6e78 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_fra.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fra +include: afrimmlu_translate +task: afrimmlu_translate_fra_prompt_3 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_hau.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_hau.yaml new file mode 100644 index 00000000..d9841db9 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: hau +include: afrimmlu_translate +task: afrimmlu_translate_hau_prompt_3 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_ibo.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_ibo.yaml new file mode 100644 index 00000000..20c58b27 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ibo +include: afrimmlu_translate +task: afrimmlu_translate_ibo_prompt_3 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_kin.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_kin.yaml new file mode 100644 index 00000000..ed3cfd16 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: kin +include: afrimmlu_translate +task: afrimmlu_translate_kin_prompt_3 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_lin.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_lin.yaml new file mode 100644 index 00000000..9eeb66ea --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_lin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lin +include: afrimmlu_translate +task: afrimmlu_translate_lin_prompt_3 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_lug.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_lug.yaml new file mode 100644 index 00000000..46f722b7 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lug +include: afrimmlu_translate +task: afrimmlu_translate_lug_prompt_3 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_orm.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_orm.yaml new file mode 100644 index 00000000..6f95b375 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_orm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: orm +include: afrimmlu_translate +task: afrimmlu_translate_orm_prompt_3 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_sna.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_sna.yaml new file mode 100644 index 00000000..aa031e5c --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sna +include: afrimmlu_translate +task: afrimmlu_translate_sna_prompt_3 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_sot.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_sot.yaml new file mode 100644 index 00000000..e55ce671 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_sot.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sot +include: afrimmlu_translate +task: afrimmlu_translate_sot_prompt_3 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_swa.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_swa.yaml new file mode 100644 index 00000000..9f507772 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrimmlu_translate +task: afrimmlu_translate_swa_prompt_3 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_twi.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_twi.yaml new file mode 100644 index 00000000..1ea25d8b --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: twi +include: afrimmlu_translate +task: afrimmlu_translate_twi_prompt_3 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_wol.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_wol.yaml new file mode 100644 index 00000000..2a077d85 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: wol +include: afrimmlu_translate +task: afrimmlu_translate_wol_prompt_3 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_xho.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_xho.yaml new file mode 100644 index 00000000..18678da8 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: xho +include: afrimmlu_translate +task: afrimmlu_translate_xho_prompt_3 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_yor.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_yor.yaml new file mode 100644 index 00000000..815d219b --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_yor.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: yor +include: afrimmlu_translate +task: afrimmlu_translate_yor_prompt_3 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_zul.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_zul.yaml new file mode 100644 index 00000000..f294a873 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_3/afrimmlu_translate_zul.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: zul +include: afrimmlu_translate +task: afrimmlu_translate_zul_prompt_3 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_3/utils.py b/lm_eval/tasks/afrimmlu/translate/prompt_3/utils.py new file mode 100644 index 00000000..a4609d97 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_3/utils.py @@ -0,0 +1,32 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_choice(doc): + choices = eval(doc["choices"]) + return choices + + +def doc_to_text(doc): + output = """You are a subject matter expert in {subject}. + + Utilizing your expertise in {subject}, answer the following multiple-choice question + by picking ''A'', ''B'', ''C'', or ''D''. + +Question: {question} +Choices: + A: {choice1} + B: {choice2} + C: {choice3} + D: {choice4} +Answer: """ + + choices = eval(doc["choices"]) + text = output.format( + subject=doc["subject"], + question=doc["question"], + choice1=choices[0], + choice2=choices[1], + choice3=choices[2], + choice4=choices[3], + ) + return text diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate b/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate new file mode 100644 index 00000000..7a974279 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate @@ -0,0 +1,32 @@ +tag: afrimmlu_tt_tasks +dataset_path: masakhane/afrimmlu-translate-test +dataset_name: null +output_type: multiple_choice +test_split: test +doc_to_text: !function utils.doc_to_text +doc_to_target: "{{['A', 'B', 'C', 'D'].index(answer)}}" +doc_to_choice: !function utils.doc_to_choice +should_decontaminate: true +doc_to_decontamination_query: "Question: {{question}}\nAnswer:" +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_amh.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_amh.yaml new file mode 100644 index 00000000..90c88024 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: amh +include: afrimmlu_translate +task: afrimmlu_translate_amh_prompt_4 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_ewe.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_ewe.yaml new file mode 100644 index 00000000..c63ccfb4 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ewe +include: afrimmlu_translate +task: afrimmlu_translate_ewe_prompt_4 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_fra.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_fra.yaml new file mode 100644 index 00000000..740f87ad --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_fra.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fra +include: afrimmlu_translate +task: afrimmlu_translate_fra_prompt_4 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_hau.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_hau.yaml new file mode 100644 index 00000000..4f73a2ed --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: hau +include: afrimmlu_translate +task: afrimmlu_translate_hau_prompt_4 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_ibo.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_ibo.yaml new file mode 100644 index 00000000..424fbab9 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ibo +include: afrimmlu_translate +task: afrimmlu_translate_ibo_prompt_4 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_kin.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_kin.yaml new file mode 100644 index 00000000..cafcae60 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: kin +include: afrimmlu_translate +task: afrimmlu_translate_kin_prompt_4 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_lin.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_lin.yaml new file mode 100644 index 00000000..618f4aaf --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_lin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lin +include: afrimmlu_translate +task: afrimmlu_translate_lin_prompt_4 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_lug.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_lug.yaml new file mode 100644 index 00000000..fe59cfd2 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lug +include: afrimmlu_translate +task: afrimmlu_translate_lug_prompt_4 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_orm.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_orm.yaml new file mode 100644 index 00000000..4f25d96c --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_orm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: orm +include: afrimmlu_translate +task: afrimmlu_translate_orm_prompt_4 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_sna.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_sna.yaml new file mode 100644 index 00000000..0943eec1 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sna +include: afrimmlu_translate +task: afrimmlu_translate_sna_prompt_4 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_sot.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_sot.yaml new file mode 100644 index 00000000..882117e4 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_sot.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sot +include: afrimmlu_translate +task: afrimmlu_translate_sot_prompt_4 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_swa.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_swa.yaml new file mode 100644 index 00000000..92928aae --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrimmlu_translate +task: afrimmlu_translate_swa_prompt_4 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_twi.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_twi.yaml new file mode 100644 index 00000000..8d093262 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: twi +include: afrimmlu_translate +task: afrimmlu_translate_twi_prompt_4 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_wol.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_wol.yaml new file mode 100644 index 00000000..66161c7d --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: wol +include: afrimmlu_translate +task: afrimmlu_translate_wol_prompt_4 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_xho.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_xho.yaml new file mode 100644 index 00000000..25ff91f0 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: xho +include: afrimmlu_translate +task: afrimmlu_translate_xho_prompt_4 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_yor.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_yor.yaml new file mode 100644 index 00000000..54743fdd --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_yor.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: yor +include: afrimmlu_translate +task: afrimmlu_translate_yor_prompt_4 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_zul.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_zul.yaml new file mode 100644 index 00000000..ebd80f22 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_4/afrimmlu_translate_zul.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: zul +include: afrimmlu_translate +task: afrimmlu_translate_zul_prompt_4 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_4/utils.py b/lm_eval/tasks/afrimmlu/translate/prompt_4/utils.py new file mode 100644 index 00000000..29c23b7f --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_4/utils.py @@ -0,0 +1,28 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_choice(doc): + choices = eval(doc["choices"]) + return choices + + +def doc_to_text(doc): + output = """Analyze each question critically and determine the most correct option based on your understanding of the subject matter + +Question: {question} +Choices: + A: {choice1} + B: {choice2} + C: {choice3} + D: {choice4} +Answer: """ + + choices = eval(doc["choices"]) + text = output.format( + question=doc["question"], + choice1=choices[0], + choice2=choices[1], + choice3=choices[2], + choice4=choices[3], + ) + return text diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate b/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate new file mode 100644 index 00000000..7a974279 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate @@ -0,0 +1,32 @@ +tag: afrimmlu_tt_tasks +dataset_path: masakhane/afrimmlu-translate-test +dataset_name: null +output_type: multiple_choice +test_split: test +doc_to_text: !function utils.doc_to_text +doc_to_target: "{{['A', 'B', 'C', 'D'].index(answer)}}" +doc_to_choice: !function utils.doc_to_choice +should_decontaminate: true +doc_to_decontamination_query: "Question: {{question}}\nAnswer:" +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_amh.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_amh.yaml new file mode 100644 index 00000000..4b5ebb38 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: amh +include: afrimmlu_translate +task: afrimmlu_translate_amh_prompt_5 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_ewe.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_ewe.yaml new file mode 100644 index 00000000..4860b79c --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ewe +include: afrimmlu_translate +task: afrimmlu_translate_ewe_prompt_5 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_fra.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_fra.yaml new file mode 100644 index 00000000..192f3423 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_fra.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fra +include: afrimmlu_translate +task: afrimmlu_translate_fra_prompt_5 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_hau.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_hau.yaml new file mode 100644 index 00000000..1fc6aafb --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: hau +include: afrimmlu_translate +task: afrimmlu_translate_hau_prompt_5 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_ibo.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_ibo.yaml new file mode 100644 index 00000000..a6cca83c --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ibo +include: afrimmlu_translate +task: afrimmlu_translate_ibo_prompt_5 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_kin.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_kin.yaml new file mode 100644 index 00000000..6acd743a --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: kin +include: afrimmlu_translate +task: afrimmlu_translate_kin_prompt_5 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_lin.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_lin.yaml new file mode 100644 index 00000000..f62c43f6 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_lin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lin +include: afrimmlu_translate +task: afrimmlu_translate_lin_prompt_5 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_lug.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_lug.yaml new file mode 100644 index 00000000..9b8a97f3 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lug +include: afrimmlu_translate +task: afrimmlu_translate_lug_prompt_5 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_orm.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_orm.yaml new file mode 100644 index 00000000..0eebc1bd --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_orm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: orm +include: afrimmlu_translate +task: afrimmlu_translate_orm_prompt_5 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_sna.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_sna.yaml new file mode 100644 index 00000000..60d4d57f --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sna +include: afrimmlu_translate +task: afrimmlu_translate_sna_prompt_5 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_sot.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_sot.yaml new file mode 100644 index 00000000..eeddbb7b --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_sot.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sot +include: afrimmlu_translate +task: afrimmlu_translate_sot_prompt_5 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_swa.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_swa.yaml new file mode 100644 index 00000000..24a3b78e --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrimmlu_translate +task: afrimmlu_translate_swa_prompt_5 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_twi.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_twi.yaml new file mode 100644 index 00000000..ab3ea80c --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: twi +include: afrimmlu_translate +task: afrimmlu_translate_twi_prompt_5 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_wol.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_wol.yaml new file mode 100644 index 00000000..2cd495e8 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: wol +include: afrimmlu_translate +task: afrimmlu_translate_wol_prompt_5 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_xho.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_xho.yaml new file mode 100644 index 00000000..a9af97c0 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: xho +include: afrimmlu_translate +task: afrimmlu_translate_xho_prompt_5 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_yor.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_yor.yaml new file mode 100644 index 00000000..6e227431 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_yor.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: yor +include: afrimmlu_translate +task: afrimmlu_translate_yor_prompt_5 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_zul.yaml b/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_zul.yaml new file mode 100644 index 00000000..a08884b8 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_5/afrimmlu_translate_zul.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: zul +include: afrimmlu_translate +task: afrimmlu_translate_zul_prompt_5 diff --git a/lm_eval/tasks/afrimmlu/translate/prompt_5/utils.py b/lm_eval/tasks/afrimmlu/translate/prompt_5/utils.py new file mode 100644 index 00000000..147225bb --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/prompt_5/utils.py @@ -0,0 +1,28 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_choice(doc): + choices = eval(doc["choices"]) + return choices + + +def doc_to_text(doc): + output = """Given your proficiency in {subject}, please answer the subsequent multiple-choice question with 'A', 'B', 'C', or 'D'. +Question: {question} +Choices: + A: {choice1} + B: {choice2} + C: {choice3} + D: {choice4} +Answer: """ + + choices = eval(doc["choices"]) + text = output.format( + subject=doc["subject"], + question=doc["question"], + choice1=choices[0], + choice2=choices[1], + choice3=choices[2], + choice4=choices[3], + ) + return text diff --git a/lm_eval/tasks/afrimmlu/translate/utils.py b/lm_eval/tasks/afrimmlu/translate/utils.py deleted file mode 100644 index 9d02b342..00000000 --- a/lm_eval/tasks/afrimmlu/translate/utils.py +++ /dev/null @@ -1,32 +0,0 @@ -from lm_eval.utils import weighted_f1_score - - -def doc_to_choice(doc): - choices = eval(doc["choices"]) - return choices - - -def doc_to_text(doc): - output = """You are a highly knowledgeable and intelligent artificial intelligence - model answers multiple-choice questions about '{subject}' - - Question: '''{question}''' - - Choices: - A: ''{choice1}''' - B: ''{choice2}''' - C: ''{choice3}''' - D: ''{choice4}''' - - Answer: """ - - choices = eval(doc["choices"]) - text = output.format( - subject=doc["subject"], - question=doc["question"], - choice1=choices[0], - choice2=choices[1], - choice3=choices[2], - choice4=choices[3], - ) - return text diff --git a/lm_eval/tasks/afrixnli/direct/afrixnli.yaml b/lm_eval/tasks/afrixnli/direct/afrixnli.yaml new file mode 100644 index 00000000..d85ccd12 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/afrixnli.yaml @@ -0,0 +1,13 @@ +group: afrixnli-irokobench +task: + - afrixnli_tasks_prompt_1 + - afrixnli_tasks_prompt_2 + - afrixnli_tasks_prompt_3 + - afrixnli_tasks_prompt_4 + - afrixnli_tasks_prompt_5 +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 2 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_amh.yaml b/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_amh.yaml new file mode 100644 index 00000000..39f727b4 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_amh.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_yaml +task: afrixnli_amh_prompt_1 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_eng.yaml b/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_eng.yaml new file mode 100644 index 00000000..593c57a3 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_eng.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: eng +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_yaml +task: afrixnli_eng_prompt_1 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_ewe.yaml b/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_ewe.yaml new file mode 100644 index 00000000..b6a10baa --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_ewe.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: ewe +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_yaml +task: afrixnli_ewe_prompt_1 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_fra.yaml b/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_fra.yaml new file mode 100644 index 00000000..08b2b524 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_fra.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: fra +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_yaml +task: afrixnli_fra_prompt_1 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_hau.yaml b/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_hau.yaml new file mode 100644 index 00000000..fe234b72 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_hau.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_yaml +task: afrixnli_hau_prompt_1 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_ibo.yaml b/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_ibo.yaml new file mode 100644 index 00000000..d282e0e5 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_ibo.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_yaml +task: afrixnli_ibo_prompt_1 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_kin.yaml b/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_kin.yaml new file mode 100644 index 00000000..cfdff6c8 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_kin.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_yaml +task: afrixnli_kin_prompt_1 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_lin.yaml b/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_lin.yaml new file mode 100644 index 00000000..410cb29f --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_lin.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_yaml +task: afrixnli_lin_prompt_1 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_lug.yaml b/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_lug.yaml new file mode 100644 index 00000000..b5665e37 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_lug.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_yaml +task: afrixnli_lug_prompt_1 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_orm.yaml b/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_orm.yaml new file mode 100644 index 00000000..12751c7f --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_orm.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_yaml +task: afrixnli_orm_prompt_1 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_sna.yaml b/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_sna.yaml new file mode 100644 index 00000000..d00bbb6f --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_sna.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_yaml +task: afrixnli_sna_prompt_1 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_sot.yaml b/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_sot.yaml new file mode 100644 index 00000000..2ae346ae --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_sot.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: sot +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_yaml +task: afrixnli_sot_prompt_1 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_swa.yaml b/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_swa.yaml new file mode 100644 index 00000000..ca6729bf --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_swa.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_yaml +task: afrixnli_swa_prompt_1 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_twi.yaml b/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_twi.yaml new file mode 100644 index 00000000..7dc85428 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_twi.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_yaml +task: afrixnli_twi_prompt_1 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_wol.yaml b/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_wol.yaml new file mode 100644 index 00000000..78ef254a --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_wol.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_yaml +task: afrixnli_wol_prompt_1 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_xho.yaml b/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_xho.yaml new file mode 100644 index 00000000..cb0a8527 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_xho.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_yaml +task: afrixnli_xho_prompt_1 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_yaml b/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_yaml new file mode 100644 index 00000000..81c9eeaa --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_yaml @@ -0,0 +1,30 @@ +tag: + - afrixnli_tasks + - afrixnli_tasks_prompt_1 +dataset_path: masakhane/afrixnli +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: !function utils.doc_to_target +doc_to_choice: + - "entailment" + - "neutral" + - "contradiction" +should_decontaminate: true +doc_to_decontamination_query: premise +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + average: weighted + higher_is_better: True + ignore_case: true + ignore_punctuation: true + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_yor.yaml b/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_yor.yaml new file mode 100644 index 00000000..473aea37 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_yor.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_yaml +task: afrixnli_yor_prompt_1 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_zul.yaml b/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_zul.yaml new file mode 100644 index 00000000..fa07a8c9 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_1/afrixnli_zul.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_yaml +task: afrixnli_zul_prompt_1 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_1/utils.py b/lm_eval/tasks/afrixnli/direct/prompt_1/utils.py new file mode 100644 index 00000000..d97a0a28 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_1/utils.py @@ -0,0 +1,19 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_text(doc): + output = """Please identify whether the premise entails or contradicts the hypothesis in the following premise + and hypothesis. The answer should be exact entailment, contradiction, or neutral. + + Premise: {premise} + Hypothesis: {hypothesis} + + Is it entailment, contradiction, or neutral?""" + + text = output.format(premise=doc["premise"], hypothesis=doc["hypothesis"]) + return text + + +def doc_to_target(doc): + replacements = {0: "entailment", 1: "neutral", 2: "contradiction"} + return replacements[doc["label"]] diff --git a/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_amh.yaml b/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_amh.yaml new file mode 100644 index 00000000..fbf916b2 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: amh +include: afrixnli_yaml +task: afrixnli_amh_prompt_2 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_eng.yaml b/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_eng.yaml new file mode 100644 index 00000000..dfa8ebfe --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_eng.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: eng +include: afrixnli_yaml +task: afrixnli_eng_prompt_2 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_ewe.yaml b/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_ewe.yaml new file mode 100644 index 00000000..995ef3e6 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ewe +include: afrixnli_yaml +task: afrixnli_ewe_prompt_2 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_fra.yaml b/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_fra.yaml new file mode 100644 index 00000000..ce72588c --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_fra.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fra +include: afrixnli_yaml +task: afrixnli_fra_prompt_2 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_hau.yaml b/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_hau.yaml new file mode 100644 index 00000000..369ee58b --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: hau +include: afrixnli_yaml +task: afrixnli_hau_prompt_2 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_ibo.yaml b/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_ibo.yaml new file mode 100644 index 00000000..e118c613 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ibo +include: afrixnli_yaml +task: afrixnli_ibo_prompt_2 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_kin.yaml b/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_kin.yaml new file mode 100644 index 00000000..81f6d803 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: kin +include: afrixnli_yaml +task: afrixnli_kin_prompt_2 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_lin.yaml b/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_lin.yaml new file mode 100644 index 00000000..2d99c2eb --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_lin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lin +include: afrixnli_yaml +task: afrixnli_lin_prompt_2 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_lug.yaml b/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_lug.yaml new file mode 100644 index 00000000..31325539 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lug +include: afrixnli_yaml +task: afrixnli_lug_prompt_2 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_orm.yaml b/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_orm.yaml new file mode 100644 index 00000000..c4ad555a --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_orm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: orm +include: afrixnli_yaml +task: afrixnli_orm_prompt_2 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_sna.yaml b/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_sna.yaml new file mode 100644 index 00000000..a780b0c4 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sna +include: afrixnli_yaml +task: afrixnli_sna_prompt_2 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_sot.yaml b/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_sot.yaml new file mode 100644 index 00000000..94e78880 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_sot.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sot +include: afrixnli_yaml +task: afrixnli_sot_prompt_2 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_swa.yaml b/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_swa.yaml new file mode 100644 index 00000000..8622e283 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrixnli_yaml +task: afrixnli_swa_prompt_2 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_twi.yaml b/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_twi.yaml new file mode 100644 index 00000000..4219b81e --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: twi +include: afrixnli_yaml +task: afrixnli_twi_prompt_2 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_wol.yaml b/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_wol.yaml new file mode 100644 index 00000000..546b1790 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: wol +include: afrixnli_yaml +task: afrixnli_wol_prompt_2 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_xho.yaml b/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_xho.yaml new file mode 100644 index 00000000..649c61df --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: xho +include: afrixnli_yaml +task: afrixnli_xho_prompt_2 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_yaml b/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_yaml new file mode 100644 index 00000000..cfab642b --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_yaml @@ -0,0 +1,34 @@ +tag: + - afrixnli_tasks + - afrixnli_tasks_prompt_2 +dataset_path: masakhane/afrixnli +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_text: "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither?\nAnswer:" +# True = entailment +# False = contradiction +# Neither = neutral +doc_to_target: !function utils.doc_to_target +doc_to_choice: + - "True" + - "Neither" + - "False" +should_decontaminate: true +doc_to_decontamination_query: premise +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + average: weighted + higher_is_better: True + ignore_case: true + ignore_punctuation: true + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_yor.yaml b/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_yor.yaml new file mode 100644 index 00000000..53f23ace --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_yor.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: yor +include: afrixnli_yaml +task: afrixnli_yor_prompt_2 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_zul.yaml b/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_zul.yaml new file mode 100644 index 00000000..dd89fe13 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_2/afrixnli_zul.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: zul +include: afrixnli_yaml +task: afrixnli_zul_prompt_2 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_2/utils.py b/lm_eval/tasks/afrixnli/direct/prompt_2/utils.py new file mode 100644 index 00000000..5d1ac19e --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_2/utils.py @@ -0,0 +1,6 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_target(doc): + replacements = {0: "True", 1: "Neither", 2: "False"} + return replacements[doc["label"]] diff --git a/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_amh.yaml b/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_amh.yaml new file mode 100644 index 00000000..3ff9f99c --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_amh.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: "Given the following premise and hypothesis in Amharic, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_amh_prompt_3 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_eng.yaml b/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_eng.yaml new file mode 100644 index 00000000..a53aea6d --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_eng.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: eng +doc_to_text: "Given the following premise and hypothesis in English, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_eng_prompt_3 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_ewe.yaml b/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_ewe.yaml new file mode 100644 index 00000000..54b58ae6 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_ewe.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: ewe +doc_to_text: "Given the following premise and hypothesis in Ewe, identify if the premise\ + \ entails, contradicts, or is neutral towards the hypothesis. Please respond with\ + \ exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \n\ + Hypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_ewe_prompt_3 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_fra.yaml b/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_fra.yaml new file mode 100644 index 00000000..fedb519e --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_fra.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: fra +doc_to_text: "Given the following premise and hypothesis in French, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_fra_prompt_3 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_hau.yaml b/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_hau.yaml new file mode 100644 index 00000000..3a9ebb95 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_hau.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: "Given the following premise and hypothesis in Hausa, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_hau_prompt_3 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_ibo.yaml b/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_ibo.yaml new file mode 100644 index 00000000..6b61f767 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_ibo.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "Given the following premise and hypothesis in Igbo, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_ibo_prompt_3 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_kin.yaml b/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_kin.yaml new file mode 100644 index 00000000..1bd0829b --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_kin.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: "Given the following premise and hypothesis in Kinyarwanda, identify\ + \ if the premise entails, contradicts, or is neutral towards the hypothesis. Please\ + \ respond with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_kin_prompt_3 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_lin.yaml b/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_lin.yaml new file mode 100644 index 00000000..697c439f --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_lin.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: "Given the following premise and hypothesis in Lingala, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_lin_prompt_3 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_lug.yaml b/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_lug.yaml new file mode 100644 index 00000000..1b5667c0 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_lug.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: "Given the following premise and hypothesis in Luganda, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_lug_prompt_3 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_orm.yaml b/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_orm.yaml new file mode 100644 index 00000000..37a6d843 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_orm.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: "Given the following premise and hypothesis in Oromo, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_orm_prompt_3 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_sna.yaml b/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_sna.yaml new file mode 100644 index 00000000..c7e0f0b0 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_sna.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: "Given the following premise and hypothesis in chiShona, identify if\ + \ the premise entails, contradicts, or is neutral towards the hypothesis. Please\ + \ respond with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_sna_prompt_3 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_sot.yaml b/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_sot.yaml new file mode 100644 index 00000000..0c0ccd9e --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_sot.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: sot +doc_to_text: "Given the following premise and hypothesis in Sesotho, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_sot_prompt_3 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_swa.yaml b/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_swa.yaml new file mode 100644 index 00000000..dabd96ef --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_swa.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: "Given the following premise and hypothesis in Swahili, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_swa_prompt_3 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_twi.yaml b/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_twi.yaml new file mode 100644 index 00000000..4d3158d4 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_twi.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: "Given the following premise and hypothesis in Twi, identify if the premise\ + \ entails, contradicts, or is neutral towards the hypothesis. Please respond with\ + \ exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \n\ + Hypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_twi_prompt_3 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_wol.yaml b/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_wol.yaml new file mode 100644 index 00000000..51fbdc79 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_wol.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: "Given the following premise and hypothesis in Wolof, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_wol_prompt_3 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_xho.yaml b/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_xho.yaml new file mode 100644 index 00000000..00ca9d17 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_xho.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: "Given the following premise and hypothesis in isiXhosa, identify if\ + \ the premise entails, contradicts, or is neutral towards the hypothesis. Please\ + \ respond with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_xho_prompt_3 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_yaml b/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_yaml new file mode 100644 index 00000000..04609ac3 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_yaml @@ -0,0 +1,30 @@ +tag: + - afrixnli_tasks + - afrixnli_tasks_prompt_3 +dataset_path: masakhane/afrixnli +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: !function utils.doc_to_target +doc_to_choice: + - "entailment" + - "neutral" + - "contradiction" +should_decontaminate: true +doc_to_decontamination_query: premise +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + average: weighted + higher_is_better: True + ignore_case: true + ignore_punctuation: true + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_yor.yaml b/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_yor.yaml new file mode 100644 index 00000000..6d8b2f84 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_yor.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: "Given the following premise and hypothesis in Yoruba, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_yor_prompt_3 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_zul.yaml b/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_zul.yaml new file mode 100644 index 00000000..83b87141 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_3/afrixnli_zul.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: "Given the following premise and hypothesis in Zulu, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_zul_prompt_3 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_3/utils.py b/lm_eval/tasks/afrixnli/direct/prompt_3/utils.py new file mode 100644 index 00000000..422ed169 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_3/utils.py @@ -0,0 +1,6 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_target(doc): + replacements = {0: "entailment", 1: "neutral", 2: "contradiction"} + return replacements[doc["label"]] diff --git a/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_amh.yaml b/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_amh.yaml new file mode 100644 index 00000000..63b05465 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_amh.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Amharic language.\nAnalyze the premise and hypothesis given in Amharic, and\ + \ determine the relationship between them.\n Respond with one of the following options:\ + \ 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis:\ + \ {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_amh_prompt_4 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_eng.yaml b/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_eng.yaml new file mode 100644 index 00000000..1ecb06d1 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_eng.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: eng +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the English language.\nAnalyze the premise and hypothesis given in English, and\ + \ determine the relationship between them.\n Respond with one of the following options:\ + \ 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis:\ + \ {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_eng_prompt_4 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_ewe.yaml b/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_ewe.yaml new file mode 100644 index 00000000..64157b54 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_ewe.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: ewe +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Ewe language.\nAnalyze the premise and hypothesis given in Ewe, and determine\ + \ the relationship between them.\n Respond with one of the following options: 'entailment',\ + \ 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_ewe_prompt_4 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_fra.yaml b/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_fra.yaml new file mode 100644 index 00000000..78da10cf --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_fra.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: fra +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the French language.\nAnalyze the premise and hypothesis given in French, and\ + \ determine the relationship between them.\n Respond with one of the following options:\ + \ 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis:\ + \ {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_fra_prompt_4 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_hau.yaml b/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_hau.yaml new file mode 100644 index 00000000..811a0fca --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_hau.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Hausa language.\nAnalyze the premise and hypothesis given in Hausa, and determine\ + \ the relationship between them.\n Respond with one of the following options: 'entailment',\ + \ 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_hau_prompt_4 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_ibo.yaml b/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_ibo.yaml new file mode 100644 index 00000000..73fdba2f --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_ibo.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Igbo language.\nAnalyze the premise and hypothesis given in Igbo, and determine\ + \ the relationship between them.\n Respond with one of the following options: 'entailment',\ + \ 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_ibo_prompt_4 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_kin.yaml b/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_kin.yaml new file mode 100644 index 00000000..f975d82b --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_kin.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Kinyarwanda language.\nAnalyze the premise and hypothesis given in Kinyarwanda,\ + \ and determine the relationship between them.\n Respond with one of the following\ + \ options: 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_kin_prompt_4 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_lin.yaml b/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_lin.yaml new file mode 100644 index 00000000..63062ac4 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_lin.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Lingala language.\nAnalyze the premise and hypothesis given in Lingala, and\ + \ determine the relationship between them.\n Respond with one of the following options:\ + \ 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis:\ + \ {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_lin_prompt_4 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_lug.yaml b/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_lug.yaml new file mode 100644 index 00000000..1553c620 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_lug.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Luganda language.\nAnalyze the premise and hypothesis given in Luganda, and\ + \ determine the relationship between them.\n Respond with one of the following options:\ + \ 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis:\ + \ {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_lug_prompt_4 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_orm.yaml b/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_orm.yaml new file mode 100644 index 00000000..ba2a377b --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_orm.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Oromo language.\nAnalyze the premise and hypothesis given in Oromo, and determine\ + \ the relationship between them.\n Respond with one of the following options: 'entailment',\ + \ 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_orm_prompt_4 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_sna.yaml b/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_sna.yaml new file mode 100644 index 00000000..afce6e95 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_sna.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the chiShona language.\nAnalyze the premise and hypothesis given in chiShona,\ + \ and determine the relationship between them.\n Respond with one of the following\ + \ options: 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_sna_prompt_4 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_sot.yaml b/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_sot.yaml new file mode 100644 index 00000000..40c7cf84 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_sot.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: sot +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Sesotho language.\nAnalyze the premise and hypothesis given in Sesotho, and\ + \ determine the relationship between them.\n Respond with one of the following options:\ + \ 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis:\ + \ {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_sot_prompt_4 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_swa.yaml b/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_swa.yaml new file mode 100644 index 00000000..1c28aaae --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_swa.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Swahili language.\nAnalyze the premise and hypothesis given in Swahili, and\ + \ determine the relationship between them.\n Respond with one of the following options:\ + \ 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis:\ + \ {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_swa_prompt_4 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_twi.yaml b/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_twi.yaml new file mode 100644 index 00000000..f9835314 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_twi.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Twi language.\nAnalyze the premise and hypothesis given in Twi, and determine\ + \ the relationship between them.\n Respond with one of the following options: 'entailment',\ + \ 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_twi_prompt_4 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_wol.yaml b/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_wol.yaml new file mode 100644 index 00000000..6b535bc2 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_wol.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Wolof language.\nAnalyze the premise and hypothesis given in Wolof, and determine\ + \ the relationship between them.\n Respond with one of the following options: 'entailment',\ + \ 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_wol_prompt_4 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_xho.yaml b/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_xho.yaml new file mode 100644 index 00000000..45f55e0e --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_xho.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the isiXhosa language.\nAnalyze the premise and hypothesis given in isiXhosa,\ + \ and determine the relationship between them.\n Respond with one of the following\ + \ options: 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_xho_prompt_4 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_yaml b/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_yaml new file mode 100644 index 00000000..fe5de1a6 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_yaml @@ -0,0 +1,30 @@ +tag: + - afrixnli_tasks + - afrixnli_tasks_prompt_4 +dataset_path: masakhane/afrixnli +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: !function utils.doc_to_target +doc_to_choice: + - "entailment" + - "neutral" + - "contradiction" +should_decontaminate: true +doc_to_decontamination_query: premise +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + average: weighted + higher_is_better: True + ignore_case: true + ignore_punctuation: true + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_yor.yaml b/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_yor.yaml new file mode 100644 index 00000000..63d4f606 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_yor.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Yoruba language.\nAnalyze the premise and hypothesis given in Yoruba, and\ + \ determine the relationship between them.\n Respond with one of the following options:\ + \ 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis:\ + \ {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_yor_prompt_4 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_zul.yaml b/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_zul.yaml new file mode 100644 index 00000000..1b4a232e --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_4/afrixnli_zul.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Zulu language.\nAnalyze the premise and hypothesis given in Zulu, and determine\ + \ the relationship between them.\n Respond with one of the following options: 'entailment',\ + \ 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_zul_prompt_4 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_4/utils.py b/lm_eval/tasks/afrixnli/direct/prompt_4/utils.py new file mode 100644 index 00000000..d97a0a28 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_4/utils.py @@ -0,0 +1,19 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_text(doc): + output = """Please identify whether the premise entails or contradicts the hypothesis in the following premise + and hypothesis. The answer should be exact entailment, contradiction, or neutral. + + Premise: {premise} + Hypothesis: {hypothesis} + + Is it entailment, contradiction, or neutral?""" + + text = output.format(premise=doc["premise"], hypothesis=doc["hypothesis"]) + return text + + +def doc_to_target(doc): + replacements = {0: "entailment", 1: "neutral", 2: "contradiction"} + return replacements[doc["label"]] diff --git a/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_amh.yaml b/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_amh.yaml new file mode 100644 index 00000000..70873a21 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_amh.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_amh_prompt_5 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_eng.yaml b/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_eng.yaml new file mode 100644 index 00000000..675264a8 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_eng.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_eng_prompt_5 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_ewe.yaml b/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_ewe.yaml new file mode 100644 index 00000000..7f60db0b --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_ewe.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ewe +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_ewe_prompt_5 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_fra.yaml b/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_fra.yaml new file mode 100644 index 00000000..2bb558da --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_fra.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: fra +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_fra_prompt_5 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_hau.yaml b/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_hau.yaml new file mode 100644 index 00000000..728ae1b8 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_hau.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_hau_prompt_5 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_ibo.yaml b/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_ibo.yaml new file mode 100644 index 00000000..3086b9b4 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_ibo.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_ibo_prompt_5 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_kin.yaml b/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_kin.yaml new file mode 100644 index 00000000..13a8845c --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_kin.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_kin_prompt_5 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_lin.yaml b/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_lin.yaml new file mode 100644 index 00000000..a0250f29 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_lin.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_lin_prompt_5 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_lug.yaml b/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_lug.yaml new file mode 100644 index 00000000..928b74ce --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_lug.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_lug_prompt_5 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_orm.yaml b/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_orm.yaml new file mode 100644 index 00000000..f7f555db --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_orm.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_orm_prompt_5 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_sna.yaml b/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_sna.yaml new file mode 100644 index 00000000..ac0ef300 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_sna.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_sna_prompt_5 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_sot.yaml b/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_sot.yaml new file mode 100644 index 00000000..21fcdde5 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_sot.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: sot +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_sot_prompt_5 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_swa.yaml b/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_swa.yaml new file mode 100644 index 00000000..5d5824ad --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_swa.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_swa_prompt_5 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_twi.yaml b/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_twi.yaml new file mode 100644 index 00000000..b519ef71 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_twi.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_twi_prompt_5 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_wol.yaml b/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_wol.yaml new file mode 100644 index 00000000..a865c8b1 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_wol.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_wol_prompt_5 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_xho.yaml b/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_xho.yaml new file mode 100644 index 00000000..1891bfd0 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_xho.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_xho_prompt_5 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_yaml b/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_yaml new file mode 100644 index 00000000..13e2b6ef --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_yaml @@ -0,0 +1,30 @@ +tag: + - afrixnli_tasks + - afrixnli_tasks_prompt_5 +dataset_path: masakhane/afrixnli +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: !function utils.doc_to_target +doc_to_choice: + - "true" + - "inconclusive" + - "false" +should_decontaminate: true +doc_to_decontamination_query: premise +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + average: weighted + higher_is_better: True + ignore_case: true + ignore_punctuation: true + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_yor.yaml b/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_yor.yaml new file mode 100644 index 00000000..4510441b --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_yor.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_yor_prompt_5 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_zul.yaml b/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_zul.yaml new file mode 100644 index 00000000..2aa872b0 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_5/afrixnli_zul.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_yaml +task: afrixnli_zul_prompt_5 diff --git a/lm_eval/tasks/afrixnli/direct/prompt_5/utils.py b/lm_eval/tasks/afrixnli/direct/prompt_5/utils.py new file mode 100644 index 00000000..6b9cb312 --- /dev/null +++ b/lm_eval/tasks/afrixnli/direct/prompt_5/utils.py @@ -0,0 +1,6 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_target(doc): + replacements = {0: "true", 1: "false", 2: "inconclusive"} + return replacements[doc["label"]] diff --git a/lm_eval/tasks/afrixnli/gen_utils.py b/lm_eval/tasks/afrixnli/gen_utils.py new file mode 100644 index 00000000..338b4f9d --- /dev/null +++ b/lm_eval/tasks/afrixnli/gen_utils.py @@ -0,0 +1,129 @@ +import argparse +import os + +import yaml + + +class FunctionTag: + def __init__(self, value): + self.value = value + + +def prompt_func(mode, lang): + prompt_map = { + "prompt_1": "Please identify whether the premise entails or contradicts the hypothesis in the following premise " + "and hypothesis. The answer should be exact entailment, contradiction, or neutral.\n\nPremise: {premise}\nHypothesis: {hypothesis}\n\n" + "Is it entailment, contradiction, or neutral?", + "prompt_3": f"Given the following premise and hypothesis in {lang}, identify if the premise entails, contradicts, " + f"or is neutral towards the hypothesis. Please respond with exact 'entailment', 'contradiction', or 'neutral'. \n\n" + "Premise: {{premise}} \nHypothesis: {{hypothesis}}", + "prompt_4": f"You are an expert in Natural Language Inference (NLI) specializing in the {lang} language.\n" + f"Analyze the premise and hypothesis given in {lang}, and determine the relationship between them.\n " + f"Respond with one of the following options: 'entailment', 'contradiction', or 'neutral'. \n\n" + "Premise: {{premise}} \nHypothesis: {{hypothesis}}", + "prompt_5": "Based on the given statement, is the following claim 'true', 'false', or 'inconclusive'. \n" + "Statement: {{premise}} \nClaim: {{hypothesis}}", + } + return prompt_map[mode] + + +def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None: + """ + Generate a yaml file for each language. + + :param output_dir: The directory to output the files to. + :param overwrite: Whether to overwrite files if they already exist. + """ + err = [] + languages = { + "eng": "English", + "amh": "Amharic", + "ibo": "Igbo", + "fra": "French", + "sna": "chiShona", + "wol": "Wolof", + "ewe": "Ewe", + "lin": "Lingala", + "lug": "Luganda", + "xho": "isiXhosa", + "kin": "Kinyarwanda", + "twi": "Twi", + "zul": "Zulu", + "orm": "Oromo", + "yor": "Yoruba", + "hau": "Hausa", + "sot": "Sesotho", + "swa": "Swahili", + } + + for lang in languages.keys(): + try: + file_name = f"afrixnli_{lang}.yaml" + task_name = f"afrixnli_{lang}_{mode}" + yaml_template = "afrixnli_yaml" + if output_dir.split("/")[-1] == "translate": + file_name = f"afrixnli_translate_{lang}.yaml" + task_name = f"afrixnli_translate_{lang}_{mode}" + yaml_template = "afrixnli_translate_yaml" + if int(mode.split("_")[-1]) == 1 or int(mode.split("_")[-1]) > 2: + yaml_details = { + "include": yaml_template, + "task": task_name, + "dataset_name": lang, + "doc_to_text": prompt_func(mode, languages[lang]), + } + else: + yaml_details = { + "include": yaml_template, + "task": task_name, + "dataset_name": lang, + } + os.makedirs(f"{output_dir}/{mode}", exist_ok=True) + with open( + f"{output_dir}/{mode}/{file_name}", + "w" if overwrite else "x", + encoding="utf8", + ) as f: + f.write("# Generated by utils.py\n") + yaml.dump( + yaml_details, + f, + allow_unicode=True, + ) + except FileExistsError: + err.append(file_name) + + if len(err) > 0: + raise FileExistsError( + "Files were not created because they already exist (use --overwrite flag):" + f" {', '.join(err)}" + ) + + +def main() -> None: + """Parse CLI args and generate language-specific yaml files.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + default=True, + action="store_true", + help="Overwrite files if they already exist", + ) + parser.add_argument( + "--output-dir", + default="./translate", + help="Directory to write yaml files to", + ) + parser.add_argument( + "--mode", + default="prompt_5", + choices=["prompt_1", "prompt_2", "prompt_3", "prompt_4", "prompt_5"], + help="Prompt number", + ) + args = parser.parse_args() + + gen_lang_yamls(output_dir=args.output_dir, overwrite=args.overwrite, mode=args.mode) + + +if __name__ == "__main__": + main() diff --git a/lm_eval/tasks/afrixnli/translate/afrixnli_tt.yaml b/lm_eval/tasks/afrixnli/translate/afrixnli_tt.yaml new file mode 100644 index 00000000..ba507b39 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/afrixnli_tt.yaml @@ -0,0 +1,9 @@ +group: afrixnli_tt-irokobench +task: + - afrixnli_tt_tasks +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 2 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_amh.yaml b/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_amh.yaml new file mode 100644 index 00000000..92ef8df7 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_amh.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_amh_prompt_1 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_ewe.yaml b/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_ewe.yaml new file mode 100644 index 00000000..fa32dd72 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_ewe.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: ewe +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_ewe_prompt_1 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_fra.yaml b/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_fra.yaml new file mode 100644 index 00000000..3dc72af6 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_fra.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: fra +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_fra_prompt_1 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_hau.yaml b/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_hau.yaml new file mode 100644 index 00000000..77f22faf --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_hau.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_hau_prompt_1 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_ibo.yaml b/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_ibo.yaml new file mode 100644 index 00000000..a7ac8793 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_ibo.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_ibo_prompt_1 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_kin.yaml b/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_kin.yaml new file mode 100644 index 00000000..3a528614 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_kin.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_kin_prompt_1 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_lin.yaml b/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_lin.yaml new file mode 100644 index 00000000..eb2a667e --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_lin.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_lin_prompt_1 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_lug.yaml b/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_lug.yaml new file mode 100644 index 00000000..bf45d957 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_lug.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_lug_prompt_1 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_orm.yaml b/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_orm.yaml new file mode 100644 index 00000000..14b20a1c --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_orm.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_orm_prompt_1 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_sna.yaml b/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_sna.yaml new file mode 100644 index 00000000..13103dd7 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_sna.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_sna_prompt_1 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_sot.yaml b/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_sot.yaml new file mode 100644 index 00000000..97cf3cba --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_sot.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: sot +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_sot_prompt_1 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_swa.yaml b/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_swa.yaml new file mode 100644 index 00000000..824bb17a --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_swa.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_swa_prompt_1 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_twi.yaml b/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_twi.yaml new file mode 100644 index 00000000..9d971c3e --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_twi.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_twi_prompt_1 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_wol.yaml b/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_wol.yaml new file mode 100644 index 00000000..877787a8 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_wol.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_wol_prompt_1 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_xho.yaml b/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_xho.yaml new file mode 100644 index 00000000..c907a2bf --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_xho.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_xho_prompt_1 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_yaml b/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_yaml new file mode 100644 index 00000000..832b5149 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_yaml @@ -0,0 +1,27 @@ +tag: afrixnli_tt_tasks +dataset_path: masakhane/afrixnli-translate-test +dataset_name: null +output_type: multiple_choice +test_split: test +fewshot_split: test +doc_to_target: !function utils.doc_to_target +doc_to_choice: + - "entailment" + - "neutral" + - "contradiction" +should_decontaminate: true +doc_to_decontamination_query: premise +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + average: weighted + higher_is_better: True + ignore_case: true + ignore_punctuation: true + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_yor.yaml b/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_yor.yaml new file mode 100644 index 00000000..4c0ec7c9 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_yor.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_yor_prompt_1 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_zul.yaml b/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_zul.yaml new file mode 100644 index 00000000..78753d1f --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_1/afrixnli_translate_zul.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: 'Please identify whether the premise entails or contradicts the hypothesis + in the following premise and hypothesis. The answer should be exact entailment, + contradiction, or neutral. + + + Premise: {premise} + + Hypothesis: {hypothesis} + + + Is it entailment, contradiction, or neutral?' +include: afrixnli_translate_yaml +task: afrixnli_translate_zul_prompt_1 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_1/utils.py b/lm_eval/tasks/afrixnli/translate/prompt_1/utils.py new file mode 100644 index 00000000..d97a0a28 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_1/utils.py @@ -0,0 +1,19 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_text(doc): + output = """Please identify whether the premise entails or contradicts the hypothesis in the following premise + and hypothesis. The answer should be exact entailment, contradiction, or neutral. + + Premise: {premise} + Hypothesis: {hypothesis} + + Is it entailment, contradiction, or neutral?""" + + text = output.format(premise=doc["premise"], hypothesis=doc["hypothesis"]) + return text + + +def doc_to_target(doc): + replacements = {0: "entailment", 1: "neutral", 2: "contradiction"} + return replacements[doc["label"]] diff --git a/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_amh.yaml b/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_amh.yaml new file mode 100644 index 00000000..0810f6b3 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: amh +include: afrixnli_translate_yaml +task: afrixnli_translate_amh_prompt_2 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_ewe.yaml b/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_ewe.yaml new file mode 100644 index 00000000..d7aec16a --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ewe +include: afrixnli_translate_yaml +task: afrixnli_translate_ewe_prompt_2 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_fra.yaml b/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_fra.yaml new file mode 100644 index 00000000..f17a4ecf --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_fra.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fra +include: afrixnli_translate_yaml +task: afrixnli_translate_fra_prompt_2 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_hau.yaml b/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_hau.yaml new file mode 100644 index 00000000..688778c3 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: hau +include: afrixnli_translate_yaml +task: afrixnli_translate_hau_prompt_2 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_ibo.yaml b/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_ibo.yaml new file mode 100644 index 00000000..5667b3d0 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ibo +include: afrixnli_translate_yaml +task: afrixnli_translate_ibo_prompt_2 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_kin.yaml b/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_kin.yaml new file mode 100644 index 00000000..a74950cc --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: kin +include: afrixnli_translate_yaml +task: afrixnli_translate_kin_prompt_2 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_lin.yaml b/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_lin.yaml new file mode 100644 index 00000000..27e88a5b --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_lin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lin +include: afrixnli_translate_yaml +task: afrixnli_translate_lin_prompt_2 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_lug.yaml b/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_lug.yaml new file mode 100644 index 00000000..63ff988c --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lug +include: afrixnli_translate_yaml +task: afrixnli_translate_lug_prompt_2 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_orm.yaml b/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_orm.yaml new file mode 100644 index 00000000..db1a3ea1 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_orm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: orm +include: afrixnli_translate_yaml +task: afrixnli_translate_orm_prompt_2 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_sna.yaml b/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_sna.yaml new file mode 100644 index 00000000..fa110774 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sna +include: afrixnli_translate_yaml +task: afrixnli_translate_sna_prompt_2 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_sot.yaml b/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_sot.yaml new file mode 100644 index 00000000..3133308b --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_sot.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sot +include: afrixnli_translate_yaml +task: afrixnli_translate_sot_prompt_2 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_swa.yaml b/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_swa.yaml new file mode 100644 index 00000000..926f9132 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrixnli_translate_yaml +task: afrixnli_translate_swa_prompt_2 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_twi.yaml b/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_twi.yaml new file mode 100644 index 00000000..c229de3d --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: twi +include: afrixnli_translate_yaml +task: afrixnli_translate_twi_prompt_2 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_wol.yaml b/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_wol.yaml new file mode 100644 index 00000000..87844c49 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: wol +include: afrixnli_translate_yaml +task: afrixnli_translate_wol_prompt_2 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_xho.yaml b/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_xho.yaml new file mode 100644 index 00000000..63fa3ffc --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: xho +include: afrixnli_translate_yaml +task: afrixnli_translate_xho_prompt_2 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_yaml b/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_yaml new file mode 100644 index 00000000..8ad87afc --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_yaml @@ -0,0 +1,31 @@ +tag: afrixnli_tt_tasks +dataset_path: masakhane/afrixnli-translate-test +dataset_name: null +output_type: multiple_choice +test_split: test +fewshot_split: test +doc_to_text: "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither?\nAnswer:" +# True = entailment +# False = contradiction +# Neither = neutral +doc_to_target: !function utils.doc_to_target +doc_to_choice: + - "True" + - "Neither" + - "False" +should_decontaminate: true +doc_to_decontamination_query: premise +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + average: weighted + higher_is_better: True + ignore_case: true + ignore_punctuation: true + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_yor.yaml b/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_yor.yaml new file mode 100644 index 00000000..7dfc9bd6 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_yor.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: yor +include: afrixnli_translate_yaml +task: afrixnli_translate_yor_prompt_2 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_zul.yaml b/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_zul.yaml new file mode 100644 index 00000000..0878c4e0 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_2/afrixnli_translate_zul.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: zul +include: afrixnli_translate_yaml +task: afrixnli_translate_zul_prompt_2 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_2/utils.py b/lm_eval/tasks/afrixnli/translate/prompt_2/utils.py new file mode 100644 index 00000000..5d1ac19e --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_2/utils.py @@ -0,0 +1,6 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_target(doc): + replacements = {0: "True", 1: "Neither", 2: "False"} + return replacements[doc["label"]] diff --git a/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_amh.yaml b/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_amh.yaml new file mode 100644 index 00000000..6fb06d0f --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_amh.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: "Given the following premise and hypothesis in Amharic, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_amh_prompt_3 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_ewe.yaml b/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_ewe.yaml new file mode 100644 index 00000000..0d550f9d --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_ewe.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: ewe +doc_to_text: "Given the following premise and hypothesis in Ewe, identify if the premise\ + \ entails, contradicts, or is neutral towards the hypothesis. Please respond with\ + \ exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \n\ + Hypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_ewe_prompt_3 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_fra.yaml b/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_fra.yaml new file mode 100644 index 00000000..3156466c --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_fra.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: fra +doc_to_text: "Given the following premise and hypothesis in French, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_fra_prompt_3 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_hau.yaml b/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_hau.yaml new file mode 100644 index 00000000..9ae3c21e --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_hau.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: "Given the following premise and hypothesis in Hausa, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_hau_prompt_3 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_ibo.yaml b/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_ibo.yaml new file mode 100644 index 00000000..28696f33 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_ibo.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "Given the following premise and hypothesis in Igbo, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_ibo_prompt_3 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_kin.yaml b/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_kin.yaml new file mode 100644 index 00000000..6981da83 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_kin.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: "Given the following premise and hypothesis in Kinyarwanda, identify\ + \ if the premise entails, contradicts, or is neutral towards the hypothesis. Please\ + \ respond with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_kin_prompt_3 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_lin.yaml b/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_lin.yaml new file mode 100644 index 00000000..1984416f --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_lin.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: "Given the following premise and hypothesis in Lingala, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_lin_prompt_3 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_lug.yaml b/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_lug.yaml new file mode 100644 index 00000000..32a7ad2a --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_lug.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: "Given the following premise and hypothesis in Luganda, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_lug_prompt_3 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_orm.yaml b/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_orm.yaml new file mode 100644 index 00000000..d3923a80 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_orm.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: "Given the following premise and hypothesis in Oromo, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_orm_prompt_3 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_sna.yaml b/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_sna.yaml new file mode 100644 index 00000000..d7dbf17e --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_sna.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: "Given the following premise and hypothesis in chiShona, identify if\ + \ the premise entails, contradicts, or is neutral towards the hypothesis. Please\ + \ respond with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_sna_prompt_3 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_sot.yaml b/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_sot.yaml new file mode 100644 index 00000000..c4e89ec9 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_sot.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: sot +doc_to_text: "Given the following premise and hypothesis in Sesotho, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_sot_prompt_3 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_swa.yaml b/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_swa.yaml new file mode 100644 index 00000000..a3c5243b --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_swa.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: "Given the following premise and hypothesis in Swahili, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_swa_prompt_3 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_twi.yaml b/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_twi.yaml new file mode 100644 index 00000000..e7e85687 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_twi.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: "Given the following premise and hypothesis in Twi, identify if the premise\ + \ entails, contradicts, or is neutral towards the hypothesis. Please respond with\ + \ exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \n\ + Hypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_twi_prompt_3 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_wol.yaml b/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_wol.yaml new file mode 100644 index 00000000..2cf0b08e --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_wol.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: "Given the following premise and hypothesis in Wolof, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_wol_prompt_3 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_xho.yaml b/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_xho.yaml new file mode 100644 index 00000000..a4dafa34 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_xho.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: "Given the following premise and hypothesis in isiXhosa, identify if\ + \ the premise entails, contradicts, or is neutral towards the hypothesis. Please\ + \ respond with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_xho_prompt_3 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_yaml b/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_yaml new file mode 100644 index 00000000..832b5149 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_yaml @@ -0,0 +1,27 @@ +tag: afrixnli_tt_tasks +dataset_path: masakhane/afrixnli-translate-test +dataset_name: null +output_type: multiple_choice +test_split: test +fewshot_split: test +doc_to_target: !function utils.doc_to_target +doc_to_choice: + - "entailment" + - "neutral" + - "contradiction" +should_decontaminate: true +doc_to_decontamination_query: premise +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + average: weighted + higher_is_better: True + ignore_case: true + ignore_punctuation: true + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_yor.yaml b/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_yor.yaml new file mode 100644 index 00000000..f5c01ca5 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_yor.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: "Given the following premise and hypothesis in Yoruba, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_yor_prompt_3 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_zul.yaml b/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_zul.yaml new file mode 100644 index 00000000..fdbbec80 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_3/afrixnli_translate_zul.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: "Given the following premise and hypothesis in Zulu, identify if the\ + \ premise entails, contradicts, or is neutral towards the hypothesis. Please respond\ + \ with exact 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_zul_prompt_3 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_3/utils.py b/lm_eval/tasks/afrixnli/translate/prompt_3/utils.py new file mode 100644 index 00000000..c455a304 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_3/utils.py @@ -0,0 +1,21 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_text(doc): + output = """You are an NLP assistant whose purpose is to solve Natural Language Inference (NLI) problems + + Please identify whether the premise entails or contradicts the hypothesis in the following premise + and hypothesis. The answer should be exact entailment, contradiction, or neutral. + + Premise: {premise} + Hypothesis: {hypothesis} + + Is it entailment, contradiction, or neutral?""" + + text = output.format(premise=doc["premise"], hypothesis=doc["hypothesis"]) + return text + + +def doc_to_target(doc): + replacements = {0: "entailment", 1: "neutral", 2: "contradiction"} + return replacements[doc["label"]] diff --git a/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_amh.yaml b/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_amh.yaml new file mode 100644 index 00000000..b5f972e7 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_amh.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Amharic language.\nAnalyze the premise and hypothesis given in Amharic, and\ + \ determine the relationship between them.\n Respond with one of the following options:\ + \ 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis:\ + \ {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_amh_prompt_4 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_ewe.yaml b/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_ewe.yaml new file mode 100644 index 00000000..ebc775dd --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_ewe.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: ewe +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Ewe language.\nAnalyze the premise and hypothesis given in Ewe, and determine\ + \ the relationship between them.\n Respond with one of the following options: 'entailment',\ + \ 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_ewe_prompt_4 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_fra.yaml b/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_fra.yaml new file mode 100644 index 00000000..2ad718c7 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_fra.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: fra +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the French language.\nAnalyze the premise and hypothesis given in French, and\ + \ determine the relationship between them.\n Respond with one of the following options:\ + \ 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis:\ + \ {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_fra_prompt_4 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_hau.yaml b/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_hau.yaml new file mode 100644 index 00000000..dd65f366 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_hau.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Hausa language.\nAnalyze the premise and hypothesis given in Hausa, and determine\ + \ the relationship between them.\n Respond with one of the following options: 'entailment',\ + \ 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_hau_prompt_4 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_ibo.yaml b/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_ibo.yaml new file mode 100644 index 00000000..13df1264 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_ibo.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Igbo language.\nAnalyze the premise and hypothesis given in Igbo, and determine\ + \ the relationship between them.\n Respond with one of the following options: 'entailment',\ + \ 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_ibo_prompt_4 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_kin.yaml b/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_kin.yaml new file mode 100644 index 00000000..198d8875 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_kin.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Kinyarwanda language.\nAnalyze the premise and hypothesis given in Kinyarwanda,\ + \ and determine the relationship between them.\n Respond with one of the following\ + \ options: 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_kin_prompt_4 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_lin.yaml b/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_lin.yaml new file mode 100644 index 00000000..b25856cf --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_lin.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Lingala language.\nAnalyze the premise and hypothesis given in Lingala, and\ + \ determine the relationship between them.\n Respond with one of the following options:\ + \ 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis:\ + \ {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_lin_prompt_4 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_lug.yaml b/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_lug.yaml new file mode 100644 index 00000000..633c173c --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_lug.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Luganda language.\nAnalyze the premise and hypothesis given in Luganda, and\ + \ determine the relationship between them.\n Respond with one of the following options:\ + \ 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis:\ + \ {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_lug_prompt_4 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_orm.yaml b/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_orm.yaml new file mode 100644 index 00000000..e63f93eb --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_orm.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Oromo language.\nAnalyze the premise and hypothesis given in Oromo, and determine\ + \ the relationship between them.\n Respond with one of the following options: 'entailment',\ + \ 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_orm_prompt_4 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_sna.yaml b/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_sna.yaml new file mode 100644 index 00000000..6fcb4e06 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_sna.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the chiShona language.\nAnalyze the premise and hypothesis given in chiShona,\ + \ and determine the relationship between them.\n Respond with one of the following\ + \ options: 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_sna_prompt_4 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_sot.yaml b/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_sot.yaml new file mode 100644 index 00000000..358e4b35 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_sot.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: sot +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Sesotho language.\nAnalyze the premise and hypothesis given in Sesotho, and\ + \ determine the relationship between them.\n Respond with one of the following options:\ + \ 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis:\ + \ {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_sot_prompt_4 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_swa.yaml b/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_swa.yaml new file mode 100644 index 00000000..8ce271ed --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_swa.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Swahili language.\nAnalyze the premise and hypothesis given in Swahili, and\ + \ determine the relationship between them.\n Respond with one of the following options:\ + \ 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis:\ + \ {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_swa_prompt_4 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_twi.yaml b/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_twi.yaml new file mode 100644 index 00000000..8171e0da --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_twi.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Twi language.\nAnalyze the premise and hypothesis given in Twi, and determine\ + \ the relationship between them.\n Respond with one of the following options: 'entailment',\ + \ 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_twi_prompt_4 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_wol.yaml b/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_wol.yaml new file mode 100644 index 00000000..b2662662 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_wol.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Wolof language.\nAnalyze the premise and hypothesis given in Wolof, and determine\ + \ the relationship between them.\n Respond with one of the following options: 'entailment',\ + \ 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_wol_prompt_4 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_xho.yaml b/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_xho.yaml new file mode 100644 index 00000000..5aa3a9d1 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_xho.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the isiXhosa language.\nAnalyze the premise and hypothesis given in isiXhosa,\ + \ and determine the relationship between them.\n Respond with one of the following\ + \ options: 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}}\ + \ \nHypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_xho_prompt_4 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_yaml b/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_yaml new file mode 100644 index 00000000..832b5149 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_yaml @@ -0,0 +1,27 @@ +tag: afrixnli_tt_tasks +dataset_path: masakhane/afrixnli-translate-test +dataset_name: null +output_type: multiple_choice +test_split: test +fewshot_split: test +doc_to_target: !function utils.doc_to_target +doc_to_choice: + - "entailment" + - "neutral" + - "contradiction" +should_decontaminate: true +doc_to_decontamination_query: premise +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + average: weighted + higher_is_better: True + ignore_case: true + ignore_punctuation: true + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_yor.yaml b/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_yor.yaml new file mode 100644 index 00000000..478e5043 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_yor.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Yoruba language.\nAnalyze the premise and hypothesis given in Yoruba, and\ + \ determine the relationship between them.\n Respond with one of the following options:\ + \ 'entailment', 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis:\ + \ {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_yor_prompt_4 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_zul.yaml b/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_zul.yaml new file mode 100644 index 00000000..c0dc06e6 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_4/afrixnli_translate_zul.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: "You are an expert in Natural Language Inference (NLI) specializing in\ + \ the Zulu language.\nAnalyze the premise and hypothesis given in Zulu, and determine\ + \ the relationship between them.\n Respond with one of the following options: 'entailment',\ + \ 'contradiction', or 'neutral'. \n\nPremise: {{premise}} \nHypothesis: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_zul_prompt_4 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_4/utils.py b/lm_eval/tasks/afrixnli/translate/prompt_4/utils.py new file mode 100644 index 00000000..d97a0a28 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_4/utils.py @@ -0,0 +1,19 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_text(doc): + output = """Please identify whether the premise entails or contradicts the hypothesis in the following premise + and hypothesis. The answer should be exact entailment, contradiction, or neutral. + + Premise: {premise} + Hypothesis: {hypothesis} + + Is it entailment, contradiction, or neutral?""" + + text = output.format(premise=doc["premise"], hypothesis=doc["hypothesis"]) + return text + + +def doc_to_target(doc): + replacements = {0: "entailment", 1: "neutral", 2: "contradiction"} + return replacements[doc["label"]] diff --git a/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_amh.yaml b/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_amh.yaml new file mode 100644 index 00000000..3079712c --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_amh.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_amh_prompt_5 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_ewe.yaml b/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_ewe.yaml new file mode 100644 index 00000000..6eb452db --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_ewe.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ewe +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_ewe_prompt_5 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_fra.yaml b/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_fra.yaml new file mode 100644 index 00000000..d6ddf493 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_fra.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: fra +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_fra_prompt_5 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_hau.yaml b/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_hau.yaml new file mode 100644 index 00000000..09d182f7 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_hau.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_hau_prompt_5 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_ibo.yaml b/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_ibo.yaml new file mode 100644 index 00000000..b5bf1555 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_ibo.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_ibo_prompt_5 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_kin.yaml b/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_kin.yaml new file mode 100644 index 00000000..f0cbe9c2 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_kin.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_kin_prompt_5 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_lin.yaml b/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_lin.yaml new file mode 100644 index 00000000..159116be --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_lin.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_lin_prompt_5 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_lug.yaml b/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_lug.yaml new file mode 100644 index 00000000..9448fa28 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_lug.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_lug_prompt_5 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_orm.yaml b/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_orm.yaml new file mode 100644 index 00000000..64621cb4 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_orm.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_orm_prompt_5 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_sna.yaml b/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_sna.yaml new file mode 100644 index 00000000..788bae30 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_sna.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_sna_prompt_5 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_sot.yaml b/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_sot.yaml new file mode 100644 index 00000000..617dd9f8 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_sot.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: sot +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_sot_prompt_5 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_swa.yaml b/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_swa.yaml new file mode 100644 index 00000000..81a15925 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_swa.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_swa_prompt_5 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_twi.yaml b/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_twi.yaml new file mode 100644 index 00000000..cb9f115f --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_twi.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_twi_prompt_5 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_wol.yaml b/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_wol.yaml new file mode 100644 index 00000000..a5f4eb0c --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_wol.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_wol_prompt_5 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_xho.yaml b/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_xho.yaml new file mode 100644 index 00000000..d085919b --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_xho.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_xho_prompt_5 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_yaml b/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_yaml new file mode 100644 index 00000000..30472384 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_yaml @@ -0,0 +1,27 @@ +tag: afrixnli_tt_tasks +dataset_path: masakhane/afrixnli-translate-test +dataset_name: null +output_type: multiple_choice +test_split: test +fewshot_split: test +doc_to_target: !function utils.doc_to_target +doc_to_choice: + - "true" + - "inconclusive" + - "false" +should_decontaminate: true +doc_to_decontamination_query: premise +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + average: weighted + higher_is_better: True + ignore_case: true + ignore_punctuation: true + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_yor.yaml b/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_yor.yaml new file mode 100644 index 00000000..107c6634 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_yor.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_yor_prompt_5 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_zul.yaml b/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_zul.yaml new file mode 100644 index 00000000..d9636460 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_5/afrixnli_translate_zul.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: "Based on the given statement, is the following claim 'true', 'false',\ + \ or 'inconclusive'. \nStatement: {{premise}} \nClaim: {{hypothesis}}" +include: afrixnli_translate_yaml +task: afrixnli_translate_zul_prompt_5 diff --git a/lm_eval/tasks/afrixnli/translate/prompt_5/utils.py b/lm_eval/tasks/afrixnli/translate/prompt_5/utils.py new file mode 100644 index 00000000..6b9cb312 --- /dev/null +++ b/lm_eval/tasks/afrixnli/translate/prompt_5/utils.py @@ -0,0 +1,6 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_target(doc): + replacements = {0: "true", 1: "false", 2: "inconclusive"} + return replacements[doc["label"]] diff --git a/lm_eval/tasks/afrobench/README.md b/lm_eval/tasks/afrobench/README.md new file mode 100644 index 00000000..a6ab3cee --- /dev/null +++ b/lm_eval/tasks/afrobench/README.md @@ -0,0 +1,72 @@ +# AfroBench + +### Paper + +Title: `AfroBench: How Good are Large Language Models on African Languages?` + +Paper Link: https://arxiv.org/abs/2311.07978 + +## Abstract +> Large-scale multilingual evaluations, such as MEGA, often include only a handful of African languages due to the scarcity of high-quality evaluation data and the limited discoverability of existing African datasets. This lack of representation hinders comprehensive LLM evaluation across a diverse range of languages and tasks. To address these challenges, we introduce AfroBench -- a multi-task benchmark for evaluating the performance of LLMs across 64 African languages, 15 tasks and 22 datasets. AfroBench consists of nine natural language understanding datasets, six text generation datasets, six knowledge and question answering tasks, and one mathematical reasoning task. We present results comparing the performance of prompting LLMs to fine-tuned baselines based on BERT and T5-style models. Our results suggest large gaps in performance between high-resource languages, such as English, and African languages across most tasks; but performance also varies based on the availability of monolingual data resources. Our findings confirm that performance on African languages continues to remain a hurdle for current LLMs, underscoring the need for additional efforts to close this gap. + +HomePage: https://mcgill-nlp.github.io/AfroBench/ + +### Groups, and Tasks +#### Groups +* `afrobench` : Runs all that tasks, datasets and prompts in this benchmark +* `afrobench_lite`: Runs the lite version of the benchmark which includes; afrimgsm, afrimmlu, afrixnli, sib, intent, adr and flores + +Dataset specific grouping that listing all prompts, allowing users to review or edit them. +* `adr` `afrihate` `afrisenti` `belebele` `african_flores` `injongointent` `mafand` `masakhaner` `masakhapos` `naijarc` `nollysenti` `african_ntrex` `openai_mmlu` `salt` `sib` `uhura` `xlsum` + + +#### Task Tags +* `adr_tasks`: all datasets in this benchmark relating to Automatic Diacritics Restoration task +* `afrihate_tasks`: all datasets in this benchmark relating to Hate Speech detection task +* `afrimgsm_tasks`: all datasets in this benchmark relating to Mathematical reasoning task +* `afrixnli_tasks`: all datasets in this benchmark relating to Natural Language Inference task +* `afrobench_xqa_tasks`: all datasets in this benchmark relating to Crosslingual QA (XQA) task +* `afrobench_sentiment_tasks`: all datasets in this benchmark relating to Sentiment Classification task +* `afrobench_MT_tasks`: all datasets in this benchmark relating to Machine Translation task +* `afrobench_TC_tasks`: all datasets in this benchmark relating to Topic Classification task +* `afrobench_mmlu_tasks`: all datasets in this benchmark relating to MMLU task +* `injongointent_tasks`: all datasets in this benchmark relating to Intent Detection task +* `masakhaner_tasks`: all datasets in this benchmark relating to Named Entity Recognition (NER) task +* `masakhapos_tasks`: all datasets in this benchmark relating to Part of Speech Tagging (POS) task +* `RC_tasks`: all datasets in this benchmark relating to Reading Comprehension task +* `uhura_arc_easy_tasks`: all datasets in this benchmark relating to Arc-Easy (XQA) task +* `xlsum_tasks`: all datasets in this benchmark relating to Summarization task + + +We've included sample run scripts for easier integration with the benchmark: [sample run scripts](./sample_run_scripts) + +For better understanding of the run interface see [interface.md](../../../docs/interface.md) + +All dataset used in this benchmark are available at [huggingface](https://huggingface.co/collections/masakhane/afrobench-67dbf553ebf5701c2207f883) + +### Citation + +``` +@misc{ojo2025afrobenchgoodlargelanguage, + title={AfroBench: How Good are Large Language Models on African Languages?}, + author={Jessica Ojo and Odunayo Ogundepo and Akintunde Oladipo and Kelechi Ogueji and Jimmy Lin and Pontus Stenetorp and David Ifeoluwa Adelani}, + year={2025}, + eprint={2311.07978}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2311.07978}, +} +``` +Please cite datasets used. Citations for individual datasets are included in their respective repository readme files within this benchmark. +### Checklist + +For adding novel benchmarks/datasets to the library: +* [ ] Is the task an existing benchmark in the literature? + * [ ] Have you referenced the original paper that introduced the task? + * [ ] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? The original paper doesn't have an associated implementation, but there is an official entry in [BigBench](https://github.com/google/BIG-bench/tree/main/bigbench/benchmark_tasks/social_iqa). I use the same prompting format as BigBench. + + +If other tasks on this dataset are already supported: +* [ ] Is the "Main" variant of this task clearly denoted? +* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/lm_eval/tasks/afrobench/adr/README.md b/lm_eval/tasks/afrobench/adr/README.md new file mode 100644 index 00000000..cb09567d --- /dev/null +++ b/lm_eval/tasks/afrobench/adr/README.md @@ -0,0 +1,7 @@ +# Automatic Diacritics Restoration (ADR) + +Automatic Diacritics Restoration (ADR) is the task of restoring diacritical marks in text where they have been omitted or removed. +This process is essential for languages where diacritics alter pronunciation, meaning, or grammatical structure. +ADR requires the model to have a deep understanding of linguistic context, syntax, and semantics to accurately predict and reinsert the appropriate diacritics. + +As part of this benchmark project, we utilise the mafand dataset to curate a dataset specifically for ADR. We focus on five languages: Gbomola, Fon, Igbo, Wolof, and Yoruba. diff --git a/lm_eval/tasks/afrobench/adr/afridiacritics.yaml b/lm_eval/tasks/afrobench/adr/afridiacritics.yaml new file mode 100644 index 00000000..34d60eef --- /dev/null +++ b/lm_eval/tasks/afrobench/adr/afridiacritics.yaml @@ -0,0 +1,13 @@ +group: adr +task: + - adr_prompt_1 + - adr_prompt_2 + - adr_prompt_3 + - adr_prompt_4 + - adr_prompt_5 +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 1 diff --git a/lm_eval/tasks/afrobench/adr/gen_utils.py b/lm_eval/tasks/afrobench/adr/gen_utils.py new file mode 100644 index 00000000..ff6e63e3 --- /dev/null +++ b/lm_eval/tasks/afrobench/adr/gen_utils.py @@ -0,0 +1,105 @@ +import argparse +import os + +import yaml + + +class FunctionTag: + def __init__(self, value): + self.value = value + + +def prompt_func(mode, lang): + prompt_map = { + "prompt_1": "Please restore the missing diacritics in the following sentence: {{text}}. Return output sentence only", + "prompt_2": "Given a sentence without diacritics, add the appropriate diacritics to make it grammatically " + "and semantically correct. \nSentence: {{text}}. Return output sentence only", + "prompt_3": f"This text is in {lang}. Restore all diacritical marks to their proper places in the " + "following sentence: {{text}}. Return output sentence only", + "prompt_4": f"You are a linguist specializing in diacritical marks for {lang}. " + f"Add the appropriate diacritics to this {lang} sentence: " + "{{text}}. Return output sentence only", + "prompt_5": f"You are a linguist specializing in diacritical marks for {lang}. Diacritics are essential for " + f"proper pronunciation and meaning in {lang}. You are tasked with converting {lang} sentences " + "without diacritics into their correctly accented forms. Here's the input: {{text}}. " + "Return output sentence only", + } + return prompt_map[mode] + + +def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None: + """ + Generate a yaml file for each language. + + :param output_dir: The directory to output the files to. + :param overwrite: Whether to overwrite files if they already exist. + """ + err = [] + languages = { + "fon": "Fon", + "bbj": "Gbomala", + "ibo": "Igbo", + "wol": "Wolof", + "yor": "Yoruba", + } + + for lang in languages.keys(): + try: + file_name = f"afridiacritics_{lang}.yaml" + task_name = f"afridiacritics_{lang}_{mode}" + yaml_template = "afridiacritics_yaml" + yaml_details = { + "include": yaml_template, + "task": task_name, + "dataset_name": lang, + "doc_to_text": prompt_func(mode, languages[lang]), + } + os.makedirs(f"{output_dir}/{mode}", exist_ok=True) + with open( + f"{output_dir}/{mode}/{file_name}", + "w" if overwrite else "x", + encoding="utf8", + ) as f: + f.write("# Generated by utils.py\n") + yaml.dump( + yaml_details, + f, + allow_unicode=True, + ) + except FileExistsError: + err.append(file_name) + + if len(err) > 0: + raise FileExistsError( + "Files were not created because they already exist (use --overwrite flag):" + f" {', '.join(err)}" + ) + + +def main() -> None: + """Parse CLI args and generate language-specific yaml files.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + default=True, + action="store_true", + help="Overwrite files if they already exist", + ) + parser.add_argument( + "--output-dir", + default="./", + help="Directory to write yaml files to", + ) + parser.add_argument( + "--mode", + default="prompt_1", + choices=["prompt_1", "prompt_2", "prompt_3", "prompt_4", "prompt_5"], + help="Prompt number", + ) + args = parser.parse_args() + + gen_lang_yamls(output_dir=args.output_dir, overwrite=args.overwrite, mode=args.mode) + + +if __name__ == "__main__": + main() diff --git a/lm_eval/tasks/afrobench/adr/prompt_1/afridiacritics_bbj.yaml b/lm_eval/tasks/afrobench/adr/prompt_1/afridiacritics_bbj.yaml new file mode 100644 index 00000000..f3eb26eb --- /dev/null +++ b/lm_eval/tasks/afrobench/adr/prompt_1/afridiacritics_bbj.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: bbj +doc_to_text: 'Please restore the missing diacritics in the following sentence: {{text}}. + Return output sentence only' +include: afridiacritics_yaml +task: afridiacritics_bbj_prompt_1 diff --git a/lm_eval/tasks/afrobench/adr/prompt_1/afridiacritics_fon.yaml b/lm_eval/tasks/afrobench/adr/prompt_1/afridiacritics_fon.yaml new file mode 100644 index 00000000..874832d5 --- /dev/null +++ b/lm_eval/tasks/afrobench/adr/prompt_1/afridiacritics_fon.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: fon +doc_to_text: 'Please restore the missing diacritics in the following sentence: {{text}}. + Return output sentence only' +include: afridiacritics_yaml +task: afridiacritics_fon_prompt_1 diff --git a/lm_eval/tasks/afrobench/adr/prompt_1/afridiacritics_ibo.yaml b/lm_eval/tasks/afrobench/adr/prompt_1/afridiacritics_ibo.yaml new file mode 100644 index 00000000..983bc391 --- /dev/null +++ b/lm_eval/tasks/afrobench/adr/prompt_1/afridiacritics_ibo.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: 'Please restore the missing diacritics in the following sentence: {{text}}. + Return output sentence only' +include: afridiacritics_yaml +task: afridiacritics_ibo_prompt_1 diff --git a/lm_eval/tasks/afrobench/adr/prompt_1/afridiacritics_wol.yaml b/lm_eval/tasks/afrobench/adr/prompt_1/afridiacritics_wol.yaml new file mode 100644 index 00000000..9067770a --- /dev/null +++ b/lm_eval/tasks/afrobench/adr/prompt_1/afridiacritics_wol.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: 'Please restore the missing diacritics in the following sentence: {{text}}. + Return output sentence only' +include: afridiacritics_yaml +task: afridiacritics_wol_prompt_1 diff --git a/lm_eval/tasks/afrobench/adr/prompt_1/afridiacritics_yaml b/lm_eval/tasks/afrobench/adr/prompt_1/afridiacritics_yaml new file mode 100644 index 00000000..53cebaee --- /dev/null +++ b/lm_eval/tasks/afrobench/adr/prompt_1/afridiacritics_yaml @@ -0,0 +1,25 @@ +tag: +- adr_tasks +- adr_prompt_1 +dataset_path: masakhane/diacritics-restoration +dataset_kwargs: {trust_remote_code: True} +doc_to_target: target +output_type: generate_until +fewshot_split: dev +test_split: test +training_split: train +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + do_sample: false + until: + - '' + - + - <|im_end|> +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/adr/prompt_1/afridiacritics_yor.yaml b/lm_eval/tasks/afrobench/adr/prompt_1/afridiacritics_yor.yaml new file mode 100644 index 00000000..8e98af10 --- /dev/null +++ b/lm_eval/tasks/afrobench/adr/prompt_1/afridiacritics_yor.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: 'Please restore the missing diacritics in the following sentence: {{text}}. + Return output sentence only' +include: afridiacritics_yaml +task: afridiacritics_yor_prompt_1 diff --git a/lm_eval/tasks/afrobench/adr/prompt_2/afridiacritics_bbj.yaml b/lm_eval/tasks/afrobench/adr/prompt_2/afridiacritics_bbj.yaml new file mode 100644 index 00000000..0f054eea --- /dev/null +++ b/lm_eval/tasks/afrobench/adr/prompt_2/afridiacritics_bbj.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: bbj +doc_to_text: "Given a sentence without diacritics, add the appropriate diacritics\ + \ to make it grammatically and semantically correct. \nSentence: {{text}}. Return\ + \ output sentence only" +include: afridiacritics_yaml +task: afridiacritics_bbj_prompt_2 diff --git a/lm_eval/tasks/afrobench/adr/prompt_2/afridiacritics_fon.yaml b/lm_eval/tasks/afrobench/adr/prompt_2/afridiacritics_fon.yaml new file mode 100644 index 00000000..07f71146 --- /dev/null +++ b/lm_eval/tasks/afrobench/adr/prompt_2/afridiacritics_fon.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fon +doc_to_text: "Given a sentence without diacritics, add the appropriate diacritics\ + \ to make it grammatically and semantically correct. \nSentence: {{text}}. Return\ + \ output sentence only" +include: afridiacritics_yaml +task: afridiacritics_fon_prompt_2 diff --git a/lm_eval/tasks/afrobench/adr/prompt_2/afridiacritics_ibo.yaml b/lm_eval/tasks/afrobench/adr/prompt_2/afridiacritics_ibo.yaml new file mode 100644 index 00000000..c1ebac10 --- /dev/null +++ b/lm_eval/tasks/afrobench/adr/prompt_2/afridiacritics_ibo.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "Given a sentence without diacritics, add the appropriate diacritics\ + \ to make it grammatically and semantically correct. \nSentence: {{text}}. Return\ + \ output sentence only" +include: afridiacritics_yaml +task: afridiacritics_ibo_prompt_2 diff --git a/lm_eval/tasks/afrobench/adr/prompt_2/afridiacritics_wol.yaml b/lm_eval/tasks/afrobench/adr/prompt_2/afridiacritics_wol.yaml new file mode 100644 index 00000000..b8448d6f --- /dev/null +++ b/lm_eval/tasks/afrobench/adr/prompt_2/afridiacritics_wol.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: "Given a sentence without diacritics, add the appropriate diacritics\ + \ to make it grammatically and semantically correct. \nSentence: {{text}}. Return\ + \ output sentence only" +include: afridiacritics_yaml +task: afridiacritics_wol_prompt_2 diff --git a/lm_eval/tasks/afrobench/adr/prompt_2/afridiacritics_yaml b/lm_eval/tasks/afrobench/adr/prompt_2/afridiacritics_yaml new file mode 100644 index 00000000..a0cc722d --- /dev/null +++ b/lm_eval/tasks/afrobench/adr/prompt_2/afridiacritics_yaml @@ -0,0 +1,25 @@ +tag: +- adr_tasks +- adr_prompt_2 +dataset_path: masakhane/diacritics-restoration +dataset_kwargs: {trust_remote_code: True} +doc_to_target: target +output_type: generate_until +fewshot_split: dev +test_split: test +training_split: train +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + do_sample: false + until: + - '' + - + - <|im_end|> +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/adr/prompt_2/afridiacritics_yor.yaml b/lm_eval/tasks/afrobench/adr/prompt_2/afridiacritics_yor.yaml new file mode 100644 index 00000000..eb95f5e2 --- /dev/null +++ b/lm_eval/tasks/afrobench/adr/prompt_2/afridiacritics_yor.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: "Given a sentence without diacritics, add the appropriate diacritics\ + \ to make it grammatically and semantically correct. \nSentence: {{text}}. Return\ + \ output sentence only" +include: afridiacritics_yaml +task: afridiacritics_yor_prompt_2 diff --git a/lm_eval/tasks/afrobench/adr/prompt_3/afridiacritics_bbj.yaml b/lm_eval/tasks/afrobench/adr/prompt_3/afridiacritics_bbj.yaml new file mode 100644 index 00000000..a50b40c5 --- /dev/null +++ b/lm_eval/tasks/afrobench/adr/prompt_3/afridiacritics_bbj.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: bbj +doc_to_text: 'This text is in Gbomala. Restore all diacritical marks to their proper + places in the following sentence: {{text}}. Return output sentence only' +include: afridiacritics_yaml +task: afridiacritics_bbj_prompt_3 diff --git a/lm_eval/tasks/afrobench/adr/prompt_3/afridiacritics_fon.yaml b/lm_eval/tasks/afrobench/adr/prompt_3/afridiacritics_fon.yaml new file mode 100644 index 00000000..5b0909ce --- /dev/null +++ b/lm_eval/tasks/afrobench/adr/prompt_3/afridiacritics_fon.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: fon +doc_to_text: 'This text is in Fon. Restore all diacritical marks to their proper places + in the following sentence: {{text}}. Return output sentence only' +include: afridiacritics_yaml +task: afridiacritics_fon_prompt_3 diff --git a/lm_eval/tasks/afrobench/adr/prompt_3/afridiacritics_ibo.yaml b/lm_eval/tasks/afrobench/adr/prompt_3/afridiacritics_ibo.yaml new file mode 100644 index 00000000..04d1df0e --- /dev/null +++ b/lm_eval/tasks/afrobench/adr/prompt_3/afridiacritics_ibo.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: 'This text is in Igbo. Restore all diacritical marks to their proper + places in the following sentence: {{text}}. Return output sentence only' +include: afridiacritics_yaml +task: afridiacritics_ibo_prompt_3 diff --git a/lm_eval/tasks/afrobench/adr/prompt_3/afridiacritics_wol.yaml b/lm_eval/tasks/afrobench/adr/prompt_3/afridiacritics_wol.yaml new file mode 100644 index 00000000..576e0845 --- /dev/null +++ b/lm_eval/tasks/afrobench/adr/prompt_3/afridiacritics_wol.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: 'This text is in Wolof. Restore all diacritical marks to their proper + places in the following sentence: {{text}}. Return output sentence only' +include: afridiacritics_yaml +task: afridiacritics_wol_prompt_3 diff --git a/lm_eval/tasks/afrobench/adr/prompt_3/afridiacritics_yaml b/lm_eval/tasks/afrobench/adr/prompt_3/afridiacritics_yaml new file mode 100644 index 00000000..0a27eeef --- /dev/null +++ b/lm_eval/tasks/afrobench/adr/prompt_3/afridiacritics_yaml @@ -0,0 +1,25 @@ +tag: +- adr_tasks +- adr_prompt_3 +dataset_path: masakhane/diacritics-restoration +dataset_kwargs: {trust_remote_code: True} +doc_to_target: target +output_type: generate_until +fewshot_split: dev +test_split: test +training_split: train +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + do_sample: false + until: + - '' + - + - <|im_end|> +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/adr/prompt_3/afridiacritics_yor.yaml b/lm_eval/tasks/afrobench/adr/prompt_3/afridiacritics_yor.yaml new file mode 100644 index 00000000..169c1108 --- /dev/null +++ b/lm_eval/tasks/afrobench/adr/prompt_3/afridiacritics_yor.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: 'This text is in Yoruba. Restore all diacritical marks to their proper + places in the following sentence: {{text}}. Return output sentence only' +include: afridiacritics_yaml +task: afridiacritics_yor_prompt_3 diff --git a/lm_eval/tasks/afrobench/adr/prompt_4/afridiacritics_bbj.yaml b/lm_eval/tasks/afrobench/adr/prompt_4/afridiacritics_bbj.yaml new file mode 100644 index 00000000..7a807b09 --- /dev/null +++ b/lm_eval/tasks/afrobench/adr/prompt_4/afridiacritics_bbj.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: bbj +doc_to_text: 'You are a linguist specializing in diacritical marks for Gbomala. Add + the appropriate diacritics to this Gbomala sentence: {{text}}. Return output sentence + only' +include: afridiacritics_yaml +task: afridiacritics_bbj_prompt_4 diff --git a/lm_eval/tasks/afrobench/adr/prompt_4/afridiacritics_fon.yaml b/lm_eval/tasks/afrobench/adr/prompt_4/afridiacritics_fon.yaml new file mode 100644 index 00000000..11076e68 --- /dev/null +++ b/lm_eval/tasks/afrobench/adr/prompt_4/afridiacritics_fon.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: fon +doc_to_text: 'You are a linguist specializing in diacritical marks for Fon. Add the + appropriate diacritics to this Fon sentence: {{text}}. Return output sentence only' +include: afridiacritics_yaml +task: afridiacritics_fon_prompt_4 diff --git a/lm_eval/tasks/afrobench/adr/prompt_4/afridiacritics_ibo.yaml b/lm_eval/tasks/afrobench/adr/prompt_4/afridiacritics_ibo.yaml new file mode 100644 index 00000000..367e387a --- /dev/null +++ b/lm_eval/tasks/afrobench/adr/prompt_4/afridiacritics_ibo.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: 'You are a linguist specializing in diacritical marks for Igbo. Add the + appropriate diacritics to this Igbo sentence: {{text}}. Return output sentence only' +include: afridiacritics_yaml +task: afridiacritics_ibo_prompt_4 diff --git a/lm_eval/tasks/afrobench/adr/prompt_4/afridiacritics_wol.yaml b/lm_eval/tasks/afrobench/adr/prompt_4/afridiacritics_wol.yaml new file mode 100644 index 00000000..23fb81e7 --- /dev/null +++ b/lm_eval/tasks/afrobench/adr/prompt_4/afridiacritics_wol.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: 'You are a linguist specializing in diacritical marks for Wolof. Add + the appropriate diacritics to this Wolof sentence: {{text}}. Return output sentence + only' +include: afridiacritics_yaml +task: afridiacritics_wol_prompt_4 diff --git a/lm_eval/tasks/afrobench/adr/prompt_4/afridiacritics_yaml b/lm_eval/tasks/afrobench/adr/prompt_4/afridiacritics_yaml new file mode 100644 index 00000000..6ae62e9d --- /dev/null +++ b/lm_eval/tasks/afrobench/adr/prompt_4/afridiacritics_yaml @@ -0,0 +1,25 @@ +tag: +- adr_tasks +- adr_prompt_4 +dataset_path: masakhane/diacritics-restoration +dataset_kwargs: {trust_remote_code: True} +doc_to_target: target +output_type: generate_until +fewshot_split: dev +test_split: test +training_split: train +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + do_sample: false + until: + - '' + - + - <|im_end|> +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/adr/prompt_4/afridiacritics_yor.yaml b/lm_eval/tasks/afrobench/adr/prompt_4/afridiacritics_yor.yaml new file mode 100644 index 00000000..21e3a53f --- /dev/null +++ b/lm_eval/tasks/afrobench/adr/prompt_4/afridiacritics_yor.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: 'You are a linguist specializing in diacritical marks for Yoruba. Add + the appropriate diacritics to this Yoruba sentence: {{text}}. Return output sentence + only' +include: afridiacritics_yaml +task: afridiacritics_yor_prompt_4 diff --git a/lm_eval/tasks/afrobench/adr/prompt_5/afridiacritics_bbj.yaml b/lm_eval/tasks/afrobench/adr/prompt_5/afridiacritics_bbj.yaml new file mode 100644 index 00000000..b1bcc833 --- /dev/null +++ b/lm_eval/tasks/afrobench/adr/prompt_5/afridiacritics_bbj.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: bbj +doc_to_text: 'You are a linguist specializing in diacritical marks for Gbomala. Diacritics + are essential for proper pronunciation and meaning in Gbomala. You are tasked with + converting Gbomala sentences without diacritics into their correctly accented forms. + Here''s the input: {{text}}. Return output sentence only' +include: afridiacritics_yaml +task: afridiacritics_bbj_prompt_5 diff --git a/lm_eval/tasks/afrobench/adr/prompt_5/afridiacritics_fon.yaml b/lm_eval/tasks/afrobench/adr/prompt_5/afridiacritics_fon.yaml new file mode 100644 index 00000000..3a1c55f8 --- /dev/null +++ b/lm_eval/tasks/afrobench/adr/prompt_5/afridiacritics_fon.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: fon +doc_to_text: 'You are a linguist specializing in diacritical marks for Fon. Diacritics + are essential for proper pronunciation and meaning in Fon. You are tasked with converting + Fon sentences without diacritics into their correctly accented forms. Here''s the + input: {{text}}. Return output sentence only' +include: afridiacritics_yaml +task: afridiacritics_fon_prompt_5 diff --git a/lm_eval/tasks/afrobench/adr/prompt_5/afridiacritics_ibo.yaml b/lm_eval/tasks/afrobench/adr/prompt_5/afridiacritics_ibo.yaml new file mode 100644 index 00000000..6cc9865d --- /dev/null +++ b/lm_eval/tasks/afrobench/adr/prompt_5/afridiacritics_ibo.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: 'You are a linguist specializing in diacritical marks for Igbo. Diacritics + are essential for proper pronunciation and meaning in Igbo. You are tasked with + converting Igbo sentences without diacritics into their correctly accented forms. + Here''s the input: {{text}}. Return output sentence only' +include: afridiacritics_yaml +task: afridiacritics_ibo_prompt_5 diff --git a/lm_eval/tasks/afrobench/adr/prompt_5/afridiacritics_wol.yaml b/lm_eval/tasks/afrobench/adr/prompt_5/afridiacritics_wol.yaml new file mode 100644 index 00000000..fed10a70 --- /dev/null +++ b/lm_eval/tasks/afrobench/adr/prompt_5/afridiacritics_wol.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: 'You are a linguist specializing in diacritical marks for Wolof. Diacritics + are essential for proper pronunciation and meaning in Wolof. You are tasked with + converting Wolof sentences without diacritics into their correctly accented forms. + Here''s the input: {{text}}. Return output sentence only' +include: afridiacritics_yaml +task: afridiacritics_wol_prompt_5 diff --git a/lm_eval/tasks/afrobench/adr/prompt_5/afridiacritics_yaml b/lm_eval/tasks/afrobench/adr/prompt_5/afridiacritics_yaml new file mode 100644 index 00000000..aaad3306 --- /dev/null +++ b/lm_eval/tasks/afrobench/adr/prompt_5/afridiacritics_yaml @@ -0,0 +1,25 @@ +tag: +- adr_tasks +- adr_prompt_5 +dataset_path: masakhane/diacritics-restoration +dataset_kwargs: {trust_remote_code: True} +doc_to_target: target +output_type: generate_until +fewshot_split: dev +test_split: test +training_split: train +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + do_sample: false + until: + - '' + - + - <|im_end|> +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/adr/prompt_5/afridiacritics_yor.yaml b/lm_eval/tasks/afrobench/adr/prompt_5/afridiacritics_yor.yaml new file mode 100644 index 00000000..bd1c9007 --- /dev/null +++ b/lm_eval/tasks/afrobench/adr/prompt_5/afridiacritics_yor.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: 'You are a linguist specializing in diacritical marks for Yoruba. Diacritics + are essential for proper pronunciation and meaning in Yoruba. You are tasked with + converting Yoruba sentences without diacritics into their correctly accented forms. + Here''s the input: {{text}}. Return output sentence only' +include: afridiacritics_yaml +task: afridiacritics_yor_prompt_5 diff --git a/lm_eval/tasks/afrobench/afriqa/README.md b/lm_eval/tasks/afrobench/afriqa/README.md new file mode 100644 index 00000000..8730d7c8 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/README.md @@ -0,0 +1,24 @@ +# + +## Paper +Title: `AfriQA: Cross-lingual Open-Retrieval Question Answering for African Languages` + +Paper Link: https://arxiv.org/abs/2305.06897 + +## Abstract +>AfriQA is the first cross-lingual question answering (QA) dataset with a focus on African languages. The dataset includes over 12,000 XOR QA examples across 10 African languages, making it an invaluable resource for developing more equitable QA technology. African languages have historically been underserved in the digital landscape, with far less in-language content available online. This makes it difficult for QA systems to provide accurate information to users in their native language. However, cross-lingual open-retrieval question answering (XOR QA) systems can help fill this gap by retrieving answer content from other languages. AfriQA focuses specifically on African languages where cross-lingual answer content is the only high-coverage source of information. Previous datasets have primarily focused on languages where cross-lingual QA augments coverage from the target language, but AfriQA highlights the importance of African languages as a realistic use case for XOR QA. + +HomePage: https://github.com/masakhane-io/afriqa + +### Citation + +``` +@misc{ogundepo2023afriqa, + title={AfriQA: Cross-lingual Open-Retrieval Question Answering for African Languages}, + author={Odunayo Ogundepo and Tajuddeen R. Gwadabe and Clara E. Rivera and Jonathan H. Clark and Sebastian Ruder and David Ifeoluwa Adelani and Bonaventure F. P. Dossou and Abdou Aziz DIOP and Claytone Sikasote and Gilles Hacheme and Happy Buzaaba and Ignatius Ezeani and Rooweither Mabuya and Salomey Osei and Chris Emezue and Albert Njoroge Kahira and Shamsuddeen H. Muhammad and Akintunde Oladipo and Abraham Toluwase Owodunni and Atnafu Lambebo Tonja and Iyanuoluwa Shode and Akari Asai and Tunde Oluwaseyi Ajayi and Clemencia Siro and Steven Arthur and Mofetoluwa Adeyemi and Orevaoghene Ahia and Aremu Anuoluwapo and Oyinkansola Awosan and Chiamaka Chukwuneke and Bernard Opoku and Awokoya Ayodele and Verrah Otiende and Christine Mwase and Boyd Sinkala and Andre Niyongabo Rubungo and Daniel A. Ajisafe and Emeka Felix Onwuegbuzia and Habib Mbow and Emile Niyomutabazi and Eunice Mukonde and Falalu Ibrahim Lawan and Ibrahim Said Ahmad and Jesujoba O. Alabi and Martin Namukombo and Mbonu Chinedu and Mofya Phiri and Neo Putini and Ndumiso Mngoma and Priscilla A. Amuok and Ruqayya Nasir Iro and Sonia Adhiambo}, + year={2023}, + eprint={2305.06897}, + archivePrefix={arXiv}, + primaryClass={cs.CL} +} +``` diff --git a/lm_eval/tasks/afrobench/afriqa/afriqa.yaml b/lm_eval/tasks/afrobench/afriqa/afriqa.yaml new file mode 100644 index 00000000..80810ca4 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/afriqa.yaml @@ -0,0 +1,13 @@ +group: afriqa +task: + - afriqa_prompt_1 + - afriqa_prompt_2 + - afriqa_prompt_3 + - afriqa_prompt_4 + - afriqa_prompt_5 +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 1 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa b/lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa new file mode 100644 index 00000000..d9b6218e --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa @@ -0,0 +1,42 @@ +tag: + - afrobench_xqa_tasks + - afriqa_prompt_1 +dataset_kwargs: {trust_remote_code: True} +dataset_path: masakhane/afriqa-gold-passages +dataset_name: null +output_type: generate_until +test_split: test +fewshot_split: train +doc_to_target: answer_pivot +should_decontaminate: true +doc_to_decontamination_query: question_lang +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +filter_list: + - name: remove_whitespace + filter: + - function: remove_whitespace + - function: take_first +target_delimiter: " " +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + - "." + - "," + - "\\$" + - metric: f1 + aggregation: !function utils.f1 + higher_is_better: true + ignore_case: true + ignore_punctuation: true + - "." + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_bem.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_bem.yaml new file mode 100644 index 00000000..a3b639a8 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_bem.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: bem +doc_to_text: 'Your task is to answer a qestion given a context.Make sure you respond + with the shortest span containing the answer in the context. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_bem_prompt_1 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_fon.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_fon.yaml new file mode 100644 index 00000000..c5119615 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_fon.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: fon +doc_to_text: 'Your task is to answer a qestion given a context.Make sure you respond + with the shortest span containing the answer in the context. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_fon_prompt_1 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_hau.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_hau.yaml new file mode 100644 index 00000000..0536590a --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_hau.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: 'Your task is to answer a qestion given a context.Make sure you respond + with the shortest span containing the answer in the context. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_hau_prompt_1 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_ibo.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_ibo.yaml new file mode 100644 index 00000000..62eb7116 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_ibo.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: 'Your task is to answer a qestion given a context.Make sure you respond + with the shortest span containing the answer in the context. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_ibo_prompt_1 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_kin.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_kin.yaml new file mode 100644 index 00000000..e632c4be --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_kin.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: 'Your task is to answer a qestion given a context.Make sure you respond + with the shortest span containing the answer in the context. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_kin_prompt_1 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_swa.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_swa.yaml new file mode 100644 index 00000000..dbdebe14 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_swa.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: 'Your task is to answer a qestion given a context.Make sure you respond + with the shortest span containing the answer in the context. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +fewshot_split: test +fewshot_config: + sampler: first_n +task: afriqa_swa_prompt_1 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_twi.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_twi.yaml new file mode 100644 index 00000000..67ba1715 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_twi.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: 'Your task is to answer a qestion given a context.Make sure you respond + with the shortest span containing the answer in the context. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_twi_prompt_1 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_yor.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_yor.yaml new file mode 100644 index 00000000..51d20e43 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_yor.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: 'Your task is to answer a qestion given a context.Make sure you respond + with the shortest span containing the answer in the context. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_yor_prompt_1 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_zul.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_zul.yaml new file mode 100644 index 00000000..1c254b96 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_1/afriqa_zul.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: 'Your task is to answer a qestion given a context.Make sure you respond + with the shortest span containing the answer in the context. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_zul_prompt_1 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_1/utils.py b/lm_eval/tasks/afrobench/afriqa/prompt_1/utils.py new file mode 100644 index 00000000..eae1d885 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_1/utils.py @@ -0,0 +1,53 @@ +import re +import string +from collections import Counter + + +def normalize_answer(s): + """ + Taken from the official evaluation script for v1.1 of the SQuAD dataset. + Lower text and remove punctuation, articles and extra whitespace. + """ + + def remove_articles(text): + return re.sub(r"\b(a|an|the)\b", " ", text) + + def white_space_fix(text): + return " ".join(text.split()) + + def remove_punc(text): + exclude = set(string.punctuation) + return "".join(ch for ch in text if ch not in exclude) + + def lower(text): + return text.lower() + + return white_space_fix(remove_articles(remove_punc(lower(s)))) + + +def f1(items): + """ + Taken from the official evaluation script for v1.1 of the SQuAD dataset. + """ + + unzipped_list = list(zip(*items)) + golds = unzipped_list[0] + preds = unzipped_list[1] + + f1_list = [] + + for i in range(len(golds)): + prediction_tokens = normalize_answer(preds[i]).split() + references_tokens = normalize_answer(golds[i]).split() + common = Counter(prediction_tokens) & Counter(references_tokens) + num_same = sum(common.values()) + if num_same == 0: + f1_score = 0 + else: + precision = 1.0 * num_same / len(prediction_tokens) + recall = 1.0 * num_same / len(references_tokens) + f1_score = (2 * precision * recall) / (precision + recall) + + f1_list.append(f1_score) + + return sum(f1_list) / len(f1_list) diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa b/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa new file mode 100644 index 00000000..d53ce05b --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa @@ -0,0 +1,42 @@ +tag: + - afrobench_xqa_tasks + - afriqa_prompt_2 +dataset_kwargs: {trust_remote_code: True} +dataset_path: masakhane/afriqa-gold-passages +dataset_name: null +output_type: generate_until +test_split: test +fewshot_split: train +doc_to_target: answer_pivot +should_decontaminate: true +doc_to_decontamination_query: question_lang +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +filter_list: + - name: remove_whitespace + filter: + - function: remove_whitespace + - function: take_first +target_delimiter: " " +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + - "." + - "," + - "\\$" + - metric: f1 + aggregation: !function utils.f1 + higher_is_better: true + ignore_case: true + ignore_punctuation: true + - "." + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_bem.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_bem.yaml new file mode 100644 index 00000000..2469c7f4 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_bem.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: bem +doc_to_text: 'Your task is to answer a question given a context. The question is in + Bemba, while the context is in English or French.Make sure you respond with the + shortest span in the context that contains the answer. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_bem_prompt_2 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_fon.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_fon.yaml new file mode 100644 index 00000000..384db449 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_fon.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: fon +doc_to_text: 'Your task is to answer a question given a context. The question is in + Fon, while the context is in English or French.Make sure you respond with the shortest + span in the context that contains the answer. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_fon_prompt_2 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_hau.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_hau.yaml new file mode 100644 index 00000000..40c942ec --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_hau.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: 'Your task is to answer a question given a context. The question is in + Hausa, while the context is in English or French.Make sure you respond with the + shortest span in the context that contains the answer. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_hau_prompt_2 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_ibo.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_ibo.yaml new file mode 100644 index 00000000..8198795d --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_ibo.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: 'Your task is to answer a question given a context. The question is in + Igbo, while the context is in English or French.Make sure you respond with the shortest + span in the context that contains the answer. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_ibo_prompt_2 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_kin.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_kin.yaml new file mode 100644 index 00000000..7a238ae5 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_kin.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: 'Your task is to answer a question given a context. The question is in + Kinyarwanda, while the context is in English or French.Make sure you respond with + the shortest span in the context that contains the answer. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_kin_prompt_2 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_swa.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_swa.yaml new file mode 100644 index 00000000..4be94d07 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_swa.yaml @@ -0,0 +1,16 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: 'Your task is to answer a question given a context. The question is in + Swahili, while the context is in English or French.Make sure you respond with the + shortest span in the context that contains the answer. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +fewshot_split: test +fewshot_config: + sampler: first_n +task: afriqa_swa_prompt_2 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_twi.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_twi.yaml new file mode 100644 index 00000000..f08487d0 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_twi.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: 'Your task is to answer a question given a context. The question is in + Twi, while the context is in English or French.Make sure you respond with the shortest + span in the context that contains the answer. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_twi_prompt_2 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_yor.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_yor.yaml new file mode 100644 index 00000000..44aee11a --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_yor.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: 'Your task is to answer a question given a context. The question is in + Yoruba, while the context is in English or French.Make sure you respond with the + shortest span in the context that contains the answer. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_yor_prompt_2 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_zul.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_zul.yaml new file mode 100644 index 00000000..99c5b18f --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_2/afriqa_zul.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: 'Your task is to answer a question given a context. The question is in + Zulu, while the context is in English or French.Make sure you respond with the shortest + span in the context that contains the answer. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_zul_prompt_2 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_2/utils.py b/lm_eval/tasks/afrobench/afriqa/prompt_2/utils.py new file mode 100644 index 00000000..eae1d885 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_2/utils.py @@ -0,0 +1,53 @@ +import re +import string +from collections import Counter + + +def normalize_answer(s): + """ + Taken from the official evaluation script for v1.1 of the SQuAD dataset. + Lower text and remove punctuation, articles and extra whitespace. + """ + + def remove_articles(text): + return re.sub(r"\b(a|an|the)\b", " ", text) + + def white_space_fix(text): + return " ".join(text.split()) + + def remove_punc(text): + exclude = set(string.punctuation) + return "".join(ch for ch in text if ch not in exclude) + + def lower(text): + return text.lower() + + return white_space_fix(remove_articles(remove_punc(lower(s)))) + + +def f1(items): + """ + Taken from the official evaluation script for v1.1 of the SQuAD dataset. + """ + + unzipped_list = list(zip(*items)) + golds = unzipped_list[0] + preds = unzipped_list[1] + + f1_list = [] + + for i in range(len(golds)): + prediction_tokens = normalize_answer(preds[i]).split() + references_tokens = normalize_answer(golds[i]).split() + common = Counter(prediction_tokens) & Counter(references_tokens) + num_same = sum(common.values()) + if num_same == 0: + f1_score = 0 + else: + precision = 1.0 * num_same / len(prediction_tokens) + recall = 1.0 * num_same / len(references_tokens) + f1_score = (2 * precision * recall) / (precision + recall) + + f1_list.append(f1_score) + + return sum(f1_list) / len(f1_list) diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa b/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa new file mode 100644 index 00000000..79a923b1 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa @@ -0,0 +1,42 @@ +tag: + - afrobench_xqa_tasks + - afriqa_prompt_3 +dataset_kwargs: {trust_remote_code: True} +dataset_path: masakhane/afriqa-gold-passages +dataset_name: null +output_type: generate_until +test_split: test +fewshot_split: train +doc_to_target: answer_pivot +should_decontaminate: true +doc_to_decontamination_query: question_lang +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +filter_list: + - name: remove_whitespace + filter: + - function: remove_whitespace + - function: take_first +target_delimiter: " " +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + - "." + - "," + - "\\$" + - metric: f1 + aggregation: !function utils.f1 + higher_is_better: true + ignore_case: true + ignore_punctuation: true + - "." + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_bem.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_bem.yaml new file mode 100644 index 00000000..3af92f5a --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_bem.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: bem +doc_to_text: 'Given the context, provide the answer to the following question.Ensure + your response is concise and directly from the context. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_bem_prompt_3 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_fon.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_fon.yaml new file mode 100644 index 00000000..73c12439 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_fon.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: fon +doc_to_text: 'Given the context, provide the answer to the following question.Ensure + your response is concise and directly from the context. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_fon_prompt_3 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_hau.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_hau.yaml new file mode 100644 index 00000000..ff08d081 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_hau.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: 'Given the context, provide the answer to the following question.Ensure + your response is concise and directly from the context. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_hau_prompt_3 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_ibo.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_ibo.yaml new file mode 100644 index 00000000..12f18a0b --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_ibo.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: 'Given the context, provide the answer to the following question.Ensure + your response is concise and directly from the context. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_ibo_prompt_3 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_kin.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_kin.yaml new file mode 100644 index 00000000..e92dec41 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_kin.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: 'Given the context, provide the answer to the following question.Ensure + your response is concise and directly from the context. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_kin_prompt_3 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_swa.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_swa.yaml new file mode 100644 index 00000000..30c574e5 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_swa.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: 'Given the context, provide the answer to the following question.Ensure + your response is concise and directly from the context. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +fewshot_split: test +fewshot_config: + sampler: first_n +task: afriqa_swa_prompt_3 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_twi.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_twi.yaml new file mode 100644 index 00000000..b08534d9 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_twi.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: 'Given the context, provide the answer to the following question.Ensure + your response is concise and directly from the context. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_twi_prompt_3 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_yor.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_yor.yaml new file mode 100644 index 00000000..d3c74ce7 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_yor.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: 'Given the context, provide the answer to the following question.Ensure + your response is concise and directly from the context. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_yor_prompt_3 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_zul.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_zul.yaml new file mode 100644 index 00000000..c54b0bd7 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_3/afriqa_zul.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: 'Given the context, provide the answer to the following question.Ensure + your response is concise and directly from the context. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_zul_prompt_3 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_3/utils.py b/lm_eval/tasks/afrobench/afriqa/prompt_3/utils.py new file mode 100644 index 00000000..eae1d885 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_3/utils.py @@ -0,0 +1,53 @@ +import re +import string +from collections import Counter + + +def normalize_answer(s): + """ + Taken from the official evaluation script for v1.1 of the SQuAD dataset. + Lower text and remove punctuation, articles and extra whitespace. + """ + + def remove_articles(text): + return re.sub(r"\b(a|an|the)\b", " ", text) + + def white_space_fix(text): + return " ".join(text.split()) + + def remove_punc(text): + exclude = set(string.punctuation) + return "".join(ch for ch in text if ch not in exclude) + + def lower(text): + return text.lower() + + return white_space_fix(remove_articles(remove_punc(lower(s)))) + + +def f1(items): + """ + Taken from the official evaluation script for v1.1 of the SQuAD dataset. + """ + + unzipped_list = list(zip(*items)) + golds = unzipped_list[0] + preds = unzipped_list[1] + + f1_list = [] + + for i in range(len(golds)): + prediction_tokens = normalize_answer(preds[i]).split() + references_tokens = normalize_answer(golds[i]).split() + common = Counter(prediction_tokens) & Counter(references_tokens) + num_same = sum(common.values()) + if num_same == 0: + f1_score = 0 + else: + precision = 1.0 * num_same / len(prediction_tokens) + recall = 1.0 * num_same / len(references_tokens) + f1_score = (2 * precision * recall) / (precision + recall) + + f1_list.append(f1_score) + + return sum(f1_list) / len(f1_list) diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa b/lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa new file mode 100644 index 00000000..e251f1e2 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa @@ -0,0 +1,42 @@ +tag: + - afrobench_xqa_tasks + - afriqa_prompt_4 +dataset_kwargs: {trust_remote_code: True} +dataset_path: masakhane/afriqa-gold-passages +dataset_name: null +output_type: generate_until +test_split: test +fewshot_split: train +doc_to_target: answer_pivot +should_decontaminate: true +doc_to_decontamination_query: question_lang +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +filter_list: + - name: remove_whitespace + filter: + - function: remove_whitespace + - function: take_first +target_delimiter: " " +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + - "." + - "," + - "\\$" + - metric: f1 + aggregation: !function utils.f1 + higher_is_better: true + ignore_case: true + ignore_punctuation: true + - "." + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa_bem.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa_bem.yaml new file mode 100644 index 00000000..db3d1c2a --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa_bem.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: bem +doc_to_text: 'You are an AI assistant and your task is to answer the question based + on the provided context.Your answer should be the shortest span that contains the + answer within the context. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_bem_prompt_4 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa_fon.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa_fon.yaml new file mode 100644 index 00000000..0c65dd07 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa_fon.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: fon +doc_to_text: 'You are an AI assistant and your task is to answer the question based + on the provided context.Your answer should be the shortest span that contains the + answer within the context. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_fon_prompt_4 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa_hau.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa_hau.yaml new file mode 100644 index 00000000..baeaf020 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa_hau.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: 'You are an AI assistant and your task is to answer the question based + on the provided context.Your answer should be the shortest span that contains the + answer within the context. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_hau_prompt_4 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa_ibo.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa_ibo.yaml new file mode 100644 index 00000000..6db1cc71 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa_ibo.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: 'You are an AI assistant and your task is to answer the question based + on the provided context.Your answer should be the shortest span that contains the + answer within the context. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_ibo_prompt_4 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa_kin.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa_kin.yaml new file mode 100644 index 00000000..dc8f3678 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa_kin.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: 'You are an AI assistant and your task is to answer the question based + on the provided context.Your answer should be the shortest span that contains the + answer within the context. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_kin_prompt_4 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa_swa.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa_swa.yaml new file mode 100644 index 00000000..4fe8fbcd --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa_swa.yaml @@ -0,0 +1,16 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: 'You are an AI assistant and your task is to answer the question based + on the provided context.Your answer should be the shortest span that contains the + answer within the context. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +fewshot_split: test +fewshot_config: + sampler: first_n +task: afriqa_swa_prompt_4 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa_twi.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa_twi.yaml new file mode 100644 index 00000000..d679cd0b --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa_twi.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: 'You are an AI assistant and your task is to answer the question based + on the provided context.Your answer should be the shortest span that contains the + answer within the context. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_twi_prompt_4 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa_yor.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa_yor.yaml new file mode 100644 index 00000000..6011dc33 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa_yor.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: 'You are an AI assistant and your task is to answer the question based + on the provided context.Your answer should be the shortest span that contains the + answer within the context. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_yor_prompt_4 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa_zul.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa_zul.yaml new file mode 100644 index 00000000..26a6ccad --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_4/afriqa_zul.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: 'You are an AI assistant and your task is to answer the question based + on the provided context.Your answer should be the shortest span that contains the + answer within the context. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_zul_prompt_4 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_4/utils.py b/lm_eval/tasks/afrobench/afriqa/prompt_4/utils.py new file mode 100644 index 00000000..eae1d885 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_4/utils.py @@ -0,0 +1,53 @@ +import re +import string +from collections import Counter + + +def normalize_answer(s): + """ + Taken from the official evaluation script for v1.1 of the SQuAD dataset. + Lower text and remove punctuation, articles and extra whitespace. + """ + + def remove_articles(text): + return re.sub(r"\b(a|an|the)\b", " ", text) + + def white_space_fix(text): + return " ".join(text.split()) + + def remove_punc(text): + exclude = set(string.punctuation) + return "".join(ch for ch in text if ch not in exclude) + + def lower(text): + return text.lower() + + return white_space_fix(remove_articles(remove_punc(lower(s)))) + + +def f1(items): + """ + Taken from the official evaluation script for v1.1 of the SQuAD dataset. + """ + + unzipped_list = list(zip(*items)) + golds = unzipped_list[0] + preds = unzipped_list[1] + + f1_list = [] + + for i in range(len(golds)): + prediction_tokens = normalize_answer(preds[i]).split() + references_tokens = normalize_answer(golds[i]).split() + common = Counter(prediction_tokens) & Counter(references_tokens) + num_same = sum(common.values()) + if num_same == 0: + f1_score = 0 + else: + precision = 1.0 * num_same / len(prediction_tokens) + recall = 1.0 * num_same / len(references_tokens) + f1_score = (2 * precision * recall) / (precision + recall) + + f1_list.append(f1_score) + + return sum(f1_list) / len(f1_list) diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa b/lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa new file mode 100644 index 00000000..fab00068 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa @@ -0,0 +1,42 @@ +tag: + - afrobench_xqa_tasks + - afriqa_prompt_5 +dataset_kwargs: {trust_remote_code: True} +dataset_path: masakhane/afriqa-gold-passages +dataset_name: null +output_type: generate_until +test_split: test +fewshot_split: train +doc_to_target: answer_pivot +should_decontaminate: true +doc_to_decontamination_query: question_lang +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +filter_list: + - name: remove_whitespace + filter: + - function: remove_whitespace + - function: take_first +target_delimiter: " " +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + - "." + - "," + - "\\$" + - metric: f1 + aggregation: !function utils.f1 + higher_is_better: true + ignore_case: true + ignore_punctuation: true + - "." + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa_bem.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa_bem.yaml new file mode 100644 index 00000000..4288845d --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa_bem.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: bem +doc_to_text: 'Using the context, find the answer to the question.Respond with the + briefest span that includes the answer from the context. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_bem_prompt_5 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa_fon.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa_fon.yaml new file mode 100644 index 00000000..c234e944 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa_fon.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: fon +doc_to_text: 'Using the context, find the answer to the question.Respond with the + briefest span that includes the answer from the context. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_fon_prompt_5 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa_hau.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa_hau.yaml new file mode 100644 index 00000000..34823c9e --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa_hau.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: 'Using the context, find the answer to the question.Respond with the + briefest span that includes the answer from the context. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_hau_prompt_5 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa_ibo.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa_ibo.yaml new file mode 100644 index 00000000..6582d2d5 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa_ibo.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: 'Using the context, find the answer to the question.Respond with the + briefest span that includes the answer from the context. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_ibo_prompt_5 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa_kin.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa_kin.yaml new file mode 100644 index 00000000..ed9d6517 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa_kin.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: 'Using the context, find the answer to the question.Respond with the + briefest span that includes the answer from the context. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_kin_prompt_5 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa_swa.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa_swa.yaml new file mode 100644 index 00000000..dfcfb147 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa_swa.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: 'Using the context, find the answer to the question.Respond with the + briefest span that includes the answer from the context. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +fewshot_split: test +fewshot_config: + sampler: first_n +task: afriqa_swa_prompt_5 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa_twi.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa_twi.yaml new file mode 100644 index 00000000..cde555cf --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa_twi.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: 'Using the context, find the answer to the question.Respond with the + briefest span that includes the answer from the context. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_twi_prompt_5 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa_yor.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa_yor.yaml new file mode 100644 index 00000000..c9fa17e8 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa_yor.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: 'Using the context, find the answer to the question.Respond with the + briefest span that includes the answer from the context. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_yor_prompt_5 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa_zul.yaml b/lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa_zul.yaml new file mode 100644 index 00000000..427e7217 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_5/afriqa_zul.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: 'Using the context, find the answer to the question.Respond with the + briefest span that includes the answer from the context. + + Question: {{question_lang}} + + Context: {{context}} + + Answer:' +include: afriqa +task: afriqa_zul_prompt_5 diff --git a/lm_eval/tasks/afrobench/afriqa/prompt_5/utils.py b/lm_eval/tasks/afrobench/afriqa/prompt_5/utils.py new file mode 100644 index 00000000..eae1d885 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/prompt_5/utils.py @@ -0,0 +1,53 @@ +import re +import string +from collections import Counter + + +def normalize_answer(s): + """ + Taken from the official evaluation script for v1.1 of the SQuAD dataset. + Lower text and remove punctuation, articles and extra whitespace. + """ + + def remove_articles(text): + return re.sub(r"\b(a|an|the)\b", " ", text) + + def white_space_fix(text): + return " ".join(text.split()) + + def remove_punc(text): + exclude = set(string.punctuation) + return "".join(ch for ch in text if ch not in exclude) + + def lower(text): + return text.lower() + + return white_space_fix(remove_articles(remove_punc(lower(s)))) + + +def f1(items): + """ + Taken from the official evaluation script for v1.1 of the SQuAD dataset. + """ + + unzipped_list = list(zip(*items)) + golds = unzipped_list[0] + preds = unzipped_list[1] + + f1_list = [] + + for i in range(len(golds)): + prediction_tokens = normalize_answer(preds[i]).split() + references_tokens = normalize_answer(golds[i]).split() + common = Counter(prediction_tokens) & Counter(references_tokens) + num_same = sum(common.values()) + if num_same == 0: + f1_score = 0 + else: + precision = 1.0 * num_same / len(prediction_tokens) + recall = 1.0 * num_same / len(references_tokens) + f1_score = (2 * precision * recall) / (precision + recall) + + f1_list.append(f1_score) + + return sum(f1_list) / len(f1_list) diff --git a/lm_eval/tasks/afrobench/afriqa/utils.py b/lm_eval/tasks/afrobench/afriqa/utils.py new file mode 100644 index 00000000..5fef58f0 --- /dev/null +++ b/lm_eval/tasks/afrobench/afriqa/utils.py @@ -0,0 +1,125 @@ +import argparse +import os + +import yaml + + +class FunctionTag: + def __init__(self, value): + self.value = value + + +def prompt_func(mode, lang): + prompt_map = { + "prompt_1": "Your task is to answer a question given a context." + "Make sure you respond with the shortest span containing the answer in the context.\n" + "Question: {{question_lang}}\n" + "Context: {{context}}\n" + "Answer:", + "prompt_2": f"Your task is to answer a question given a context. The question is in {lang}, while the context is in English or French." + "Make sure you respond with the shortest span in the context that contains the answer.\n" + "Question: {{question_lang}}\n" + "Context: {{context}}\n" + "Answer:", + "prompt_3": "Given the context, provide the answer to the following question." + "Ensure your response is concise and directly from the context.\n" + "Question: {{question_lang}}\n" + "Context: {{context}}\n" + "Answer:", + "prompt_4": "You are an AI assistant and your task is to answer the question based on the provided context." + "Your answer should be the shortest span that contains the answer within the context.\n" + "Question: {{question_lang}}\n" + "Context: {{context}}\n" + "Answer:", + "prompt_5": "Using the context, find the answer to the question." + "Respond with the briefest span that includes the answer from the context.\n" + "Question: {{question_lang}}\n" + "Context: {{context}}\n" + "Answer:", + } + return prompt_map[mode] + + +def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None: + """ + Generate a yaml file for each language. + + :param output_dir: The directory to output the files to. + :param overwrite: Whether to overwrite files if they already exist. + """ + err = [] + languages = { + "bem": "Bemba", + "fon": "Fon", + "hau": "Hausa", + "ibo": "Igbo", + "kin": "Kinyarwanda", + "swa": "Swahili", + "twi": "Twi", + "wol": "Wolof", + "yor": "Yoruba", + "zul": "Zulu", + } + + for lang in languages.keys(): + try: + file_name = f"afriqa_{lang}.yaml" + task_name = f"afriqa_{lang}_{mode}" + yaml_template = "afriqa" + yaml_details = { + "include": yaml_template, + "task": task_name, + "dataset_name": lang, + "doc_to_text": prompt_func(mode, languages[lang]), + } + file_path = os.path.join(output_dir, mode) + os.makedirs(file_path, exist_ok=True) + + with open( + f"{output_dir}/{mode}/{file_name}", + "w" if overwrite else "x", + encoding="utf8", + ) as f: + f.write("# Generated by utils.py\n") + yaml.dump( + yaml_details, + f, + allow_unicode=True, + ) + except FileExistsError: + err.append(file_name) + + if len(err) > 0: + raise FileExistsError( + "Files were not created because they already exist (use --overwrite flag):" + f" {', '.join(err)}" + ) + + +def main() -> None: + """Parse CLI args and generate language-specific yaml files.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + default=True, + action="store_true", + help="Overwrite files if they already exist", + ) + parser.add_argument( + "--output-dir", + default="./", + help="Directory to write yaml files to", + ) + parser.add_argument( + "--mode", + default="prompt_1", + choices=["prompt_1", "prompt_2", "prompt_3", "prompt_4", "prompt_5"], + help="Prompt number", + ) + args = parser.parse_args() + + gen_lang_yamls(output_dir=args.output_dir, overwrite=args.overwrite, mode=args.mode) + + +if __name__ == "__main__": + main() diff --git a/lm_eval/tasks/afrobench/afrisenti/README.md b/lm_eval/tasks/afrobench/afrisenti/README.md new file mode 100644 index 00000000..99bd489e --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/README.md @@ -0,0 +1,58 @@ +# + +## Paper +Title: `AfriSenti: A Twitter Sentiment Analysis Benchmark for African Languages` + +Paper Link: https://aclanthology.org/2023.emnlp-main.862/ + +## Abstract +>Africa is home to over 2,000 languages from over six language families and has the highest linguistic diversity among all continents. This includes 75 languages with at least one million speakers each. Yet, there is little NLP research conducted on African languages. Crucial in enabling such research is the availability of high-quality annotated datasets. In this paper, we introduce AfriSenti, a sentiment analysis benchmark that contains a total of >110,000 tweets in 14 African languages (Amharic, Algerian Arabic, Hausa, Igbo, Kinyarwanda, Moroccan Arabic, Mozambican Portuguese, Nigerian Pidgin, Oromo, Swahili, Tigrinya, Twi, Xitsonga, and Yoruba) from four language families. The tweets were annotated by native speakers and used in the AfriSenti-SemEval shared task (with over 200 participants, see website: https://afrisenti-semeval.github.io). We describe the data collection methodology, annotation process, and the challenges we dealt with when curating each dataset. We further report baseline experiments conducted on the AfriSenti datasets and discuss their usefulness. + +HomePage: https://github.com/afrisenti-semeval/afrisent-semeval-2023 + +### Citation + +``` +@inproceedings{muhammad-etal-2023-afrisenti, + title = "{A}fri{S}enti: A {T}witter Sentiment Analysis Benchmark for {A}frican Languages", + author = "Muhammad, Shamsuddeen Hassan and + Abdulmumin, Idris and + Ayele, Abinew Ali and + Ousidhoum, Nedjma and + Adelani, David Ifeoluwa and + Yimam, Seid Muhie and + Ahmad, Ibrahim Sa'id and + Beloucif, Meriem and + Mohammad, Saif M. and + Ruder, Sebastian and + Hourrane, Oumaima and + Brazdil, Pavel and + Jorge, Alipio and + Ali, Felermino D{\'a}rio M{\'a}rio Ant{\'o}nio and + David, Davis and + Osei, Salomey and + Shehu Bello, Bello and + Ibrahim, Falalu and + Gwadabe, Tajuddeen and + Rutunda, Samuel and + Belay, Tadesse and + Messelle, Wendimu Baye and + Balcha, Hailu Beshada and + Chala, Sisay Adugna and + Gebremichael, Hagos Tesfahun and + Opoku, Bernard and + Arthur, Stephen", + editor = "Bouamor, Houda and + Pino, Juan and + Bali, Kalika", + booktitle = "Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing", + month = dec, + year = "2023", + address = "Singapore", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2023.emnlp-main.862/", + doi = "10.18653/v1/2023.emnlp-main.862", + pages = "13968--13981", + abstract = "Africa is home to over 2,000 languages from over six language families and has the highest linguistic diversity among all continents. This includes 75 languages with at least one million speakers each. Yet, there is little NLP research conducted on African languages. Crucial in enabling such research is the availability of high-quality annotated datasets. In this paper, we introduce AfriSenti, a sentiment analysis benchmark that contains a total of {\ensuremath{>}}110,000 tweets in 14 African languages (Amharic, Algerian Arabic, Hausa, Igbo, Kinyarwanda, Moroccan Arabic, Mozambican Portuguese, Nigerian Pidgin, Oromo, Swahili, Tigrinya, Twi, Xitsonga, and Yoruba) from four language families. The tweets were annotated by native speakers and used in the AfriSenti-SemEval shared task (with over 200 participants, see website: https://afrisenti-semeval.github.io). We describe the data collection methodology, annotation process, and the challenges we dealt with when curating each dataset. We further report baseline experiments conducted on the AfriSenti datasets and discuss their usefulness." +} +``` diff --git a/lm_eval/tasks/afrobench/afrisenti/afrisenti.yaml b/lm_eval/tasks/afrobench/afrisenti/afrisenti.yaml new file mode 100644 index 00000000..36a1efdb --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/afrisenti.yaml @@ -0,0 +1,13 @@ +group: afrisenti +task: + - afrisenti_prompt_1 + - afrisenti_prompt_2 + - afrisenti_prompt_3 + - afrisenti_prompt_4 + - afrisenti_prompt_5 +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 1 diff --git a/lm_eval/tasks/afrobench/afrisenti/fewshot.sh b/lm_eval/tasks/afrobench/afrisenti/fewshot.sh new file mode 100644 index 00000000..428d455b --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/fewshot.sh @@ -0,0 +1,109 @@ +lm_eval --model hf \ + --model_args pretrained=masakhane/African-ultrachat-alpaca \ + --tasks afrimmlu_direct_amh,afrimmlu_direct_eng,afrimmlu_direct_ewe,afrimmlu_direct_fra,afrimmlu_direct_hau,afrimmlu_direct_ibo,afrimmlu_direct_kin,afrimmlu_direct_lin,afrimmlu_direct_lug,afrimmlu_direct_orm,afrimmlu_direct_sna,afrimmlu_direct_sot,afrimmlu_direct_twi,afrimmlu_direct_wol,afrimmlu_direct_xho,afrimmlu_direct_yor,afrimmlu_direct_zul \ + --device cuda:0 \ + --batch_size 1 \ + --num_fewshot 0 \ + --verbosity DEBUG \ + --wandb_args project=afrimmlu + + +lm_eval --model hf \ + --model_args pretrained=bigscience/mt0-small,parallelize=true \ + --tasks afrisenti_amh_prompt_1,afrisenti_arq_prompt_1,afrisenti_ary_prompt_1,afrisenti_hau_prompt_1,afrisenti_ibo_prompt_1,afrisenti_kin_prompt_1,afrisenti_orm_prompt_1,afrisenti_pcm_prompt_1,afrisenti_por_prompt_1,afrisenti_swa_prompt_1,afrisenti_tir_prompt_1,afrisenti_tso_prompt_1,afrisenti_twi_prompt_1,afrisenti_yor_prompt_1\ + --device cuda:0 \ + --batch_size 1 \ + --num_fewshot 0 \ + --verbosity DEBUG \ + --limit 5 + + +lm_eval --model hf \ + --model_args pretrained=bigscience/mt0-xxl,parallelize=true \ + --tasks afrisenti_amh_prompt_1,afrisenti_arq_prompt_1,afrisenti_ary_prompt_1,afrisenti_hau_prompt_1,afrisenti_ibo_prompt_1,afrisenti_kin_prompt_1,afrisenti_orm_prompt_1,afrisenti_pcm_prompt_1,afrisenti_por_prompt_1,afrisenti_swa_prompt_1,afrisenti_tir_prompt_1,afrisenti_tso_prompt_1,afrisenti_twi_prompt_1,afrisenti_yor_prompt_1\ + --batch_size 128 \ + --num_fewshot 0 \ + --verbosity DEBUG + +lm_eval --model hf \ + --model_args pretrained=google/gemma-2-27b-it,parallelize=true,trust_remote_code=True \ + --tasks afriqa_wol_prompt_2\ + --batch_size 1 \ + --device 'cuda' \ + --num_fewshot 5 \ + --verbosity DEBUG \ + --output_path './afriqa_results/' \ + --log_samples + +lm_eval --model vllm \ + --model_args pretrained=meta-llama/Llama-2-7b-chat-hf,tensor_parallel_size=2,dtype='auto',gpu_memory_utilization=0.8,data_parallel_size=1 \ + --tasks masakhapos_pcm_prompt_1,masakhapos_pcm_prompt_2,masakhapos_pcm_prompt_3,masakhapos_pcm_prompt_4,masakhapos_pcm_prompt_5 \ + --batch_size 'auto' \ + --device 'cuda' \ + --num_fewshot 0 \ + --verbosity DEBUG \ + --limit 2 + + +lm_eval --model vllm \ + --model_args pretrained=meta-llama/Llama-2-7b-chat-hf,tensor_parallel_size=2,dtype='auto',gpu_memory_utilization=0.8,data_parallel_size=1 \ + --tasks masakhapos_pcm_prompt_1,masakhapos_pcm_prompt_2,masakhapos_pcm_prompt_3,masakhapos_bam_prompt_2,masakhapos_bbj_prompt_3 \ + --batch_size 'auto' \ + --device 'cuda' \ + --num_fewshot 0 \ + --verbosity DEBUG + +lm_eval --model vllm \ + --model_args pretrained=google/gemma-1.1-7b-it,tensor_parallel_size=2,dtype='auto',gpu_memory_utilization=0.8,data_parallel_size=1 \ + --tasks masakhaner_pcm_prompt_1\ + --batch_size 'auto' \ + --device 'cuda' \ + --num_fewshot 0 \ + --verbosity DEBUG \ + --limit 5 + +lm_eval --model vllm \ + --model_args pretrained=google/gemma-2-9b-it,tensor_parallel_size=2,dtype='auto',gpu_memory_utilization=0.8,data_parallel_size=1 \ + --tasks masakhaner_pcm_prompt_1,masakhaner_pcm_prompt_2,masakhaner_pcm_prompt_3,masakhaner_pcm_prompt_4,masakhaner_pcm_prompt_5\ + --batch_size 'auto' \ + --device 'cuda' \ + --num_fewshot 0 \ + --verbosity DEBUG \ + --limit 5 + +lm_eval --model vllm \ + --model_args pretrained=google/gemma-1.1-7b-it,tensor_parallel_size=2,dtype='auto',gpu_memory_utilization=0.8,data_parallel_size=1 \ + --tasks flores_eng_Latn-fuv_Latn_prompt_1,flores_eng_Latn-fuv_Latn_prompt_2,flores_eng_Latn-fuv_Latn_prompt_3,flores_fuv_Latn-eng_Latn_prompt_1,flores_fuv_Latn-eng_Latn_prompt_2,flores_fuv_Latn-eng_Latn_prompt_3 \ + --batch_size 'auto' \ + --device 'cuda' \ + --num_fewshot 0 \ + --verbosity DEBUG \ + --limit 2 + +lm_eval --model vllm \ + --model_args pretrained=google/gemma-2-27b-it,tensor_parallel_size=2,dtype='auto',gpu_memory_utilization=0.9,data_parallel_size=1 \ + --tasks masakhapos_twi_prompt_3,masakhapos_wol_prompt_3,masakhapos_xho_prompt_3,masakhapos_yor_prompt_3,masakhapos_zul_prompt_3\ + --batch_size 'auto' \ + --num_fewshot 5 \ + --verbosity DEBUG \ + --output_path './masakhapos_results/' \ + --log_samples + +lm_eval --model hf \ + --model_args pretrained=bigscience/mt0-small,parallelize=true \ + --tasks injongointent_amh_prompt_1,injongointent_eng_prompt_1,injongointent_yor_prompt_1,injongointent_ibo_prompt_1,injongointent_wol_prompt_1\ + --device 'mps' \ + --batch_size 1 \ + --num_fewshot 0 \ + --verbosity DEBUG \ + --limit 5 + +lm_eval --model hf \ + --model_args pretrained=google/gemma-3-27b-it,parallelize=true \ + --tasks afrobench_sentiment_tasks\ + --device 'cuda' \ + --batch_size 1 \ + --num_fewshot 0 \ + --verbosity DEBUG \ + --output_path './senti_results/' \ + --log_samples diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti b/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti new file mode 100644 index 00000000..69ef6b2b --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti @@ -0,0 +1,41 @@ +tag: + - afrobench_sentiment_tasks + - afrisenti_prompt_1 +task: null +dataset_path: masakhane/afrisenti +dataset_name: null +dataset_kwargs: {trust_remote_code: True} +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: train +doc_to_text: 'Does this statement; "{{tweet}}" have a Neutral, Positive or Negative sentiment? Labels only' +doc_to_target: label +doc_to_choice: + - "negative" + - "positive" + - "neutral" +should_decontaminate: true +doc_to_decontamination_query: tweet +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_amh.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_amh.yaml new file mode 100644 index 00000000..7eefbe86 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: amh +include: afrisenti +task: afrisenti_amh_prompt_1 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_arq.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_arq.yaml new file mode 100644 index 00000000..8b2e2522 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_arq.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: arq +include: afrisenti +task: afrisenti_arq_prompt_1 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_ary.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_ary.yaml new file mode 100644 index 00000000..8f9ef3f2 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_ary.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ary +include: afrisenti +task: afrisenti_ary_prompt_1 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_hau.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_hau.yaml new file mode 100644 index 00000000..f0ab9071 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: hau +include: afrisenti +task: afrisenti_hau_prompt_1 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_ibo.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_ibo.yaml new file mode 100644 index 00000000..b0176d08 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ibo +include: afrisenti +task: afrisenti_ibo_prompt_1 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_kin.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_kin.yaml new file mode 100644 index 00000000..75bb717a --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: kin +include: afrisenti +task: afrisenti_kin_prompt_1 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_orm.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_orm.yaml new file mode 100644 index 00000000..65c63b06 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_orm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: orm +include: afrisenti +task: afrisenti_orm_prompt_1 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_pcm.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_pcm.yaml new file mode 100644 index 00000000..0f24fe9f --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_pcm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: pcm +include: afrisenti +task: afrisenti_pcm_prompt_1 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_por.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_por.yaml new file mode 100644 index 00000000..1e1b4cd6 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_por.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: por +include: afrisenti +task: afrisenti_por_prompt_1 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_swa.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_swa.yaml new file mode 100644 index 00000000..3386948c --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrisenti +task: afrisenti_swa_prompt_1 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_tir.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_tir.yaml new file mode 100644 index 00000000..c4942628 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_tir.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: tir +include: afrisenti +task: afrisenti_tir_prompt_1 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_tso.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_tso.yaml new file mode 100644 index 00000000..d5006932 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_tso.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: tso +include: afrisenti +task: afrisenti_tso_prompt_1 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_twi.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_twi.yaml new file mode 100644 index 00000000..a68bb23d --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: twi +include: afrisenti +task: afrisenti_twi_prompt_1 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_yor.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_yor.yaml new file mode 100644 index 00000000..fda98c2c --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_1/afrisenti_yor.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: yor +include: afrisenti +task: afrisenti_yor_prompt_1 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_1/run.sh b/lm_eval/tasks/afrobench/afrisenti/prompt_1/run.sh new file mode 100644 index 00000000..50d1a133 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_1/run.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +models=( + + "google/gemma-1.1-7b-it" + "CohereForAI/aya-101" + "meta-llama/Llama-2-7b-chat-hf" + "meta-llama/Meta-Llama-3-8B-Instruct" + "google/gemma-2-9b-it" + "bigscience/mt0-xxl" + "google/gemma-2-27b-it" + "meta-llama/Meta-Llama-3-70B-Instruct" +) +task=afrisenti_amh_prompt_1,afrisenti_arq_prompt_1,afrisenti_ary_prompt_1,afrisenti_hau_prompt_1,afrisenti_ibo_prompt_1,afrisenti_kin_prompt_1,afrisenti_pcm_prompt_1,afrisenti_por_prompt_1,afrisenti_swa_prompt_1,afrisenti_tir_prompt_1,afrisenti_tso_prompt_1,afrisenti_twi_prompt_1,afrisenti_yor_prompt_1 + +for model in "${models[@]}" +do + echo "Evaluating model: $model" + for fewshot in 0 5 + do + export OUTPUT_DIR=results/$fewshot + + mkdir -p "$OUTPUT_DIR" + + lm_eval --model hf \ + --model_args "pretrained=${model}" \ + --tasks $task\ + --device cuda:0 \ + --batch_size 16 \ + --output_path "$OUTPUT_DIR" \ + --num_fewshot $fewshot \ + --verbosity DEBUG + done +done diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_1/utils.py b/lm_eval/tasks/afrobench/afrisenti/prompt_1/utils.py new file mode 100644 index 00000000..3e735e2d --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_1/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_1/xx.py b/lm_eval/tasks/afrobench/afrisenti/prompt_1/xx.py new file mode 100644 index 00000000..ca0e325e --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_1/xx.py @@ -0,0 +1,13 @@ +from datasets import load_dataset + + +# ['amh', 'hau', 'ibo', 'arq', 'ary', 'yor', 'por', 'twi', 'tso', 'tir', 'orm', 'pcm', 'kin', 'swa'] + +data = load_dataset("masakhane/afrisenti", "pcm", trust_remote_code=True) +print(data) +print(data["test"][:5]) +# +# ['Naija', 'Pipo', 'wey', 'dey', 'for', 'inside', 'social', 'Media', 'sef', 'don', 'put', 'hand', 'for', 'ear', 'give', +# 'federal', 'goment', 'and', 'polical', 'leader', 'dem', 'ova', 'di', 'kilin', '.'] +# +# [6, 0, 14, 17, 2, 2, 6, 0, 7, 17, 16, 0, 2, 0, 16, 0, 0, 9, 0, 0, 11, 2, 8, 0, 1] diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti b/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti new file mode 100644 index 00000000..879f2826 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti @@ -0,0 +1,39 @@ +tag: + - afrobench_sentiment_tasks + - afrisent_prompt_2 +dataset_path: masakhane/afrisenti +dataset_name: null +dataset_kwargs: {trust_remote_code: True} +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: train +doc_to_target: label +doc_to_choice: + - "negative" + - "positive" + - "neutral" +should_decontaminate: true +doc_to_decontamination_query: 'text: {{tweet}} \nlabel: ' +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_amh.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_amh.yaml new file mode 100644 index 00000000..d97b2c25 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_amh.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: Does this Amharic statement; '{{tweet}}' have a Neutral, Positive or + Negative sentiment? Labels only +include: afrisenti +task: afrisenti_amh_prompt_2 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_arq.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_arq.yaml new file mode 100644 index 00000000..c61e310d --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_arq.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: arq +doc_to_text: Does this Algerian Arabic statement; '{{tweet}}' have a Neutral, Positive + or Negative sentiment? Labels only +include: afrisenti +task: afrisenti_arq_prompt_2 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_ary.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_ary.yaml new file mode 100644 index 00000000..4e76d385 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_ary.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ary +doc_to_text: Does this Moroccan Arabic statement; '{{tweet}}' have a Neutral, Positive + or Negative sentiment? Labels only +include: afrisenti +task: afrisenti_ary_prompt_2 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_hau.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_hau.yaml new file mode 100644 index 00000000..f7b0ccb2 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_hau.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: Does this Hausa statement; '{{tweet}}' have a Neutral, Positive or Negative + sentiment? Labels only +include: afrisenti +task: afrisenti_hau_prompt_2 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_ibo.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_ibo.yaml new file mode 100644 index 00000000..d4d6c6c8 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_ibo.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: Does this Igbo statement; '{{tweet}}' have a Neutral, Positive or Negative + sentiment? Labels only +include: afrisenti +task: afrisenti_ibo_prompt_2 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_kin.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_kin.yaml new file mode 100644 index 00000000..5067b9fb --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_kin.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: Does this Kinyarwanda statement; '{{tweet}}' have a Neutral, Positive + or Negative sentiment? Labels only +include: afrisenti +task: afrisenti_kin_prompt_2 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_orm.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_orm.yaml new file mode 100644 index 00000000..e8abbbfb --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_orm.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: Does this Oromo statement; '{{tweet}}' have a Neutral, Positive or Negative + sentiment? Labels only +include: afrisenti +task: afrisenti_orm_prompt_2 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_pcm.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_pcm.yaml new file mode 100644 index 00000000..4dd98925 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_pcm.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: pcm +doc_to_text: Does this Nigerian Pidgin statement; '{{tweet}}' have a Neutral, Positive + or Negative sentiment? Labels only +include: afrisenti +task: afrisenti_pcm_prompt_2 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_por.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_por.yaml new file mode 100644 index 00000000..4b8beecf --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_por.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: por +doc_to_text: Does this Mozambique Portuguese statement; '{{tweet}}' have a Neutral, + Positive or Negative sentiment? Labels only +include: afrisenti +task: afrisenti_por_prompt_2 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_swa.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_swa.yaml new file mode 100644 index 00000000..496da1a1 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_swa.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: Does this Swahili statement; '{{tweet}}' have a Neutral, Positive or + Negative sentiment? Labels only +include: afrisenti +task: afrisenti_swa_prompt_2 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_tir.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_tir.yaml new file mode 100644 index 00000000..3899c992 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_tir.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: tir +doc_to_text: Does this Tigrinya statement; '{{tweet}}' have a Neutral, Positive or + Negative sentiment? Labels only +include: afrisenti +task: afrisenti_tir_prompt_2 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_tso.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_tso.yaml new file mode 100644 index 00000000..b371b747 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_tso.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: tso +doc_to_text: Does this Xithonga statement; '{{tweet}}' have a Neutral, Positive or + Negative sentiment? Labels only +include: afrisenti +task: afrisenti_tso_prompt_2 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_twi.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_twi.yaml new file mode 100644 index 00000000..c985efc4 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_twi.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: Does this Twi statement; '{{tweet}}' have a Neutral, Positive or Negative + sentiment? Labels only +include: afrisenti +task: afrisenti_twi_prompt_2 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_yor.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_yor.yaml new file mode 100644 index 00000000..78932ed4 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_2/afrisenti_yor.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: Does this Yoruba statement; '{{tweet}}' have a Neutral, Positive or Negative + sentiment? Labels only +include: afrisenti +task: afrisenti_yor_prompt_2 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_2/run.sh b/lm_eval/tasks/afrobench/afrisenti/prompt_2/run.sh new file mode 100755 index 00000000..48797912 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_2/run.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +models=( + + "google/gemma-1.1-7b-it" + "CohereForAI/aya-101" + "meta-llama/Llama-2-7b-chat-hf" + "meta-llama/Meta-Llama-3-8B-Instruct" + "google/gemma-2-9b-it" + "bigscience/mt0-xxl" + "google/gemma-2-27b-it" + "meta-llama/Meta-Llama-3-70B-Instruct" +) + +for model in "${models[@]}" +do + echo "Evaluating model: $model" + for fewshot in 0 5 + do + export OUTPUT_DIR=./results/$fewshot + + mkdir -p "$OUTPUT_DIR" + + lm_eval --model hf \ + --model_args "pretrained=${model},parallelize: true" \ + --tasks afribench\ + --batch_size 256 \ + --output_path "$OUTPUT_DIR" \ + --num_fewshot $fewshot \ + --verbosity DEBUG \ + --limit 2 + done +done diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_2/utils.py b/lm_eval/tasks/afrobench/afrisenti/prompt_2/utils.py new file mode 100644 index 00000000..3e735e2d --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_2/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_2/xx.py b/lm_eval/tasks/afrobench/afrisenti/prompt_2/xx.py new file mode 100644 index 00000000..4aa0db7a --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_2/xx.py @@ -0,0 +1,5 @@ +from datasets import load_dataset + + +data = load_dataset("HausaNLP/AfriSenti-Twitter", "yor", trust_remote_code=True) +print(data) diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti b/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti new file mode 100644 index 00000000..53cb7777 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti @@ -0,0 +1,39 @@ +tag: + - afrobench_sentiment_tasks + - afrisenti_prompt_3 +dataset_path: masakhane/afrisenti +dataset_name: null +dataset_kwargs: {trust_remote_code: True} +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: train +doc_to_target: label +doc_to_choice: + - "negative" + - "positive" + - "neutral" +should_decontaminate: true +doc_to_decontamination_query: 'text: {{tweet}} \nlabel: ' +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_amh.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_amh.yaml new file mode 100644 index 00000000..2645b72b --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_amh.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: "You are an assistant able to detect sentiments in tweets. \n\nGiven\ + \ the sentiment labels Neutral, Positive or Negative; what is the sentiment of the\ + \ Amharic statement below? Return only the labels. \n\ntext: {{tweet}} \nlabel:" +include: afrisenti +task: afrisenti_amh_prompt_3 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_arq.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_arq.yaml new file mode 100644 index 00000000..0b90f690 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_arq.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: arq +doc_to_text: "You are an assistant able to detect sentiments in tweets. \n\nGiven\ + \ the sentiment labels Neutral, Positive or Negative; what is the sentiment of the\ + \ Algerian Arabic statement below? Return only the labels. \n\ntext: {{tweet}} \n\ + label:" +include: afrisenti +task: afrisenti_arq_prompt_3 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_ary.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_ary.yaml new file mode 100644 index 00000000..ba11ee3e --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_ary.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: ary +doc_to_text: "You are an assistant able to detect sentiments in tweets. \n\nGiven\ + \ the sentiment labels Neutral, Positive or Negative; what is the sentiment of the\ + \ Moroccan Arabic statement below? Return only the labels. \n\ntext: {{tweet}} \n\ + label:" +include: afrisenti +task: afrisenti_ary_prompt_3 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_hau.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_hau.yaml new file mode 100644 index 00000000..4f4e6b3f --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_hau.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: "You are an assistant able to detect sentiments in tweets. \n\nGiven\ + \ the sentiment labels Neutral, Positive or Negative; what is the sentiment of the\ + \ Hausa statement below? Return only the labels. \n\ntext: {{tweet}} \nlabel:" +include: afrisenti +task: afrisenti_hau_prompt_3 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_ibo.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_ibo.yaml new file mode 100644 index 00000000..bbcc88d7 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_ibo.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "You are an assistant able to detect sentiments in tweets. \n\nGiven\ + \ the sentiment labels Neutral, Positive or Negative; what is the sentiment of the\ + \ Igbo statement below? Return only the labels. \n\ntext: {{tweet}} \nlabel:" +include: afrisenti +task: afrisenti_ibo_prompt_3 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_kin.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_kin.yaml new file mode 100644 index 00000000..52d84b26 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_kin.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: "You are an assistant able to detect sentiments in tweets. \n\nGiven\ + \ the sentiment labels Neutral, Positive or Negative; what is the sentiment of the\ + \ Kinyarwanda statement below? Return only the labels. \n\ntext: {{tweet}} \nlabel:" +include: afrisenti +task: afrisenti_kin_prompt_3 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_orm.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_orm.yaml new file mode 100644 index 00000000..e2d524bf --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_orm.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: "You are an assistant able to detect sentiments in tweets. \n\nGiven\ + \ the sentiment labels Neutral, Positive or Negative; what is the sentiment of the\ + \ Oromo statement below? Return only the labels. \n\ntext: {{tweet}} \nlabel:" +include: afrisenti +task: afrisenti_orm_prompt_3 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_pcm.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_pcm.yaml new file mode 100644 index 00000000..eb0ac8ff --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_pcm.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: pcm +doc_to_text: "You are an assistant able to detect sentiments in tweets. \n\nGiven\ + \ the sentiment labels Neutral, Positive or Negative; what is the sentiment of the\ + \ Nigerian Pidgin statement below? Return only the labels. \n\ntext: {{tweet}} \n\ + label:" +include: afrisenti +task: afrisenti_pcm_prompt_3 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_por.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_por.yaml new file mode 100644 index 00000000..821a4355 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_por.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: por +doc_to_text: "You are an assistant able to detect sentiments in tweets. \n\nGiven\ + \ the sentiment labels Neutral, Positive or Negative; what is the sentiment of the\ + \ Mozambique Portuguese statement below? Return only the labels. \n\ntext: {{tweet}}\ + \ \nlabel:" +include: afrisenti +task: afrisenti_por_prompt_3 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_swa.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_swa.yaml new file mode 100644 index 00000000..d8e92842 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_swa.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: "You are an assistant able to detect sentiments in tweets. \n\nGiven\ + \ the sentiment labels Neutral, Positive or Negative; what is the sentiment of the\ + \ Swahili statement below? Return only the labels. \n\ntext: {{tweet}} \nlabel:" +include: afrisenti +task: afrisenti_swa_prompt_3 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_tir.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_tir.yaml new file mode 100644 index 00000000..c0f96c24 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_tir.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tir +doc_to_text: "You are an assistant able to detect sentiments in tweets. \n\nGiven\ + \ the sentiment labels Neutral, Positive or Negative; what is the sentiment of the\ + \ Tigrinya statement below? Return only the labels. \n\ntext: {{tweet}} \nlabel:" +include: afrisenti +task: afrisenti_tir_prompt_3 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_tso.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_tso.yaml new file mode 100644 index 00000000..8355035e --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_tso.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tso +doc_to_text: "You are an assistant able to detect sentiments in tweets. \n\nGiven\ + \ the sentiment labels Neutral, Positive or Negative; what is the sentiment of the\ + \ Xithonga statement below? Return only the labels. \n\ntext: {{tweet}} \nlabel:" +include: afrisenti +task: afrisenti_tso_prompt_3 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_twi.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_twi.yaml new file mode 100644 index 00000000..98809176 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_twi.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: "You are an assistant able to detect sentiments in tweets. \n\nGiven\ + \ the sentiment labels Neutral, Positive or Negative; what is the sentiment of the\ + \ Twi statement below? Return only the labels. \n\ntext: {{tweet}} \nlabel:" +include: afrisenti +task: afrisenti_twi_prompt_3 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_yor.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_yor.yaml new file mode 100644 index 00000000..9d1b7ac3 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_3/afrisenti_yor.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: "You are an assistant able to detect sentiments in tweets. \n\nGiven\ + \ the sentiment labels Neutral, Positive or Negative; what is the sentiment of the\ + \ Yoruba statement below? Return only the labels. \n\ntext: {{tweet}} \nlabel:" +include: afrisenti +task: afrisenti_yor_prompt_3 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_3/utils.py b/lm_eval/tasks/afrobench/afrisenti/prompt_3/utils.py new file mode 100644 index 00000000..3e735e2d --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_3/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_3/xx.py b/lm_eval/tasks/afrobench/afrisenti/prompt_3/xx.py new file mode 100644 index 00000000..2133cfa0 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_3/xx.py @@ -0,0 +1,5 @@ +from datasets import load_dataset + + +data = load_dataset("masakhane/afrisenti", "por", trust_remote_code=True) +print(data) diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti b/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti new file mode 100644 index 00000000..6464d7b2 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti @@ -0,0 +1,39 @@ +tag: + - afrobench_sentiment_tasks + - afrisenti_prompt_4 +dataset_path: masakhane/afrisenti +dataset_name: null +dataset_kwargs: {trust_remote_code: True} +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: train +doc_to_target: label +doc_to_choice: + - "negative" + - "positive" + - "neutral" +should_decontaminate: true +doc_to_decontamination_query: 'text: {{tweet}} \nlabel: ' +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_amh.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_amh.yaml new file mode 100644 index 00000000..8a30a72a --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_amh.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: "Label the following text as Neutral, Positive, or Negative. Provide\ + \ only the label as your response. \n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_amh_prompt_4 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_arq.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_arq.yaml new file mode 100644 index 00000000..125771f5 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_arq.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: arq +doc_to_text: "Label the following text as Neutral, Positive, or Negative. Provide\ + \ only the label as your response. \n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_arq_prompt_4 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_ary.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_ary.yaml new file mode 100644 index 00000000..7868fbf3 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_ary.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ary +doc_to_text: "Label the following text as Neutral, Positive, or Negative. Provide\ + \ only the label as your response. \n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_ary_prompt_4 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_hau.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_hau.yaml new file mode 100644 index 00000000..5e7e9a44 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_hau.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: "Label the following text as Neutral, Positive, or Negative. Provide\ + \ only the label as your response. \n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_hau_prompt_4 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_ibo.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_ibo.yaml new file mode 100644 index 00000000..686e16c2 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_ibo.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "Label the following text as Neutral, Positive, or Negative. Provide\ + \ only the label as your response. \n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_ibo_prompt_4 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_kin.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_kin.yaml new file mode 100644 index 00000000..7061ab63 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_kin.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: "Label the following text as Neutral, Positive, or Negative. Provide\ + \ only the label as your response. \n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_kin_prompt_4 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_orm.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_orm.yaml new file mode 100644 index 00000000..8f745ebb --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_orm.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: "Label the following text as Neutral, Positive, or Negative. Provide\ + \ only the label as your response. \n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_orm_prompt_4 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_pcm.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_pcm.yaml new file mode 100644 index 00000000..b5071134 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_pcm.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: pcm +doc_to_text: "Label the following text as Neutral, Positive, or Negative. Provide\ + \ only the label as your response. \n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_pcm_prompt_4 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_por.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_por.yaml new file mode 100644 index 00000000..f5196bcf --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_por.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: por +doc_to_text: "Label the following text as Neutral, Positive, or Negative. Provide\ + \ only the label as your response. \n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_por_prompt_4 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_swa.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_swa.yaml new file mode 100644 index 00000000..97b9e4f1 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_swa.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: "Label the following text as Neutral, Positive, or Negative. Provide\ + \ only the label as your response. \n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_swa_prompt_4 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_tir.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_tir.yaml new file mode 100644 index 00000000..02dfca85 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_tir.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: tir +doc_to_text: "Label the following text as Neutral, Positive, or Negative. Provide\ + \ only the label as your response. \n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_tir_prompt_4 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_tso.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_tso.yaml new file mode 100644 index 00000000..fa83c137 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_tso.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: tso +doc_to_text: "Label the following text as Neutral, Positive, or Negative. Provide\ + \ only the label as your response. \n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_tso_prompt_4 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_twi.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_twi.yaml new file mode 100644 index 00000000..b4366d1f --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_twi.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: "Label the following text as Neutral, Positive, or Negative. Provide\ + \ only the label as your response. \n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_twi_prompt_4 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_yor.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_yor.yaml new file mode 100644 index 00000000..8394706c --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_4/afrisenti_yor.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: "Label the following text as Neutral, Positive, or Negative. Provide\ + \ only the label as your response. \n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_yor_prompt_4 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_4/utils.py b/lm_eval/tasks/afrobench/afrisenti/prompt_4/utils.py new file mode 100644 index 00000000..3e735e2d --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_4/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_4/xx.py b/lm_eval/tasks/afrobench/afrisenti/prompt_4/xx.py new file mode 100644 index 00000000..4515053c --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_4/xx.py @@ -0,0 +1,5 @@ +from datasets import load_dataset + + +data = load_dataset("masakhane/afrisenti", "orm", trust_remote_code=True) +print(data) diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti b/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti new file mode 100644 index 00000000..5107bb80 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti @@ -0,0 +1,39 @@ +tag: + - afrobench_sentiment_tasks + - afrisenti_prompt_5 +dataset_path: masakhane/afrisenti +dataset_name: null +dataset_kwargs: {trust_remote_code: True} +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: train +doc_to_target: label +doc_to_choice: + - "negative" + - "positive" + - "neutral" +should_decontaminate: true +doc_to_decontamination_query: 'Text: {{tweet}} \nlabel:' +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_amh.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_amh.yaml new file mode 100644 index 00000000..866ffbe9 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_amh.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: "You are tasked with performing sentiment classification on the following\ + \ Amharic text. For each input, classify the sentiment as positive, negative, or\ + \ neutral. Use the following guidelines: \n\n Positive: The text expresses happiness,\ + \ satisfaction, or optimism. \nNegative: The text conveys disappointment, dissatisfaction,\ + \ or pessimism. \nNeutral: The text is factual, objective, or without strong emotional\ + \ undertones. \n\nIf the text contains both positive and negative sentiments, choose\ + \ the dominant sentiment. For ambiguous or unclear sentiments, select the label\ + \ that best reflects the overall tone. Please provide a single classification for\ + \ each input.\n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_amh_prompt_5 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_arq.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_arq.yaml new file mode 100644 index 00000000..783785c0 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_arq.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: arq +doc_to_text: "You are tasked with performing sentiment classification on the following\ + \ Algerian Arabic text. For each input, classify the sentiment as positive, negative,\ + \ or neutral. Use the following guidelines: \n\n Positive: The text expresses happiness,\ + \ satisfaction, or optimism. \nNegative: The text conveys disappointment, dissatisfaction,\ + \ or pessimism. \nNeutral: The text is factual, objective, or without strong emotional\ + \ undertones. \n\nIf the text contains both positive and negative sentiments, choose\ + \ the dominant sentiment. For ambiguous or unclear sentiments, select the label\ + \ that best reflects the overall tone. Please provide a single classification for\ + \ each input.\n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_arq_prompt_5 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_ary.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_ary.yaml new file mode 100644 index 00000000..e601dc19 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_ary.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: ary +doc_to_text: "You are tasked with performing sentiment classification on the following\ + \ Moroccan Arabic text. For each input, classify the sentiment as positive, negative,\ + \ or neutral. Use the following guidelines: \n\n Positive: The text expresses happiness,\ + \ satisfaction, or optimism. \nNegative: The text conveys disappointment, dissatisfaction,\ + \ or pessimism. \nNeutral: The text is factual, objective, or without strong emotional\ + \ undertones. \n\nIf the text contains both positive and negative sentiments, choose\ + \ the dominant sentiment. For ambiguous or unclear sentiments, select the label\ + \ that best reflects the overall tone. Please provide a single classification for\ + \ each input.\n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_ary_prompt_5 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_hau.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_hau.yaml new file mode 100644 index 00000000..2ab2adc1 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_hau.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: "You are tasked with performing sentiment classification on the following\ + \ Hausa text. For each input, classify the sentiment as positive, negative, or neutral.\ + \ Use the following guidelines: \n\n Positive: The text expresses happiness, satisfaction,\ + \ or optimism. \nNegative: The text conveys disappointment, dissatisfaction, or\ + \ pessimism. \nNeutral: The text is factual, objective, or without strong emotional\ + \ undertones. \n\nIf the text contains both positive and negative sentiments, choose\ + \ the dominant sentiment. For ambiguous or unclear sentiments, select the label\ + \ that best reflects the overall tone. Please provide a single classification for\ + \ each input.\n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_hau_prompt_5 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_ibo.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_ibo.yaml new file mode 100644 index 00000000..dba7b179 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_ibo.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "You are tasked with performing sentiment classification on the following\ + \ Igbo text. For each input, classify the sentiment as positive, negative, or neutral.\ + \ Use the following guidelines: \n\n Positive: The text expresses happiness, satisfaction,\ + \ or optimism. \nNegative: The text conveys disappointment, dissatisfaction, or\ + \ pessimism. \nNeutral: The text is factual, objective, or without strong emotional\ + \ undertones. \n\nIf the text contains both positive and negative sentiments, choose\ + \ the dominant sentiment. For ambiguous or unclear sentiments, select the label\ + \ that best reflects the overall tone. Please provide a single classification for\ + \ each input.\n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_ibo_prompt_5 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_kin.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_kin.yaml new file mode 100644 index 00000000..16ea6f0c --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_kin.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: "You are tasked with performing sentiment classification on the following\ + \ Kinyarwanda text. For each input, classify the sentiment as positive, negative,\ + \ or neutral. Use the following guidelines: \n\n Positive: The text expresses happiness,\ + \ satisfaction, or optimism. \nNegative: The text conveys disappointment, dissatisfaction,\ + \ or pessimism. \nNeutral: The text is factual, objective, or without strong emotional\ + \ undertones. \n\nIf the text contains both positive and negative sentiments, choose\ + \ the dominant sentiment. For ambiguous or unclear sentiments, select the label\ + \ that best reflects the overall tone. Please provide a single classification for\ + \ each input.\n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_kin_prompt_5 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_orm.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_orm.yaml new file mode 100644 index 00000000..c61ea75e --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_orm.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: "You are tasked with performing sentiment classification on the following\ + \ Oromo text. For each input, classify the sentiment as positive, negative, or neutral.\ + \ Use the following guidelines: \n\n Positive: The text expresses happiness, satisfaction,\ + \ or optimism. \nNegative: The text conveys disappointment, dissatisfaction, or\ + \ pessimism. \nNeutral: The text is factual, objective, or without strong emotional\ + \ undertones. \n\nIf the text contains both positive and negative sentiments, choose\ + \ the dominant sentiment. For ambiguous or unclear sentiments, select the label\ + \ that best reflects the overall tone. Please provide a single classification for\ + \ each input.\n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_orm_prompt_5 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_pcm.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_pcm.yaml new file mode 100644 index 00000000..6de78061 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_pcm.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: pcm +doc_to_text: "You are tasked with performing sentiment classification on the following\ + \ Nigerian Pidgin text. For each input, classify the sentiment as positive, negative,\ + \ or neutral. Use the following guidelines: \n\n Positive: The text expresses happiness,\ + \ satisfaction, or optimism. \nNegative: The text conveys disappointment, dissatisfaction,\ + \ or pessimism. \nNeutral: The text is factual, objective, or without strong emotional\ + \ undertones. \n\nIf the text contains both positive and negative sentiments, choose\ + \ the dominant sentiment. For ambiguous or unclear sentiments, select the label\ + \ that best reflects the overall tone. Please provide a single classification for\ + \ each input.\n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_pcm_prompt_5 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_por.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_por.yaml new file mode 100644 index 00000000..48b728d5 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_por.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: por +doc_to_text: "You are tasked with performing sentiment classification on the following\ + \ Mozambique Portuguese text. For each input, classify the sentiment as positive,\ + \ negative, or neutral. Use the following guidelines: \n\n Positive: The text expresses\ + \ happiness, satisfaction, or optimism. \nNegative: The text conveys disappointment,\ + \ dissatisfaction, or pessimism. \nNeutral: The text is factual, objective, or without\ + \ strong emotional undertones. \n\nIf the text contains both positive and negative\ + \ sentiments, choose the dominant sentiment. For ambiguous or unclear sentiments,\ + \ select the label that best reflects the overall tone. Please provide a single\ + \ classification for each input.\n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_por_prompt_5 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_swa.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_swa.yaml new file mode 100644 index 00000000..fee357ab --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_swa.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: "You are tasked with performing sentiment classification on the following\ + \ Swahili text. For each input, classify the sentiment as positive, negative, or\ + \ neutral. Use the following guidelines: \n\n Positive: The text expresses happiness,\ + \ satisfaction, or optimism. \nNegative: The text conveys disappointment, dissatisfaction,\ + \ or pessimism. \nNeutral: The text is factual, objective, or without strong emotional\ + \ undertones. \n\nIf the text contains both positive and negative sentiments, choose\ + \ the dominant sentiment. For ambiguous or unclear sentiments, select the label\ + \ that best reflects the overall tone. Please provide a single classification for\ + \ each input.\n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_swa_prompt_5 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_tir.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_tir.yaml new file mode 100644 index 00000000..47a67e1c --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_tir.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: tir +doc_to_text: "You are tasked with performing sentiment classification on the following\ + \ Tigrinya text. For each input, classify the sentiment as positive, negative, or\ + \ neutral. Use the following guidelines: \n\n Positive: The text expresses happiness,\ + \ satisfaction, or optimism. \nNegative: The text conveys disappointment, dissatisfaction,\ + \ or pessimism. \nNeutral: The text is factual, objective, or without strong emotional\ + \ undertones. \n\nIf the text contains both positive and negative sentiments, choose\ + \ the dominant sentiment. For ambiguous or unclear sentiments, select the label\ + \ that best reflects the overall tone. Please provide a single classification for\ + \ each input.\n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_tir_prompt_5 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_tso.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_tso.yaml new file mode 100644 index 00000000..5f570528 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_tso.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: tso +doc_to_text: "You are tasked with performing sentiment classification on the following\ + \ Xithonga text. For each input, classify the sentiment as positive, negative, or\ + \ neutral. Use the following guidelines: \n\n Positive: The text expresses happiness,\ + \ satisfaction, or optimism. \nNegative: The text conveys disappointment, dissatisfaction,\ + \ or pessimism. \nNeutral: The text is factual, objective, or without strong emotional\ + \ undertones. \n\nIf the text contains both positive and negative sentiments, choose\ + \ the dominant sentiment. For ambiguous or unclear sentiments, select the label\ + \ that best reflects the overall tone. Please provide a single classification for\ + \ each input.\n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_tso_prompt_5 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_twi.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_twi.yaml new file mode 100644 index 00000000..c0b4fe03 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_twi.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: "You are tasked with performing sentiment classification on the following\ + \ Twi text. For each input, classify the sentiment as positive, negative, or neutral.\ + \ Use the following guidelines: \n\n Positive: The text expresses happiness, satisfaction,\ + \ or optimism. \nNegative: The text conveys disappointment, dissatisfaction, or\ + \ pessimism. \nNeutral: The text is factual, objective, or without strong emotional\ + \ undertones. \n\nIf the text contains both positive and negative sentiments, choose\ + \ the dominant sentiment. For ambiguous or unclear sentiments, select the label\ + \ that best reflects the overall tone. Please provide a single classification for\ + \ each input.\n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_twi_prompt_5 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_yor.yaml b/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_yor.yaml new file mode 100644 index 00000000..b96edb41 --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_5/afrisenti_yor.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: "You are tasked with performing sentiment classification on the following\ + \ Yoruba text. For each input, classify the sentiment as positive, negative, or\ + \ neutral. Use the following guidelines: \n\n Positive: The text expresses happiness,\ + \ satisfaction, or optimism. \nNegative: The text conveys disappointment, dissatisfaction,\ + \ or pessimism. \nNeutral: The text is factual, objective, or without strong emotional\ + \ undertones. \n\nIf the text contains both positive and negative sentiments, choose\ + \ the dominant sentiment. For ambiguous or unclear sentiments, select the label\ + \ that best reflects the overall tone. Please provide a single classification for\ + \ each input.\n\ntext: {{tweet}} \nlabel: " +include: afrisenti +task: afrisenti_yor_prompt_5 diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_5/utils.py b/lm_eval/tasks/afrobench/afrisenti/prompt_5/utils.py new file mode 100644 index 00000000..3e735e2d --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_5/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm_eval/tasks/afrobench/afrisenti/prompt_5/xx.py b/lm_eval/tasks/afrobench/afrisenti/prompt_5/xx.py new file mode 100644 index 00000000..375facff --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/prompt_5/xx.py @@ -0,0 +1,8 @@ +# data = load_dataset('HausaNLP/AfriSenti-Twitter', 'yor', trust_remote_code=True) +# print(data) + +import torch + + +print(torch.cuda.is_available()) # Should return True +print(torch.cuda.device_count()) diff --git a/lm_eval/tasks/afrobench/afrisenti/utils.py b/lm_eval/tasks/afrobench/afrisenti/utils.py new file mode 100644 index 00000000..b5f9b74e --- /dev/null +++ b/lm_eval/tasks/afrobench/afrisenti/utils.py @@ -0,0 +1,124 @@ +import argparse + +import yaml + + +class FunctionTag: + def __init__(self, value): + self.value = value + + +def prompt_func(mode, lang): + prompt_map = { + "prompt_1": "Does this statement; {{tweet}} have a Neutral, Positive or Negative sentiment? Labels only", + "prompt_2": f"Does this {lang} statement; " + "'{{tweet}}' have a Neutral, Positive or Negative sentiment? Labels only", + "prompt_3": f"You are an assistant able to detect sentiments in tweets. \n\n" + f"Given the sentiment labels Neutral, Positive or Negative; what is " + f"the sentiment of the {lang} statement below? Return only the labels. " + "\n\ntext: {{tweet}} \nlabel:", + "prompt_4": "Label the following text as Neutral, Positive, or Negative. Provide only the label as your " + "response. \n\ntext: {{tweet}} \nlabel: ", + "prompt_5": f"You are tasked with performing sentiment classification on the following {lang} text. " + f"For each input, classify the sentiment as positive, negative, or neutral. " + f"Use the following guidelines: \n\n " + f"Positive: The text expresses happiness, satisfaction, or optimism. \n" + f"Negative: The text conveys disappointment, dissatisfaction, or pessimism. \n" + f"Neutral: The text is factual, objective, or without strong emotional undertones. \n\n" + f"If the text contains both positive and negative sentiments, choose the dominant sentiment. " + f"For ambiguous or unclear sentiments, select the label that best reflects the overall tone. " + "Please provide a single classification for each input.\n\ntext: {{tweet}} \nlabel: ", + } + return prompt_map[mode] + + +def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None: + """ + Generate a yaml file for each language. + + :param output_dir: The directory to output the files to. + :param overwrite: Whether to overwrite files if they already exist. + """ + err = [] + languages = { + "amh": "Amharic", + "arq": "Algerian Arabic", + "ary": "Moroccan Arabic", + "hau": "Hausa", + "ibo": "Igbo", + "kin": "Kinyarwanda", + "orm": "Oromo", + "pcm": "Nigerian Pidgin", + "por": "Mozambique Portuguese", + "swa": "Swahili", + "tir": "Tigrinya", + "tso": "Xithonga", + "twi": "Twi", + "yor": "Yoruba", + } + for lang in languages.keys(): + try: + file_name = f"afrisenti_{lang}.yaml" + task_name = f"afrisenti_{lang}_{mode}" + yaml_template = "afrisenti" + if int(mode.split("_")[-1]) > 1: + yaml_details = { + "include": yaml_template, + "task": task_name, + "dataset_name": lang, + "doc_to_text": prompt_func(mode, languages[lang]), + } + else: + yaml_details = { + "include": yaml_template, + "task": task_name, + "dataset_name": lang, + } + with open( + f"{output_dir}/{mode}/{file_name}", + "w" if overwrite else "x", + encoding="utf8", + ) as f: + f.write("# Generated by utils.py\n") + yaml.dump( + yaml_details, + f, + allow_unicode=True, + ) + except FileExistsError: + err.append(file_name) + + if len(err) > 0: + raise FileExistsError( + "Files were not created because they already exist (use --overwrite flag):" + f" {', '.join(err)}" + ) + + +def main() -> None: + """Parse CLI args and generate language-specific yaml files.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + default=True, + action="store_true", + help="Overwrite files if they already exist", + ) + parser.add_argument( + "--output-dir", + default="./", + help="Directory to write yaml files to", + ) + parser.add_argument( + "--mode", + default="prompt_1", + choices=["prompt_1", "prompt_2", "prompt_3", "prompt_4", "prompt_5"], + help="Prompt number", + ) + args = parser.parse_args() + + gen_lang_yamls(output_dir=args.output_dir, overwrite=args.overwrite, mode=args.mode) + + +if __name__ == "__main__": + main() diff --git a/lm_eval/tasks/afrobench/afrobench-lite.yaml b/lm_eval/tasks/afrobench/afrobench-lite.yaml new file mode 100644 index 00000000..a23c050a --- /dev/null +++ b/lm_eval/tasks/afrobench/afrobench-lite.yaml @@ -0,0 +1,15 @@ +group: afrobench_lite +task: + - afrimgsm_cot_tasks + - afrimmlu_tasks + - afrixnli_tasks + - belebele_tasks + - sib_tasks + - african_flores_tasks + - injongointent_tasks +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 1 diff --git a/lm_eval/tasks/afrobench/afrobench.yaml b/lm_eval/tasks/afrobench/afrobench.yaml new file mode 100644 index 00000000..52234bef --- /dev/null +++ b/lm_eval/tasks/afrobench/afrobench.yaml @@ -0,0 +1,23 @@ +group: afrobench +task: +# - adr_tasks +## - afrihate_tasks #dataset not publicly available yet +# - afrimgsm_cot_tasks +# - afrixnli_tasks +# - afrobench_xqa_tasks +# - afrobench_sentiment_tasks + - afrobench_MT_tasks +# - afrobench_TC_tasks +# - afrobench_mmlu_tasks +# - injongointent_tasks +# - masakhaner_tasks +# - masakhapos_tasks +# - RC_tasks +# - uhura_arc_easy_tasks +# - xlsum_tasks +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 1 diff --git a/lm_eval/tasks/afrobench/belebele/README.md b/lm_eval/tasks/afrobench/belebele/README.md new file mode 100644 index 00000000..10d46a44 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/README.md @@ -0,0 +1,41 @@ +# + +## Paper +Title: `The Belebele Benchmark: a Parallel Reading Comprehension Dataset in 122 Language Variants` + +Paper Link: https://aclanthology.org/2023.emnlp-main.862/ + +## Abstract +>Belebele is a multiple-choice machine reading comprehension (MRC) dataset spanning 122 language variants. This dataset enables the evaluation of mono- and multi-lingual models in high-, medium-, and low-resource languages. Each question has four multiple-choice answers and is linked to a short passage from the FLORES-200 dataset. The human annotation procedure was carefully curated to create questions that discriminate between different levels of generalizable language comprehension and is reinforced by extensive quality checks. While all questions directly relate to the passage, the English dataset on its own proves difficult enough to challenge state-of-the-art language models. Being fully parallel, this dataset enables direct comparison of model performance across all languages. Belebele opens up new avenues for evaluating and analyzing the multilingual abilities of language models and NLP systems. + +HomePage: https://github.com/facebookresearch/belebele + +### Citation + +``` +@inproceedings{bandarkar-etal-2024-belebele, + title = "The Belebele Benchmark: a Parallel Reading Comprehension Dataset in 122 Language Variants", + author = "Bandarkar, Lucas and + Liang, Davis and + Muller, Benjamin and + Artetxe, Mikel and + Shukla, Satya Narayan and + Husa, Donald and + Goyal, Naman and + Krishnan, Abhinandan and + Zettlemoyer, Luke and + Khabsa, Madian", + editor = "Ku, Lun-Wei and + Martins, Andre and + Srikumar, Vivek", + booktitle = "Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)", + month = aug, + year = "2024", + address = "Bangkok, Thailand", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2024.acl-long.44/", + doi = "10.18653/v1/2024.acl-long.44", + pages = "749--775", + abstract = "We present Belebele, a multiple-choice machine reading comprehension (MRC) dataset spanning 122 language variants. Significantly expanding the language coverage of natural language understanding (NLU) benchmarks, this dataset enables the evaluation of text models in high-, medium-, and low-resource languages. Each question is based on a short passage from the FLORES-200 dataset and has four multiple-choice answers. The questions were carefully curated to discriminate between models with different levels of general language comprehension. The English dataset on its own proves difficult enough to challenge state-of-the-art language models. Being fully parallel, this dataset enables direct comparison of model performance across all languages. We use this dataset to evaluate the capabilities of multilingual masked language models (MLMs) and large language models (LLMs). We present extensive results and findings, notably that despite significant cross-lingual transfer in English-centric LLMs, much smaller MLMs pretrained on balanced multilingual data still understand far more languages. Overall, Belebele opens up new avenues for evaluating and analyzing the multilingual capabilities of NLP systems." +} +``` diff --git a/lm_eval/tasks/afrobench/belebele/belebele.yaml b/lm_eval/tasks/afrobench/belebele/belebele.yaml new file mode 100644 index 00000000..5c7d3a9d --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/belebele.yaml @@ -0,0 +1,13 @@ +group: belebele +task: + - belebele_prompt_1 + - belebele_prompt_2 + - belebele_prompt_3 + - belebele_prompt_4 + - belebele_prompt_5 +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 2 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_1/belebele b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele new file mode 100644 index 00000000..51553e0e --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele @@ -0,0 +1,23 @@ +tag: + - belebele_tasks + - belebele_prompt_1 + - RC_tasks +dataset_path: facebook/belebele +dataset_name: null +output_type: multiple_choice +test_split: test +fewshot_config: + sampler: first_n +doc_to_target: "{{['1', '2', '3', '4'].index(correct_answer_num)}}" +should_decontaminate: true +doc_to_decontamination_query: "{{question}}" +doc_to_choice: ["A", "B", "C", "D"] +metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_afr.yaml b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_afr.yaml new file mode 100644 index 00000000..e3a7c2b9 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_afr.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: afr_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_afr_prompt_1 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_amh.yaml b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_amh.yaml new file mode 100644 index 00000000..7ee55e8e --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_amh.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: amh_Ethi +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_amh_prompt_1 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_ary.yaml b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_ary.yaml new file mode 100644 index 00000000..82f0d523 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_ary.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: ary_Arab +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_ary_prompt_1 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_arz.yaml b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_arz.yaml new file mode 100644 index 00000000..38f8c3ed --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_arz.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: arz_Arab +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_arz_prompt_1 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_bam.yaml b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_bam.yaml new file mode 100644 index 00000000..f2bc2d49 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_bam.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: bam_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_bam_prompt_1 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_eng.yaml b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_eng.yaml new file mode 100644 index 00000000..ef1f0463 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_eng.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: eng_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_eng_prompt_1 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_fra.yaml b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_fra.yaml new file mode 100644 index 00000000..5f251382 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_fra.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: fra_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_fra_prompt_1 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_fuv.yaml b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_fuv.yaml new file mode 100644 index 00000000..b24422c0 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_fuv.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: fuv_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_fuv_prompt_1 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_gaz.yaml b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_gaz.yaml new file mode 100644 index 00000000..b999f4a8 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_gaz.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: gaz_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_gaz_prompt_1 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_hau.yaml b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_hau.yaml new file mode 100644 index 00000000..933e90b5 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_hau.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: hau_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_hau_prompt_1 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_ibo.yaml b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_ibo.yaml new file mode 100644 index 00000000..fa17935c --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_ibo.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: ibo_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_ibo_prompt_1 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_kea.yaml b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_kea.yaml new file mode 100644 index 00000000..ad535d49 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_kea.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: kea_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_kea_prompt_1 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_kin.yaml b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_kin.yaml new file mode 100644 index 00000000..de957a59 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_kin.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: kin_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_kin_prompt_1 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_lin.yaml b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_lin.yaml new file mode 100644 index 00000000..b3247f06 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_lin.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: lin_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_lin_prompt_1 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_lug.yaml b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_lug.yaml new file mode 100644 index 00000000..8b2ef7a1 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_lug.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: lug_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_lug_prompt_1 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_luo.yaml b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_luo.yaml new file mode 100644 index 00000000..b667c1d6 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_luo.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: luo_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_luo_prompt_1 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_nya.yaml b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_nya.yaml new file mode 100644 index 00000000..c220c773 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_nya.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: nya_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_nya_prompt_1 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_plt.yaml b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_plt.yaml new file mode 100644 index 00000000..d5c286e8 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_plt.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: plt_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_plt_prompt_1 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_por.yaml b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_por.yaml new file mode 100644 index 00000000..ceba2310 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_por.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: por_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_por_prompt_1 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_sna.yaml b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_sna.yaml new file mode 100644 index 00000000..eec0b1e1 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_sna.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: sna_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_sna_prompt_1 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_som.yaml b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_som.yaml new file mode 100644 index 00000000..24af5558 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_som.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: som_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_som_prompt_1 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_sot.yaml b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_sot.yaml new file mode 100644 index 00000000..10cde5be --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_sot.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: sot_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_sot_prompt_1 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_ssw.yaml b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_ssw.yaml new file mode 100644 index 00000000..032b4629 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_ssw.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: ssw_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_ssw_prompt_1 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_swa.yaml b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_swa.yaml new file mode 100644 index 00000000..4c4ae7b7 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_swa.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: swh_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_swa_prompt_1 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_tir.yaml b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_tir.yaml new file mode 100644 index 00000000..9b62e848 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_tir.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: tir_Ethi +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_tir_prompt_1 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_tsn.yaml b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_tsn.yaml new file mode 100644 index 00000000..147a1c98 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_tsn.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: tsn_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_tsn_prompt_1 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_tso.yaml b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_tso.yaml new file mode 100644 index 00000000..869c5015 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_tso.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: tso_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_tso_prompt_1 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_wol.yaml b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_wol.yaml new file mode 100644 index 00000000..1aed1e5a --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_wol.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: wol_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_wol_prompt_1 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_xho.yaml b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_xho.yaml new file mode 100644 index 00000000..549560ac --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_xho.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: xho_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_xho_prompt_1 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_yor.yaml b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_yor.yaml new file mode 100644 index 00000000..70c55eba --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_yor.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: yor_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_yor_prompt_1 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_zul.yaml b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_zul.yaml new file mode 100644 index 00000000..25739692 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_1/belebele_zul.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: zul_Latn +doc_to_text: 'P: {{flores_passage}} + + Q: {{question.strip()}} + + A: {{mc_answer1}} + + B: {{mc_answer2}} + + C: {{mc_answer3}} + + D: {{mc_answer4}} + + Please choose the correct answer from the options above:' +include: belebele +task: belebele_zul_prompt_1 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_2/belebele b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele new file mode 100644 index 00000000..75f673a4 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele @@ -0,0 +1,23 @@ +tag: + - belebele_tasks + - belebele_prompt_2 + - RC_tasks +dataset_path: facebook/belebele +dataset_name: null +output_type: multiple_choice +test_split: test +fewshot_config: + sampler: first_n +doc_to_target: "{{['1', '2', '3', '4'].index(correct_answer_num)}}" +should_decontaminate: true +doc_to_decontamination_query: "{{question}}" +doc_to_choice: ["A", "B", "C", "D"] +metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_afr.yaml b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_afr.yaml new file mode 100644 index 00000000..d7c71809 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_afr.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: afr_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_afr_prompt_2 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_amh.yaml b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_amh.yaml new file mode 100644 index 00000000..f8f95cac --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_amh.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: amh_Ethi +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_amh_prompt_2 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_ary.yaml b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_ary.yaml new file mode 100644 index 00000000..12a78490 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_ary.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: ary_Arab +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_ary_prompt_2 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_arz.yaml b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_arz.yaml new file mode 100644 index 00000000..a975b485 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_arz.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: arz_Arab +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_arz_prompt_2 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_bam.yaml b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_bam.yaml new file mode 100644 index 00000000..814d32b5 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_bam.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: bam_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_bam_prompt_2 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_eng.yaml b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_eng.yaml new file mode 100644 index 00000000..510f1fbb --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_eng.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: eng_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_eng_prompt_2 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_fra.yaml b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_fra.yaml new file mode 100644 index 00000000..15743372 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_fra.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: fra_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_fra_prompt_2 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_fuv.yaml b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_fuv.yaml new file mode 100644 index 00000000..3bf02ff0 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_fuv.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: fuv_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_fuv_prompt_2 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_gaz.yaml b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_gaz.yaml new file mode 100644 index 00000000..bc2b2704 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_gaz.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: gaz_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_gaz_prompt_2 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_hau.yaml b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_hau.yaml new file mode 100644 index 00000000..7af70e03 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_hau.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: hau_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_hau_prompt_2 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_ibo.yaml b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_ibo.yaml new file mode 100644 index 00000000..92d89580 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_ibo.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: ibo_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_ibo_prompt_2 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_kea.yaml b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_kea.yaml new file mode 100644 index 00000000..7f1dcf91 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_kea.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: kea_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_kea_prompt_2 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_kin.yaml b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_kin.yaml new file mode 100644 index 00000000..e686e477 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_kin.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: kin_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_kin_prompt_2 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_lin.yaml b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_lin.yaml new file mode 100644 index 00000000..544eb9dd --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_lin.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: lin_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_lin_prompt_2 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_lug.yaml b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_lug.yaml new file mode 100644 index 00000000..fe97881b --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_lug.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: lug_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_lug_prompt_2 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_luo.yaml b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_luo.yaml new file mode 100644 index 00000000..d7bdde48 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_luo.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: luo_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_luo_prompt_2 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_nya.yaml b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_nya.yaml new file mode 100644 index 00000000..212c0635 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_nya.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: nya_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_nya_prompt_2 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_plt.yaml b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_plt.yaml new file mode 100644 index 00000000..57e71ac9 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_plt.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: plt_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_plt_prompt_2 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_por.yaml b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_por.yaml new file mode 100644 index 00000000..f9be02a8 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_por.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: por_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_por_prompt_2 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_sna.yaml b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_sna.yaml new file mode 100644 index 00000000..8a5ad43a --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_sna.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: sna_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_sna_prompt_2 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_som.yaml b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_som.yaml new file mode 100644 index 00000000..d551d1d5 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_som.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: som_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_som_prompt_2 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_sot.yaml b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_sot.yaml new file mode 100644 index 00000000..18780c90 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_sot.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: sot_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_sot_prompt_2 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_ssw.yaml b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_ssw.yaml new file mode 100644 index 00000000..e6bd0a69 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_ssw.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: ssw_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_ssw_prompt_2 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_swa.yaml b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_swa.yaml new file mode 100644 index 00000000..9bdfd132 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_swa.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: swh_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_swa_prompt_2 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_tir.yaml b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_tir.yaml new file mode 100644 index 00000000..1fba28cb --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_tir.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: tir_Ethi +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_tir_prompt_2 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_tsn.yaml b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_tsn.yaml new file mode 100644 index 00000000..337e08ce --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_tsn.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: tsn_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_tsn_prompt_2 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_tso.yaml b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_tso.yaml new file mode 100644 index 00000000..3a0e24e4 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_tso.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: tso_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_tso_prompt_2 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_wol.yaml b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_wol.yaml new file mode 100644 index 00000000..43914810 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_wol.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: wol_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_wol_prompt_2 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_xho.yaml b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_xho.yaml new file mode 100644 index 00000000..20776148 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_xho.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: xho_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_xho_prompt_2 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_yor.yaml b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_yor.yaml new file mode 100644 index 00000000..f9684f05 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_yor.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: yor_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_yor_prompt_2 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_zul.yaml b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_zul.yaml new file mode 100644 index 00000000..c81180e1 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_2/belebele_zul.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: zul_Latn +doc_to_text: 'Passage: {{flores_passage}} + + Question: {{question.strip()}} + + 1: {{mc_answer1}} + + 2: {{mc_answer2}} + + 3: {{mc_answer3}} + + 4: {{mc_answer4}} + + Please select the correct answer from the given choices:' +include: belebele +task: belebele_zul_prompt_2 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_3/belebele b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele new file mode 100644 index 00000000..a27ea5fb --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele @@ -0,0 +1,23 @@ +tag: + - belebele_tasks + - belebele_prompt_3 + - RC_tasks +dataset_path: facebook/belebele +dataset_name: null +output_type: multiple_choice +test_split: test +fewshot_config: + sampler: first_n +doc_to_target: "{{['1', '2', '3', '4'].index(correct_answer_num)}}" +should_decontaminate: true +doc_to_decontamination_query: "{{question}}" +doc_to_choice: ["A", "B", "C", "D"] +metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_afr.yaml b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_afr.yaml new file mode 100644 index 00000000..8c296cc9 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_afr.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: afr_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_afr_prompt_3 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_amh.yaml b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_amh.yaml new file mode 100644 index 00000000..0e6f2fd7 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_amh.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: amh_Ethi +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_amh_prompt_3 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_ary.yaml b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_ary.yaml new file mode 100644 index 00000000..203bf1c9 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_ary.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: ary_Arab +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_ary_prompt_3 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_arz.yaml b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_arz.yaml new file mode 100644 index 00000000..97f13672 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_arz.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: arz_Arab +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_arz_prompt_3 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_bam.yaml b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_bam.yaml new file mode 100644 index 00000000..9b5d3415 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_bam.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: bam_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_bam_prompt_3 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_eng.yaml b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_eng.yaml new file mode 100644 index 00000000..ceb5270e --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_eng.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: eng_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_eng_prompt_3 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_fra.yaml b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_fra.yaml new file mode 100644 index 00000000..affc5d12 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_fra.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: fra_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_fra_prompt_3 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_fuv.yaml b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_fuv.yaml new file mode 100644 index 00000000..8ff7bfda --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_fuv.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: fuv_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_fuv_prompt_3 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_gaz.yaml b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_gaz.yaml new file mode 100644 index 00000000..c067e7c1 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_gaz.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: gaz_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_gaz_prompt_3 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_hau.yaml b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_hau.yaml new file mode 100644 index 00000000..689724b4 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_hau.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: hau_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_hau_prompt_3 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_ibo.yaml b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_ibo.yaml new file mode 100644 index 00000000..c5eaacad --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_ibo.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: ibo_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_ibo_prompt_3 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_kea.yaml b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_kea.yaml new file mode 100644 index 00000000..c24b2ae7 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_kea.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: kea_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_kea_prompt_3 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_kin.yaml b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_kin.yaml new file mode 100644 index 00000000..ae0a821f --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_kin.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: kin_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_kin_prompt_3 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_lin.yaml b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_lin.yaml new file mode 100644 index 00000000..93e1a5b5 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_lin.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: lin_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_lin_prompt_3 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_lug.yaml b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_lug.yaml new file mode 100644 index 00000000..724947d4 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_lug.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: lug_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_lug_prompt_3 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_luo.yaml b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_luo.yaml new file mode 100644 index 00000000..21b4b8f7 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_luo.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: luo_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_luo_prompt_3 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_nya.yaml b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_nya.yaml new file mode 100644 index 00000000..db045f72 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_nya.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: nya_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_nya_prompt_3 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_plt.yaml b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_plt.yaml new file mode 100644 index 00000000..946e4179 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_plt.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: plt_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_plt_prompt_3 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_por.yaml b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_por.yaml new file mode 100644 index 00000000..72ca651b --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_por.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: por_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_por_prompt_3 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_sna.yaml b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_sna.yaml new file mode 100644 index 00000000..2f5d810a --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_sna.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: sna_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_sna_prompt_3 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_som.yaml b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_som.yaml new file mode 100644 index 00000000..3d3a7c4e --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_som.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: som_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_som_prompt_3 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_sot.yaml b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_sot.yaml new file mode 100644 index 00000000..3db32d81 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_sot.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: sot_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_sot_prompt_3 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_ssw.yaml b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_ssw.yaml new file mode 100644 index 00000000..888ecf84 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_ssw.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: ssw_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_ssw_prompt_3 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_swa.yaml b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_swa.yaml new file mode 100644 index 00000000..ec8127aa --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_swa.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: swh_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_swa_prompt_3 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_tir.yaml b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_tir.yaml new file mode 100644 index 00000000..ab354544 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_tir.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: tir_Ethi +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_tir_prompt_3 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_tsn.yaml b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_tsn.yaml new file mode 100644 index 00000000..019a95fe --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_tsn.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: tsn_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_tsn_prompt_3 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_tso.yaml b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_tso.yaml new file mode 100644 index 00000000..fcc97c4f --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_tso.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: tso_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_tso_prompt_3 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_wol.yaml b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_wol.yaml new file mode 100644 index 00000000..20af7b3c --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_wol.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: wol_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_wol_prompt_3 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_xho.yaml b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_xho.yaml new file mode 100644 index 00000000..a205da90 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_xho.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: xho_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_xho_prompt_3 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_yor.yaml b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_yor.yaml new file mode 100644 index 00000000..cdcbb8c2 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_yor.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: yor_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_yor_prompt_3 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_zul.yaml b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_zul.yaml new file mode 100644 index 00000000..da1ef423 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_3/belebele_zul.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: zul_Latn +doc_to_text: 'Context: {{flores_passage}} + + Query: {{question.strip()}} + + Option A: {{mc_answer1}} + + Option B: {{mc_answer2}} + + Option C: {{mc_answer3}} + + Option D: {{mc_answer4}} + + Please indicate the correct option from the list above:' +include: belebele +task: belebele_zul_prompt_3 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_4/belebele b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele new file mode 100644 index 00000000..cc28101b --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele @@ -0,0 +1,23 @@ +tag: + - belebele_tasks + - belebele_prompt_4 + - RC_tasks +dataset_path: facebook/belebele +dataset_name: null +output_type: multiple_choice +test_split: test +fewshot_config: + sampler: first_n +doc_to_target: "{{['1', '2', '3', '4'].index(correct_answer_num)}}" +should_decontaminate: true +doc_to_decontamination_query: "{{question}}" +doc_to_choice: ["A", "B", "C", "D"] +metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_afr.yaml b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_afr.yaml new file mode 100644 index 00000000..325cb85f --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_afr.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: afr_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_afr_prompt_4 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_amh.yaml b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_amh.yaml new file mode 100644 index 00000000..02eb0683 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_amh.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: amh_Ethi +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_amh_prompt_4 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_ary.yaml b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_ary.yaml new file mode 100644 index 00000000..4c7899d2 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_ary.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: ary_Arab +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_ary_prompt_4 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_arz.yaml b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_arz.yaml new file mode 100644 index 00000000..5acc3222 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_arz.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: arz_Arab +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_arz_prompt_4 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_bam.yaml b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_bam.yaml new file mode 100644 index 00000000..466dddff --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_bam.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: bam_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_bam_prompt_4 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_eng.yaml b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_eng.yaml new file mode 100644 index 00000000..21dfa3ea --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_eng.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: eng_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_eng_prompt_4 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_fra.yaml b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_fra.yaml new file mode 100644 index 00000000..c7fea6f1 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_fra.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: fra_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_fra_prompt_4 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_fuv.yaml b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_fuv.yaml new file mode 100644 index 00000000..77fa7798 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_fuv.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: fuv_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_fuv_prompt_4 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_gaz.yaml b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_gaz.yaml new file mode 100644 index 00000000..a9e54eb9 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_gaz.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: gaz_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_gaz_prompt_4 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_hau.yaml b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_hau.yaml new file mode 100644 index 00000000..45dbfc57 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_hau.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: hau_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_hau_prompt_4 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_ibo.yaml b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_ibo.yaml new file mode 100644 index 00000000..eb58d8a0 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_ibo.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: ibo_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_ibo_prompt_4 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_kea.yaml b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_kea.yaml new file mode 100644 index 00000000..c8ce8300 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_kea.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: kea_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_kea_prompt_4 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_kin.yaml b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_kin.yaml new file mode 100644 index 00000000..028de73a --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_kin.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: kin_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_kin_prompt_4 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_lin.yaml b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_lin.yaml new file mode 100644 index 00000000..95cad4e2 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_lin.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: lin_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_lin_prompt_4 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_lug.yaml b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_lug.yaml new file mode 100644 index 00000000..4e7b6a20 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_lug.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: lug_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_lug_prompt_4 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_luo.yaml b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_luo.yaml new file mode 100644 index 00000000..ce5ec04a --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_luo.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: luo_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_luo_prompt_4 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_nya.yaml b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_nya.yaml new file mode 100644 index 00000000..26d2f699 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_nya.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: nya_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_nya_prompt_4 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_plt.yaml b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_plt.yaml new file mode 100644 index 00000000..ffdf1460 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_plt.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: plt_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_plt_prompt_4 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_por.yaml b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_por.yaml new file mode 100644 index 00000000..e8c06382 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_por.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: por_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_por_prompt_4 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_sna.yaml b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_sna.yaml new file mode 100644 index 00000000..3869a679 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_sna.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: sna_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_sna_prompt_4 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_som.yaml b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_som.yaml new file mode 100644 index 00000000..6d7be50c --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_som.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: som_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_som_prompt_4 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_sot.yaml b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_sot.yaml new file mode 100644 index 00000000..ec30bccc --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_sot.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: sot_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_sot_prompt_4 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_ssw.yaml b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_ssw.yaml new file mode 100644 index 00000000..510e7b8f --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_ssw.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: ssw_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_ssw_prompt_4 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_swa.yaml b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_swa.yaml new file mode 100644 index 00000000..afbdcad2 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_swa.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: swh_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_swa_prompt_4 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_tir.yaml b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_tir.yaml new file mode 100644 index 00000000..827f1f36 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_tir.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: tir_Ethi +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_tir_prompt_4 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_tsn.yaml b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_tsn.yaml new file mode 100644 index 00000000..a8f0a28d --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_tsn.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: tsn_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_tsn_prompt_4 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_tso.yaml b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_tso.yaml new file mode 100644 index 00000000..4f1a87fa --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_tso.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: tso_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_tso_prompt_4 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_wol.yaml b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_wol.yaml new file mode 100644 index 00000000..0e0f6a62 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_wol.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: wol_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_wol_prompt_4 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_xho.yaml b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_xho.yaml new file mode 100644 index 00000000..c3510a4d --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_xho.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: xho_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_xho_prompt_4 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_yor.yaml b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_yor.yaml new file mode 100644 index 00000000..526e24ef --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_yor.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: yor_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_yor_prompt_4 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_zul.yaml b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_zul.yaml new file mode 100644 index 00000000..7472e521 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_4/belebele_zul.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: zul_Latn +doc_to_text: '{{flores_passage}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{mc_answer1}} + + B) {{mc_answer2}} + + C) {{mc_answer3}} + + D) {{mc_answer4}} + + Please provide the correct answer from the choices given:' +include: belebele +task: belebele_zul_prompt_4 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_5/belebele b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele new file mode 100644 index 00000000..0d85bf51 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele @@ -0,0 +1,23 @@ +tag: + - belebele_tasks + - belebele_prompt_5 + - RC_tasks +dataset_path: facebook/belebele +dataset_name: null +output_type: multiple_choice +test_split: test +fewshot_config: + sampler: first_n +doc_to_target: "{{['1', '2', '3', '4'].index(correct_answer_num)}}" +should_decontaminate: true +doc_to_decontamination_query: "{{question}}" +doc_to_choice: ["A", "B", "C", "D"] +metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_afr.yaml b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_afr.yaml new file mode 100644 index 00000000..01a72471 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_afr.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: afr_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_afr_prompt_5 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_amh.yaml b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_amh.yaml new file mode 100644 index 00000000..f707d7c3 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_amh.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: amh_Ethi +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_amh_prompt_5 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_ary.yaml b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_ary.yaml new file mode 100644 index 00000000..2cf68405 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_ary.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: ary_Arab +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_ary_prompt_5 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_arz.yaml b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_arz.yaml new file mode 100644 index 00000000..4c0314a9 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_arz.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: arz_Arab +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_arz_prompt_5 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_bam.yaml b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_bam.yaml new file mode 100644 index 00000000..704c41a5 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_bam.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: bam_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_bam_prompt_5 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_eng.yaml b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_eng.yaml new file mode 100644 index 00000000..62617bf1 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_eng.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: eng_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_eng_prompt_5 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_fra.yaml b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_fra.yaml new file mode 100644 index 00000000..05131046 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_fra.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: fra_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_fra_prompt_5 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_fuv.yaml b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_fuv.yaml new file mode 100644 index 00000000..35103b5c --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_fuv.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: fuv_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_fuv_prompt_5 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_gaz.yaml b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_gaz.yaml new file mode 100644 index 00000000..3822a588 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_gaz.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: gaz_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_gaz_prompt_5 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_hau.yaml b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_hau.yaml new file mode 100644 index 00000000..9a0a5311 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_hau.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: hau_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_hau_prompt_5 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_ibo.yaml b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_ibo.yaml new file mode 100644 index 00000000..f5a8e29b --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_ibo.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: ibo_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_ibo_prompt_5 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_kea.yaml b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_kea.yaml new file mode 100644 index 00000000..45fb47ad --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_kea.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: kea_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_kea_prompt_5 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_kin.yaml b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_kin.yaml new file mode 100644 index 00000000..8bd9a07b --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_kin.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: kin_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_kin_prompt_5 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_lin.yaml b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_lin.yaml new file mode 100644 index 00000000..ff6493b7 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_lin.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: lin_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_lin_prompt_5 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_lug.yaml b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_lug.yaml new file mode 100644 index 00000000..1b64c68b --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_lug.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: lug_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_lug_prompt_5 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_luo.yaml b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_luo.yaml new file mode 100644 index 00000000..f81859aa --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_luo.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: luo_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_luo_prompt_5 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_nya.yaml b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_nya.yaml new file mode 100644 index 00000000..c957760a --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_nya.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: nya_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_nya_prompt_5 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_plt.yaml b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_plt.yaml new file mode 100644 index 00000000..baad68ab --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_plt.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: plt_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_plt_prompt_5 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_por.yaml b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_por.yaml new file mode 100644 index 00000000..13b4e639 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_por.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: por_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_por_prompt_5 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_sna.yaml b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_sna.yaml new file mode 100644 index 00000000..fd4fc080 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_sna.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: sna_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_sna_prompt_5 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_som.yaml b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_som.yaml new file mode 100644 index 00000000..3dfa4066 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_som.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: som_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_som_prompt_5 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_sot.yaml b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_sot.yaml new file mode 100644 index 00000000..c78c862a --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_sot.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: sot_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_sot_prompt_5 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_ssw.yaml b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_ssw.yaml new file mode 100644 index 00000000..d2e8b96f --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_ssw.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: ssw_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_ssw_prompt_5 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_swa.yaml b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_swa.yaml new file mode 100644 index 00000000..5a44af34 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_swa.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: swh_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_swa_prompt_5 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_tir.yaml b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_tir.yaml new file mode 100644 index 00000000..4ef9af2a --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_tir.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: tir_Ethi +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_tir_prompt_5 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_tsn.yaml b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_tsn.yaml new file mode 100644 index 00000000..0de5669b --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_tsn.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: tsn_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_tsn_prompt_5 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_tso.yaml b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_tso.yaml new file mode 100644 index 00000000..92def0f4 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_tso.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: tso_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_tso_prompt_5 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_wol.yaml b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_wol.yaml new file mode 100644 index 00000000..10192b8a --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_wol.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: wol_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_wol_prompt_5 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_xho.yaml b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_xho.yaml new file mode 100644 index 00000000..9ea12584 --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_xho.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: xho_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_xho_prompt_5 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_yor.yaml b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_yor.yaml new file mode 100644 index 00000000..c69e05ce --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_yor.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: yor_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_yor_prompt_5 diff --git a/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_zul.yaml b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_zul.yaml new file mode 100644 index 00000000..c3c6905f --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/prompt_5/belebele_zul.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: zul_Latn +doc_to_text: 'Read the passage: {{flores_passage}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{mc_answer1}} + + B. {{mc_answer2}} + + C. {{mc_answer3}} + + D. {{mc_answer4}} + + Please choose the correct option from the above list:' +include: belebele +task: belebele_zul_prompt_5 diff --git a/lm_eval/tasks/afrobench/belebele/utils.py b/lm_eval/tasks/afrobench/belebele/utils.py new file mode 100644 index 00000000..7654a6cf --- /dev/null +++ b/lm_eval/tasks/afrobench/belebele/utils.py @@ -0,0 +1,155 @@ +import argparse +import os + +import yaml + + +def prompt_func(mode, lang): + prompt_map = { + "prompt_1": "P: {{flores_passage}}\nQ: {{question.strip()}}\nA: {{mc_answer1}}\nB: {{mc_answer2}}\nC: {{mc_answer3}}\nD: {{mc_answer4}}\nPlease choose the correct answer from the options above:", + "prompt_2": "Passage: {{flores_passage}}\nQuestion: {{question.strip()}}\n1: {{mc_answer1}}\n2: {{mc_answer2}}\n3: {{mc_answer3}}\n4: {{mc_answer4}}\nPlease select the correct answer from the given choices:", + "prompt_3": "Context: {{flores_passage}}\nQuery: {{question.strip()}}\nOption A: {{mc_answer1}}\nOption B: {{mc_answer2}}\nOption C: {{mc_answer3}}\nOption D: {{mc_answer4}}\nPlease indicate the correct option from the list above:", + "prompt_4": "{{flores_passage}}\nBased on the above passage, answer the following question:\n{{question.strip()}}\nChoices:\nA) {{mc_answer1}}\nB) {{mc_answer2}}\nC) {{mc_answer3}}\nD) {{mc_answer4}}\nPlease provide the correct answer from the choices given:", + "prompt_5": "Read the passage: {{flores_passage}}\nThen answer the question: {{question.strip()}}\nOptions:\nA. {{mc_answer1}}\nB. {{mc_answer2}}\nC. {{mc_answer3}}\nD. {{mc_answer4}}\nPlease choose the correct option from the above list:", + } + return prompt_map[mode] + + +def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None: + """ + Generate a yaml file for each language. + + :param output_dir: The directory to output the files to. + :param overwrite: Whether to overwrite files if they already exist. + """ + err = [] + languages = { + "afr": "Afrikaans", + "amh": "Amharic", + "ary": "Moroccan Arabic", + "arz": "Egyptian Arabic", + "bam": "Bambara", + "eng": "English", + "fra": "French", + "hau": "Hausa", + "ibo": "Igbo", + "lin": "Lingala", + "por": "Portuguese", + "sna": "Shona", + "swa": "Swahili", + "tir": "Tigrinya", + "tso": "Tsonga", + "tsn": "Tswana", + "wol": "Wolof", + "xho": "Xhosa", + "yor": "Yoruba", + "zul": "Zulu", + "ssw": "Swati", + "sot": "Southern Sotho", + "som": "Somali", + "plt": "Plateau Malagasy", + "nya": "Nyanja", + "luo": "Luo", + "lug": "Luganda", + "kin": "Kinyarwanda", + "kea": "Kabuverdianu", + "gaz": "Oromo", + "fuv": "Nigerian Fulfulde", + } + + lang_2_dataset_lang_code = { + "afr": "afr_Latn", + "amh": "amh_Ethi", + "ary": "ary_Arab", + "arz": "arz_Arab", + "bam": "bam_Latn", + "eng": "eng_Latn", + "fra": "fra_Latn", + "hau": "hau_Latn", + "ibo": "ibo_Latn", + "lin": "lin_Latn", + "por": "por_Latn", + "sna": "sna_Latn", + "swa": "swh_Latn", + "tir": "tir_Ethi", + "tso": "tso_Latn", + "tsn": "tsn_Latn", + "wol": "wol_Latn", + "xho": "xho_Latn", + "yor": "yor_Latn", + "zul": "zul_Latn", + "ssw": "ssw_Latn", + "sot": "sot_Latn", + "som": "som_Latn", + "plt": "plt_Latn", + "nya": "nya_Latn", + "luo": "luo_Latn", + "lug": "lug_Latn", + "kin": "kin_Latn", + "kea": "kea_Latn", + "gaz": "gaz_Latn", + "fuv": "fuv_Latn", + } + + for lang in languages.keys(): + try: + file_name = f"belebele_{lang}.yaml" + task_name = f"belebele_{lang}_{mode}" + yaml_template = "belebele" + yaml_details = { + "include": yaml_template, + "task": task_name, + "dataset_name": lang_2_dataset_lang_code[lang], + "doc_to_text": prompt_func(mode, languages[lang]), + } + file_path = os.path.join(output_dir, mode) + os.makedirs(file_path, exist_ok=True) + + with open( + f"{output_dir}/{mode}/{file_name}", + "w" if overwrite else "x", + encoding="utf8", + ) as f: + f.write("# Generated by utils.py\n") + yaml.dump( + yaml_details, + f, + allow_unicode=True, + ) + except FileExistsError: + err.append(file_name) + + if len(err) > 0: + raise FileExistsError( + "Files were not created because they already exist (use --overwrite flag):" + f" {', '.join(err)}" + ) + + +def main() -> None: + """Parse CLI args and generate language-specific yaml files.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + default=True, + action="store_true", + help="Overwrite files if they already exist", + ) + parser.add_argument( + "--output-dir", + default="./", + help="Directory to write yaml files to", + ) + parser.add_argument( + "--mode", + default="prompt_5", + choices=["prompt_1", "prompt_2", "prompt_3", "prompt_4", "prompt_5"], + help="Prompt number", + ) + args = parser.parse_args() + + gen_lang_yamls(output_dir=args.output_dir, overwrite=args.overwrite, mode=args.mode) + + +if __name__ == "__main__": + main() diff --git a/lm_eval/tasks/afrobench/flores/README.md b/lm_eval/tasks/afrobench/flores/README.md new file mode 100644 index 00000000..ccf433a9 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/README.md @@ -0,0 +1,31 @@ +# + +## Paper +Title: `The FLORES-200 Evaluation Benchmark for Low-Resource and Multilingual Machine Translation` + +Paper Link: https://arxiv.org/abs/2207.04672 + +HomePage: https://huggingface.co/datasets/facebook/flores + +### Citation + +``` +@article{nllb2022, + author = {NLLB Team, Marta R. Costa-jussà, James Cross, Onur Çelebi, Maha Elbayad, Kenneth Heafield, Kevin Heffernan, Elahe Kalbassi, Janice Lam, Daniel Licht, Jean Maillard, Anna Sun, Skyler Wang, Guillaume Wenzek, Al Youngblood, Bapi Akula, Loic Barrault, Gabriel Mejia Gonzalez, Prangthip Hansanti, John Hoffman, Semarley Jarrett, Kaushik Ram Sadagopan, Dirk Rowe, Shannon Spruit, Chau Tran, Pierre Andrews, Necip Fazil Ayan, Shruti Bhosale, Sergey Edunov, Angela Fan, Cynthia Gao, Vedanuj Goswami, Francisco Guzmán, Philipp Koehn, Alexandre Mourachko, Christophe Ropers, Safiyyah Saleem, Holger Schwenk, Jeff Wang}, + title = {No Language Left Behind: Scaling Human-Centered Machine Translation}, + year = {2022} +} + +@inproceedings{, + title={The FLORES-101 Evaluation Benchmark for Low-Resource and Multilingual Machine Translation}, + author={Goyal, Naman and Gao, Cynthia and Chaudhary, Vishrav and Chen, Peng-Jen and Wenzek, Guillaume and Ju, Da and Krishnan, Sanjana and Ranzato, Marc'Aurelio and Guzm\'{a}n, Francisco and Fan, Angela}, + year={2021} +} + +@inproceedings{, + title={Two New Evaluation Datasets for Low-Resource Machine Translation: Nepali-English and Sinhala-English}, + author={Guzm\'{a}n, Francisco and Chen, Peng-Jen and Ott, Myle and Pino, Juan and Lample, Guillaume and Koehn, Philipp and Chaudhary, Vishrav and Ranzato, Marc'Aurelio}, + journal={arXiv preprint arXiv:1902.01382}, + year={2019} +} +``` diff --git a/lm_eval/tasks/afrobench/flores/flores.yaml b/lm_eval/tasks/afrobench/flores/flores.yaml new file mode 100644 index 00000000..09b6e392 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/flores.yaml @@ -0,0 +1,14 @@ +group: african_flores +task: + - flores_eng-afr_prompt_1 + - flores_eng-afr_prompt_2 + - flores_eng-afr_prompt_3 + - flores_afr-eng_prompt_1 + - flores_afr-eng_prompt_2 + - flores_afr-eng_prompt_3 +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 1 diff --git a/lm_eval/tasks/afrobench/flores/gen_utils.py b/lm_eval/tasks/afrobench/flores/gen_utils.py new file mode 100644 index 00000000..37e22e13 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/gen_utils.py @@ -0,0 +1,202 @@ +import argparse +import os + +import yaml + + +class FunctionTag: + def __init__(self, value): + self.value = value + + +def prompt_func(mode, lang, lang_dict): + language_column_name = f"sentence_{lang}" + prompt_map = { + "prompt_1": f"{lang_dict[lang]}: {{{{{language_column_name}}}}} \nEnglish: ", + "prompt_1_reverse": f"English: {{{{sentence_eng_Latn}}}} \n{lang_dict[lang]}: ", + "prompt_2": f"You are a translation expert. Translate the following {lang_dict[lang]} sentences to English \n" + f"{lang_dict[lang]}: {{{{{language_column_name}}}}}\nEnglish: ", + "prompt_2_reverse": f"You are a translation expert. Translate the following English sentences to " + f"{lang_dict[lang]} " + "\nEnglish: {{sentence_eng_Latn}} " + f"\n{lang_dict[lang]}: ", + "prompt_3": f"As a {lang_dict[lang]} and English linguist, translate the following {lang_dict[lang]} sentences " + f"to English \n{lang_dict[lang]}: {{{{{language_column_name}}}}}\nEnglish: ", + "prompt_3_reverse": f"As a {lang_dict[lang]} and English linguist, translate the following English sentences to " + f"{lang_dict[lang]} " + "\nEnglish: {{sentence_eng_Latn}} " + f"\n{lang_dict[lang]}: ", + } + return prompt_map[mode] + + +def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str, reverse: bool) -> None: + """ + Generate a yaml file for each language. + + :param output_dir: The directory to output the files to. + :param overwrite: Whether to overwrite files if they already exist. + """ + err = [] + languages = { + "ace_Latn": "Acehnese (Latin script)", + "ace_Arab": "Acehnese (Arabic script)", + "acq_Arab": "Ta’izzi-Adeni Arabic", + "aeb_Arab": "Tunisian Arabic", + "afr_Latn": "Afrikaans", + "aka_Latn": "Akan", + "amh_Ethi": "Amharic", + "ary_Arab": "Moroccan Arabic", + "arz_Arab": "Egyptian Arabic", + "bam_Latn": "Bambara", + "ban_Latn": "Balinese", + "bem_Latn": "Bemba", + "cjk_Latn": "Chokwe", + "dik_Latn": "Southwestern Dinka", + "dyu_Latn": "Dyula", + "ewe_Latn": "Ewe", + "fon_Latn": "Fon", + "fra_Latn": "French", + "fuv_Latn": "Nigerian Fulfulde", + "hau_Latn": "Hausa", + "ibo_Latn": "Igbo", + "kab_Latn": "Kabyle", + "kam_Latn": "Kamba", + "knc_Arab": "Central Kanuri (Arabic script)", + "knc_Latn": "Central Kanuri (Latin script)", + "kbp_Latn": "Kabiyè", + "kea_Latn": "Kabuverdianu", + "kik_Latn": "Kikuyu", + "kin_Latn": "Kinyarwanda", + "kmb_Latn": "Kimbundu", + "kon_Latn": "Kikongo", + "lin_Latn": "Lingala", + "lua_Latn": "Luba-Kasai", + "lug_Latn": "Luganda", + "luo_Latn": "Luo", + "plt_Latn": "Plateau Malagasy", + "mos_Latn": "Mossi", + "nso_Latn": "Northern Sotho", + "nus_Latn": "Nuer", + "nya_Latn": "Nyanja", + "gaz_Latn": "Oromo", + "run_Latn": "Rundi", + "sag_Latn": "Sango", + "sna_Latn": "Shona", + "som_Latn": "Somali", + "sot_Latn": "Southern Sotho", + "ssw_Latn": "Swati", + "sun_Latn": "Sundanese", + "swh_Latn": "Swahili", + "tir_Ethi": "Tigrinya", + "taq_Latn": "Tamasheq", + "taq_Tfng": "Tamasheq (Tifinagh script)", + "tsn_Latn": "Setswana", + "tso_Latn": "Tsonga", + "tum_Latn": "Tumbuka", + "twi_Latn": "Twi", + "tzm_Tfng": "Central Atlas Tamazight", + "umb_Latn": "Umbundu", + "wol_Latn": "Wolof", + "xho_Latn": "Xhosa", + "yor_Latn": "Yoruba", + "zul_Latn": "Zulu", + } + + for lang in languages.keys(): + try: + if not reverse: + file_name = f"flores_{lang}-eng_Latn.yaml" + task_name = f"flores_{lang}-eng_Latn_{mode}" + yaml_template = "flores" + yaml_details = { + "include": yaml_template, + "task": task_name, + "dataset_name": f"{lang}-eng_Latn", + "doc_to_target": "sentence_eng_Latn", + "doc_to_text": prompt_func(mode, lang, languages), + } + os.makedirs(f"{output_dir}/{mode}/african-english", exist_ok=True) + with open( + f"{output_dir}/{mode}/african-english/{file_name}", + "w" if overwrite else "x", + encoding="utf8", + ) as f: + f.write("# Generated by utils.py\n") + yaml.dump( + yaml_details, + f, + allow_unicode=True, + ) + else: + file_name = f"flores_eng_Latn-{lang}.yaml" + task_name = f"flores_eng_Latn-{lang}_{mode}" + yaml_template = "flores" + # mode_reverse = f"{mode}_reverse" + yaml_details = { + "include": yaml_template, + "task": task_name, + "dataset_name": f"eng_Latn-{lang}", + "doc_to_target": f"sentence_{lang}", + "doc_to_text": prompt_func(f"{mode}_reverse", lang, languages), + } + os.makedirs(f"{output_dir}/{mode}/english-african", exist_ok=True) + with open( + f"{output_dir}/{mode}/english-african/{file_name}", + "w" if overwrite else "x", + encoding="utf8", + ) as f: + f.write("# Generated by utils.py\n") + yaml.dump( + yaml_details, + f, + allow_unicode=True, + ) + except FileExistsError: + err.append(file_name) + + if len(err) > 0: + raise FileExistsError( + "Files were not created because they already exist (use --overwrite flag):" + f" {', '.join(err)}" + ) + + +def main() -> None: + """Parse CLI args and generate language-specific yaml files.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + default=True, + action="store_true", + help="Overwrite files if they already exist", + ) + parser.add_argument( + "--output-dir", + default="./", + help="Directory to write yaml files to", + ) + parser.add_argument( + "--mode", + default="prompt_1", + choices=["prompt_1", "prompt_2", "prompt_3"], + help="Prompt number", + ) + parser.add_argument( + "--reverse", + default=True, + choices=[True, False], + help="Reverse the translation direction", + ) + args = parser.parse_args() + + gen_lang_yamls( + output_dir=args.output_dir, + overwrite=args.overwrite, + mode=args.mode, + reverse=args.reverse, + ) + + +if __name__ == "__main__": + main() diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores new file mode 100644 index 00000000..c25cf195 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores @@ -0,0 +1,27 @@ +tag: +- african_flores_tasks +- flores_afr-eng +- flores_afr-eng_prompt_1 +- afrobench_MT_tasks +dataset_path: facebook/flores +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: dev +fewshot_split: dev +test_split: devtest +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "**" + - + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_ace_Arab-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_ace_Arab-eng_Latn.yaml new file mode 100644 index 00000000..c5582497 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_ace_Arab-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ace_Arab-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Acehnese (Arabic script): {{sentence_ace_Arab}} \nEnglish: " +include: flores +task: flores_ace_Arab-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_ace_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_ace_Latn-eng_Latn.yaml new file mode 100644 index 00000000..1f0a6ee2 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_ace_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ace_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Acehnese (Latin script): {{sentence_ace_Latn}} \nEnglish: " +include: flores +task: flores_ace_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_acq_Arab-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_acq_Arab-eng_Latn.yaml new file mode 100644 index 00000000..3634e7a6 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_acq_Arab-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: acq_Arab-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Ta’izzi-Adeni Arabic: {{sentence_acq_Arab}} \nEnglish: " +include: flores +task: flores_acq_Arab-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_aeb_Arab-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_aeb_Arab-eng_Latn.yaml new file mode 100644 index 00000000..53636d7c --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_aeb_Arab-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: aeb_Arab-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Tunisian Arabic: {{sentence_aeb_Arab}} \nEnglish: " +include: flores +task: flores_aeb_Arab-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_afr_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_afr_Latn-eng_Latn.yaml new file mode 100644 index 00000000..2ac14a0c --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_afr_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: afr_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Afrikaans: {{sentence_afr_Latn}} \nEnglish: " +include: flores +task: flores_afr_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_aka_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_aka_Latn-eng_Latn.yaml new file mode 100644 index 00000000..c3caf192 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_aka_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: aka_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Akan: {{sentence_aka_Latn}} \nEnglish: " +include: flores +task: flores_aka_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_amh_Ethi-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_amh_Ethi-eng_Latn.yaml new file mode 100644 index 00000000..6c0be082 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_amh_Ethi-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: amh_Ethi-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Amharic: {{sentence_amh_Ethi}} \nEnglish: " +include: flores +task: flores_amh_Ethi-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_ary_Arab-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_ary_Arab-eng_Latn.yaml new file mode 100644 index 00000000..8bcd452d --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_ary_Arab-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ary_Arab-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Moroccan Arabic: {{sentence_ary_Arab}} \nEnglish: " +include: flores +task: flores_ary_Arab-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_arz_Arab-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_arz_Arab-eng_Latn.yaml new file mode 100644 index 00000000..72552bab --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_arz_Arab-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: arz_Arab-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Egyptian Arabic: {{sentence_arz_Arab}} \nEnglish: " +include: flores +task: flores_arz_Arab-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_bam_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_bam_Latn-eng_Latn.yaml new file mode 100644 index 00000000..14e8a1c7 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_bam_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: bam_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Bambara: {{sentence_bam_Latn}} \nEnglish: " +include: flores +task: flores_bam_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_ban_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_ban_Latn-eng_Latn.yaml new file mode 100644 index 00000000..54a58244 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_ban_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ban_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Balinese: {{sentence_ban_Latn}} \nEnglish: " +include: flores +task: flores_ban_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_bem_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_bem_Latn-eng_Latn.yaml new file mode 100644 index 00000000..53bbe221 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_bem_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: bem_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Bemba: {{sentence_bem_Latn}} \nEnglish: " +include: flores +task: flores_bem_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_cjk_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_cjk_Latn-eng_Latn.yaml new file mode 100644 index 00000000..63994d04 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_cjk_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: cjk_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Chokwe: {{sentence_cjk_Latn}} \nEnglish: " +include: flores +task: flores_cjk_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_dik_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_dik_Latn-eng_Latn.yaml new file mode 100644 index 00000000..fd9022b5 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_dik_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: dik_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Southwestern Dinka: {{sentence_dik_Latn}} \nEnglish: " +include: flores +task: flores_dik_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_dyu_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_dyu_Latn-eng_Latn.yaml new file mode 100644 index 00000000..e25e23d8 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_dyu_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: dyu_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Dyula: {{sentence_dyu_Latn}} \nEnglish: " +include: flores +task: flores_dyu_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_ewe_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_ewe_Latn-eng_Latn.yaml new file mode 100644 index 00000000..fffa31fc --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_ewe_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ewe_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Ewe: {{sentence_ewe_Latn}} \nEnglish: " +include: flores +task: flores_ewe_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_fon_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_fon_Latn-eng_Latn.yaml new file mode 100644 index 00000000..70c9bfbe --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_fon_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: fon_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Fon: {{sentence_fon_Latn}} \nEnglish: " +include: flores +task: flores_fon_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_fra_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_fra_Latn-eng_Latn.yaml new file mode 100644 index 00000000..2c515a8f --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_fra_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: fra_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "French: {{sentence_fra_Latn}} \nEnglish: " +include: flores +task: flores_fra_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_fuv_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_fuv_Latn-eng_Latn.yaml new file mode 100644 index 00000000..4a162567 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_fuv_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: fuv_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Nigerian Fulfulde: {{sentence_fuv_Latn}} \nEnglish: " +include: flores +task: flores_fuv_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_gaz_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_gaz_Latn-eng_Latn.yaml new file mode 100644 index 00000000..ec443459 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_gaz_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: gaz_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Oromo: {{sentence_gaz_Latn}} \nEnglish: " +include: flores +task: flores_gaz_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_hau_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_hau_Latn-eng_Latn.yaml new file mode 100644 index 00000000..8d518b51 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_hau_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: hau_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Hausa: {{sentence_hau_Latn}} \nEnglish: " +include: flores +task: flores_hau_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_ibo_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_ibo_Latn-eng_Latn.yaml new file mode 100644 index 00000000..9c121ae7 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_ibo_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ibo_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Igbo: {{sentence_ibo_Latn}} \nEnglish: " +include: flores +task: flores_ibo_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_kab_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_kab_Latn-eng_Latn.yaml new file mode 100644 index 00000000..42c62548 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_kab_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: kab_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Kabyle: {{sentence_kab_Latn}} \nEnglish: " +include: flores +task: flores_kab_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_kam_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_kam_Latn-eng_Latn.yaml new file mode 100644 index 00000000..d7d10cc5 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_kam_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: kam_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Kamba: {{sentence_kam_Latn}} \nEnglish: " +include: flores +task: flores_kam_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_kbp_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_kbp_Latn-eng_Latn.yaml new file mode 100644 index 00000000..43cc5e32 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_kbp_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: kbp_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Kabiyè: {{sentence_kbp_Latn}} \nEnglish: " +include: flores +task: flores_kbp_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_kea_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_kea_Latn-eng_Latn.yaml new file mode 100644 index 00000000..4c894681 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_kea_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: kea_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Kabuverdianu: {{sentence_kea_Latn}} \nEnglish: " +include: flores +task: flores_kea_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_kik_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_kik_Latn-eng_Latn.yaml new file mode 100644 index 00000000..dbdff8e2 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_kik_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: kik_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Kikuyu: {{sentence_kik_Latn}} \nEnglish: " +include: flores +task: flores_kik_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_kin_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_kin_Latn-eng_Latn.yaml new file mode 100644 index 00000000..b11194a9 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_kin_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: kin_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Kinyarwanda: {{sentence_kin_Latn}} \nEnglish: " +include: flores +task: flores_kin_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_kmb_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_kmb_Latn-eng_Latn.yaml new file mode 100644 index 00000000..258b847d --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_kmb_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: kmb_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Kimbundu: {{sentence_kmb_Latn}} \nEnglish: " +include: flores +task: flores_kmb_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_knc_Arab-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_knc_Arab-eng_Latn.yaml new file mode 100644 index 00000000..642dfc6f --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_knc_Arab-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: knc_Arab-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Central Kanuri (Arabic script): {{sentence_knc_Arab}} \nEnglish: " +include: flores +task: flores_knc_Arab-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_knc_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_knc_Latn-eng_Latn.yaml new file mode 100644 index 00000000..7f904da7 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_knc_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: knc_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Central Kanuri (Latin script): {{sentence_knc_Latn}} \nEnglish: " +include: flores +task: flores_knc_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_kon_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_kon_Latn-eng_Latn.yaml new file mode 100644 index 00000000..54fce1f8 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_kon_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: kon_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Kikongo: {{sentence_kon_Latn}} \nEnglish: " +include: flores +task: flores_kon_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_lin_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_lin_Latn-eng_Latn.yaml new file mode 100644 index 00000000..41494a72 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_lin_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: lin_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Lingala: {{sentence_lin_Latn}} \nEnglish: " +include: flores +task: flores_lin_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_lua_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_lua_Latn-eng_Latn.yaml new file mode 100644 index 00000000..9d54350a --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_lua_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: lua_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Luba-Kasai: {{sentence_lua_Latn}} \nEnglish: " +include: flores +task: flores_lua_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_lug_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_lug_Latn-eng_Latn.yaml new file mode 100644 index 00000000..35d8e31b --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_lug_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: lug_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Luganda: {{sentence_lug_Latn}} \nEnglish: " +include: flores +task: flores_lug_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_luo_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_luo_Latn-eng_Latn.yaml new file mode 100644 index 00000000..7a22ec7d --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_luo_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: luo_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Luo: {{sentence_luo_Latn}} \nEnglish: " +include: flores +task: flores_luo_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_mos_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_mos_Latn-eng_Latn.yaml new file mode 100644 index 00000000..4a4c1009 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_mos_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: mos_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Mossi: {{sentence_mos_Latn}} \nEnglish: " +include: flores +task: flores_mos_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_nso_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_nso_Latn-eng_Latn.yaml new file mode 100644 index 00000000..2409753c --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_nso_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: nso_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Northern Sotho: {{sentence_nso_Latn}} \nEnglish: " +include: flores +task: flores_nso_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_nus_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_nus_Latn-eng_Latn.yaml new file mode 100644 index 00000000..f7738095 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_nus_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: nus_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Nuer: {{sentence_nus_Latn}} \nEnglish: " +include: flores +task: flores_nus_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_nya_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_nya_Latn-eng_Latn.yaml new file mode 100644 index 00000000..def5625d --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_nya_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: nya_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Nyanja: {{sentence_nya_Latn}} \nEnglish: " +include: flores +task: flores_nya_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_plt_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_plt_Latn-eng_Latn.yaml new file mode 100644 index 00000000..f877a307 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_plt_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: plt_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Plateau Malagasy: {{sentence_plt_Latn}} \nEnglish: " +include: flores +task: flores_plt_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_run_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_run_Latn-eng_Latn.yaml new file mode 100644 index 00000000..e00eb857 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_run_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: run_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Rundi: {{sentence_run_Latn}} \nEnglish: " +include: flores +task: flores_run_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_sag_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_sag_Latn-eng_Latn.yaml new file mode 100644 index 00000000..e7f43c6b --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_sag_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: sag_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Sango: {{sentence_sag_Latn}} \nEnglish: " +include: flores +task: flores_sag_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_sna_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_sna_Latn-eng_Latn.yaml new file mode 100644 index 00000000..d63b4c6b --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_sna_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: sna_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Shona: {{sentence_sna_Latn}} \nEnglish: " +include: flores +task: flores_sna_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_som_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_som_Latn-eng_Latn.yaml new file mode 100644 index 00000000..f625c559 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_som_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: som_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Somali: {{sentence_som_Latn}} \nEnglish: " +include: flores +task: flores_som_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_sot_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_sot_Latn-eng_Latn.yaml new file mode 100644 index 00000000..11653e60 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_sot_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: sot_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Southern Sotho: {{sentence_sot_Latn}} \nEnglish: " +include: flores +task: flores_sot_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_ssw_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_ssw_Latn-eng_Latn.yaml new file mode 100644 index 00000000..fe3ceb9a --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_ssw_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ssw_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Swati: {{sentence_ssw_Latn}} \nEnglish: " +include: flores +task: flores_ssw_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_sun_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_sun_Latn-eng_Latn.yaml new file mode 100644 index 00000000..c3f605f9 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_sun_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: sun_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Sundanese: {{sentence_sun_Latn}} \nEnglish: " +include: flores +task: flores_sun_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_swh_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_swh_Latn-eng_Latn.yaml new file mode 100644 index 00000000..7651ac31 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_swh_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: swh_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Swahili: {{sentence_swh_Latn}} \nEnglish: " +include: flores +task: flores_swh_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_taq_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_taq_Latn-eng_Latn.yaml new file mode 100644 index 00000000..d3fca390 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_taq_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: taq_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Tamasheq: {{sentence_taq_Latn}} \nEnglish: " +include: flores +task: flores_taq_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_taq_Tfng-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_taq_Tfng-eng_Latn.yaml new file mode 100644 index 00000000..7152867e --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_taq_Tfng-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: taq_Tfng-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Tamasheq (Tifinagh script): {{sentence_taq_Tfng}} \nEnglish: " +include: flores +task: flores_taq_Tfng-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_tir_Ethi-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_tir_Ethi-eng_Latn.yaml new file mode 100644 index 00000000..cc13ae04 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_tir_Ethi-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: tir_Ethi-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Tigrinya: {{sentence_tir_Ethi}} \nEnglish: " +include: flores +task: flores_tir_Ethi-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_tsn_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_tsn_Latn-eng_Latn.yaml new file mode 100644 index 00000000..2a6c4e1c --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_tsn_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: tsn_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Setswana: {{sentence_tsn_Latn}} \nEnglish: " +include: flores +task: flores_tsn_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_tso_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_tso_Latn-eng_Latn.yaml new file mode 100644 index 00000000..0d473ab0 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_tso_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: tso_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Tsonga: {{sentence_tso_Latn}} \nEnglish: " +include: flores +task: flores_tso_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_tum_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_tum_Latn-eng_Latn.yaml new file mode 100644 index 00000000..c491f25b --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_tum_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: tum_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Tumbuka: {{sentence_tum_Latn}} \nEnglish: " +include: flores +task: flores_tum_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_twi_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_twi_Latn-eng_Latn.yaml new file mode 100644 index 00000000..2d8ad29e --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_twi_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: twi_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Twi: {{sentence_twi_Latn}} \nEnglish: " +include: flores +task: flores_twi_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_tzm_Tfng-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_tzm_Tfng-eng_Latn.yaml new file mode 100644 index 00000000..ba462465 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_tzm_Tfng-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: tzm_Tfng-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Central Atlas Tamazight: {{sentence_tzm_Tfng}} \nEnglish: " +include: flores +task: flores_tzm_Tfng-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_umb_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_umb_Latn-eng_Latn.yaml new file mode 100644 index 00000000..0758003a --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_umb_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: umb_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Umbundu: {{sentence_umb_Latn}} \nEnglish: " +include: flores +task: flores_umb_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_wol_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_wol_Latn-eng_Latn.yaml new file mode 100644 index 00000000..914e6c12 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_wol_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: wol_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Wolof: {{sentence_wol_Latn}} \nEnglish: " +include: flores +task: flores_wol_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_xho_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_xho_Latn-eng_Latn.yaml new file mode 100644 index 00000000..bc130fb0 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_xho_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: xho_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Xhosa: {{sentence_xho_Latn}} \nEnglish: " +include: flores +task: flores_xho_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_yor_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_yor_Latn-eng_Latn.yaml new file mode 100644 index 00000000..0ea0fbc4 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_yor_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: yor_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Yoruba: {{sentence_yor_Latn}} \nEnglish: " +include: flores +task: flores_yor_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_zul_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_zul_Latn-eng_Latn.yaml new file mode 100644 index 00000000..ea070b30 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/african-english/flores_zul_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: zul_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Zulu: {{sentence_zul_Latn}} \nEnglish: " +include: flores +task: flores_zul_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores new file mode 100644 index 00000000..e6f4d051 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores @@ -0,0 +1,27 @@ +tag: +- african_flores_tasks +- flores_eng-afr +- flores_eng-afr_prompt_1 +- afrobench_MT_tasks +dataset_path: facebook/flores +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: dev +fewshot_split: dev +test_split: devtest +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "**" + - + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-ace_Arab.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-ace_Arab.yaml new file mode 100644 index 00000000..a9da0648 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-ace_Arab.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-ace_Arab +doc_to_target: sentence_ace_Arab +doc_to_text: "English: {{sentence_eng_Latn}} \nAcehnese (Arabic script): " +include: flores +task: flores_eng_Latn-ace_Arab_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-ace_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-ace_Latn.yaml new file mode 100644 index 00000000..d2ed6066 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-ace_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-ace_Latn +doc_to_target: sentence_ace_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nAcehnese (Latin script): " +include: flores +task: flores_eng_Latn-ace_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-acq_Arab.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-acq_Arab.yaml new file mode 100644 index 00000000..e61bb247 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-acq_Arab.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-acq_Arab +doc_to_target: sentence_acq_Arab +doc_to_text: "English: {{sentence_eng_Latn}} \nTa’izzi-Adeni Arabic: " +include: flores +task: flores_eng_Latn-acq_Arab_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-aeb_Arab.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-aeb_Arab.yaml new file mode 100644 index 00000000..d5900062 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-aeb_Arab.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-aeb_Arab +doc_to_target: sentence_aeb_Arab +doc_to_text: "English: {{sentence_eng_Latn}} \nTunisian Arabic: " +include: flores +task: flores_eng_Latn-aeb_Arab_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-afr_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-afr_Latn.yaml new file mode 100644 index 00000000..3b4c4d46 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-afr_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-afr_Latn +doc_to_target: sentence_afr_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nAfrikaans: " +include: flores +task: flores_eng_Latn-afr_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-aka_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-aka_Latn.yaml new file mode 100644 index 00000000..d66a637f --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-aka_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-aka_Latn +doc_to_target: sentence_aka_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nAkan: " +include: flores +task: flores_eng_Latn-aka_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-amh_Ethi.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-amh_Ethi.yaml new file mode 100644 index 00000000..e648d332 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-amh_Ethi.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-amh_Ethi +doc_to_target: sentence_amh_Ethi +doc_to_text: "English: {{sentence_eng_Latn}} \nAmharic: " +include: flores +task: flores_eng_Latn-amh_Ethi_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-ary_Arab.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-ary_Arab.yaml new file mode 100644 index 00000000..54f9a2ad --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-ary_Arab.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-ary_Arab +doc_to_target: sentence_ary_Arab +doc_to_text: "English: {{sentence_eng_Latn}} \nMoroccan Arabic: " +include: flores +task: flores_eng_Latn-ary_Arab_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-arz_Arab.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-arz_Arab.yaml new file mode 100644 index 00000000..a42fa079 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-arz_Arab.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-arz_Arab +doc_to_target: sentence_arz_Arab +doc_to_text: "English: {{sentence_eng_Latn}} \nEgyptian Arabic: " +include: flores +task: flores_eng_Latn-arz_Arab_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-bam_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-bam_Latn.yaml new file mode 100644 index 00000000..4c85b7db --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-bam_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-bam_Latn +doc_to_target: sentence_bam_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nBambara: " +include: flores +task: flores_eng_Latn-bam_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-ban_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-ban_Latn.yaml new file mode 100644 index 00000000..f43a4b71 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-ban_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-ban_Latn +doc_to_target: sentence_ban_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nBalinese: " +include: flores +task: flores_eng_Latn-ban_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-bem_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-bem_Latn.yaml new file mode 100644 index 00000000..252117ef --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-bem_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-bem_Latn +doc_to_target: sentence_bem_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nBemba: " +include: flores +task: flores_eng_Latn-bem_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-cjk_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-cjk_Latn.yaml new file mode 100644 index 00000000..eb4e3566 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-cjk_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-cjk_Latn +doc_to_target: sentence_cjk_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nChokwe: " +include: flores +task: flores_eng_Latn-cjk_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-dik_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-dik_Latn.yaml new file mode 100644 index 00000000..36dea9d3 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-dik_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-dik_Latn +doc_to_target: sentence_dik_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nSouthwestern Dinka: " +include: flores +task: flores_eng_Latn-dik_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-dyu_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-dyu_Latn.yaml new file mode 100644 index 00000000..c32be8ac --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-dyu_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-dyu_Latn +doc_to_target: sentence_dyu_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nDyula: " +include: flores +task: flores_eng_Latn-dyu_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-ewe_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-ewe_Latn.yaml new file mode 100644 index 00000000..0a71b455 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-ewe_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-ewe_Latn +doc_to_target: sentence_ewe_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nEwe: " +include: flores +task: flores_eng_Latn-ewe_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-fon_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-fon_Latn.yaml new file mode 100644 index 00000000..1000e13a --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-fon_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-fon_Latn +doc_to_target: sentence_fon_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nFon: " +include: flores +task: flores_eng_Latn-fon_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-fra_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-fra_Latn.yaml new file mode 100644 index 00000000..47b99a08 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-fra_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-fra_Latn +doc_to_target: sentence_fra_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nFrench: " +include: flores +task: flores_eng_Latn-fra_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-fuv_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-fuv_Latn.yaml new file mode 100644 index 00000000..88553787 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-fuv_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-fuv_Latn +doc_to_target: sentence_fuv_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nNigerian Fulfulde: " +include: flores +task: flores_eng_Latn-fuv_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-gaz_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-gaz_Latn.yaml new file mode 100644 index 00000000..8e124ae1 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-gaz_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-gaz_Latn +doc_to_target: sentence_gaz_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nOromo: " +include: flores +task: flores_eng_Latn-gaz_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-hau_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-hau_Latn.yaml new file mode 100644 index 00000000..a9aaf537 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-hau_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-hau_Latn +doc_to_target: sentence_hau_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nHausa: " +include: flores +task: flores_eng_Latn-hau_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-ibo_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-ibo_Latn.yaml new file mode 100644 index 00000000..ebf8f517 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-ibo_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-ibo_Latn +doc_to_target: sentence_ibo_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nIgbo: " +include: flores +task: flores_eng_Latn-ibo_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kab_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kab_Latn.yaml new file mode 100644 index 00000000..fd22cb7d --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kab_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-kab_Latn +doc_to_target: sentence_kab_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nKabyle: " +include: flores +task: flores_eng_Latn-kab_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kam_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kam_Latn.yaml new file mode 100644 index 00000000..802ae7dc --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kam_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-kam_Latn +doc_to_target: sentence_kam_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nKamba: " +include: flores +task: flores_eng_Latn-kam_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kbp_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kbp_Latn.yaml new file mode 100644 index 00000000..9cc1afd5 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kbp_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-kbp_Latn +doc_to_target: sentence_kbp_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nKabiyè: " +include: flores +task: flores_eng_Latn-kbp_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kea_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kea_Latn.yaml new file mode 100644 index 00000000..55d3e776 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kea_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-kea_Latn +doc_to_target: sentence_kea_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nKabuverdianu: " +include: flores +task: flores_eng_Latn-kea_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kik_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kik_Latn.yaml new file mode 100644 index 00000000..bee435fe --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kik_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-kik_Latn +doc_to_target: sentence_kik_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nKikuyu: " +include: flores +task: flores_eng_Latn-kik_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kin_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kin_Latn.yaml new file mode 100644 index 00000000..28530541 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kin_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-kin_Latn +doc_to_target: sentence_kin_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nKinyarwanda: " +include: flores +task: flores_eng_Latn-kin_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kmb_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kmb_Latn.yaml new file mode 100644 index 00000000..5619209f --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kmb_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-kmb_Latn +doc_to_target: sentence_kmb_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nKimbundu: " +include: flores +task: flores_eng_Latn-kmb_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-knc_Arab.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-knc_Arab.yaml new file mode 100644 index 00000000..fba5e257 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-knc_Arab.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-knc_Arab +doc_to_target: sentence_knc_Arab +doc_to_text: "English: {{sentence_eng_Latn}} \nCentral Kanuri (Arabic script): " +include: flores +task: flores_eng_Latn-knc_Arab_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-knc_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-knc_Latn.yaml new file mode 100644 index 00000000..c1f84d57 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-knc_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-knc_Latn +doc_to_target: sentence_knc_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nCentral Kanuri (Latin script): " +include: flores +task: flores_eng_Latn-knc_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kon_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kon_Latn.yaml new file mode 100644 index 00000000..d6d8ef32 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-kon_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-kon_Latn +doc_to_target: sentence_kon_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nKikongo: " +include: flores +task: flores_eng_Latn-kon_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-lin_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-lin_Latn.yaml new file mode 100644 index 00000000..f998f359 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-lin_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-lin_Latn +doc_to_target: sentence_lin_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nLingala: " +include: flores +task: flores_eng_Latn-lin_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-lua_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-lua_Latn.yaml new file mode 100644 index 00000000..246fc135 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-lua_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-lua_Latn +doc_to_target: sentence_lua_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nLuba-Kasai: " +include: flores +task: flores_eng_Latn-lua_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-lug_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-lug_Latn.yaml new file mode 100644 index 00000000..3416989f --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-lug_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-lug_Latn +doc_to_target: sentence_lug_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nLuganda: " +include: flores +task: flores_eng_Latn-lug_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-luo_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-luo_Latn.yaml new file mode 100644 index 00000000..8a56e148 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-luo_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-luo_Latn +doc_to_target: sentence_luo_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nLuo: " +include: flores +task: flores_eng_Latn-luo_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-mos_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-mos_Latn.yaml new file mode 100644 index 00000000..39386268 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-mos_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-mos_Latn +doc_to_target: sentence_mos_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nMossi: " +include: flores +task: flores_eng_Latn-mos_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-nso_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-nso_Latn.yaml new file mode 100644 index 00000000..86bd9c6b --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-nso_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-nso_Latn +doc_to_target: sentence_nso_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nNorthern Sotho: " +include: flores +task: flores_eng_Latn-nso_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-nus_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-nus_Latn.yaml new file mode 100644 index 00000000..5ac91489 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-nus_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-nus_Latn +doc_to_target: sentence_nus_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nNuer: " +include: flores +task: flores_eng_Latn-nus_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-nya_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-nya_Latn.yaml new file mode 100644 index 00000000..f4e35d78 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-nya_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-nya_Latn +doc_to_target: sentence_nya_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nNyanja: " +include: flores +task: flores_eng_Latn-nya_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-plt_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-plt_Latn.yaml new file mode 100644 index 00000000..e07ffcd2 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-plt_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-plt_Latn +doc_to_target: sentence_plt_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nPlateau Malagasy: " +include: flores +task: flores_eng_Latn-plt_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-run_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-run_Latn.yaml new file mode 100644 index 00000000..cad3666b --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-run_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-run_Latn +doc_to_target: sentence_run_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nRundi: " +include: flores +task: flores_eng_Latn-run_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-sag_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-sag_Latn.yaml new file mode 100644 index 00000000..9eaa3c89 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-sag_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-sag_Latn +doc_to_target: sentence_sag_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nSango: " +include: flores +task: flores_eng_Latn-sag_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-sna_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-sna_Latn.yaml new file mode 100644 index 00000000..16f70ba7 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-sna_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-sna_Latn +doc_to_target: sentence_sna_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nShona: " +include: flores +task: flores_eng_Latn-sna_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-som_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-som_Latn.yaml new file mode 100644 index 00000000..b628b7a4 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-som_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-som_Latn +doc_to_target: sentence_som_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nSomali: " +include: flores +task: flores_eng_Latn-som_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-sot_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-sot_Latn.yaml new file mode 100644 index 00000000..62655dff --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-sot_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-sot_Latn +doc_to_target: sentence_sot_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nSouthern Sotho: " +include: flores +task: flores_eng_Latn-sot_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-ssw_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-ssw_Latn.yaml new file mode 100644 index 00000000..c247e565 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-ssw_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-ssw_Latn +doc_to_target: sentence_ssw_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nSwati: " +include: flores +task: flores_eng_Latn-ssw_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-sun_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-sun_Latn.yaml new file mode 100644 index 00000000..ee3c4a57 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-sun_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-sun_Latn +doc_to_target: sentence_sun_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nSundanese: " +include: flores +task: flores_eng_Latn-sun_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-swh_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-swh_Latn.yaml new file mode 100644 index 00000000..2b464c16 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-swh_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-swh_Latn +doc_to_target: sentence_swh_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nSwahili: " +include: flores +task: flores_eng_Latn-swh_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-taq_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-taq_Latn.yaml new file mode 100644 index 00000000..cc50d54f --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-taq_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-taq_Latn +doc_to_target: sentence_taq_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nTamasheq: " +include: flores +task: flores_eng_Latn-taq_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-taq_Tfng.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-taq_Tfng.yaml new file mode 100644 index 00000000..c8c00453 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-taq_Tfng.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-taq_Tfng +doc_to_target: sentence_taq_Tfng +doc_to_text: "English: {{sentence_eng_Latn}} \nTamasheq (Tifinagh script): " +include: flores +task: flores_eng_Latn-taq_Tfng_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-tir_Ethi.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-tir_Ethi.yaml new file mode 100644 index 00000000..2d311069 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-tir_Ethi.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-tir_Ethi +doc_to_target: sentence_tir_Ethi +doc_to_text: "English: {{sentence_eng_Latn}} \nTigrinya: " +include: flores +task: flores_eng_Latn-tir_Ethi_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-tsn_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-tsn_Latn.yaml new file mode 100644 index 00000000..d782a2af --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-tsn_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-tsn_Latn +doc_to_target: sentence_tsn_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nSetswana: " +include: flores +task: flores_eng_Latn-tsn_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-tso_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-tso_Latn.yaml new file mode 100644 index 00000000..85bca5e9 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-tso_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-tso_Latn +doc_to_target: sentence_tso_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nTsonga: " +include: flores +task: flores_eng_Latn-tso_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-tum_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-tum_Latn.yaml new file mode 100644 index 00000000..a9036f3b --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-tum_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-tum_Latn +doc_to_target: sentence_tum_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nTumbuka: " +include: flores +task: flores_eng_Latn-tum_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-twi_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-twi_Latn.yaml new file mode 100644 index 00000000..96586159 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-twi_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-twi_Latn +doc_to_target: sentence_twi_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nTwi: " +include: flores +task: flores_eng_Latn-twi_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-tzm_Tfng.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-tzm_Tfng.yaml new file mode 100644 index 00000000..28728f41 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-tzm_Tfng.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-tzm_Tfng +doc_to_target: sentence_tzm_Tfng +doc_to_text: "English: {{sentence_eng_Latn}} \nCentral Atlas Tamazight: " +include: flores +task: flores_eng_Latn-tzm_Tfng_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-umb_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-umb_Latn.yaml new file mode 100644 index 00000000..bd95ac31 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-umb_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-umb_Latn +doc_to_target: sentence_umb_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nUmbundu: " +include: flores +task: flores_eng_Latn-umb_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-wol_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-wol_Latn.yaml new file mode 100644 index 00000000..eb696524 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-wol_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-wol_Latn +doc_to_target: sentence_wol_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nWolof: " +include: flores +task: flores_eng_Latn-wol_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-xho_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-xho_Latn.yaml new file mode 100644 index 00000000..08480361 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-xho_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-xho_Latn +doc_to_target: sentence_xho_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nXhosa: " +include: flores +task: flores_eng_Latn-xho_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-yor_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-yor_Latn.yaml new file mode 100644 index 00000000..d29e9a9c --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-yor_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-yor_Latn +doc_to_target: sentence_yor_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nYoruba: " +include: flores +task: flores_eng_Latn-yor_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-zul_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-zul_Latn.yaml new file mode 100644 index 00000000..62de5460 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/english-african/flores_eng_Latn-zul_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng_Latn-zul_Latn +doc_to_target: sentence_zul_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nZulu: " +include: flores +task: flores_eng_Latn-zul_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/flores/prompt_1/flores b/lm_eval/tasks/afrobench/flores/prompt_1/flores new file mode 100644 index 00000000..74f9f33e --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_1/flores @@ -0,0 +1,24 @@ +tag: +- flores_tasks +- flores_afr-eng +dataset_path: facebook/flores +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: dev +fewshot_split: dev +test_split: devtest +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores new file mode 100644 index 00000000..e0fa69a2 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores @@ -0,0 +1,27 @@ +tag: +- african_flores_tasks +- flores_afr-eng +- flores_afr-eng_prompt_2 +- afrobench_MT_tasks +dataset_path: facebook/flores +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: dev +fewshot_split: dev +test_split: devtest +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "**" + - + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ace_Arab-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ace_Arab-eng_Latn.yaml new file mode 100644 index 00000000..dd54b6c8 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ace_Arab-eng_Latn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: ace_Arab-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Acehnese (Arabic\ + \ script) sentences to English \nAcehnese (Arabic script): {{sentence_ace_Arab}}\n\ + English: " +include: flores +task: flores_ace_Arab-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ace_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ace_Latn-eng_Latn.yaml new file mode 100644 index 00000000..b0814b27 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ace_Latn-eng_Latn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: ace_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Acehnese (Latin\ + \ script) sentences to English \nAcehnese (Latin script): {{sentence_ace_Latn}}\n\ + English: " +include: flores +task: flores_ace_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_acq_Arab-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_acq_Arab-eng_Latn.yaml new file mode 100644 index 00000000..d1464b49 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_acq_Arab-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: acq_Arab-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Ta’izzi-Adeni\ + \ Arabic sentences to English \nTa’izzi-Adeni Arabic: {{sentence_acq_Arab}}\nEnglish: " +include: flores +task: flores_acq_Arab-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_aeb_Arab-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_aeb_Arab-eng_Latn.yaml new file mode 100644 index 00000000..2bbded5f --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_aeb_Arab-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: aeb_Arab-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Tunisian Arabic\ + \ sentences to English \nTunisian Arabic: {{sentence_aeb_Arab}}\nEnglish: " +include: flores +task: flores_aeb_Arab-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_afr_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_afr_Latn-eng_Latn.yaml new file mode 100644 index 00000000..7b5847d8 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_afr_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: afr_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Afrikaans sentences\ + \ to English \nAfrikaans: {{sentence_afr_Latn}}\nEnglish: " +include: flores +task: flores_afr_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_aka_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_aka_Latn-eng_Latn.yaml new file mode 100644 index 00000000..5f9493c5 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_aka_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: aka_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Akan sentences\ + \ to English \nAkan: {{sentence_aka_Latn}}\nEnglish: " +include: flores +task: flores_aka_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_amh_Ethi-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_amh_Ethi-eng_Latn.yaml new file mode 100644 index 00000000..d615bfc3 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_amh_Ethi-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: amh_Ethi-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Amharic sentences\ + \ to English \nAmharic: {{sentence_amh_Ethi}}\nEnglish: " +include: flores +task: flores_amh_Ethi-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ary_Arab-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ary_Arab-eng_Latn.yaml new file mode 100644 index 00000000..feecf451 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ary_Arab-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ary_Arab-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Moroccan Arabic\ + \ sentences to English \nMoroccan Arabic: {{sentence_ary_Arab}}\nEnglish: " +include: flores +task: flores_ary_Arab-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_arz_Arab-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_arz_Arab-eng_Latn.yaml new file mode 100644 index 00000000..13f3e18b --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_arz_Arab-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: arz_Arab-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Egyptian Arabic\ + \ sentences to English \nEgyptian Arabic: {{sentence_arz_Arab}}\nEnglish: " +include: flores +task: flores_arz_Arab-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_bam_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_bam_Latn-eng_Latn.yaml new file mode 100644 index 00000000..a258d264 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_bam_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: bam_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Bambara sentences\ + \ to English \nBambara: {{sentence_bam_Latn}}\nEnglish: " +include: flores +task: flores_bam_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ban_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ban_Latn-eng_Latn.yaml new file mode 100644 index 00000000..c19cf008 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ban_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ban_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Balinese sentences\ + \ to English \nBalinese: {{sentence_ban_Latn}}\nEnglish: " +include: flores +task: flores_ban_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_bem_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_bem_Latn-eng_Latn.yaml new file mode 100644 index 00000000..9500a3b3 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_bem_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: bem_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Bemba sentences\ + \ to English \nBemba: {{sentence_bem_Latn}}\nEnglish: " +include: flores +task: flores_bem_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_cjk_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_cjk_Latn-eng_Latn.yaml new file mode 100644 index 00000000..58185199 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_cjk_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: cjk_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Chokwe sentences\ + \ to English \nChokwe: {{sentence_cjk_Latn}}\nEnglish: " +include: flores +task: flores_cjk_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_dik_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_dik_Latn-eng_Latn.yaml new file mode 100644 index 00000000..4c9090a5 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_dik_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: dik_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Southwestern Dinka\ + \ sentences to English \nSouthwestern Dinka: {{sentence_dik_Latn}}\nEnglish: " +include: flores +task: flores_dik_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_dyu_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_dyu_Latn-eng_Latn.yaml new file mode 100644 index 00000000..47187fb0 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_dyu_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: dyu_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Dyula sentences\ + \ to English \nDyula: {{sentence_dyu_Latn}}\nEnglish: " +include: flores +task: flores_dyu_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ewe_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ewe_Latn-eng_Latn.yaml new file mode 100644 index 00000000..8838bc3a --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ewe_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ewe_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Ewe sentences\ + \ to English \nEwe: {{sentence_ewe_Latn}}\nEnglish: " +include: flores +task: flores_ewe_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_fon_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_fon_Latn-eng_Latn.yaml new file mode 100644 index 00000000..7874a87c --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_fon_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fon_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Fon sentences\ + \ to English \nFon: {{sentence_fon_Latn}}\nEnglish: " +include: flores +task: flores_fon_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_fra_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_fra_Latn-eng_Latn.yaml new file mode 100644 index 00000000..bb84246e --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_fra_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fra_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following French sentences\ + \ to English \nFrench: {{sentence_fra_Latn}}\nEnglish: " +include: flores +task: flores_fra_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_fuv_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_fuv_Latn-eng_Latn.yaml new file mode 100644 index 00000000..0686706d --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_fuv_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fuv_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Nigerian Fulfulde\ + \ sentences to English \nNigerian Fulfulde: {{sentence_fuv_Latn}}\nEnglish: " +include: flores +task: flores_fuv_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_gaz_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_gaz_Latn-eng_Latn.yaml new file mode 100644 index 00000000..f0ba07a6 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_gaz_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: gaz_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Oromo sentences\ + \ to English \nOromo: {{sentence_gaz_Latn}}\nEnglish: " +include: flores +task: flores_gaz_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_hau_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_hau_Latn-eng_Latn.yaml new file mode 100644 index 00000000..85647455 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_hau_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: hau_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Hausa sentences\ + \ to English \nHausa: {{sentence_hau_Latn}}\nEnglish: " +include: flores +task: flores_hau_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ibo_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ibo_Latn-eng_Latn.yaml new file mode 100644 index 00000000..c401f1e7 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ibo_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ibo_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Igbo sentences\ + \ to English \nIgbo: {{sentence_ibo_Latn}}\nEnglish: " +include: flores +task: flores_ibo_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kab_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kab_Latn-eng_Latn.yaml new file mode 100644 index 00000000..c82946b9 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kab_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kab_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Kabyle sentences\ + \ to English \nKabyle: {{sentence_kab_Latn}}\nEnglish: " +include: flores +task: flores_kab_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kam_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kam_Latn-eng_Latn.yaml new file mode 100644 index 00000000..8661bf6e --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kam_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kam_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Kamba sentences\ + \ to English \nKamba: {{sentence_kam_Latn}}\nEnglish: " +include: flores +task: flores_kam_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kbp_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kbp_Latn-eng_Latn.yaml new file mode 100644 index 00000000..7e20af31 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kbp_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kbp_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Kabiyè sentences\ + \ to English \nKabiyè: {{sentence_kbp_Latn}}\nEnglish: " +include: flores +task: flores_kbp_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kea_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kea_Latn-eng_Latn.yaml new file mode 100644 index 00000000..d078c293 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kea_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kea_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Kabuverdianu sentences\ + \ to English \nKabuverdianu: {{sentence_kea_Latn}}\nEnglish: " +include: flores +task: flores_kea_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kik_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kik_Latn-eng_Latn.yaml new file mode 100644 index 00000000..346dcb98 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kik_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kik_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Kikuyu sentences\ + \ to English \nKikuyu: {{sentence_kik_Latn}}\nEnglish: " +include: flores +task: flores_kik_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kin_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kin_Latn-eng_Latn.yaml new file mode 100644 index 00000000..7210e7e6 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kin_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kin_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Kinyarwanda sentences\ + \ to English \nKinyarwanda: {{sentence_kin_Latn}}\nEnglish: " +include: flores +task: flores_kin_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kmb_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kmb_Latn-eng_Latn.yaml new file mode 100644 index 00000000..b3dc8d5a --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kmb_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kmb_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Kimbundu sentences\ + \ to English \nKimbundu: {{sentence_kmb_Latn}}\nEnglish: " +include: flores +task: flores_kmb_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_knc_Arab-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_knc_Arab-eng_Latn.yaml new file mode 100644 index 00000000..37d5d624 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_knc_Arab-eng_Latn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: knc_Arab-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Central Kanuri\ + \ (Arabic script) sentences to English \nCentral Kanuri (Arabic script): {{sentence_knc_Arab}}\n\ + English: " +include: flores +task: flores_knc_Arab-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_knc_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_knc_Latn-eng_Latn.yaml new file mode 100644 index 00000000..1d60408c --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_knc_Latn-eng_Latn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: knc_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Central Kanuri\ + \ (Latin script) sentences to English \nCentral Kanuri (Latin script): {{sentence_knc_Latn}}\n\ + English: " +include: flores +task: flores_knc_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kon_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kon_Latn-eng_Latn.yaml new file mode 100644 index 00000000..63b3539c --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_kon_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kon_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Kikongo sentences\ + \ to English \nKikongo: {{sentence_kon_Latn}}\nEnglish: " +include: flores +task: flores_kon_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_lin_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_lin_Latn-eng_Latn.yaml new file mode 100644 index 00000000..82543c70 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_lin_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lin_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Lingala sentences\ + \ to English \nLingala: {{sentence_lin_Latn}}\nEnglish: " +include: flores +task: flores_lin_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_lua_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_lua_Latn-eng_Latn.yaml new file mode 100644 index 00000000..af0796cc --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_lua_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lua_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Luba-Kasai sentences\ + \ to English \nLuba-Kasai: {{sentence_lua_Latn}}\nEnglish: " +include: flores +task: flores_lua_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_lug_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_lug_Latn-eng_Latn.yaml new file mode 100644 index 00000000..eb9f47bc --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_lug_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lug_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Luganda sentences\ + \ to English \nLuganda: {{sentence_lug_Latn}}\nEnglish: " +include: flores +task: flores_lug_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_luo_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_luo_Latn-eng_Latn.yaml new file mode 100644 index 00000000..6000ab87 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_luo_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: luo_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Luo sentences\ + \ to English \nLuo: {{sentence_luo_Latn}}\nEnglish: " +include: flores +task: flores_luo_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_mos_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_mos_Latn-eng_Latn.yaml new file mode 100644 index 00000000..b72acf36 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_mos_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: mos_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Mossi sentences\ + \ to English \nMossi: {{sentence_mos_Latn}}\nEnglish: " +include: flores +task: flores_mos_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_nso_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_nso_Latn-eng_Latn.yaml new file mode 100644 index 00000000..028aa75c --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_nso_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nso_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Northern Sotho\ + \ sentences to English \nNorthern Sotho: {{sentence_nso_Latn}}\nEnglish: " +include: flores +task: flores_nso_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_nus_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_nus_Latn-eng_Latn.yaml new file mode 100644 index 00000000..e1f9ca54 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_nus_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nus_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Nuer sentences\ + \ to English \nNuer: {{sentence_nus_Latn}}\nEnglish: " +include: flores +task: flores_nus_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_nya_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_nya_Latn-eng_Latn.yaml new file mode 100644 index 00000000..a5ceb017 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_nya_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nya_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Nyanja sentences\ + \ to English \nNyanja: {{sentence_nya_Latn}}\nEnglish: " +include: flores +task: flores_nya_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_plt_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_plt_Latn-eng_Latn.yaml new file mode 100644 index 00000000..a2cdace5 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_plt_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: plt_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Plateau Malagasy\ + \ sentences to English \nPlateau Malagasy: {{sentence_plt_Latn}}\nEnglish: " +include: flores +task: flores_plt_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_run_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_run_Latn-eng_Latn.yaml new file mode 100644 index 00000000..aa4b5bc9 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_run_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: run_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Rundi sentences\ + \ to English \nRundi: {{sentence_run_Latn}}\nEnglish: " +include: flores +task: flores_run_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_sag_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_sag_Latn-eng_Latn.yaml new file mode 100644 index 00000000..b20eef56 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_sag_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sag_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Sango sentences\ + \ to English \nSango: {{sentence_sag_Latn}}\nEnglish: " +include: flores +task: flores_sag_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_sna_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_sna_Latn-eng_Latn.yaml new file mode 100644 index 00000000..f0c98f03 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_sna_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sna_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Shona sentences\ + \ to English \nShona: {{sentence_sna_Latn}}\nEnglish: " +include: flores +task: flores_sna_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_som_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_som_Latn-eng_Latn.yaml new file mode 100644 index 00000000..b862c759 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_som_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: som_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Somali sentences\ + \ to English \nSomali: {{sentence_som_Latn}}\nEnglish: " +include: flores +task: flores_som_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_sot_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_sot_Latn-eng_Latn.yaml new file mode 100644 index 00000000..d5d4e247 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_sot_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sot_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Southern Sotho\ + \ sentences to English \nSouthern Sotho: {{sentence_sot_Latn}}\nEnglish: " +include: flores +task: flores_sot_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ssw_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ssw_Latn-eng_Latn.yaml new file mode 100644 index 00000000..5ae236e5 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_ssw_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ssw_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Swati sentences\ + \ to English \nSwati: {{sentence_ssw_Latn}}\nEnglish: " +include: flores +task: flores_ssw_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_sun_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_sun_Latn-eng_Latn.yaml new file mode 100644 index 00000000..5a697a21 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_sun_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sun_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Sundanese sentences\ + \ to English \nSundanese: {{sentence_sun_Latn}}\nEnglish: " +include: flores +task: flores_sun_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_swh_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_swh_Latn-eng_Latn.yaml new file mode 100644 index 00000000..06dd9fcc --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_swh_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: swh_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Swahili sentences\ + \ to English \nSwahili: {{sentence_swh_Latn}}\nEnglish: " +include: flores +task: flores_swh_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_taq_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_taq_Latn-eng_Latn.yaml new file mode 100644 index 00000000..5380298e --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_taq_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: taq_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Tamasheq sentences\ + \ to English \nTamasheq: {{sentence_taq_Latn}}\nEnglish: " +include: flores +task: flores_taq_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_taq_Tfng-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_taq_Tfng-eng_Latn.yaml new file mode 100644 index 00000000..7cfb5419 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_taq_Tfng-eng_Latn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: taq_Tfng-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Tamasheq (Tifinagh\ + \ script) sentences to English \nTamasheq (Tifinagh script): {{sentence_taq_Tfng}}\n\ + English: " +include: flores +task: flores_taq_Tfng-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_tir_Ethi-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_tir_Ethi-eng_Latn.yaml new file mode 100644 index 00000000..56607b6a --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_tir_Ethi-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tir_Ethi-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Tigrinya sentences\ + \ to English \nTigrinya: {{sentence_tir_Ethi}}\nEnglish: " +include: flores +task: flores_tir_Ethi-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_tsn_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_tsn_Latn-eng_Latn.yaml new file mode 100644 index 00000000..b8d04feb --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_tsn_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tsn_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Setswana sentences\ + \ to English \nSetswana: {{sentence_tsn_Latn}}\nEnglish: " +include: flores +task: flores_tsn_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_tso_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_tso_Latn-eng_Latn.yaml new file mode 100644 index 00000000..3c357e9d --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_tso_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tso_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Tsonga sentences\ + \ to English \nTsonga: {{sentence_tso_Latn}}\nEnglish: " +include: flores +task: flores_tso_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_tum_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_tum_Latn-eng_Latn.yaml new file mode 100644 index 00000000..d70a89b2 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_tum_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tum_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Tumbuka sentences\ + \ to English \nTumbuka: {{sentence_tum_Latn}}\nEnglish: " +include: flores +task: flores_tum_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_twi_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_twi_Latn-eng_Latn.yaml new file mode 100644 index 00000000..d9dc9577 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_twi_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: twi_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Twi sentences\ + \ to English \nTwi: {{sentence_twi_Latn}}\nEnglish: " +include: flores +task: flores_twi_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_tzm_Tfng-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_tzm_Tfng-eng_Latn.yaml new file mode 100644 index 00000000..81f9c721 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_tzm_Tfng-eng_Latn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: tzm_Tfng-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Central Atlas\ + \ Tamazight sentences to English \nCentral Atlas Tamazight: {{sentence_tzm_Tfng}}\n\ + English: " +include: flores +task: flores_tzm_Tfng-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_umb_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_umb_Latn-eng_Latn.yaml new file mode 100644 index 00000000..983675b0 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_umb_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: umb_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Umbundu sentences\ + \ to English \nUmbundu: {{sentence_umb_Latn}}\nEnglish: " +include: flores +task: flores_umb_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_wol_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_wol_Latn-eng_Latn.yaml new file mode 100644 index 00000000..0f1210fe --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_wol_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: wol_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Wolof sentences\ + \ to English \nWolof: {{sentence_wol_Latn}}\nEnglish: " +include: flores +task: flores_wol_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_xho_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_xho_Latn-eng_Latn.yaml new file mode 100644 index 00000000..f28e1bb3 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_xho_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: xho_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Xhosa sentences\ + \ to English \nXhosa: {{sentence_xho_Latn}}\nEnglish: " +include: flores +task: flores_xho_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_yor_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_yor_Latn-eng_Latn.yaml new file mode 100644 index 00000000..e0665926 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_yor_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: yor_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Yoruba sentences\ + \ to English \nYoruba: {{sentence_yor_Latn}}\nEnglish: " +include: flores +task: flores_yor_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_zul_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_zul_Latn-eng_Latn.yaml new file mode 100644 index 00000000..f3b2fef4 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/african-english/flores_zul_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: zul_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Zulu sentences\ + \ to English \nZulu: {{sentence_zul_Latn}}\nEnglish: " +include: flores +task: flores_zul_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores new file mode 100644 index 00000000..ab71d656 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores @@ -0,0 +1,27 @@ +tag: +- african_flores_tasks +- flores_eng-afr +- flores_eng-afr_prompt_2 +- afrobench_MT_tasks +dataset_path: facebook/flores +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: dev +fewshot_split: dev +test_split: devtest +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "**" + - + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ace_Arab.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ace_Arab.yaml new file mode 100644 index 00000000..30890d91 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ace_Arab.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: eng_Latn-ace_Arab +doc_to_target: sentence_ace_Arab +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Acehnese (Arabic script) \nEnglish: {{sentence_eng_Latn}} \nAcehnese (Arabic\ + \ script): " +include: flores +task: flores_eng_Latn-ace_Arab_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ace_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ace_Latn.yaml new file mode 100644 index 00000000..4356785a --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ace_Latn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: eng_Latn-ace_Latn +doc_to_target: sentence_ace_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Acehnese (Latin script) \nEnglish: {{sentence_eng_Latn}} \nAcehnese (Latin\ + \ script): " +include: flores +task: flores_eng_Latn-ace_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-acq_Arab.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-acq_Arab.yaml new file mode 100644 index 00000000..630c824e --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-acq_Arab.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-acq_Arab +doc_to_target: sentence_acq_Arab +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Ta’izzi-Adeni Arabic \nEnglish: {{sentence_eng_Latn}} \nTa’izzi-Adeni Arabic: " +include: flores +task: flores_eng_Latn-acq_Arab_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-aeb_Arab.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-aeb_Arab.yaml new file mode 100644 index 00000000..0df4f642 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-aeb_Arab.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-aeb_Arab +doc_to_target: sentence_aeb_Arab +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Tunisian Arabic \nEnglish: {{sentence_eng_Latn}} \nTunisian Arabic: " +include: flores +task: flores_eng_Latn-aeb_Arab_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-afr_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-afr_Latn.yaml new file mode 100644 index 00000000..e2769adf --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-afr_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-afr_Latn +doc_to_target: sentence_afr_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Afrikaans \nEnglish: {{sentence_eng_Latn}} \nAfrikaans: " +include: flores +task: flores_eng_Latn-afr_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-aka_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-aka_Latn.yaml new file mode 100644 index 00000000..624149c7 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-aka_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-aka_Latn +doc_to_target: sentence_aka_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Akan \nEnglish: {{sentence_eng_Latn}} \nAkan: " +include: flores +task: flores_eng_Latn-aka_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-amh_Ethi.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-amh_Ethi.yaml new file mode 100644 index 00000000..0a53e8c2 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-amh_Ethi.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-amh_Ethi +doc_to_target: sentence_amh_Ethi +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Amharic \nEnglish: {{sentence_eng_Latn}} \nAmharic: " +include: flores +task: flores_eng_Latn-amh_Ethi_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ary_Arab.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ary_Arab.yaml new file mode 100644 index 00000000..bb814d76 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ary_Arab.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-ary_Arab +doc_to_target: sentence_ary_Arab +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Moroccan Arabic \nEnglish: {{sentence_eng_Latn}} \nMoroccan Arabic: " +include: flores +task: flores_eng_Latn-ary_Arab_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-arz_Arab.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-arz_Arab.yaml new file mode 100644 index 00000000..0362666c --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-arz_Arab.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-arz_Arab +doc_to_target: sentence_arz_Arab +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Egyptian Arabic \nEnglish: {{sentence_eng_Latn}} \nEgyptian Arabic: " +include: flores +task: flores_eng_Latn-arz_Arab_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-bam_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-bam_Latn.yaml new file mode 100644 index 00000000..b3845921 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-bam_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-bam_Latn +doc_to_target: sentence_bam_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Bambara \nEnglish: {{sentence_eng_Latn}} \nBambara: " +include: flores +task: flores_eng_Latn-bam_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ban_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ban_Latn.yaml new file mode 100644 index 00000000..cff3c15b --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ban_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-ban_Latn +doc_to_target: sentence_ban_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Balinese \nEnglish: {{sentence_eng_Latn}} \nBalinese: " +include: flores +task: flores_eng_Latn-ban_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-bem_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-bem_Latn.yaml new file mode 100644 index 00000000..ef6552a2 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-bem_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-bem_Latn +doc_to_target: sentence_bem_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Bemba \nEnglish: {{sentence_eng_Latn}} \nBemba: " +include: flores +task: flores_eng_Latn-bem_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-cjk_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-cjk_Latn.yaml new file mode 100644 index 00000000..38c4ea6f --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-cjk_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-cjk_Latn +doc_to_target: sentence_cjk_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Chokwe \nEnglish: {{sentence_eng_Latn}} \nChokwe: " +include: flores +task: flores_eng_Latn-cjk_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-dik_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-dik_Latn.yaml new file mode 100644 index 00000000..bfcf7180 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-dik_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-dik_Latn +doc_to_target: sentence_dik_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Southwestern Dinka \nEnglish: {{sentence_eng_Latn}} \nSouthwestern Dinka: " +include: flores +task: flores_eng_Latn-dik_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-dyu_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-dyu_Latn.yaml new file mode 100644 index 00000000..a9fab72b --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-dyu_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-dyu_Latn +doc_to_target: sentence_dyu_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Dyula \nEnglish: {{sentence_eng_Latn}} \nDyula: " +include: flores +task: flores_eng_Latn-dyu_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ewe_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ewe_Latn.yaml new file mode 100644 index 00000000..5ecc34e5 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ewe_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-ewe_Latn +doc_to_target: sentence_ewe_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Ewe \nEnglish: {{sentence_eng_Latn}} \nEwe: " +include: flores +task: flores_eng_Latn-ewe_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-fon_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-fon_Latn.yaml new file mode 100644 index 00000000..ed029237 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-fon_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-fon_Latn +doc_to_target: sentence_fon_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Fon \nEnglish: {{sentence_eng_Latn}} \nFon: " +include: flores +task: flores_eng_Latn-fon_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-fra_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-fra_Latn.yaml new file mode 100644 index 00000000..9d54e66c --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-fra_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-fra_Latn +doc_to_target: sentence_fra_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to French \nEnglish: {{sentence_eng_Latn}} \nFrench: " +include: flores +task: flores_eng_Latn-fra_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-fuv_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-fuv_Latn.yaml new file mode 100644 index 00000000..a821f58f --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-fuv_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-fuv_Latn +doc_to_target: sentence_fuv_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Nigerian Fulfulde \nEnglish: {{sentence_eng_Latn}} \nNigerian Fulfulde: " +include: flores +task: flores_eng_Latn-fuv_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-gaz_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-gaz_Latn.yaml new file mode 100644 index 00000000..36fa1d6c --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-gaz_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-gaz_Latn +doc_to_target: sentence_gaz_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Oromo \nEnglish: {{sentence_eng_Latn}} \nOromo: " +include: flores +task: flores_eng_Latn-gaz_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-hau_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-hau_Latn.yaml new file mode 100644 index 00000000..aad0a48b --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-hau_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-hau_Latn +doc_to_target: sentence_hau_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Hausa \nEnglish: {{sentence_eng_Latn}} \nHausa: " +include: flores +task: flores_eng_Latn-hau_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ibo_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ibo_Latn.yaml new file mode 100644 index 00000000..b31e37cd --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ibo_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-ibo_Latn +doc_to_target: sentence_ibo_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Igbo \nEnglish: {{sentence_eng_Latn}} \nIgbo: " +include: flores +task: flores_eng_Latn-ibo_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kab_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kab_Latn.yaml new file mode 100644 index 00000000..1d6cfd8c --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kab_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-kab_Latn +doc_to_target: sentence_kab_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Kabyle \nEnglish: {{sentence_eng_Latn}} \nKabyle: " +include: flores +task: flores_eng_Latn-kab_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kam_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kam_Latn.yaml new file mode 100644 index 00000000..dd2da95c --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kam_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-kam_Latn +doc_to_target: sentence_kam_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Kamba \nEnglish: {{sentence_eng_Latn}} \nKamba: " +include: flores +task: flores_eng_Latn-kam_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kbp_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kbp_Latn.yaml new file mode 100644 index 00000000..b04cbdf1 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kbp_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-kbp_Latn +doc_to_target: sentence_kbp_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Kabiyè \nEnglish: {{sentence_eng_Latn}} \nKabiyè: " +include: flores +task: flores_eng_Latn-kbp_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kea_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kea_Latn.yaml new file mode 100644 index 00000000..4a67cb9f --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kea_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-kea_Latn +doc_to_target: sentence_kea_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Kabuverdianu \nEnglish: {{sentence_eng_Latn}} \nKabuverdianu: " +include: flores +task: flores_eng_Latn-kea_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kik_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kik_Latn.yaml new file mode 100644 index 00000000..1519f36e --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kik_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-kik_Latn +doc_to_target: sentence_kik_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Kikuyu \nEnglish: {{sentence_eng_Latn}} \nKikuyu: " +include: flores +task: flores_eng_Latn-kik_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kin_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kin_Latn.yaml new file mode 100644 index 00000000..2b33033f --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kin_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-kin_Latn +doc_to_target: sentence_kin_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Kinyarwanda \nEnglish: {{sentence_eng_Latn}} \nKinyarwanda: " +include: flores +task: flores_eng_Latn-kin_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kmb_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kmb_Latn.yaml new file mode 100644 index 00000000..80398917 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kmb_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-kmb_Latn +doc_to_target: sentence_kmb_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Kimbundu \nEnglish: {{sentence_eng_Latn}} \nKimbundu: " +include: flores +task: flores_eng_Latn-kmb_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-knc_Arab.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-knc_Arab.yaml new file mode 100644 index 00000000..c0d26241 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-knc_Arab.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: eng_Latn-knc_Arab +doc_to_target: sentence_knc_Arab +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Central Kanuri (Arabic script) \nEnglish: {{sentence_eng_Latn}} \nCentral Kanuri\ + \ (Arabic script): " +include: flores +task: flores_eng_Latn-knc_Arab_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-knc_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-knc_Latn.yaml new file mode 100644 index 00000000..61ea7a2c --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-knc_Latn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: eng_Latn-knc_Latn +doc_to_target: sentence_knc_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Central Kanuri (Latin script) \nEnglish: {{sentence_eng_Latn}} \nCentral Kanuri\ + \ (Latin script): " +include: flores +task: flores_eng_Latn-knc_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kon_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kon_Latn.yaml new file mode 100644 index 00000000..1967452e --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-kon_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-kon_Latn +doc_to_target: sentence_kon_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Kikongo \nEnglish: {{sentence_eng_Latn}} \nKikongo: " +include: flores +task: flores_eng_Latn-kon_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-lin_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-lin_Latn.yaml new file mode 100644 index 00000000..05e2593b --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-lin_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-lin_Latn +doc_to_target: sentence_lin_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Lingala \nEnglish: {{sentence_eng_Latn}} \nLingala: " +include: flores +task: flores_eng_Latn-lin_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-lua_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-lua_Latn.yaml new file mode 100644 index 00000000..5f4fe01e --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-lua_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-lua_Latn +doc_to_target: sentence_lua_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Luba-Kasai \nEnglish: {{sentence_eng_Latn}} \nLuba-Kasai: " +include: flores +task: flores_eng_Latn-lua_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-lug_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-lug_Latn.yaml new file mode 100644 index 00000000..0cfc3556 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-lug_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-lug_Latn +doc_to_target: sentence_lug_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Luganda \nEnglish: {{sentence_eng_Latn}} \nLuganda: " +include: flores +task: flores_eng_Latn-lug_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-luo_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-luo_Latn.yaml new file mode 100644 index 00000000..05c027bb --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-luo_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-luo_Latn +doc_to_target: sentence_luo_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Luo \nEnglish: {{sentence_eng_Latn}} \nLuo: " +include: flores +task: flores_eng_Latn-luo_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-mos_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-mos_Latn.yaml new file mode 100644 index 00000000..0a676522 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-mos_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-mos_Latn +doc_to_target: sentence_mos_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Mossi \nEnglish: {{sentence_eng_Latn}} \nMossi: " +include: flores +task: flores_eng_Latn-mos_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-nso_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-nso_Latn.yaml new file mode 100644 index 00000000..c681b492 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-nso_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-nso_Latn +doc_to_target: sentence_nso_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Northern Sotho \nEnglish: {{sentence_eng_Latn}} \nNorthern Sotho: " +include: flores +task: flores_eng_Latn-nso_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-nus_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-nus_Latn.yaml new file mode 100644 index 00000000..7ae37505 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-nus_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-nus_Latn +doc_to_target: sentence_nus_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Nuer \nEnglish: {{sentence_eng_Latn}} \nNuer: " +include: flores +task: flores_eng_Latn-nus_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-nya_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-nya_Latn.yaml new file mode 100644 index 00000000..13502902 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-nya_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-nya_Latn +doc_to_target: sentence_nya_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Nyanja \nEnglish: {{sentence_eng_Latn}} \nNyanja: " +include: flores +task: flores_eng_Latn-nya_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-plt_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-plt_Latn.yaml new file mode 100644 index 00000000..faa85197 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-plt_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-plt_Latn +doc_to_target: sentence_plt_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Plateau Malagasy \nEnglish: {{sentence_eng_Latn}} \nPlateau Malagasy: " +include: flores +task: flores_eng_Latn-plt_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-run_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-run_Latn.yaml new file mode 100644 index 00000000..8b670e3f --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-run_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-run_Latn +doc_to_target: sentence_run_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Rundi \nEnglish: {{sentence_eng_Latn}} \nRundi: " +include: flores +task: flores_eng_Latn-run_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-sag_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-sag_Latn.yaml new file mode 100644 index 00000000..32f39939 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-sag_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-sag_Latn +doc_to_target: sentence_sag_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Sango \nEnglish: {{sentence_eng_Latn}} \nSango: " +include: flores +task: flores_eng_Latn-sag_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-sna_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-sna_Latn.yaml new file mode 100644 index 00000000..0e219c40 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-sna_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-sna_Latn +doc_to_target: sentence_sna_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Shona \nEnglish: {{sentence_eng_Latn}} \nShona: " +include: flores +task: flores_eng_Latn-sna_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-som_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-som_Latn.yaml new file mode 100644 index 00000000..f87466dc --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-som_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-som_Latn +doc_to_target: sentence_som_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Somali \nEnglish: {{sentence_eng_Latn}} \nSomali: " +include: flores +task: flores_eng_Latn-som_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-sot_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-sot_Latn.yaml new file mode 100644 index 00000000..674d162b --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-sot_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-sot_Latn +doc_to_target: sentence_sot_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Southern Sotho \nEnglish: {{sentence_eng_Latn}} \nSouthern Sotho: " +include: flores +task: flores_eng_Latn-sot_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ssw_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ssw_Latn.yaml new file mode 100644 index 00000000..23b9216f --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-ssw_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-ssw_Latn +doc_to_target: sentence_ssw_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Swati \nEnglish: {{sentence_eng_Latn}} \nSwati: " +include: flores +task: flores_eng_Latn-ssw_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-sun_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-sun_Latn.yaml new file mode 100644 index 00000000..9f51ced5 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-sun_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-sun_Latn +doc_to_target: sentence_sun_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Sundanese \nEnglish: {{sentence_eng_Latn}} \nSundanese: " +include: flores +task: flores_eng_Latn-sun_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-swh_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-swh_Latn.yaml new file mode 100644 index 00000000..1558af98 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-swh_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-swh_Latn +doc_to_target: sentence_swh_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Swahili \nEnglish: {{sentence_eng_Latn}} \nSwahili: " +include: flores +task: flores_eng_Latn-swh_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-taq_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-taq_Latn.yaml new file mode 100644 index 00000000..0b09b52f --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-taq_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-taq_Latn +doc_to_target: sentence_taq_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Tamasheq \nEnglish: {{sentence_eng_Latn}} \nTamasheq: " +include: flores +task: flores_eng_Latn-taq_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-taq_Tfng.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-taq_Tfng.yaml new file mode 100644 index 00000000..b69f1dbd --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-taq_Tfng.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: eng_Latn-taq_Tfng +doc_to_target: sentence_taq_Tfng +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Tamasheq (Tifinagh script) \nEnglish: {{sentence_eng_Latn}} \nTamasheq (Tifinagh\ + \ script): " +include: flores +task: flores_eng_Latn-taq_Tfng_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-tir_Ethi.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-tir_Ethi.yaml new file mode 100644 index 00000000..4340591d --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-tir_Ethi.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-tir_Ethi +doc_to_target: sentence_tir_Ethi +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Tigrinya \nEnglish: {{sentence_eng_Latn}} \nTigrinya: " +include: flores +task: flores_eng_Latn-tir_Ethi_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-tsn_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-tsn_Latn.yaml new file mode 100644 index 00000000..1e592366 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-tsn_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-tsn_Latn +doc_to_target: sentence_tsn_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Setswana \nEnglish: {{sentence_eng_Latn}} \nSetswana: " +include: flores +task: flores_eng_Latn-tsn_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-tso_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-tso_Latn.yaml new file mode 100644 index 00000000..0d027a2a --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-tso_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-tso_Latn +doc_to_target: sentence_tso_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Tsonga \nEnglish: {{sentence_eng_Latn}} \nTsonga: " +include: flores +task: flores_eng_Latn-tso_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-tum_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-tum_Latn.yaml new file mode 100644 index 00000000..1accaeaf --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-tum_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-tum_Latn +doc_to_target: sentence_tum_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Tumbuka \nEnglish: {{sentence_eng_Latn}} \nTumbuka: " +include: flores +task: flores_eng_Latn-tum_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-twi_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-twi_Latn.yaml new file mode 100644 index 00000000..4a45df82 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-twi_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-twi_Latn +doc_to_target: sentence_twi_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Twi \nEnglish: {{sentence_eng_Latn}} \nTwi: " +include: flores +task: flores_eng_Latn-twi_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-tzm_Tfng.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-tzm_Tfng.yaml new file mode 100644 index 00000000..6a3faa15 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-tzm_Tfng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-tzm_Tfng +doc_to_target: sentence_tzm_Tfng +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Central Atlas Tamazight \nEnglish: {{sentence_eng_Latn}} \nCentral Atlas Tamazight: " +include: flores +task: flores_eng_Latn-tzm_Tfng_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-umb_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-umb_Latn.yaml new file mode 100644 index 00000000..3f21c6fe --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-umb_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-umb_Latn +doc_to_target: sentence_umb_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Umbundu \nEnglish: {{sentence_eng_Latn}} \nUmbundu: " +include: flores +task: flores_eng_Latn-umb_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-wol_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-wol_Latn.yaml new file mode 100644 index 00000000..263ded27 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-wol_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-wol_Latn +doc_to_target: sentence_wol_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Wolof \nEnglish: {{sentence_eng_Latn}} \nWolof: " +include: flores +task: flores_eng_Latn-wol_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-xho_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-xho_Latn.yaml new file mode 100644 index 00000000..a92e46f9 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-xho_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-xho_Latn +doc_to_target: sentence_xho_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Xhosa \nEnglish: {{sentence_eng_Latn}} \nXhosa: " +include: flores +task: flores_eng_Latn-xho_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-yor_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-yor_Latn.yaml new file mode 100644 index 00000000..80ec895c --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-yor_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-yor_Latn +doc_to_target: sentence_yor_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Yoruba \nEnglish: {{sentence_eng_Latn}} \nYoruba: " +include: flores +task: flores_eng_Latn-yor_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-zul_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-zul_Latn.yaml new file mode 100644 index 00000000..593cdfe3 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/english-african/flores_eng_Latn-zul_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-zul_Latn +doc_to_target: sentence_zul_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Zulu \nEnglish: {{sentence_eng_Latn}} \nZulu: " +include: flores +task: flores_eng_Latn-zul_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/flores/prompt_2/flores b/lm_eval/tasks/afrobench/flores/prompt_2/flores new file mode 100644 index 00000000..74f9f33e --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_2/flores @@ -0,0 +1,24 @@ +tag: +- flores_tasks +- flores_afr-eng +dataset_path: facebook/flores +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: dev +fewshot_split: dev +test_split: devtest +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores new file mode 100644 index 00000000..60bf4111 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores @@ -0,0 +1,27 @@ +tag: +- african_flores_tasks +- flores_afr-eng +- flores_afr-eng_prompt_3 +- afrobench_MT_tasks +dataset_path: facebook/flores +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: dev +fewshot_split: dev +test_split: devtest +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "**" + - + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ace_Arab-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ace_Arab-eng_Latn.yaml new file mode 100644 index 00000000..ee5f1270 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ace_Arab-eng_Latn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: ace_Arab-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Acehnese (Arabic script) and English linguist, translate the following\ + \ Acehnese (Arabic script) sentences to English \nAcehnese (Arabic script): {{sentence_ace_Arab}}\n\ + English: " +include: flores +task: flores_ace_Arab-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ace_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ace_Latn-eng_Latn.yaml new file mode 100644 index 00000000..e1d70ba3 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ace_Latn-eng_Latn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: ace_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Acehnese (Latin script) and English linguist, translate the following\ + \ Acehnese (Latin script) sentences to English \nAcehnese (Latin script): {{sentence_ace_Latn}}\n\ + English: " +include: flores +task: flores_ace_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_acq_Arab-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_acq_Arab-eng_Latn.yaml new file mode 100644 index 00000000..8cda3962 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_acq_Arab-eng_Latn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: acq_Arab-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Ta’izzi-Adeni Arabic and English linguist, translate the following\ + \ Ta’izzi-Adeni Arabic sentences to English \nTa’izzi-Adeni Arabic: {{sentence_acq_Arab}}\n\ + English: " +include: flores +task: flores_acq_Arab-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_aeb_Arab-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_aeb_Arab-eng_Latn.yaml new file mode 100644 index 00000000..97f8ef2c --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_aeb_Arab-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: aeb_Arab-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Tunisian Arabic and English linguist, translate the following Tunisian\ + \ Arabic sentences to English \nTunisian Arabic: {{sentence_aeb_Arab}}\nEnglish: " +include: flores +task: flores_aeb_Arab-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_afr_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_afr_Latn-eng_Latn.yaml new file mode 100644 index 00000000..e228cb9c --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_afr_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: afr_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Afrikaans and English linguist, translate the following Afrikaans\ + \ sentences to English \nAfrikaans: {{sentence_afr_Latn}}\nEnglish: " +include: flores +task: flores_afr_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_aka_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_aka_Latn-eng_Latn.yaml new file mode 100644 index 00000000..6d6fc385 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_aka_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: aka_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Akan and English linguist, translate the following Akan sentences\ + \ to English \nAkan: {{sentence_aka_Latn}}\nEnglish: " +include: flores +task: flores_aka_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_amh_Ethi-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_amh_Ethi-eng_Latn.yaml new file mode 100644 index 00000000..58f33f9a --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_amh_Ethi-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: amh_Ethi-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Amharic and English linguist, translate the following Amharic sentences\ + \ to English \nAmharic: {{sentence_amh_Ethi}}\nEnglish: " +include: flores +task: flores_amh_Ethi-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ary_Arab-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ary_Arab-eng_Latn.yaml new file mode 100644 index 00000000..3006ebf7 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ary_Arab-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ary_Arab-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Moroccan Arabic and English linguist, translate the following Moroccan\ + \ Arabic sentences to English \nMoroccan Arabic: {{sentence_ary_Arab}}\nEnglish: " +include: flores +task: flores_ary_Arab-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_arz_Arab-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_arz_Arab-eng_Latn.yaml new file mode 100644 index 00000000..46cc0a18 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_arz_Arab-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: arz_Arab-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Egyptian Arabic and English linguist, translate the following Egyptian\ + \ Arabic sentences to English \nEgyptian Arabic: {{sentence_arz_Arab}}\nEnglish: " +include: flores +task: flores_arz_Arab-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_bam_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_bam_Latn-eng_Latn.yaml new file mode 100644 index 00000000..c50a8dfa --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_bam_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: bam_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Bambara and English linguist, translate the following Bambara sentences\ + \ to English \nBambara: {{sentence_bam_Latn}}\nEnglish: " +include: flores +task: flores_bam_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ban_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ban_Latn-eng_Latn.yaml new file mode 100644 index 00000000..86f2eed3 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ban_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ban_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Balinese and English linguist, translate the following Balinese\ + \ sentences to English \nBalinese: {{sentence_ban_Latn}}\nEnglish: " +include: flores +task: flores_ban_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_bem_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_bem_Latn-eng_Latn.yaml new file mode 100644 index 00000000..55c32fe9 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_bem_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: bem_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Bemba and English linguist, translate the following Bemba sentences\ + \ to English \nBemba: {{sentence_bem_Latn}}\nEnglish: " +include: flores +task: flores_bem_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_cjk_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_cjk_Latn-eng_Latn.yaml new file mode 100644 index 00000000..642cd4dd --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_cjk_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: cjk_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Chokwe and English linguist, translate the following Chokwe sentences\ + \ to English \nChokwe: {{sentence_cjk_Latn}}\nEnglish: " +include: flores +task: flores_cjk_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_dik_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_dik_Latn-eng_Latn.yaml new file mode 100644 index 00000000..8005a642 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_dik_Latn-eng_Latn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: dik_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Southwestern Dinka and English linguist, translate the following\ + \ Southwestern Dinka sentences to English \nSouthwestern Dinka: {{sentence_dik_Latn}}\n\ + English: " +include: flores +task: flores_dik_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_dyu_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_dyu_Latn-eng_Latn.yaml new file mode 100644 index 00000000..a99efc08 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_dyu_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: dyu_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Dyula and English linguist, translate the following Dyula sentences\ + \ to English \nDyula: {{sentence_dyu_Latn}}\nEnglish: " +include: flores +task: flores_dyu_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ewe_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ewe_Latn-eng_Latn.yaml new file mode 100644 index 00000000..77133ad6 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ewe_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ewe_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Ewe and English linguist, translate the following Ewe sentences\ + \ to English \nEwe: {{sentence_ewe_Latn}}\nEnglish: " +include: flores +task: flores_ewe_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_fon_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_fon_Latn-eng_Latn.yaml new file mode 100644 index 00000000..436bf4ac --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_fon_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fon_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Fon and English linguist, translate the following Fon sentences\ + \ to English \nFon: {{sentence_fon_Latn}}\nEnglish: " +include: flores +task: flores_fon_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_fra_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_fra_Latn-eng_Latn.yaml new file mode 100644 index 00000000..b10c46e3 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_fra_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fra_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a French and English linguist, translate the following French sentences\ + \ to English \nFrench: {{sentence_fra_Latn}}\nEnglish: " +include: flores +task: flores_fra_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_fuv_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_fuv_Latn-eng_Latn.yaml new file mode 100644 index 00000000..ffcbd3c0 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_fuv_Latn-eng_Latn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: fuv_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Nigerian Fulfulde and English linguist, translate the following\ + \ Nigerian Fulfulde sentences to English \nNigerian Fulfulde: {{sentence_fuv_Latn}}\n\ + English: " +include: flores +task: flores_fuv_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_gaz_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_gaz_Latn-eng_Latn.yaml new file mode 100644 index 00000000..703cd351 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_gaz_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: gaz_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Oromo and English linguist, translate the following Oromo sentences\ + \ to English \nOromo: {{sentence_gaz_Latn}}\nEnglish: " +include: flores +task: flores_gaz_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_hau_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_hau_Latn-eng_Latn.yaml new file mode 100644 index 00000000..7527bf78 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_hau_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: hau_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Hausa and English linguist, translate the following Hausa sentences\ + \ to English \nHausa: {{sentence_hau_Latn}}\nEnglish: " +include: flores +task: flores_hau_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ibo_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ibo_Latn-eng_Latn.yaml new file mode 100644 index 00000000..7705911a --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ibo_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ibo_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Igbo and English linguist, translate the following Igbo sentences\ + \ to English \nIgbo: {{sentence_ibo_Latn}}\nEnglish: " +include: flores +task: flores_ibo_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kab_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kab_Latn-eng_Latn.yaml new file mode 100644 index 00000000..ec406c5e --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kab_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kab_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Kabyle and English linguist, translate the following Kabyle sentences\ + \ to English \nKabyle: {{sentence_kab_Latn}}\nEnglish: " +include: flores +task: flores_kab_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kam_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kam_Latn-eng_Latn.yaml new file mode 100644 index 00000000..ed27b6d7 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kam_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kam_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Kamba and English linguist, translate the following Kamba sentences\ + \ to English \nKamba: {{sentence_kam_Latn}}\nEnglish: " +include: flores +task: flores_kam_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kbp_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kbp_Latn-eng_Latn.yaml new file mode 100644 index 00000000..5c1a0961 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kbp_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kbp_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Kabiyè and English linguist, translate the following Kabiyè sentences\ + \ to English \nKabiyè: {{sentence_kbp_Latn}}\nEnglish: " +include: flores +task: flores_kbp_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kea_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kea_Latn-eng_Latn.yaml new file mode 100644 index 00000000..67dd9e73 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kea_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kea_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Kabuverdianu and English linguist, translate the following Kabuverdianu\ + \ sentences to English \nKabuverdianu: {{sentence_kea_Latn}}\nEnglish: " +include: flores +task: flores_kea_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kik_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kik_Latn-eng_Latn.yaml new file mode 100644 index 00000000..14a6be5d --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kik_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kik_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Kikuyu and English linguist, translate the following Kikuyu sentences\ + \ to English \nKikuyu: {{sentence_kik_Latn}}\nEnglish: " +include: flores +task: flores_kik_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kin_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kin_Latn-eng_Latn.yaml new file mode 100644 index 00000000..8bb14aed --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kin_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kin_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Kinyarwanda and English linguist, translate the following Kinyarwanda\ + \ sentences to English \nKinyarwanda: {{sentence_kin_Latn}}\nEnglish: " +include: flores +task: flores_kin_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kmb_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kmb_Latn-eng_Latn.yaml new file mode 100644 index 00000000..c31ede4d --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kmb_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kmb_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Kimbundu and English linguist, translate the following Kimbundu\ + \ sentences to English \nKimbundu: {{sentence_kmb_Latn}}\nEnglish: " +include: flores +task: flores_kmb_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_knc_Arab-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_knc_Arab-eng_Latn.yaml new file mode 100644 index 00000000..c8c7f809 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_knc_Arab-eng_Latn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: knc_Arab-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Central Kanuri (Arabic script) and English linguist, translate\ + \ the following Central Kanuri (Arabic script) sentences to English \nCentral Kanuri\ + \ (Arabic script): {{sentence_knc_Arab}}\nEnglish: " +include: flores +task: flores_knc_Arab-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_knc_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_knc_Latn-eng_Latn.yaml new file mode 100644 index 00000000..9621de73 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_knc_Latn-eng_Latn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: knc_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Central Kanuri (Latin script) and English linguist, translate the\ + \ following Central Kanuri (Latin script) sentences to English \nCentral Kanuri\ + \ (Latin script): {{sentence_knc_Latn}}\nEnglish: " +include: flores +task: flores_knc_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kon_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kon_Latn-eng_Latn.yaml new file mode 100644 index 00000000..54ede3a6 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_kon_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kon_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Kikongo and English linguist, translate the following Kikongo sentences\ + \ to English \nKikongo: {{sentence_kon_Latn}}\nEnglish: " +include: flores +task: flores_kon_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_lin_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_lin_Latn-eng_Latn.yaml new file mode 100644 index 00000000..ea7e736d --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_lin_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lin_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Lingala and English linguist, translate the following Lingala sentences\ + \ to English \nLingala: {{sentence_lin_Latn}}\nEnglish: " +include: flores +task: flores_lin_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_lua_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_lua_Latn-eng_Latn.yaml new file mode 100644 index 00000000..327f0144 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_lua_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lua_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Luba-Kasai and English linguist, translate the following Luba-Kasai\ + \ sentences to English \nLuba-Kasai: {{sentence_lua_Latn}}\nEnglish: " +include: flores +task: flores_lua_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_lug_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_lug_Latn-eng_Latn.yaml new file mode 100644 index 00000000..9bfa92fa --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_lug_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lug_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Luganda and English linguist, translate the following Luganda sentences\ + \ to English \nLuganda: {{sentence_lug_Latn}}\nEnglish: " +include: flores +task: flores_lug_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_luo_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_luo_Latn-eng_Latn.yaml new file mode 100644 index 00000000..a66fded3 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_luo_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: luo_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Luo and English linguist, translate the following Luo sentences\ + \ to English \nLuo: {{sentence_luo_Latn}}\nEnglish: " +include: flores +task: flores_luo_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_mos_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_mos_Latn-eng_Latn.yaml new file mode 100644 index 00000000..e428853b --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_mos_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: mos_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Mossi and English linguist, translate the following Mossi sentences\ + \ to English \nMossi: {{sentence_mos_Latn}}\nEnglish: " +include: flores +task: flores_mos_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_nso_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_nso_Latn-eng_Latn.yaml new file mode 100644 index 00000000..054aa409 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_nso_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nso_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Northern Sotho and English linguist, translate the following Northern\ + \ Sotho sentences to English \nNorthern Sotho: {{sentence_nso_Latn}}\nEnglish: " +include: flores +task: flores_nso_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_nus_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_nus_Latn-eng_Latn.yaml new file mode 100644 index 00000000..a3e0d1e3 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_nus_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nus_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Nuer and English linguist, translate the following Nuer sentences\ + \ to English \nNuer: {{sentence_nus_Latn}}\nEnglish: " +include: flores +task: flores_nus_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_nya_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_nya_Latn-eng_Latn.yaml new file mode 100644 index 00000000..e23c57c6 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_nya_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nya_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Nyanja and English linguist, translate the following Nyanja sentences\ + \ to English \nNyanja: {{sentence_nya_Latn}}\nEnglish: " +include: flores +task: flores_nya_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_plt_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_plt_Latn-eng_Latn.yaml new file mode 100644 index 00000000..3ddfd864 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_plt_Latn-eng_Latn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: plt_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Plateau Malagasy and English linguist, translate the following\ + \ Plateau Malagasy sentences to English \nPlateau Malagasy: {{sentence_plt_Latn}}\n\ + English: " +include: flores +task: flores_plt_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_run_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_run_Latn-eng_Latn.yaml new file mode 100644 index 00000000..64a82f71 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_run_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: run_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Rundi and English linguist, translate the following Rundi sentences\ + \ to English \nRundi: {{sentence_run_Latn}}\nEnglish: " +include: flores +task: flores_run_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_sag_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_sag_Latn-eng_Latn.yaml new file mode 100644 index 00000000..48408f94 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_sag_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sag_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Sango and English linguist, translate the following Sango sentences\ + \ to English \nSango: {{sentence_sag_Latn}}\nEnglish: " +include: flores +task: flores_sag_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_sna_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_sna_Latn-eng_Latn.yaml new file mode 100644 index 00000000..ff162641 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_sna_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sna_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Shona and English linguist, translate the following Shona sentences\ + \ to English \nShona: {{sentence_sna_Latn}}\nEnglish: " +include: flores +task: flores_sna_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_som_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_som_Latn-eng_Latn.yaml new file mode 100644 index 00000000..7e27e2a5 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_som_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: som_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Somali and English linguist, translate the following Somali sentences\ + \ to English \nSomali: {{sentence_som_Latn}}\nEnglish: " +include: flores +task: flores_som_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_sot_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_sot_Latn-eng_Latn.yaml new file mode 100644 index 00000000..cc70b6f6 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_sot_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sot_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Southern Sotho and English linguist, translate the following Southern\ + \ Sotho sentences to English \nSouthern Sotho: {{sentence_sot_Latn}}\nEnglish: " +include: flores +task: flores_sot_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ssw_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ssw_Latn-eng_Latn.yaml new file mode 100644 index 00000000..0cd61ae8 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_ssw_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ssw_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Swati and English linguist, translate the following Swati sentences\ + \ to English \nSwati: {{sentence_ssw_Latn}}\nEnglish: " +include: flores +task: flores_ssw_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_sun_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_sun_Latn-eng_Latn.yaml new file mode 100644 index 00000000..000108f7 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_sun_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sun_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Sundanese and English linguist, translate the following Sundanese\ + \ sentences to English \nSundanese: {{sentence_sun_Latn}}\nEnglish: " +include: flores +task: flores_sun_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_swh_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_swh_Latn-eng_Latn.yaml new file mode 100644 index 00000000..1c81805c --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_swh_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: swh_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Swahili and English linguist, translate the following Swahili sentences\ + \ to English \nSwahili: {{sentence_swh_Latn}}\nEnglish: " +include: flores +task: flores_swh_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_taq_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_taq_Latn-eng_Latn.yaml new file mode 100644 index 00000000..6febb300 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_taq_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: taq_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Tamasheq and English linguist, translate the following Tamasheq\ + \ sentences to English \nTamasheq: {{sentence_taq_Latn}}\nEnglish: " +include: flores +task: flores_taq_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_taq_Tfng-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_taq_Tfng-eng_Latn.yaml new file mode 100644 index 00000000..6290ab94 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_taq_Tfng-eng_Latn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: taq_Tfng-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Tamasheq (Tifinagh script) and English linguist, translate the\ + \ following Tamasheq (Tifinagh script) sentences to English \nTamasheq (Tifinagh\ + \ script): {{sentence_taq_Tfng}}\nEnglish: " +include: flores +task: flores_taq_Tfng-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_tir_Ethi-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_tir_Ethi-eng_Latn.yaml new file mode 100644 index 00000000..60133a3b --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_tir_Ethi-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tir_Ethi-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Tigrinya and English linguist, translate the following Tigrinya\ + \ sentences to English \nTigrinya: {{sentence_tir_Ethi}}\nEnglish: " +include: flores +task: flores_tir_Ethi-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_tsn_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_tsn_Latn-eng_Latn.yaml new file mode 100644 index 00000000..40417bde --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_tsn_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tsn_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Setswana and English linguist, translate the following Setswana\ + \ sentences to English \nSetswana: {{sentence_tsn_Latn}}\nEnglish: " +include: flores +task: flores_tsn_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_tso_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_tso_Latn-eng_Latn.yaml new file mode 100644 index 00000000..56d46325 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_tso_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tso_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Tsonga and English linguist, translate the following Tsonga sentences\ + \ to English \nTsonga: {{sentence_tso_Latn}}\nEnglish: " +include: flores +task: flores_tso_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_tum_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_tum_Latn-eng_Latn.yaml new file mode 100644 index 00000000..cc4bb541 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_tum_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tum_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Tumbuka and English linguist, translate the following Tumbuka sentences\ + \ to English \nTumbuka: {{sentence_tum_Latn}}\nEnglish: " +include: flores +task: flores_tum_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_twi_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_twi_Latn-eng_Latn.yaml new file mode 100644 index 00000000..4cc0d674 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_twi_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: twi_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Twi and English linguist, translate the following Twi sentences\ + \ to English \nTwi: {{sentence_twi_Latn}}\nEnglish: " +include: flores +task: flores_twi_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_tzm_Tfng-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_tzm_Tfng-eng_Latn.yaml new file mode 100644 index 00000000..d3575ccb --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_tzm_Tfng-eng_Latn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: tzm_Tfng-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Central Atlas Tamazight and English linguist, translate the following\ + \ Central Atlas Tamazight sentences to English \nCentral Atlas Tamazight: {{sentence_tzm_Tfng}}\n\ + English: " +include: flores +task: flores_tzm_Tfng-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_umb_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_umb_Latn-eng_Latn.yaml new file mode 100644 index 00000000..e7df76cf --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_umb_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: umb_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Umbundu and English linguist, translate the following Umbundu sentences\ + \ to English \nUmbundu: {{sentence_umb_Latn}}\nEnglish: " +include: flores +task: flores_umb_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_wol_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_wol_Latn-eng_Latn.yaml new file mode 100644 index 00000000..22275ca1 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_wol_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: wol_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Wolof and English linguist, translate the following Wolof sentences\ + \ to English \nWolof: {{sentence_wol_Latn}}\nEnglish: " +include: flores +task: flores_wol_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_xho_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_xho_Latn-eng_Latn.yaml new file mode 100644 index 00000000..85ae368b --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_xho_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: xho_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Xhosa and English linguist, translate the following Xhosa sentences\ + \ to English \nXhosa: {{sentence_xho_Latn}}\nEnglish: " +include: flores +task: flores_xho_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_yor_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_yor_Latn-eng_Latn.yaml new file mode 100644 index 00000000..5bbd8eb9 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_yor_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: yor_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Yoruba and English linguist, translate the following Yoruba sentences\ + \ to English \nYoruba: {{sentence_yor_Latn}}\nEnglish: " +include: flores +task: flores_yor_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_zul_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_zul_Latn-eng_Latn.yaml new file mode 100644 index 00000000..ea2c2edb --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/african-english/flores_zul_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: zul_Latn-eng_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Zulu and English linguist, translate the following Zulu sentences\ + \ to English \nZulu: {{sentence_zul_Latn}}\nEnglish: " +include: flores +task: flores_zul_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores new file mode 100644 index 00000000..ac7dc165 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores @@ -0,0 +1,27 @@ +tag: +- african_flores_tasks +- flores_eng-afr +- flores_eng-afr_prompt_3 +- afrobench_MT_tasks +dataset_path: facebook/flores +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: dev +fewshot_split: dev +test_split: devtest +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "**" + - + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-ace_Arab.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-ace_Arab.yaml new file mode 100644 index 00000000..53cf711f --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-ace_Arab.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: eng_Latn-ace_Arab +doc_to_target: sentence_ace_Arab +doc_to_text: "As a Acehnese (Arabic script) and English linguist, translate the following\ + \ English sentences to Acehnese (Arabic script) \nEnglish: {{sentence_eng_Latn}}\ + \ \nAcehnese (Arabic script): " +include: flores +task: flores_eng_Latn-ace_Arab_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-ace_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-ace_Latn.yaml new file mode 100644 index 00000000..766b7c30 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-ace_Latn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: eng_Latn-ace_Latn +doc_to_target: sentence_ace_Latn +doc_to_text: "As a Acehnese (Latin script) and English linguist, translate the following\ + \ English sentences to Acehnese (Latin script) \nEnglish: {{sentence_eng_Latn}}\ + \ \nAcehnese (Latin script): " +include: flores +task: flores_eng_Latn-ace_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-acq_Arab.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-acq_Arab.yaml new file mode 100644 index 00000000..e809c866 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-acq_Arab.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: eng_Latn-acq_Arab +doc_to_target: sentence_acq_Arab +doc_to_text: "As a Ta’izzi-Adeni Arabic and English linguist, translate the following\ + \ English sentences to Ta’izzi-Adeni Arabic \nEnglish: {{sentence_eng_Latn}} \n\ + Ta’izzi-Adeni Arabic: " +include: flores +task: flores_eng_Latn-acq_Arab_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-aeb_Arab.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-aeb_Arab.yaml new file mode 100644 index 00000000..9e8263fe --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-aeb_Arab.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-aeb_Arab +doc_to_target: sentence_aeb_Arab +doc_to_text: "As a Tunisian Arabic and English linguist, translate the following English\ + \ sentences to Tunisian Arabic \nEnglish: {{sentence_eng_Latn}} \nTunisian Arabic: " +include: flores +task: flores_eng_Latn-aeb_Arab_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-afr_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-afr_Latn.yaml new file mode 100644 index 00000000..86421c26 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-afr_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-afr_Latn +doc_to_target: sentence_afr_Latn +doc_to_text: "As a Afrikaans and English linguist, translate the following English\ + \ sentences to Afrikaans \nEnglish: {{sentence_eng_Latn}} \nAfrikaans: " +include: flores +task: flores_eng_Latn-afr_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-aka_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-aka_Latn.yaml new file mode 100644 index 00000000..33733905 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-aka_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-aka_Latn +doc_to_target: sentence_aka_Latn +doc_to_text: "As a Akan and English linguist, translate the following English sentences\ + \ to Akan \nEnglish: {{sentence_eng_Latn}} \nAkan: " +include: flores +task: flores_eng_Latn-aka_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-amh_Ethi.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-amh_Ethi.yaml new file mode 100644 index 00000000..ba3e0116 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-amh_Ethi.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-amh_Ethi +doc_to_target: sentence_amh_Ethi +doc_to_text: "As a Amharic and English linguist, translate the following English sentences\ + \ to Amharic \nEnglish: {{sentence_eng_Latn}} \nAmharic: " +include: flores +task: flores_eng_Latn-amh_Ethi_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-ary_Arab.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-ary_Arab.yaml new file mode 100644 index 00000000..c732756a --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-ary_Arab.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-ary_Arab +doc_to_target: sentence_ary_Arab +doc_to_text: "As a Moroccan Arabic and English linguist, translate the following English\ + \ sentences to Moroccan Arabic \nEnglish: {{sentence_eng_Latn}} \nMoroccan Arabic: " +include: flores +task: flores_eng_Latn-ary_Arab_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-arz_Arab.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-arz_Arab.yaml new file mode 100644 index 00000000..f11bc38a --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-arz_Arab.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-arz_Arab +doc_to_target: sentence_arz_Arab +doc_to_text: "As a Egyptian Arabic and English linguist, translate the following English\ + \ sentences to Egyptian Arabic \nEnglish: {{sentence_eng_Latn}} \nEgyptian Arabic: " +include: flores +task: flores_eng_Latn-arz_Arab_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-bam_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-bam_Latn.yaml new file mode 100644 index 00000000..c7629628 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-bam_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-bam_Latn +doc_to_target: sentence_bam_Latn +doc_to_text: "As a Bambara and English linguist, translate the following English sentences\ + \ to Bambara \nEnglish: {{sentence_eng_Latn}} \nBambara: " +include: flores +task: flores_eng_Latn-bam_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-ban_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-ban_Latn.yaml new file mode 100644 index 00000000..601aecf5 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-ban_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-ban_Latn +doc_to_target: sentence_ban_Latn +doc_to_text: "As a Balinese and English linguist, translate the following English\ + \ sentences to Balinese \nEnglish: {{sentence_eng_Latn}} \nBalinese: " +include: flores +task: flores_eng_Latn-ban_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-bem_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-bem_Latn.yaml new file mode 100644 index 00000000..fadabdb9 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-bem_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-bem_Latn +doc_to_target: sentence_bem_Latn +doc_to_text: "As a Bemba and English linguist, translate the following English sentences\ + \ to Bemba \nEnglish: {{sentence_eng_Latn}} \nBemba: " +include: flores +task: flores_eng_Latn-bem_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-cjk_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-cjk_Latn.yaml new file mode 100644 index 00000000..c5228313 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-cjk_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-cjk_Latn +doc_to_target: sentence_cjk_Latn +doc_to_text: "As a Chokwe and English linguist, translate the following English sentences\ + \ to Chokwe \nEnglish: {{sentence_eng_Latn}} \nChokwe: " +include: flores +task: flores_eng_Latn-cjk_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-dik_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-dik_Latn.yaml new file mode 100644 index 00000000..acfeb83a --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-dik_Latn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: eng_Latn-dik_Latn +doc_to_target: sentence_dik_Latn +doc_to_text: "As a Southwestern Dinka and English linguist, translate the following\ + \ English sentences to Southwestern Dinka \nEnglish: {{sentence_eng_Latn}} \nSouthwestern\ + \ Dinka: " +include: flores +task: flores_eng_Latn-dik_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-dyu_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-dyu_Latn.yaml new file mode 100644 index 00000000..796dc6d2 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-dyu_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-dyu_Latn +doc_to_target: sentence_dyu_Latn +doc_to_text: "As a Dyula and English linguist, translate the following English sentences\ + \ to Dyula \nEnglish: {{sentence_eng_Latn}} \nDyula: " +include: flores +task: flores_eng_Latn-dyu_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-ewe_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-ewe_Latn.yaml new file mode 100644 index 00000000..31a07891 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-ewe_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-ewe_Latn +doc_to_target: sentence_ewe_Latn +doc_to_text: "As a Ewe and English linguist, translate the following English sentences\ + \ to Ewe \nEnglish: {{sentence_eng_Latn}} \nEwe: " +include: flores +task: flores_eng_Latn-ewe_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-fon_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-fon_Latn.yaml new file mode 100644 index 00000000..6cdc7308 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-fon_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-fon_Latn +doc_to_target: sentence_fon_Latn +doc_to_text: "As a Fon and English linguist, translate the following English sentences\ + \ to Fon \nEnglish: {{sentence_eng_Latn}} \nFon: " +include: flores +task: flores_eng_Latn-fon_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-fra_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-fra_Latn.yaml new file mode 100644 index 00000000..3896879d --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-fra_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-fra_Latn +doc_to_target: sentence_fra_Latn +doc_to_text: "As a French and English linguist, translate the following English sentences\ + \ to French \nEnglish: {{sentence_eng_Latn}} \nFrench: " +include: flores +task: flores_eng_Latn-fra_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-fuv_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-fuv_Latn.yaml new file mode 100644 index 00000000..6b63249b --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-fuv_Latn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: eng_Latn-fuv_Latn +doc_to_target: sentence_fuv_Latn +doc_to_text: "As a Nigerian Fulfulde and English linguist, translate the following\ + \ English sentences to Nigerian Fulfulde \nEnglish: {{sentence_eng_Latn}} \nNigerian\ + \ Fulfulde: " +include: flores +task: flores_eng_Latn-fuv_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-gaz_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-gaz_Latn.yaml new file mode 100644 index 00000000..95cde87c --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-gaz_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-gaz_Latn +doc_to_target: sentence_gaz_Latn +doc_to_text: "As a Oromo and English linguist, translate the following English sentences\ + \ to Oromo \nEnglish: {{sentence_eng_Latn}} \nOromo: " +include: flores +task: flores_eng_Latn-gaz_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-hau_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-hau_Latn.yaml new file mode 100644 index 00000000..eec82e34 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-hau_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-hau_Latn +doc_to_target: sentence_hau_Latn +doc_to_text: "As a Hausa and English linguist, translate the following English sentences\ + \ to Hausa \nEnglish: {{sentence_eng_Latn}} \nHausa: " +include: flores +task: flores_eng_Latn-hau_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-ibo_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-ibo_Latn.yaml new file mode 100644 index 00000000..838990b3 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-ibo_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-ibo_Latn +doc_to_target: sentence_ibo_Latn +doc_to_text: "As a Igbo and English linguist, translate the following English sentences\ + \ to Igbo \nEnglish: {{sentence_eng_Latn}} \nIgbo: " +include: flores +task: flores_eng_Latn-ibo_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-kab_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-kab_Latn.yaml new file mode 100644 index 00000000..16888ad8 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-kab_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-kab_Latn +doc_to_target: sentence_kab_Latn +doc_to_text: "As a Kabyle and English linguist, translate the following English sentences\ + \ to Kabyle \nEnglish: {{sentence_eng_Latn}} \nKabyle: " +include: flores +task: flores_eng_Latn-kab_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-kam_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-kam_Latn.yaml new file mode 100644 index 00000000..d48c52d1 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-kam_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-kam_Latn +doc_to_target: sentence_kam_Latn +doc_to_text: "As a Kamba and English linguist, translate the following English sentences\ + \ to Kamba \nEnglish: {{sentence_eng_Latn}} \nKamba: " +include: flores +task: flores_eng_Latn-kam_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-kbp_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-kbp_Latn.yaml new file mode 100644 index 00000000..c992a28f --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-kbp_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-kbp_Latn +doc_to_target: sentence_kbp_Latn +doc_to_text: "As a Kabiyè and English linguist, translate the following English sentences\ + \ to Kabiyè \nEnglish: {{sentence_eng_Latn}} \nKabiyè: " +include: flores +task: flores_eng_Latn-kbp_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-kea_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-kea_Latn.yaml new file mode 100644 index 00000000..d8ce1b50 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-kea_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-kea_Latn +doc_to_target: sentence_kea_Latn +doc_to_text: "As a Kabuverdianu and English linguist, translate the following English\ + \ sentences to Kabuverdianu \nEnglish: {{sentence_eng_Latn}} \nKabuverdianu: " +include: flores +task: flores_eng_Latn-kea_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-kik_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-kik_Latn.yaml new file mode 100644 index 00000000..fc7975c2 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-kik_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-kik_Latn +doc_to_target: sentence_kik_Latn +doc_to_text: "As a Kikuyu and English linguist, translate the following English sentences\ + \ to Kikuyu \nEnglish: {{sentence_eng_Latn}} \nKikuyu: " +include: flores +task: flores_eng_Latn-kik_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-kin_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-kin_Latn.yaml new file mode 100644 index 00000000..1e2b91d4 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-kin_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-kin_Latn +doc_to_target: sentence_kin_Latn +doc_to_text: "As a Kinyarwanda and English linguist, translate the following English\ + \ sentences to Kinyarwanda \nEnglish: {{sentence_eng_Latn}} \nKinyarwanda: " +include: flores +task: flores_eng_Latn-kin_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-kmb_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-kmb_Latn.yaml new file mode 100644 index 00000000..270f29b6 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-kmb_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-kmb_Latn +doc_to_target: sentence_kmb_Latn +doc_to_text: "As a Kimbundu and English linguist, translate the following English\ + \ sentences to Kimbundu \nEnglish: {{sentence_eng_Latn}} \nKimbundu: " +include: flores +task: flores_eng_Latn-kmb_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-knc_Arab.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-knc_Arab.yaml new file mode 100644 index 00000000..bd2994d3 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-knc_Arab.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: eng_Latn-knc_Arab +doc_to_target: sentence_knc_Arab +doc_to_text: "As a Central Kanuri (Arabic script) and English linguist, translate\ + \ the following English sentences to Central Kanuri (Arabic script) \nEnglish: {{sentence_eng_Latn}}\ + \ \nCentral Kanuri (Arabic script): " +include: flores +task: flores_eng_Latn-knc_Arab_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-knc_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-knc_Latn.yaml new file mode 100644 index 00000000..262d0c1f --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-knc_Latn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: eng_Latn-knc_Latn +doc_to_target: sentence_knc_Latn +doc_to_text: "As a Central Kanuri (Latin script) and English linguist, translate the\ + \ following English sentences to Central Kanuri (Latin script) \nEnglish: {{sentence_eng_Latn}}\ + \ \nCentral Kanuri (Latin script): " +include: flores +task: flores_eng_Latn-knc_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-kon_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-kon_Latn.yaml new file mode 100644 index 00000000..ae9e1201 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-kon_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-kon_Latn +doc_to_target: sentence_kon_Latn +doc_to_text: "As a Kikongo and English linguist, translate the following English sentences\ + \ to Kikongo \nEnglish: {{sentence_eng_Latn}} \nKikongo: " +include: flores +task: flores_eng_Latn-kon_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-lin_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-lin_Latn.yaml new file mode 100644 index 00000000..0945c697 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-lin_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-lin_Latn +doc_to_target: sentence_lin_Latn +doc_to_text: "As a Lingala and English linguist, translate the following English sentences\ + \ to Lingala \nEnglish: {{sentence_eng_Latn}} \nLingala: " +include: flores +task: flores_eng_Latn-lin_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-lua_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-lua_Latn.yaml new file mode 100644 index 00000000..ff92a2cf --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-lua_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-lua_Latn +doc_to_target: sentence_lua_Latn +doc_to_text: "As a Luba-Kasai and English linguist, translate the following English\ + \ sentences to Luba-Kasai \nEnglish: {{sentence_eng_Latn}} \nLuba-Kasai: " +include: flores +task: flores_eng_Latn-lua_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-lug_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-lug_Latn.yaml new file mode 100644 index 00000000..4dfc626b --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-lug_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-lug_Latn +doc_to_target: sentence_lug_Latn +doc_to_text: "As a Luganda and English linguist, translate the following English sentences\ + \ to Luganda \nEnglish: {{sentence_eng_Latn}} \nLuganda: " +include: flores +task: flores_eng_Latn-lug_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-luo_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-luo_Latn.yaml new file mode 100644 index 00000000..803ed75d --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-luo_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-luo_Latn +doc_to_target: sentence_luo_Latn +doc_to_text: "As a Luo and English linguist, translate the following English sentences\ + \ to Luo \nEnglish: {{sentence_eng_Latn}} \nLuo: " +include: flores +task: flores_eng_Latn-luo_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-mos_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-mos_Latn.yaml new file mode 100644 index 00000000..0e959db1 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-mos_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-mos_Latn +doc_to_target: sentence_mos_Latn +doc_to_text: "As a Mossi and English linguist, translate the following English sentences\ + \ to Mossi \nEnglish: {{sentence_eng_Latn}} \nMossi: " +include: flores +task: flores_eng_Latn-mos_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-nso_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-nso_Latn.yaml new file mode 100644 index 00000000..44839d82 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-nso_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-nso_Latn +doc_to_target: sentence_nso_Latn +doc_to_text: "As a Northern Sotho and English linguist, translate the following English\ + \ sentences to Northern Sotho \nEnglish: {{sentence_eng_Latn}} \nNorthern Sotho: " +include: flores +task: flores_eng_Latn-nso_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-nus_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-nus_Latn.yaml new file mode 100644 index 00000000..387e4341 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-nus_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-nus_Latn +doc_to_target: sentence_nus_Latn +doc_to_text: "As a Nuer and English linguist, translate the following English sentences\ + \ to Nuer \nEnglish: {{sentence_eng_Latn}} \nNuer: " +include: flores +task: flores_eng_Latn-nus_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-nya_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-nya_Latn.yaml new file mode 100644 index 00000000..9311e264 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-nya_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-nya_Latn +doc_to_target: sentence_nya_Latn +doc_to_text: "As a Nyanja and English linguist, translate the following English sentences\ + \ to Nyanja \nEnglish: {{sentence_eng_Latn}} \nNyanja: " +include: flores +task: flores_eng_Latn-nya_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-plt_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-plt_Latn.yaml new file mode 100644 index 00000000..afc81158 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-plt_Latn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: eng_Latn-plt_Latn +doc_to_target: sentence_plt_Latn +doc_to_text: "As a Plateau Malagasy and English linguist, translate the following\ + \ English sentences to Plateau Malagasy \nEnglish: {{sentence_eng_Latn}} \nPlateau\ + \ Malagasy: " +include: flores +task: flores_eng_Latn-plt_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-run_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-run_Latn.yaml new file mode 100644 index 00000000..519700cd --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-run_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-run_Latn +doc_to_target: sentence_run_Latn +doc_to_text: "As a Rundi and English linguist, translate the following English sentences\ + \ to Rundi \nEnglish: {{sentence_eng_Latn}} \nRundi: " +include: flores +task: flores_eng_Latn-run_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-sag_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-sag_Latn.yaml new file mode 100644 index 00000000..fa99b161 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-sag_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-sag_Latn +doc_to_target: sentence_sag_Latn +doc_to_text: "As a Sango and English linguist, translate the following English sentences\ + \ to Sango \nEnglish: {{sentence_eng_Latn}} \nSango: " +include: flores +task: flores_eng_Latn-sag_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-sna_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-sna_Latn.yaml new file mode 100644 index 00000000..fd7ac49a --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-sna_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-sna_Latn +doc_to_target: sentence_sna_Latn +doc_to_text: "As a Shona and English linguist, translate the following English sentences\ + \ to Shona \nEnglish: {{sentence_eng_Latn}} \nShona: " +include: flores +task: flores_eng_Latn-sna_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-som_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-som_Latn.yaml new file mode 100644 index 00000000..17870add --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-som_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-som_Latn +doc_to_target: sentence_som_Latn +doc_to_text: "As a Somali and English linguist, translate the following English sentences\ + \ to Somali \nEnglish: {{sentence_eng_Latn}} \nSomali: " +include: flores +task: flores_eng_Latn-som_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-sot_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-sot_Latn.yaml new file mode 100644 index 00000000..a45cf383 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-sot_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-sot_Latn +doc_to_target: sentence_sot_Latn +doc_to_text: "As a Southern Sotho and English linguist, translate the following English\ + \ sentences to Southern Sotho \nEnglish: {{sentence_eng_Latn}} \nSouthern Sotho: " +include: flores +task: flores_eng_Latn-sot_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-ssw_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-ssw_Latn.yaml new file mode 100644 index 00000000..0dbd1627 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-ssw_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-ssw_Latn +doc_to_target: sentence_ssw_Latn +doc_to_text: "As a Swati and English linguist, translate the following English sentences\ + \ to Swati \nEnglish: {{sentence_eng_Latn}} \nSwati: " +include: flores +task: flores_eng_Latn-ssw_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-sun_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-sun_Latn.yaml new file mode 100644 index 00000000..0f8f6339 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-sun_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-sun_Latn +doc_to_target: sentence_sun_Latn +doc_to_text: "As a Sundanese and English linguist, translate the following English\ + \ sentences to Sundanese \nEnglish: {{sentence_eng_Latn}} \nSundanese: " +include: flores +task: flores_eng_Latn-sun_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-swh_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-swh_Latn.yaml new file mode 100644 index 00000000..20971c53 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-swh_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-swh_Latn +doc_to_target: sentence_swh_Latn +doc_to_text: "As a Swahili and English linguist, translate the following English sentences\ + \ to Swahili \nEnglish: {{sentence_eng_Latn}} \nSwahili: " +include: flores +task: flores_eng_Latn-swh_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-taq_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-taq_Latn.yaml new file mode 100644 index 00000000..bdb06f77 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-taq_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-taq_Latn +doc_to_target: sentence_taq_Latn +doc_to_text: "As a Tamasheq and English linguist, translate the following English\ + \ sentences to Tamasheq \nEnglish: {{sentence_eng_Latn}} \nTamasheq: " +include: flores +task: flores_eng_Latn-taq_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-taq_Tfng.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-taq_Tfng.yaml new file mode 100644 index 00000000..d690651d --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-taq_Tfng.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: eng_Latn-taq_Tfng +doc_to_target: sentence_taq_Tfng +doc_to_text: "As a Tamasheq (Tifinagh script) and English linguist, translate the\ + \ following English sentences to Tamasheq (Tifinagh script) \nEnglish: {{sentence_eng_Latn}}\ + \ \nTamasheq (Tifinagh script): " +include: flores +task: flores_eng_Latn-taq_Tfng_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-tir_Ethi.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-tir_Ethi.yaml new file mode 100644 index 00000000..c6b3ba34 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-tir_Ethi.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-tir_Ethi +doc_to_target: sentence_tir_Ethi +doc_to_text: "As a Tigrinya and English linguist, translate the following English\ + \ sentences to Tigrinya \nEnglish: {{sentence_eng_Latn}} \nTigrinya: " +include: flores +task: flores_eng_Latn-tir_Ethi_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-tsn_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-tsn_Latn.yaml new file mode 100644 index 00000000..845626f5 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-tsn_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-tsn_Latn +doc_to_target: sentence_tsn_Latn +doc_to_text: "As a Setswana and English linguist, translate the following English\ + \ sentences to Setswana \nEnglish: {{sentence_eng_Latn}} \nSetswana: " +include: flores +task: flores_eng_Latn-tsn_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-tso_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-tso_Latn.yaml new file mode 100644 index 00000000..958411f8 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-tso_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-tso_Latn +doc_to_target: sentence_tso_Latn +doc_to_text: "As a Tsonga and English linguist, translate the following English sentences\ + \ to Tsonga \nEnglish: {{sentence_eng_Latn}} \nTsonga: " +include: flores +task: flores_eng_Latn-tso_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-tum_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-tum_Latn.yaml new file mode 100644 index 00000000..95e6efa7 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-tum_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-tum_Latn +doc_to_target: sentence_tum_Latn +doc_to_text: "As a Tumbuka and English linguist, translate the following English sentences\ + \ to Tumbuka \nEnglish: {{sentence_eng_Latn}} \nTumbuka: " +include: flores +task: flores_eng_Latn-tum_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-twi_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-twi_Latn.yaml new file mode 100644 index 00000000..0dcb2054 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-twi_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-twi_Latn +doc_to_target: sentence_twi_Latn +doc_to_text: "As a Twi and English linguist, translate the following English sentences\ + \ to Twi \nEnglish: {{sentence_eng_Latn}} \nTwi: " +include: flores +task: flores_eng_Latn-twi_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-tzm_Tfng.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-tzm_Tfng.yaml new file mode 100644 index 00000000..887344c6 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-tzm_Tfng.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: eng_Latn-tzm_Tfng +doc_to_target: sentence_tzm_Tfng +doc_to_text: "As a Central Atlas Tamazight and English linguist, translate the following\ + \ English sentences to Central Atlas Tamazight \nEnglish: {{sentence_eng_Latn}}\ + \ \nCentral Atlas Tamazight: " +include: flores +task: flores_eng_Latn-tzm_Tfng_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-umb_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-umb_Latn.yaml new file mode 100644 index 00000000..b8c4adc0 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-umb_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-umb_Latn +doc_to_target: sentence_umb_Latn +doc_to_text: "As a Umbundu and English linguist, translate the following English sentences\ + \ to Umbundu \nEnglish: {{sentence_eng_Latn}} \nUmbundu: " +include: flores +task: flores_eng_Latn-umb_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-wol_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-wol_Latn.yaml new file mode 100644 index 00000000..66ad2579 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-wol_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-wol_Latn +doc_to_target: sentence_wol_Latn +doc_to_text: "As a Wolof and English linguist, translate the following English sentences\ + \ to Wolof \nEnglish: {{sentence_eng_Latn}} \nWolof: " +include: flores +task: flores_eng_Latn-wol_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-xho_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-xho_Latn.yaml new file mode 100644 index 00000000..8cd2fe08 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-xho_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-xho_Latn +doc_to_target: sentence_xho_Latn +doc_to_text: "As a Xhosa and English linguist, translate the following English sentences\ + \ to Xhosa \nEnglish: {{sentence_eng_Latn}} \nXhosa: " +include: flores +task: flores_eng_Latn-xho_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-yor_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-yor_Latn.yaml new file mode 100644 index 00000000..09562458 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-yor_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-yor_Latn +doc_to_target: sentence_yor_Latn +doc_to_text: "As a Yoruba and English linguist, translate the following English sentences\ + \ to Yoruba \nEnglish: {{sentence_eng_Latn}} \nYoruba: " +include: flores +task: flores_eng_Latn-yor_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-zul_Latn.yaml b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-zul_Latn.yaml new file mode 100644 index 00000000..15b41952 --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/english-african/flores_eng_Latn-zul_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn-zul_Latn +doc_to_target: sentence_zul_Latn +doc_to_text: "As a Zulu and English linguist, translate the following English sentences\ + \ to Zulu \nEnglish: {{sentence_eng_Latn}} \nZulu: " +include: flores +task: flores_eng_Latn-zul_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/flores/prompt_3/flores b/lm_eval/tasks/afrobench/flores/prompt_3/flores new file mode 100644 index 00000000..74f9f33e --- /dev/null +++ b/lm_eval/tasks/afrobench/flores/prompt_3/flores @@ -0,0 +1,24 @@ +tag: +- flores_tasks +- flores_afr-eng +dataset_path: facebook/flores +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: dev +fewshot_split: dev +test_split: devtest +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/injongointent/README.md b/lm_eval/tasks/afrobench/injongointent/README.md new file mode 100644 index 00000000..641877cb --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/README.md @@ -0,0 +1,23 @@ +# + +## Paper +Title: `INJONGO: A Multicultural Intent Detection and Slot-filling Dataset for 16 African Languages` + +Paper Link: https://arxiv.org/abs/2502.09814 + +## Abstract +>Slot-filling and intent detection are well-established tasks in Conversational AI. However, current large-scale benchmarks for these tasks often exclude evaluations of low-resource languages and rely on translations from English benchmarks, thereby predominantly reflecting Western-centric concepts. In this paper, we introduce Injongo -- a multicultural, open-source benchmark dataset for 16 African languages with utterances generated by native speakers across diverse domains, including banking, travel, home, and dining. Through extensive experiments, we benchmark the fine-tuning multilingual transformer models and the prompting large language models (LLMs), and show the advantage of leveraging African-cultural utterances over Western-centric utterances for improving cross-lingual transfer from the English language. Experimental results reveal that current LLMs struggle with the slot-filling task, with GPT-4o achieving an average performance of 26 F1-score. In contrast, intent detection performance is notably better, with an average accuracy of 70.6%, though it still falls behind the fine-tuning baselines. Compared to the English language, GPT-4o and fine-tuning baselines perform similarly on intent detection, achieving an accuracy of approximately 81%. Our findings suggest that the performance of LLMs is still behind for many low-resource African languages, and more work is needed to further improve their downstream performance. + +### Citation + +``` +@misc{yu2025injongomulticulturalintentdetection, + title={INJONGO: A Multicultural Intent Detection and Slot-filling Dataset for 16 African Languages}, + author={Hao Yu and Jesujoba O. Alabi and Andiswa Bukula and Jian Yun Zhuang and En-Shiun Annie Lee and Tadesse Kebede Guge and Israel Abebe Azime and Happy Buzaaba and Blessing Kudzaishe Sibanda and Godson K. Kalipe and Jonathan Mukiibi and Salomon Kabongo Kabenamualu and Mmasibidi Setaka and Lolwethu Ndolela and Nkiruka Odu and Rooweither Mabuya and Shamsuddeen Hassan Muhammad and Salomey Osei and Sokhar Samb and Juliet W. Murage and Dietrich Klakow and David Ifeoluwa Adelani}, + year={2025}, + eprint={2502.09814}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2502.09814}, +} +``` diff --git a/lm_eval/tasks/afrobench/injongointent/gen_utils.py b/lm_eval/tasks/afrobench/injongointent/gen_utils.py new file mode 100644 index 00000000..11204199 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/gen_utils.py @@ -0,0 +1,159 @@ +import argparse +import os + +import yaml + + +class FunctionTag: + def __init__(self, value): + self.value = value + + +def prompt_func(mode, lang, intent): + prompt_map = { + "prompt_1": "Given the text: '{{text}}', determine the correct intent from the following list: " + f"[{', '.join(intent)}]. Only output one intent from the list.", + "prompt_2": "Analyze the text: '{{text}}'. Choose the most appropriate intent from these options: " + f"[{', '.join(intent)}]. Respond with only the selected intent.", + "prompt_3": "You are a linguistic analyst trained to understand user intent. Based on the text: '{{text}}', " + f"choose the intent that best matches from this list: [{', '.join(intent)}]. Return only the intent.", + "prompt_4": f"You are a {lang} linguistic analyst trained to understand {lang} user intent. Based on the {lang}" + "text: '{{text}}', choose the intent that best matches from this list: " + f"[{', '.join(intent)}]. Return only the intent.", + "prompt_5": f"The following text is in {lang}: '{{{{text}}}}'. Given the list of intents: [{', '.join(intent)}], " + "identify the intent expressed in the text. Return only the identified intent.", + } + return prompt_map[mode] + + +def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None: + """ + Generate a yaml file for each language. + + :param output_dir: The directory to output the files to. + :param overwrite: Whether to overwrite files if they already exist. + """ + err = [] + languages = { + "amh": "Amharic", + "ewe": "Ewe", + "hau": "Hausa", + "ibo": "Igbo", + "kin": "Kinyarwanda", + "lin": "Lingala", + "lug": "Luganda", + "orm": "Oromo", + "sna": "Shona", + "sot": "Sotho", + "swa": "Swahili", + "twi": "Twi", + "wol": "Wolof", + "xho": "Xhosa", + "yor": "Yoruba", + "zul": "Zulu", + "eng": "English", + } + + intents = [ + "alarm", + "balance", + "bill_balance", + "book_flight", + "book_hotel", + "calendar_update", + "cancel_reservation", + "car_rental", + "confirm_reservation", + "cook_time", + "exchange_rate", + "food_last", + "freeze_account", + "ingredients_list", + "interest_rate", + "international_visa", + "make_call", + "meal_suggestion", + "min_payment", + "pay_bill", + "pin_change", + "play_music", + "plug_type", + "recipe", + "restaurant_reservation", + "restaurant_reviews", + "restaurant_suggestion", + "share_location", + "shopping_list_update", + "spending_history", + "text", + "time", + "timezone", + "transactions", + "transfer", + "translate", + "travel_notification", + "travel_suggestion", + "update_playlist", + "weather", + ] + + for lang in languages.keys(): + try: + file_name = f"injongointent_{lang}.yaml" + task_name = f"injongointent_{lang}_{mode}" + yaml_template = "injongointent" + yaml_details = { + "include": yaml_template, + "task": task_name, + "dataset_name": lang, + "doc_to_text": prompt_func(mode, languages[lang], intents), + } + os.makedirs(f"{output_dir}/{mode}", exist_ok=True) + with open( + f"{output_dir}/{mode}/{file_name}", + "w" if overwrite else "x", + encoding="utf8", + ) as f: + f.write("# Generated by utils.py\n") + yaml.dump( + yaml_details, + f, + allow_unicode=True, + ) + except FileExistsError: + err.append(file_name) + + if len(err) > 0: + raise FileExistsError( + "Files were not created because they already exist (use --overwrite flag):" + f" {', '.join(err)}" + ) + + +def main() -> None: + """Parse CLI args and generate language-specific yaml files.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + default=True, + action="store_true", + help="Overwrite files if they already exist", + ) + parser.add_argument( + "--output-dir", + default="./", + help="Directory to write yaml files to", + ) + parser.add_argument( + "--mode", + default="prompt_3", + choices=["prompt_1", "prompt_2", "prompt_3", "prompt_4", "prompt_5"], + help="Prompt number", + ) + args = parser.parse_args() + + gen_lang_yamls(output_dir=args.output_dir, overwrite=args.overwrite, mode=args.mode) + + +if __name__ == "__main__": + main() diff --git a/lm_eval/tasks/afrobench/injongointent/injongointent.yaml b/lm_eval/tasks/afrobench/injongointent/injongointent.yaml new file mode 100644 index 00000000..220f4c51 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/injongointent.yaml @@ -0,0 +1,13 @@ +group: injongointent +task: + - injongointent_prompt_1 + - injongointent_prompt_2 + - injongointent_prompt_3 + - injongointent_prompt_4 + - injongointent_prompt_5 +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 1 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent b/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent new file mode 100644 index 00000000..a77bc5c9 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent @@ -0,0 +1,75 @@ +tag: +- injongointent_tasks +- injongointent_prompt_1 +dataset_path: masakhane/InjongoIntent +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: intent +doc_to_choice: + - alarm + - balance + - bill_balance + - book_flight + - book_hotel + - calendar_update + - cancel_reservation + - car_rental + - confirm_reservation + - cook_time + - exchange_rate + - food_last + - freeze_account + - ingredients_list + - interest_rate + - international_visa + - make_call + - meal_suggestion + - min_payment + - pay_bill + - pin_change + - play_music + - plug_type + - recipe + - restaurant_reservation + - restaurant_reviews + - restaurant_suggestion + - share_location + - shopping_list_update + - spending_history + - text + - time + - timezone + - transactions + - transfer + - translate + - travel_notification + - travel_suggestion + - update_playlist + - weather +should_decontaminate: true +doc_to_decontamination_query: text +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_amh.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_amh.yaml new file mode 100644 index 00000000..0b3a3ee2 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_amh.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: 'Given the text: ''{{text}}'', determine the correct intent from the + following list: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Only + output one intent from the list.' +include: injongointent +task: injongointent_amh_prompt_1 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_eng.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_eng.yaml new file mode 100644 index 00000000..240c37d5 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_eng.yaml @@ -0,0 +1,16 @@ +# Generated by utils.py +dataset_name: eng +validation_split: train +test_split: test +fewshot_split: train +doc_to_text: 'Given the text: ''{{text}}'', determine the correct intent from the + following list: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Only + output one intent from the list.' +include: injongointent +task: injongointent_eng_prompt_1 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_ewe.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_ewe.yaml new file mode 100644 index 00000000..c08d8bb0 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_ewe.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: ewe +doc_to_text: 'Given the text: ''{{text}}'', determine the correct intent from the + following list: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Only + output one intent from the list.' +include: injongointent +task: injongointent_ewe_prompt_1 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_hau.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_hau.yaml new file mode 100644 index 00000000..9e1338c7 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_hau.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: 'Given the text: ''{{text}}'', determine the correct intent from the + following list: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Only + output one intent from the list.' +include: injongointent +task: injongointent_hau_prompt_1 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_ibo.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_ibo.yaml new file mode 100644 index 00000000..e4a956d2 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_ibo.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: 'Given the text: ''{{text}}'', determine the correct intent from the + following list: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Only + output one intent from the list.' +include: injongointent +task: injongointent_ibo_prompt_1 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_kin.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_kin.yaml new file mode 100644 index 00000000..f55d787a --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_kin.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: 'Given the text: ''{{text}}'', determine the correct intent from the + following list: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Only + output one intent from the list.' +include: injongointent +task: injongointent_kin_prompt_1 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_lin.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_lin.yaml new file mode 100644 index 00000000..2cc08df4 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_lin.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: 'Given the text: ''{{text}}'', determine the correct intent from the + following list: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Only + output one intent from the list.' +include: injongointent +task: injongointent_lin_prompt_1 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_lug.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_lug.yaml new file mode 100644 index 00000000..b1a42157 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_lug.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: 'Given the text: ''{{text}}'', determine the correct intent from the + following list: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Only + output one intent from the list.' +include: injongointent +task: injongointent_lug_prompt_1 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_orm.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_orm.yaml new file mode 100644 index 00000000..2b95a4e7 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_orm.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: 'Given the text: ''{{text}}'', determine the correct intent from the + following list: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Only + output one intent from the list.' +include: injongointent +task: injongointent_orm_prompt_1 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_sna.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_sna.yaml new file mode 100644 index 00000000..6cbf0105 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_sna.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: 'Given the text: ''{{text}}'', determine the correct intent from the + following list: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Only + output one intent from the list.' +include: injongointent +task: injongointent_sna_prompt_1 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_sot.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_sot.yaml new file mode 100644 index 00000000..ad3b4497 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_sot.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: sot +doc_to_text: 'Given the text: ''{{text}}'', determine the correct intent from the + following list: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Only + output one intent from the list.' +include: injongointent +task: injongointent_sot_prompt_1 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_swa.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_swa.yaml new file mode 100644 index 00000000..fc3d797c --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_swa.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: 'Given the text: ''{{text}}'', determine the correct intent from the + following list: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Only + output one intent from the list.' +include: injongointent +task: injongointent_swa_prompt_1 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_twi.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_twi.yaml new file mode 100644 index 00000000..73fc61c7 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_twi.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: 'Given the text: ''{{text}}'', determine the correct intent from the + following list: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Only + output one intent from the list.' +include: injongointent +task: injongointent_twi_prompt_1 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_wol.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_wol.yaml new file mode 100644 index 00000000..7d359d2f --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_wol.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: 'Given the text: ''{{text}}'', determine the correct intent from the + following list: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Only + output one intent from the list.' +include: injongointent +task: injongointent_wol_prompt_1 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_xho.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_xho.yaml new file mode 100644 index 00000000..4d9c173a --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_xho.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: 'Given the text: ''{{text}}'', determine the correct intent from the + following list: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Only + output one intent from the list.' +include: injongointent +task: injongointent_xho_prompt_1 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_yor.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_yor.yaml new file mode 100644 index 00000000..682e01c1 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_yor.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: 'Given the text: ''{{text}}'', determine the correct intent from the + following list: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Only + output one intent from the list.' +include: injongointent +task: injongointent_yor_prompt_1 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_zul.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_zul.yaml new file mode 100644 index 00000000..1d38a781 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_1/injongointent_zul.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: 'Given the text: ''{{text}}'', determine the correct intent from the + following list: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Only + output one intent from the list.' +include: injongointent +task: injongointent_zul_prompt_1 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_1/utils.py b/lm_eval/tasks/afrobench/injongointent/prompt_1/utils.py new file mode 100644 index 00000000..3e735e2d --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_1/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent b/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent new file mode 100644 index 00000000..dfcb8267 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent @@ -0,0 +1,75 @@ +tag: +- injongointent_tasks +- injongointent_prompt_2 +dataset_path: masakhane/InjongoIntent +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: intent +doc_to_choice: + - alarm + - balance + - bill_balance + - book_flight + - book_hotel + - calendar_update + - cancel_reservation + - car_rental + - confirm_reservation + - cook_time + - exchange_rate + - food_last + - freeze_account + - ingredients_list + - interest_rate + - international_visa + - make_call + - meal_suggestion + - min_payment + - pay_bill + - pin_change + - play_music + - plug_type + - recipe + - restaurant_reservation + - restaurant_reviews + - restaurant_suggestion + - share_location + - shopping_list_update + - spending_history + - text + - time + - timezone + - transactions + - transfer + - translate + - travel_notification + - travel_suggestion + - update_playlist + - weather +should_decontaminate: true +doc_to_decontamination_query: text +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_amh.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_amh.yaml new file mode 100644 index 00000000..7c1b2189 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_amh.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: 'Analyze the text: ''{{text}}''. Choose the most appropriate intent from + these options: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Respond + with only the selected intent.' +include: injongointent +task: injongointent_amh_prompt_2 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_eng.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_eng.yaml new file mode 100644 index 00000000..cc03705d --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_eng.yaml @@ -0,0 +1,16 @@ +# Generated by utils.py +dataset_name: eng +validation_split: train +test_split: test +fewshot_split: train +doc_to_text: 'Analyze the text: ''{{text}}''. Choose the most appropriate intent from + these options: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Respond + with only the selected intent.' +include: injongointent +task: injongointent_eng_prompt_2 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_ewe.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_ewe.yaml new file mode 100644 index 00000000..58eb9144 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_ewe.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: ewe +doc_to_text: 'Analyze the text: ''{{text}}''. Choose the most appropriate intent from + these options: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Respond + with only the selected intent.' +include: injongointent +task: injongointent_ewe_prompt_2 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_hau.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_hau.yaml new file mode 100644 index 00000000..e7745369 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_hau.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: 'Analyze the text: ''{{text}}''. Choose the most appropriate intent from + these options: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Respond + with only the selected intent.' +include: injongointent +task: injongointent_hau_prompt_2 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_ibo.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_ibo.yaml new file mode 100644 index 00000000..b47052d7 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_ibo.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: 'Analyze the text: ''{{text}}''. Choose the most appropriate intent from + these options: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Respond + with only the selected intent.' +include: injongointent +task: injongointent_ibo_prompt_2 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_kin.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_kin.yaml new file mode 100644 index 00000000..45793567 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_kin.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: 'Analyze the text: ''{{text}}''. Choose the most appropriate intent from + these options: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Respond + with only the selected intent.' +include: injongointent +task: injongointent_kin_prompt_2 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_lin.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_lin.yaml new file mode 100644 index 00000000..54e7fcb7 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_lin.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: 'Analyze the text: ''{{text}}''. Choose the most appropriate intent from + these options: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Respond + with only the selected intent.' +include: injongointent +task: injongointent_lin_prompt_2 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_lug.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_lug.yaml new file mode 100644 index 00000000..96aa42fc --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_lug.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: 'Analyze the text: ''{{text}}''. Choose the most appropriate intent from + these options: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Respond + with only the selected intent.' +include: injongointent +task: injongointent_lug_prompt_2 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_orm.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_orm.yaml new file mode 100644 index 00000000..872f96c5 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_orm.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: 'Analyze the text: ''{{text}}''. Choose the most appropriate intent from + these options: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Respond + with only the selected intent.' +include: injongointent +task: injongointent_orm_prompt_2 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_sna.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_sna.yaml new file mode 100644 index 00000000..a62dfe34 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_sna.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: 'Analyze the text: ''{{text}}''. Choose the most appropriate intent from + these options: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Respond + with only the selected intent.' +include: injongointent +task: injongointent_sna_prompt_2 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_sot.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_sot.yaml new file mode 100644 index 00000000..e9ca6a56 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_sot.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: sot +doc_to_text: 'Analyze the text: ''{{text}}''. Choose the most appropriate intent from + these options: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Respond + with only the selected intent.' +include: injongointent +task: injongointent_sot_prompt_2 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_swa.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_swa.yaml new file mode 100644 index 00000000..339f66ac --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_swa.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: 'Analyze the text: ''{{text}}''. Choose the most appropriate intent from + these options: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Respond + with only the selected intent.' +include: injongointent +task: injongointent_swa_prompt_2 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_twi.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_twi.yaml new file mode 100644 index 00000000..b758bce0 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_twi.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: 'Analyze the text: ''{{text}}''. Choose the most appropriate intent from + these options: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Respond + with only the selected intent.' +include: injongointent +task: injongointent_twi_prompt_2 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_wol.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_wol.yaml new file mode 100644 index 00000000..9b573a44 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_wol.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: 'Analyze the text: ''{{text}}''. Choose the most appropriate intent from + these options: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Respond + with only the selected intent.' +include: injongointent +task: injongointent_wol_prompt_2 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_xho.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_xho.yaml new file mode 100644 index 00000000..f2c02205 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_xho.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: 'Analyze the text: ''{{text}}''. Choose the most appropriate intent from + these options: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Respond + with only the selected intent.' +include: injongointent +task: injongointent_xho_prompt_2 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_yor.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_yor.yaml new file mode 100644 index 00000000..4c821736 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_yor.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: 'Analyze the text: ''{{text}}''. Choose the most appropriate intent from + these options: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Respond + with only the selected intent.' +include: injongointent +task: injongointent_yor_prompt_2 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_zul.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_zul.yaml new file mode 100644 index 00000000..c8a541b6 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_2/injongointent_zul.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: 'Analyze the text: ''{{text}}''. Choose the most appropriate intent from + these options: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Respond + with only the selected intent.' +include: injongointent +task: injongointent_zul_prompt_2 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_2/utils.py b/lm_eval/tasks/afrobench/injongointent/prompt_2/utils.py new file mode 100644 index 00000000..3e735e2d --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_2/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent b/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent new file mode 100644 index 00000000..afdf43cf --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent @@ -0,0 +1,75 @@ +tag: +- injongointent_tasks +- injongointent_prompt_3 +dataset_path: masakhane/InjongoIntent +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: intent +doc_to_choice: + - alarm + - balance + - bill_balance + - book_flight + - book_hotel + - calendar_update + - cancel_reservation + - car_rental + - confirm_reservation + - cook_time + - exchange_rate + - food_last + - freeze_account + - ingredients_list + - interest_rate + - international_visa + - make_call + - meal_suggestion + - min_payment + - pay_bill + - pin_change + - play_music + - plug_type + - recipe + - restaurant_reservation + - restaurant_reviews + - restaurant_suggestion + - share_location + - shopping_list_update + - spending_history + - text + - time + - timezone + - transactions + - transfer + - translate + - travel_notification + - travel_suggestion + - update_playlist + - weather +should_decontaminate: true +doc_to_decontamination_query: text +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_amh.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_amh.yaml new file mode 100644 index 00000000..7bd62c5b --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_amh.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: 'You are a linguistic analyst trained to understand user intent. Based + on the text: ''{{text}}'', choose the intent that best matches from this list: [alarm, + balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather]. Return only the intent.' +include: injongointent +task: injongointent_amh_prompt_3 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_eng.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_eng.yaml new file mode 100644 index 00000000..258f0cfa --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_eng.yaml @@ -0,0 +1,16 @@ +# Generated by utils.py +dataset_name: eng +validation_split: train +test_split: test +fewshot_split: train +doc_to_text: 'You are a linguistic analyst trained to understand user intent. Based + on the text: ''{{text}}'', choose the intent that best matches from this list: [alarm, + balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather]. Return only the intent.' +include: injongointent +task: injongointent_eng_prompt_3 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_ewe.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_ewe.yaml new file mode 100644 index 00000000..12688cc9 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_ewe.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: ewe +doc_to_text: 'You are a linguistic analyst trained to understand user intent. Based + on the text: ''{{text}}'', choose the intent that best matches from this list: [alarm, + balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather]. Return only the intent.' +include: injongointent +task: injongointent_ewe_prompt_3 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_hau.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_hau.yaml new file mode 100644 index 00000000..8414a09b --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_hau.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: 'You are a linguistic analyst trained to understand user intent. Based + on the text: ''{{text}}'', choose the intent that best matches from this list: [alarm, + balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather]. Return only the intent.' +include: injongointent +task: injongointent_hau_prompt_3 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_ibo.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_ibo.yaml new file mode 100644 index 00000000..8f254438 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_ibo.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: 'You are a linguistic analyst trained to understand user intent. Based + on the text: ''{{text}}'', choose the intent that best matches from this list: [alarm, + balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather]. Return only the intent.' +include: injongointent +task: injongointent_ibo_prompt_3 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_kin.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_kin.yaml new file mode 100644 index 00000000..b946cf00 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_kin.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: 'You are a linguistic analyst trained to understand user intent. Based + on the text: ''{{text}}'', choose the intent that best matches from this list: [alarm, + balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather]. Return only the intent.' +include: injongointent +task: injongointent_kin_prompt_3 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_lin.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_lin.yaml new file mode 100644 index 00000000..4a485d5c --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_lin.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: 'You are a linguistic analyst trained to understand user intent. Based + on the text: ''{{text}}'', choose the intent that best matches from this list: [alarm, + balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather]. Return only the intent.' +include: injongointent +task: injongointent_lin_prompt_3 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_lug.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_lug.yaml new file mode 100644 index 00000000..71376ec3 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_lug.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: 'You are a linguistic analyst trained to understand user intent. Based + on the text: ''{{text}}'', choose the intent that best matches from this list: [alarm, + balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather]. Return only the intent.' +include: injongointent +task: injongointent_lug_prompt_3 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_orm.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_orm.yaml new file mode 100644 index 00000000..706f3a90 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_orm.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: 'You are a linguistic analyst trained to understand user intent. Based + on the text: ''{{text}}'', choose the intent that best matches from this list: [alarm, + balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather]. Return only the intent.' +include: injongointent +task: injongointent_orm_prompt_3 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_sna.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_sna.yaml new file mode 100644 index 00000000..f4aca737 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_sna.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: 'You are a linguistic analyst trained to understand user intent. Based + on the text: ''{{text}}'', choose the intent that best matches from this list: [alarm, + balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather]. Return only the intent.' +include: injongointent +task: injongointent_sna_prompt_3 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_sot.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_sot.yaml new file mode 100644 index 00000000..57e27afa --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_sot.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: sot +doc_to_text: 'You are a linguistic analyst trained to understand user intent. Based + on the text: ''{{text}}'', choose the intent that best matches from this list: [alarm, + balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather]. Return only the intent.' +include: injongointent +task: injongointent_sot_prompt_3 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_swa.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_swa.yaml new file mode 100644 index 00000000..6cb4886d --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_swa.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: 'You are a linguistic analyst trained to understand user intent. Based + on the text: ''{{text}}'', choose the intent that best matches from this list: [alarm, + balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather]. Return only the intent.' +include: injongointent +task: injongointent_swa_prompt_3 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_twi.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_twi.yaml new file mode 100644 index 00000000..e8623bf3 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_twi.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: 'You are a linguistic analyst trained to understand user intent. Based + on the text: ''{{text}}'', choose the intent that best matches from this list: [alarm, + balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather]. Return only the intent.' +include: injongointent +task: injongointent_twi_prompt_3 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_wol.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_wol.yaml new file mode 100644 index 00000000..afc3cf4a --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_wol.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: 'You are a linguistic analyst trained to understand user intent. Based + on the text: ''{{text}}'', choose the intent that best matches from this list: [alarm, + balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather]. Return only the intent.' +include: injongointent +task: injongointent_wol_prompt_3 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_xho.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_xho.yaml new file mode 100644 index 00000000..9f41aa56 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_xho.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: 'You are a linguistic analyst trained to understand user intent. Based + on the text: ''{{text}}'', choose the intent that best matches from this list: [alarm, + balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather]. Return only the intent.' +include: injongointent +task: injongointent_xho_prompt_3 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_yor.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_yor.yaml new file mode 100644 index 00000000..3a5d5686 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_yor.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: 'You are a linguistic analyst trained to understand user intent. Based + on the text: ''{{text}}'', choose the intent that best matches from this list: [alarm, + balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather]. Return only the intent.' +include: injongointent +task: injongointent_yor_prompt_3 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_zul.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_zul.yaml new file mode 100644 index 00000000..f857ff06 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_3/injongointent_zul.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: 'You are a linguistic analyst trained to understand user intent. Based + on the text: ''{{text}}'', choose the intent that best matches from this list: [alarm, + balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather]. Return only the intent.' +include: injongointent +task: injongointent_zul_prompt_3 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_3/utils.py b/lm_eval/tasks/afrobench/injongointent/prompt_3/utils.py new file mode 100644 index 00000000..3e735e2d --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_3/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent b/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent new file mode 100644 index 00000000..5d5c05ae --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent @@ -0,0 +1,75 @@ +tag: +- injongointent_tasks +- injongointent_prompt_4 +dataset_path: masakhane/InjongoIntent +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: intent +doc_to_choice: + - alarm + - balance + - bill_balance + - book_flight + - book_hotel + - calendar_update + - cancel_reservation + - car_rental + - confirm_reservation + - cook_time + - exchange_rate + - food_last + - freeze_account + - ingredients_list + - interest_rate + - international_visa + - make_call + - meal_suggestion + - min_payment + - pay_bill + - pin_change + - play_music + - plug_type + - recipe + - restaurant_reservation + - restaurant_reviews + - restaurant_suggestion + - share_location + - shopping_list_update + - spending_history + - text + - time + - timezone + - transactions + - transfer + - translate + - travel_notification + - travel_suggestion + - update_playlist + - weather +should_decontaminate: true +doc_to_decontamination_query: text +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_amh.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_amh.yaml new file mode 100644 index 00000000..aa14ee5b --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_amh.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: 'You are a Amharic linguistic analyst trained to understand Amharic user + intent. Based on the Amharictext: ''{{text}}'', choose the intent that best matches + from this list: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Return + only the intent.' +include: injongointent +task: injongointent_amh_prompt_4 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_eng.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_eng.yaml new file mode 100644 index 00000000..853e6496 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_eng.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: eng +validation_split: train +test_split: test +fewshot_split: train +doc_to_text: 'You are a English linguistic analyst trained to understand English user + intent. Based on the English text: ''{{text}}'', choose the intent that best matches + from this list: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Return + only the intent.' +include: injongointent +task: injongointent_eng_prompt_4 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_ewe.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_ewe.yaml new file mode 100644 index 00000000..f61a3db5 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_ewe.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: ewe +doc_to_text: 'You are a Ewe linguistic analyst trained to understand Ewe user intent. + Based on the Ewetext: ''{{text}}'', choose the intent that best matches from this + list: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather]. Return only the intent.' +include: injongointent +task: injongointent_ewe_prompt_4 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_hau.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_hau.yaml new file mode 100644 index 00000000..fdef34cb --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_hau.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: 'You are a Hausa linguistic analyst trained to understand Hausa user + intent. Based on the Hausatext: ''{{text}}'', choose the intent that best matches + from this list: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Return + only the intent.' +include: injongointent +task: injongointent_hau_prompt_4 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_ibo.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_ibo.yaml new file mode 100644 index 00000000..23b59831 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_ibo.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: 'You are a Igbo linguistic analyst trained to understand Igbo user intent. + Based on the Igbotext: ''{{text}}'', choose the intent that best matches from this + list: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather]. Return only the intent.' +include: injongointent +task: injongointent_ibo_prompt_4 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_kin.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_kin.yaml new file mode 100644 index 00000000..28f05aeb --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_kin.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: 'You are a Kinyarwanda linguistic analyst trained to understand Kinyarwanda + user intent. Based on the Kinyarwandatext: ''{{text}}'', choose the intent that + best matches from this list: [alarm, balance, bill_balance, book_flight, book_hotel, + calendar_update, cancel_reservation, car_rental, confirm_reservation, cook_time, + exchange_rate, food_last, freeze_account, ingredients_list, interest_rate, international_visa, + make_call, meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, + recipe, restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Return + only the intent.' +include: injongointent +task: injongointent_kin_prompt_4 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_lin.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_lin.yaml new file mode 100644 index 00000000..df991d89 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_lin.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: 'You are a Lingala linguistic analyst trained to understand Lingala user + intent. Based on the Lingalatext: ''{{text}}'', choose the intent that best matches + from this list: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Return + only the intent.' +include: injongointent +task: injongointent_lin_prompt_4 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_lug.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_lug.yaml new file mode 100644 index 00000000..c1abb66e --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_lug.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: 'You are a Luganda linguistic analyst trained to understand Luganda user + intent. Based on the Lugandatext: ''{{text}}'', choose the intent that best matches + from this list: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Return + only the intent.' +include: injongointent +task: injongointent_lug_prompt_4 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_orm.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_orm.yaml new file mode 100644 index 00000000..195ff4a2 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_orm.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: 'You are a Oromo linguistic analyst trained to understand Oromo user + intent. Based on the Oromotext: ''{{text}}'', choose the intent that best matches + from this list: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Return + only the intent.' +include: injongointent +task: injongointent_orm_prompt_4 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_sna.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_sna.yaml new file mode 100644 index 00000000..23d066c3 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_sna.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: 'You are a Shona linguistic analyst trained to understand Shona user + intent. Based on the Shonatext: ''{{text}}'', choose the intent that best matches + from this list: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Return + only the intent.' +include: injongointent +task: injongointent_sna_prompt_4 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_sot.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_sot.yaml new file mode 100644 index 00000000..82102a21 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_sot.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: sot +doc_to_text: 'You are a Sotho linguistic analyst trained to understand Sotho user + intent. Based on the Sothotext: ''{{text}}'', choose the intent that best matches + from this list: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Return + only the intent.' +include: injongointent +task: injongointent_sot_prompt_4 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_swa.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_swa.yaml new file mode 100644 index 00000000..031ffbb4 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_swa.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: 'You are a Swahili linguistic analyst trained to understand Swahili user + intent. Based on the Swahilitext: ''{{text}}'', choose the intent that best matches + from this list: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Return + only the intent.' +include: injongointent +task: injongointent_swa_prompt_4 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_twi.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_twi.yaml new file mode 100644 index 00000000..a569b3ce --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_twi.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: 'You are a Twi linguistic analyst trained to understand Twi user intent. + Based on the Twitext: ''{{text}}'', choose the intent that best matches from this + list: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather]. Return only the intent.' +include: injongointent +task: injongointent_twi_prompt_4 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_wol.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_wol.yaml new file mode 100644 index 00000000..a55398ab --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_wol.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: 'You are a Wolof linguistic analyst trained to understand Wolof user + intent. Based on the Woloftext: ''{{text}}'', choose the intent that best matches + from this list: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Return + only the intent.' +include: injongointent +task: injongointent_wol_prompt_4 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_xho.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_xho.yaml new file mode 100644 index 00000000..d773a175 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_xho.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: 'You are a Xhosa linguistic analyst trained to understand Xhosa user + intent. Based on the Xhosatext: ''{{text}}'', choose the intent that best matches + from this list: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Return + only the intent.' +include: injongointent +task: injongointent_xho_prompt_4 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_yor.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_yor.yaml new file mode 100644 index 00000000..af01d9f3 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_yor.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: 'You are a Yoruba linguistic analyst trained to understand Yoruba user + intent. Based on the Yorubatext: ''{{text}}'', choose the intent that best matches + from this list: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather]. Return + only the intent.' +include: injongointent +task: injongointent_yor_prompt_4 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_zul.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_zul.yaml new file mode 100644 index 00000000..3b6e5aac --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_4/injongointent_zul.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: 'You are a Zulu linguistic analyst trained to understand Zulu user intent. + Based on the Zulutext: ''{{text}}'', choose the intent that best matches from this + list: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather]. Return only the intent.' +include: injongointent +task: injongointent_zul_prompt_4 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_4/utils.py b/lm_eval/tasks/afrobench/injongointent/prompt_4/utils.py new file mode 100644 index 00000000..3e735e2d --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_4/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent b/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent new file mode 100644 index 00000000..0012857b --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent @@ -0,0 +1,75 @@ +tag: +- injongointent_tasks +- injongointent_prompt_5 +dataset_path: masakhane/InjongoIntent +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: intent +doc_to_choice: + - alarm + - balance + - bill_balance + - book_flight + - book_hotel + - calendar_update + - cancel_reservation + - car_rental + - confirm_reservation + - cook_time + - exchange_rate + - food_last + - freeze_account + - ingredients_list + - interest_rate + - international_visa + - make_call + - meal_suggestion + - min_payment + - pay_bill + - pin_change + - play_music + - plug_type + - recipe + - restaurant_reservation + - restaurant_reviews + - restaurant_suggestion + - share_location + - shopping_list_update + - spending_history + - text + - time + - timezone + - transactions + - transfer + - translate + - travel_notification + - travel_suggestion + - update_playlist + - weather +should_decontaminate: true +doc_to_decontamination_query: text +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_amh.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_amh.yaml new file mode 100644 index 00000000..2a6623a9 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_amh.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: 'The following text is in Amharic: ''{{text}}''. Given the list of intents: + [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather], identify the intent expressed in the + text. Return only the identified intent.' +include: injongointent +task: injongointent_amh_prompt_5 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_eng.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_eng.yaml new file mode 100644 index 00000000..4dcbebbb --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_eng.yaml @@ -0,0 +1,16 @@ +# Generated by utils.py +dataset_name: eng +validation_split: train +test_split: test +fewshot_split: train +doc_to_text: 'The following text is in English: ''{{text}}''. Given the list of intents: + [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather], identify the intent expressed in the + text. Return only the identified intent.' +include: injongointent +task: injongointent_eng_prompt_5 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_ewe.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_ewe.yaml new file mode 100644 index 00000000..0cab8425 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_ewe.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: ewe +doc_to_text: 'The following text is in Ewe: ''{{text}}''. Given the list of intents: + [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather], identify the intent expressed in the + text. Return only the identified intent.' +include: injongointent +task: injongointent_ewe_prompt_5 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_hau.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_hau.yaml new file mode 100644 index 00000000..b6275db8 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_hau.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: 'The following text is in Hausa: ''{{text}}''. Given the list of intents: + [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather], identify the intent expressed in the + text. Return only the identified intent.' +include: injongointent +task: injongointent_hau_prompt_5 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_ibo.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_ibo.yaml new file mode 100644 index 00000000..518ec898 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_ibo.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: 'The following text is in Igbo: ''{{text}}''. Given the list of intents: + [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather], identify the intent expressed in the + text. Return only the identified intent.' +include: injongointent +task: injongointent_ibo_prompt_5 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_kin.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_kin.yaml new file mode 100644 index 00000000..348535c6 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_kin.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: 'The following text is in Kinyarwanda: ''{{text}}''. Given the list of + intents: [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, + cancel_reservation, car_rental, confirm_reservation, cook_time, exchange_rate, food_last, + freeze_account, ingredients_list, interest_rate, international_visa, make_call, + meal_suggestion, min_payment, pay_bill, pin_change, play_music, plug_type, recipe, + restaurant_reservation, restaurant_reviews, restaurant_suggestion, share_location, + shopping_list_update, spending_history, text, time, timezone, transactions, transfer, + translate, travel_notification, travel_suggestion, update_playlist, weather], identify + the intent expressed in the text. Return only the identified intent.' +include: injongointent +task: injongointent_kin_prompt_5 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_lin.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_lin.yaml new file mode 100644 index 00000000..75bbf4ec --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_lin.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: 'The following text is in Lingala: ''{{text}}''. Given the list of intents: + [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather], identify the intent expressed in the + text. Return only the identified intent.' +include: injongointent +task: injongointent_lin_prompt_5 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_lug.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_lug.yaml new file mode 100644 index 00000000..49b7f6fa --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_lug.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: 'The following text is in Luganda: ''{{text}}''. Given the list of intents: + [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather], identify the intent expressed in the + text. Return only the identified intent.' +include: injongointent +task: injongointent_lug_prompt_5 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_orm.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_orm.yaml new file mode 100644 index 00000000..72a76869 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_orm.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: 'The following text is in Oromo: ''{{text}}''. Given the list of intents: + [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather], identify the intent expressed in the + text. Return only the identified intent.' +include: injongointent +task: injongointent_orm_prompt_5 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_sna.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_sna.yaml new file mode 100644 index 00000000..8931b65c --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_sna.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: 'The following text is in Shona: ''{{text}}''. Given the list of intents: + [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather], identify the intent expressed in the + text. Return only the identified intent.' +include: injongointent +task: injongointent_sna_prompt_5 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_sot.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_sot.yaml new file mode 100644 index 00000000..5a8d0328 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_sot.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: sot +doc_to_text: 'The following text is in Sotho: ''{{text}}''. Given the list of intents: + [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather], identify the intent expressed in the + text. Return only the identified intent.' +include: injongointent +task: injongointent_sot_prompt_5 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_swa.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_swa.yaml new file mode 100644 index 00000000..1da6be32 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_swa.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: 'The following text is in Swahili: ''{{text}}''. Given the list of intents: + [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather], identify the intent expressed in the + text. Return only the identified intent.' +include: injongointent +task: injongointent_swa_prompt_5 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_twi.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_twi.yaml new file mode 100644 index 00000000..cc78ae4f --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_twi.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: 'The following text is in Twi: ''{{text}}''. Given the list of intents: + [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather], identify the intent expressed in the + text. Return only the identified intent.' +include: injongointent +task: injongointent_twi_prompt_5 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_wol.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_wol.yaml new file mode 100644 index 00000000..9c71483e --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_wol.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: 'The following text is in Wolof: ''{{text}}''. Given the list of intents: + [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather], identify the intent expressed in the + text. Return only the identified intent.' +include: injongointent +task: injongointent_wol_prompt_5 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_xho.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_xho.yaml new file mode 100644 index 00000000..8d8b543f --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_xho.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: 'The following text is in Xhosa: ''{{text}}''. Given the list of intents: + [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather], identify the intent expressed in the + text. Return only the identified intent.' +include: injongointent +task: injongointent_xho_prompt_5 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_yor.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_yor.yaml new file mode 100644 index 00000000..cbe28568 --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_yor.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: 'The following text is in Yoruba: ''{{text}}''. Given the list of intents: + [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather], identify the intent expressed in the + text. Return only the identified intent.' +include: injongointent +task: injongointent_yor_prompt_5 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_zul.yaml b/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_zul.yaml new file mode 100644 index 00000000..7ba384db --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_5/injongointent_zul.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: 'The following text is in Zulu: ''{{text}}''. Given the list of intents: + [alarm, balance, bill_balance, book_flight, book_hotel, calendar_update, cancel_reservation, + car_rental, confirm_reservation, cook_time, exchange_rate, food_last, freeze_account, + ingredients_list, interest_rate, international_visa, make_call, meal_suggestion, + min_payment, pay_bill, pin_change, play_music, plug_type, recipe, restaurant_reservation, + restaurant_reviews, restaurant_suggestion, share_location, shopping_list_update, + spending_history, text, time, timezone, transactions, transfer, translate, travel_notification, + travel_suggestion, update_playlist, weather], identify the intent expressed in the + text. Return only the identified intent.' +include: injongointent +task: injongointent_zul_prompt_5 diff --git a/lm_eval/tasks/afrobench/injongointent/prompt_5/utils.py b/lm_eval/tasks/afrobench/injongointent/prompt_5/utils.py new file mode 100644 index 00000000..3e735e2d --- /dev/null +++ b/lm_eval/tasks/afrobench/injongointent/prompt_5/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm_eval/tasks/afrobench/mafand/README.md b/lm_eval/tasks/afrobench/mafand/README.md new file mode 100644 index 00000000..9e7eea17 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/README.md @@ -0,0 +1,73 @@ +# + +## Paper +Title: `A Few Thousand Translations Go a Long Way! Leveraging Pre-trained Models for African News Translation` + +Paper Link: https://aclanthology.org/2022.naacl-main.223/ + +## Abstract +>Recent advances in the pre-training of language models leverage large-scale datasets to create multilingual models. However, low-resource languages are mostly left out in these datasets. This is primarily because many widely spoken languages are not well represented on the web and therefore excluded from the large-scale crawls used to create datasets. Furthermore, downstream users of these models are restricted to the selection of languages originally chosen for pre-training. This work investigates how to optimally leverage existing pre-trained models to create low-resource translation systems for 16 African languages. We focus on two questions: 1) How can pre-trained models be used for languages not included in the initial pre-training? and 2) How can the resulting translation models effectively transfer to new domains? To answer these questions, we create a new African news corpus covering 16 languages, of which eight languages are not part of any existing evaluation dataset. We demonstrate that the most effective strategy for transferring both to additional languages and to additional domains is to fine-tune large pre-trained models on small quantities of high-quality translation data. + +HomePage: https://github.com/masakhane-io/lafand-mt + +### Citation + +``` +@inproceedings{adelani-etal-2022-thousand, + title = "A Few Thousand Translations Go a Long Way! Leveraging Pre-trained Models for {A}frican News Translation", + author = "Adelani, David and + Alabi, Jesujoba and + Fan, Angela and + Kreutzer, Julia and + Shen, Xiaoyu and + Reid, Machel and + Ruiter, Dana and + Klakow, Dietrich and + Nabende, Peter and + Chang, Ernie and + Gwadabe, Tajuddeen and + Sackey, Freshia and + Dossou, Bonaventure F. P. and + Emezue, Chris and + Leong, Colin and + Beukman, Michael and + Muhammad, Shamsuddeen and + Jarso, Guyo and + Yousuf, Oreen and + Niyongabo Rubungo, Andre and + Hacheme, Gilles and + Wairagala, Eric Peter and + Nasir, Muhammad Umair and + Ajibade, Benjamin and + Ajayi, Tunde and + Gitau, Yvonne and + Abbott, Jade and + Ahmed, Mohamed and + Ochieng, Millicent and + Aremu, Anuoluwapo and + Ogayo, Perez and + Mukiibi, Jonathan and + Ouoba Kabore, Fatoumata and + Kalipe, Godson and + Mbaye, Derguene and + Tapo, Allahsera Auguste and + Memdjokam Koagne, Victoire and + Munkoh-Buabeng, Edwin and + Wagner, Valencia and + Abdulmumin, Idris and + Awokoya, Ayodele and + Buzaaba, Happy and + Sibanda, Blessing and + Bukula, Andiswa and + Manthalu, Sam", + booktitle = "Proceedings of the 2022 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies", + month = jul, + year = "2022", + address = "Seattle, United States", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2022.naacl-main.223", + doi = "10.18653/v1/2022.naacl-main.223", + pages = "3053--3070", + abstract = "Recent advances in the pre-training for language models leverage large-scale datasets to create multilingual models. However, low-resource languages are mostly left out in these datasets. This is primarily because many widely spoken languages that are not well represented on the web and therefore excluded from the large-scale crawls for datasets. Furthermore, downstream users of these models are restricted to the selection of languages originally chosen for pre-training. This work investigates how to optimally leverage existing pre-trained models to create low-resource translation systems for 16 African languages. We focus on two questions: 1) How can pre-trained models be used for languages not included in the initial pretraining? and 2) How can the resulting translation models effectively transfer to new domains? To answer these questions, we create a novel African news corpus covering 16 languages, of which eight languages are not part of any existing evaluation dataset. We demonstrate that the most effective strategy for transferring both additional languages and additional domains is to leverage small quantities of high-quality translation data to fine-tune large pre-trained models.", +} +``` diff --git a/lm_eval/tasks/afrobench/mafand/gen_utils.py b/lm_eval/tasks/afrobench/mafand/gen_utils.py new file mode 100644 index 00000000..c260a321 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/gen_utils.py @@ -0,0 +1,147 @@ +import argparse +import os + +import yaml + + +class FunctionTag: + def __init__(self, value): + self.value = value + + +def prompt_func(mode, lang, lang_dict): + language_column_name = f"{lang}_text" + prompt_map = { + "prompt_1": "You are an advanced Translator, a specialized assistant designed to translate documents from " + f"{lang_dict[lang]} into English. \nYour main goal is to ensure translations are grammatically " + f"correct and human-oriented. \n{lang_dict[lang]}: {{{{{language_column_name}}}}} \nEnglish: ", + "prompt_1_reverse": "You are an advanced Translator, a specialized assistant designed to translate documents " + f"from English into {lang_dict[lang]}. \nYour main goal is to ensure translations are " + f"grammatically correct and human-oriented. " + f"\nEnglish: {{eng_text}} \n{lang_dict[lang]}: ", + "prompt_2": f"{lang_dict[lang]} sentence: {{{{{language_column_name}}}}} \nEnglish sentence: ", + "prompt_2_reverse": "English sentence: {{eng_text}} " + f"\n{lang_dict[lang]} sentence: ", + "prompt_3": f"You are a translation expert. Translate the following {lang_dict[lang]} sentences to English \n" + f"{lang_dict[lang]} sentence: {{{{{language_column_name}}}}}\nEnglish sentence: ", + "prompt_3_reverse": f"You are a translation expert. Translate the following English sentences to " + f"{lang_dict[lang]} " + "\nEnglish sentence: {{eng_text}} " + f"\n{lang_dict[lang]} sentence: ", + } + return prompt_map[mode] + + +def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str, reverse: bool) -> None: + """ + Generate a yaml file for each language. + + :param output_dir: The directory to output the files to. + :param overwrite: Whether to overwrite files if they already exist. + """ + err = [] + languages = { + "amh": "Amharic", + "bam": "Bambara", + "bbj": "Gbomala", + "ewe": "Ewe", + "fon": "Fon", + "hau": "Hausa", + "ibo": "Igbo", + "kin": "Kinyarwanda", + "lug": "Luganda", + "luo": "Luo", + "mos": "Mossi", + "nya": "Chichewa", + "pcm": "Nigerian Pidgin", + "sna": "Shona", + "swa": "Swahili", + "tsn": "Setswana", + "twi": "Twi", + "wol": "Wolof", + "xho": "Xhosa", + "yor": "Yoruba", + "zul": "Zulu", + } + + french_langs = ["bam", "bbj", "ewe", "fon", "wol", "mos"] + + for lang in languages.keys(): + try: + norm_lang = f"{lang}-en" if lang not in french_langs else f"{lang}-fr" + reverse_lang = f"en-{lang}" if lang not in french_langs else f"fr-{lang}" + dataset_name = norm_lang if reverse else reverse_lang + file_name = f"mafand_{dataset_name}.yaml" + task_name = f"mafand_{dataset_name}_{mode}" + yaml_template = "mafand" + yaml_details = { + "include": yaml_template, + "task": task_name, + "dataset_name": reverse_lang, + } + file_dir = ( + f"{output_dir}/{mode}/african-english" + if reverse + else f"{output_dir}/{mode}/english-african" + ) + os.makedirs(file_dir, exist_ok=True) + with open( + f"{file_dir}/{file_name}", + "w" if overwrite else "x", + encoding="utf8", + ) as f: + f.write("# Generated by utils.py\n") + yaml.dump( + yaml_details, + f, + allow_unicode=True, + ) + + except FileExistsError: + err.append(file_name) + + if len(err) > 0: + raise FileExistsError( + "Files were not created because they already exist (use --overwrite flag):" + f" {', '.join(err)}" + ) + + +def main() -> None: + """Parse CLI args and generate language-specific yaml files.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + default=True, + action="store_true", + help="Overwrite files if they already exist", + ) + parser.add_argument( + "--output-dir", + default="./", + help="Directory to write yaml files to", + ) + parser.add_argument( + "--mode", + default="prompt_3", + choices=["prompt_1", "prompt_2", "prompt_3"], + help="Prompt number", + ) + parser.add_argument( + "--reverse", + default=True, + choices=[True, False], + help="Reverse the translation direction", + ) + args = parser.parse_args() + + gen_lang_yamls( + output_dir=args.output_dir, + overwrite=args.overwrite, + mode=args.mode, + reverse=args.reverse, + ) + + +if __name__ == "__main__": + main() diff --git a/lm_eval/tasks/afrobench/mafand/mafand.yaml b/lm_eval/tasks/afrobench/mafand/mafand.yaml new file mode 100644 index 00000000..ef8619ad --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/mafand.yaml @@ -0,0 +1,14 @@ +group: mafand +task: + - mafand_eng-afr_prompt_1 + - mafand_eng-afr_prompt_2 + - mafand_eng-afr_prompt_3 + - mafand_afr-eng_prompt_1 + - mafand_afr-eng_prompt_2 + - mafand_afr-eng_prompt_3 +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand new file mode 100644 index 00000000..4f2047be --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand @@ -0,0 +1,28 @@ +tag: +- mafand_tasks +- mafand_afr-eng +- mafand_afr-eng_prompt_1 +- afrobench_MT_tasks +dataset_path: masakhane/mafand +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: validation +fewshot_split: validation +test_split: test +doc_to_target: !function utils.get_target +doc_to_text: !function utils.create_text_prompt_1 +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_amh-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_amh-en.yaml new file mode 100644 index 00000000..95e87fd8 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_amh-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-amh +include: mafand +task: mafand_amh-en_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_bam-fr.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_bam-fr.yaml new file mode 100644 index 00000000..dbc612ac --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_bam-fr.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-bam +include: mafand +task: mafand_bam-fr_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_bbj-fr.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_bbj-fr.yaml new file mode 100644 index 00000000..abe64f9b --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_bbj-fr.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-bbj +include: mafand +task: mafand_bbj-fr_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_ewe-fr.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_ewe-fr.yaml new file mode 100644 index 00000000..ecd9b38b --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_ewe-fr.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-ewe +include: mafand +task: mafand_ewe-fr_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_fon-fr.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_fon-fr.yaml new file mode 100644 index 00000000..705cfbb8 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_fon-fr.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-fon +include: mafand +task: mafand_fon-fr_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_hau-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_hau-en.yaml new file mode 100644 index 00000000..3b84d9ce --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_hau-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-hau +include: mafand +task: mafand_hau-en_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_ibo-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_ibo-en.yaml new file mode 100644 index 00000000..d78c91bb --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_ibo-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-ibo +include: mafand +task: mafand_ibo-en_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_kin-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_kin-en.yaml new file mode 100644 index 00000000..954c036e --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_kin-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-kin +include: mafand +task: mafand_kin-en_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_lug-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_lug-en.yaml new file mode 100644 index 00000000..671c072a --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_lug-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-lug +include: mafand +task: mafand_lug-en_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_luo-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_luo-en.yaml new file mode 100644 index 00000000..a1d5965f --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_luo-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-luo +include: mafand +task: mafand_luo-en_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_mos-fr.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_mos-fr.yaml new file mode 100644 index 00000000..da085707 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_mos-fr.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-mos +include: mafand +task: mafand_mos-fr_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_nya-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_nya-en.yaml new file mode 100644 index 00000000..3bc24266 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_nya-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-nya +include: mafand +task: mafand_nya-en_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_pcm-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_pcm-en.yaml new file mode 100644 index 00000000..f6fb5dee --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_pcm-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-pcm +include: mafand +task: mafand_pcm-en_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_sna-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_sna-en.yaml new file mode 100644 index 00000000..283517d6 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_sna-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-sna +include: mafand +task: mafand_sna-en_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_swa-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_swa-en.yaml new file mode 100644 index 00000000..476bba42 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_swa-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-swa +include: mafand +task: mafand_swa-en_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_tsn-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_tsn-en.yaml new file mode 100644 index 00000000..a94c5b6e --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_tsn-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-tsn +include: mafand +task: mafand_tsn-en_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_twi-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_twi-en.yaml new file mode 100644 index 00000000..7f5883b1 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_twi-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-twi +include: mafand +task: mafand_twi-en_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_wol-fr.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_wol-fr.yaml new file mode 100644 index 00000000..bb887188 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_wol-fr.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-wol +include: mafand +task: mafand_wol-fr_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_xho-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_xho-en.yaml new file mode 100644 index 00000000..a0561b41 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_xho-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-xho +include: mafand +task: mafand_xho-en_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_yor-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_yor-en.yaml new file mode 100644 index 00000000..ec97ae7d --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_yor-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-yor +include: mafand +task: mafand_yor-en_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_zul-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_zul-en.yaml new file mode 100644 index 00000000..9649d772 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/mafand_zul-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-zul +include: mafand +task: mafand_zul-en_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/utils.py b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/utils.py new file mode 100644 index 00000000..0df3a329 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/african-english/utils.py @@ -0,0 +1,121 @@ +languages = { + "amh": "Amharic", + "bam": "Bambara", + "bbj": "Gbomala", + "ewe": "Ewe", + "fon": "Fon", + "hau": "Hausa", + "ibo": "Igbo", + "kin": "Kinyarwanda", + "lug": "Luganda", + "luo": "Luo", + "mos": "Mossi", + "nya": "Chichewa", + "pcm": "Nigerian Pidgin", + "sna": "Shona", + "swa": "Swahili", + "tsn": "Setswana", + "twi": "Twi", + "wol": "Wolof", + "xho": "Xhosa", + "yor": "Yoruba", + "zul": "Zulu", +} + + +def get_target(doc): + target = ( + doc["translation"]["en"] + if "en" in doc["translation"].keys() + else doc["translation"]["fr"] + ) + return target + + +def get_target_reverse(doc): + target_key = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + target = doc["translation"][target_key] + return target + + +def create_text_prompt_1(doc): + source_key = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_sentence = doc["translation"][source_key] + source_lang = "English" if "en" in doc["translation"].keys() else "French" + prompt = ( + "You are an advanced Translator, a specialized assistant designed to translate documents from " + f"{languages[source_key]} into {source_lang}. \nYour main goal is to ensure translations are grammatically " + f"correct and human-oriented. \n{languages[source_key]}: {source_sentence} \n{source_lang}: " + ) + return prompt + + +def create_reverse_prompt_1(doc): + target_lang = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_key = "en" if "en" in doc["translation"].keys() else "fr" + source_lang = "English" if source_key == "en" else "French" + source_sentence = doc["translation"][source_key] + prompt = ( + "You are an advanced Translator, a specialized assistant designed to translate documents from " + f"{source_lang} into {languages[target_lang]}. \nYour main goal is to ensure translations are " + f"grammatically correct and human-oriented. \n{source_lang}: {source_sentence} \n{languages[target_lang]}: " + ) + return prompt + + +def create_text_prompt_2(doc): + source_key = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_sentence = doc["translation"][source_key] + source_lang = "English" if "en" in doc["translation"].keys() else "French" + prompt = ( + f"{languages[source_key]} sentence: {source_sentence} \n{source_lang} sentence: ", + ) + return prompt + + +def create_reverse_prompt_2(doc): + target_lang = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_key = "en" if "en" in doc["translation"].keys() else "fr" + source_lang = "English" if source_key == "en" else "French" + source_sentence = doc["translation"][source_key] + prompt = ( + f"{source_lang} sentence: {source_sentence} \n{languages[target_lang]} sentence: \n", + ) + return prompt + + +def create_text_prompt_3(doc): + source_key = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_sentence = doc["translation"][source_key] + source_lang = "English" if "en" in doc["translation"].keys() else "French" + prompt = ( + f"You are a translation expert. Translate the following {languages[source_key]} sentences " + f"to {source_lang}. \n{languages[source_key]} sentence: {source_sentence}\n{source_lang} sentence: " + ) + return prompt + + +def create_reverse_prompt_3(doc): + target_lang = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_key = "en" if "en" in doc["translation"].keys() else "fr" + source_lang = "English" if source_key == "en" else "French" + source_sentence = doc["translation"][source_key] + prompt = ( + f"You are a translation expert. Translate the following {source_lang} sentence into {languages[target_lang]}\n" + f"{source_lang} sentence: {source_sentence}\n{languages[target_lang]} sentence: " + ) + return prompt diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand new file mode 100644 index 00000000..1d004556 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand @@ -0,0 +1,28 @@ +tag: +- mafand_tasks +- mafand_eng-afr +- mafand_eng-afr_prompt_1 +- afrobench_MT_tasks +dataset_path: masakhane/mafand +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: validation +fewshot_split: validation +test_split: test +doc_to_target: !function utils.get_target_reverse +doc_to_text: !function utils.create_reverse_prompt_1 +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-amh.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-amh.yaml new file mode 100644 index 00000000..8ef9ab28 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-amh +include: mafand +task: mafand_en-amh_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-hau.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-hau.yaml new file mode 100644 index 00000000..0ea57778 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-hau +include: mafand +task: mafand_en-hau_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-ibo.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-ibo.yaml new file mode 100644 index 00000000..88af221f --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-ibo +include: mafand +task: mafand_en-ibo_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-kin.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-kin.yaml new file mode 100644 index 00000000..0c415f51 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-kin +include: mafand +task: mafand_en-kin_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-lug.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-lug.yaml new file mode 100644 index 00000000..94070e89 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-lug +include: mafand +task: mafand_en-lug_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-luo.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-luo.yaml new file mode 100644 index 00000000..fc6b15c6 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-luo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-luo +include: mafand +task: mafand_en-luo_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-nya.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-nya.yaml new file mode 100644 index 00000000..225a4647 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-nya.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-nya +include: mafand +task: mafand_en-nya_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-pcm.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-pcm.yaml new file mode 100644 index 00000000..69380c71 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-pcm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-pcm +include: mafand +task: mafand_en-pcm_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-sna.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-sna.yaml new file mode 100644 index 00000000..634d988f --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-sna +include: mafand +task: mafand_en-sna_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-swa.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-swa.yaml new file mode 100644 index 00000000..bfbf259c --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-swa +include: mafand +task: mafand_en-swa_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-tsn.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-tsn.yaml new file mode 100644 index 00000000..faa99ddf --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-tsn.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-tsn +include: mafand +task: mafand_en-tsn_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-twi.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-twi.yaml new file mode 100644 index 00000000..a9294975 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-twi +include: mafand +task: mafand_en-twi_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-xho.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-xho.yaml new file mode 100644 index 00000000..244f5cab --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-xho +include: mafand +task: mafand_en-xho_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-yor.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-yor.yaml new file mode 100644 index 00000000..aa318977 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-yor.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-yor +include: mafand +task: mafand_en-yor_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-zul.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-zul.yaml new file mode 100644 index 00000000..6afdc0c5 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_en-zul.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-zul +include: mafand +task: mafand_en-zul_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-bam.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-bam.yaml new file mode 100644 index 00000000..7c21d96f --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-bam.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-bam +include: mafand +task: mafand_fr-bam_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-bbj.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-bbj.yaml new file mode 100644 index 00000000..76cf0750 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-bbj.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-bbj +include: mafand +task: mafand_fr-bbj_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-ewe.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-ewe.yaml new file mode 100644 index 00000000..0c7bd667 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-ewe +include: mafand +task: mafand_fr-ewe_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-fon.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-fon.yaml new file mode 100644 index 00000000..737d68eb --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-fon.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-fon +include: mafand +task: mafand_fr-fon_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-mos.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-mos.yaml new file mode 100644 index 00000000..9186a5b9 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-mos.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-mos +include: mafand +task: mafand_fr-mos_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-wol.yaml b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-wol.yaml new file mode 100644 index 00000000..6e29f5fb --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/mafand_fr-wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-wol +include: mafand +task: mafand_fr-wol_prompt_1 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/utils.py b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/utils.py new file mode 100644 index 00000000..0df3a329 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_1/english-african/utils.py @@ -0,0 +1,121 @@ +languages = { + "amh": "Amharic", + "bam": "Bambara", + "bbj": "Gbomala", + "ewe": "Ewe", + "fon": "Fon", + "hau": "Hausa", + "ibo": "Igbo", + "kin": "Kinyarwanda", + "lug": "Luganda", + "luo": "Luo", + "mos": "Mossi", + "nya": "Chichewa", + "pcm": "Nigerian Pidgin", + "sna": "Shona", + "swa": "Swahili", + "tsn": "Setswana", + "twi": "Twi", + "wol": "Wolof", + "xho": "Xhosa", + "yor": "Yoruba", + "zul": "Zulu", +} + + +def get_target(doc): + target = ( + doc["translation"]["en"] + if "en" in doc["translation"].keys() + else doc["translation"]["fr"] + ) + return target + + +def get_target_reverse(doc): + target_key = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + target = doc["translation"][target_key] + return target + + +def create_text_prompt_1(doc): + source_key = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_sentence = doc["translation"][source_key] + source_lang = "English" if "en" in doc["translation"].keys() else "French" + prompt = ( + "You are an advanced Translator, a specialized assistant designed to translate documents from " + f"{languages[source_key]} into {source_lang}. \nYour main goal is to ensure translations are grammatically " + f"correct and human-oriented. \n{languages[source_key]}: {source_sentence} \n{source_lang}: " + ) + return prompt + + +def create_reverse_prompt_1(doc): + target_lang = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_key = "en" if "en" in doc["translation"].keys() else "fr" + source_lang = "English" if source_key == "en" else "French" + source_sentence = doc["translation"][source_key] + prompt = ( + "You are an advanced Translator, a specialized assistant designed to translate documents from " + f"{source_lang} into {languages[target_lang]}. \nYour main goal is to ensure translations are " + f"grammatically correct and human-oriented. \n{source_lang}: {source_sentence} \n{languages[target_lang]}: " + ) + return prompt + + +def create_text_prompt_2(doc): + source_key = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_sentence = doc["translation"][source_key] + source_lang = "English" if "en" in doc["translation"].keys() else "French" + prompt = ( + f"{languages[source_key]} sentence: {source_sentence} \n{source_lang} sentence: ", + ) + return prompt + + +def create_reverse_prompt_2(doc): + target_lang = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_key = "en" if "en" in doc["translation"].keys() else "fr" + source_lang = "English" if source_key == "en" else "French" + source_sentence = doc["translation"][source_key] + prompt = ( + f"{source_lang} sentence: {source_sentence} \n{languages[target_lang]} sentence: \n", + ) + return prompt + + +def create_text_prompt_3(doc): + source_key = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_sentence = doc["translation"][source_key] + source_lang = "English" if "en" in doc["translation"].keys() else "French" + prompt = ( + f"You are a translation expert. Translate the following {languages[source_key]} sentences " + f"to {source_lang}. \n{languages[source_key]} sentence: {source_sentence}\n{source_lang} sentence: " + ) + return prompt + + +def create_reverse_prompt_3(doc): + target_lang = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_key = "en" if "en" in doc["translation"].keys() else "fr" + source_lang = "English" if source_key == "en" else "French" + source_sentence = doc["translation"][source_key] + prompt = ( + f"You are a translation expert. Translate the following {source_lang} sentence into {languages[target_lang]}\n" + f"{source_lang} sentence: {source_sentence}\n{languages[target_lang]} sentence: " + ) + return prompt diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand new file mode 100644 index 00000000..eb7ad988 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand @@ -0,0 +1,28 @@ +tag: +- mafand_tasks +- mafand_afr-eng +- mafand_afr-eng_prompt_3 +- afrobench_MT_tasks +dataset_path: masakhane/mafand +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: validation +fewshot_split: validation +test_split: test +doc_to_target: !function utils.get_target +doc_to_text: !function utils.create_text_prompt_2 +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_amh-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_amh-en.yaml new file mode 100644 index 00000000..6db544cb --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_amh-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-amh +include: mafand +task: mafand_amh-en_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_bam-fr.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_bam-fr.yaml new file mode 100644 index 00000000..0a9f3b3a --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_bam-fr.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-bam +include: mafand +task: mafand_bam-fr_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_bbj-fr.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_bbj-fr.yaml new file mode 100644 index 00000000..b0b42b23 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_bbj-fr.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-bbj +include: mafand +task: mafand_bbj-fr_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_ewe-fr.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_ewe-fr.yaml new file mode 100644 index 00000000..457c0d19 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_ewe-fr.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-ewe +include: mafand +task: mafand_ewe-fr_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_fon-fr.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_fon-fr.yaml new file mode 100644 index 00000000..84263d5a --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_fon-fr.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-fon +include: mafand +task: mafand_fon-fr_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_hau-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_hau-en.yaml new file mode 100644 index 00000000..05c31a46 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_hau-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-hau +include: mafand +task: mafand_hau-en_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_ibo-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_ibo-en.yaml new file mode 100644 index 00000000..3cb4a5b8 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_ibo-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-ibo +include: mafand +task: mafand_ibo-en_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_kin-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_kin-en.yaml new file mode 100644 index 00000000..e3e1acf9 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_kin-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-kin +include: mafand +task: mafand_kin-en_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_lug-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_lug-en.yaml new file mode 100644 index 00000000..eb68279d --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_lug-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-lug +include: mafand +task: mafand_lug-en_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_luo-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_luo-en.yaml new file mode 100644 index 00000000..12f19947 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_luo-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-luo +include: mafand +task: mafand_luo-en_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_mos-fr.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_mos-fr.yaml new file mode 100644 index 00000000..a723701d --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_mos-fr.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-mos +include: mafand +task: mafand_mos-fr_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_nya-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_nya-en.yaml new file mode 100644 index 00000000..24569f00 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_nya-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-nya +include: mafand +task: mafand_nya-en_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_pcm-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_pcm-en.yaml new file mode 100644 index 00000000..7ec0c66a --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_pcm-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-pcm +include: mafand +task: mafand_pcm-en_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_sna-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_sna-en.yaml new file mode 100644 index 00000000..3bf99b99 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_sna-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-sna +include: mafand +task: mafand_sna-en_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_swa-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_swa-en.yaml new file mode 100644 index 00000000..eb2ada0b --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_swa-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-swa +include: mafand +task: mafand_swa-en_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_tsn-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_tsn-en.yaml new file mode 100644 index 00000000..d16e7e94 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_tsn-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-tsn +include: mafand +task: mafand_tsn-en_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_twi-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_twi-en.yaml new file mode 100644 index 00000000..267337c1 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_twi-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-twi +include: mafand +task: mafand_twi-en_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_wol-fr.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_wol-fr.yaml new file mode 100644 index 00000000..f6c67bd8 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_wol-fr.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-wol +include: mafand +task: mafand_wol-fr_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_xho-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_xho-en.yaml new file mode 100644 index 00000000..fd1960d0 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_xho-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-xho +include: mafand +task: mafand_xho-en_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_yor-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_yor-en.yaml new file mode 100644 index 00000000..cb7241ad --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_yor-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-yor +include: mafand +task: mafand_yor-en_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_zul-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_zul-en.yaml new file mode 100644 index 00000000..d44db7a2 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/mafand_zul-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-zul +include: mafand +task: mafand_zul-en_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/utils.py b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/utils.py new file mode 100644 index 00000000..0df3a329 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/african-english/utils.py @@ -0,0 +1,121 @@ +languages = { + "amh": "Amharic", + "bam": "Bambara", + "bbj": "Gbomala", + "ewe": "Ewe", + "fon": "Fon", + "hau": "Hausa", + "ibo": "Igbo", + "kin": "Kinyarwanda", + "lug": "Luganda", + "luo": "Luo", + "mos": "Mossi", + "nya": "Chichewa", + "pcm": "Nigerian Pidgin", + "sna": "Shona", + "swa": "Swahili", + "tsn": "Setswana", + "twi": "Twi", + "wol": "Wolof", + "xho": "Xhosa", + "yor": "Yoruba", + "zul": "Zulu", +} + + +def get_target(doc): + target = ( + doc["translation"]["en"] + if "en" in doc["translation"].keys() + else doc["translation"]["fr"] + ) + return target + + +def get_target_reverse(doc): + target_key = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + target = doc["translation"][target_key] + return target + + +def create_text_prompt_1(doc): + source_key = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_sentence = doc["translation"][source_key] + source_lang = "English" if "en" in doc["translation"].keys() else "French" + prompt = ( + "You are an advanced Translator, a specialized assistant designed to translate documents from " + f"{languages[source_key]} into {source_lang}. \nYour main goal is to ensure translations are grammatically " + f"correct and human-oriented. \n{languages[source_key]}: {source_sentence} \n{source_lang}: " + ) + return prompt + + +def create_reverse_prompt_1(doc): + target_lang = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_key = "en" if "en" in doc["translation"].keys() else "fr" + source_lang = "English" if source_key == "en" else "French" + source_sentence = doc["translation"][source_key] + prompt = ( + "You are an advanced Translator, a specialized assistant designed to translate documents from " + f"{source_lang} into {languages[target_lang]}. \nYour main goal is to ensure translations are " + f"grammatically correct and human-oriented. \n{source_lang}: {source_sentence} \n{languages[target_lang]}: " + ) + return prompt + + +def create_text_prompt_2(doc): + source_key = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_sentence = doc["translation"][source_key] + source_lang = "English" if "en" in doc["translation"].keys() else "French" + prompt = ( + f"{languages[source_key]} sentence: {source_sentence} \n{source_lang} sentence: ", + ) + return prompt + + +def create_reverse_prompt_2(doc): + target_lang = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_key = "en" if "en" in doc["translation"].keys() else "fr" + source_lang = "English" if source_key == "en" else "French" + source_sentence = doc["translation"][source_key] + prompt = ( + f"{source_lang} sentence: {source_sentence} \n{languages[target_lang]} sentence: \n", + ) + return prompt + + +def create_text_prompt_3(doc): + source_key = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_sentence = doc["translation"][source_key] + source_lang = "English" if "en" in doc["translation"].keys() else "French" + prompt = ( + f"You are a translation expert. Translate the following {languages[source_key]} sentences " + f"to {source_lang}. \n{languages[source_key]} sentence: {source_sentence}\n{source_lang} sentence: " + ) + return prompt + + +def create_reverse_prompt_3(doc): + target_lang = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_key = "en" if "en" in doc["translation"].keys() else "fr" + source_lang = "English" if source_key == "en" else "French" + source_sentence = doc["translation"][source_key] + prompt = ( + f"You are a translation expert. Translate the following {source_lang} sentence into {languages[target_lang]}\n" + f"{source_lang} sentence: {source_sentence}\n{languages[target_lang]} sentence: " + ) + return prompt diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand new file mode 100644 index 00000000..35548392 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand @@ -0,0 +1,28 @@ +tag: +- mafand_tasks +- mafand_eng-afr +- mafand_eng-afr_prompt_2 +- afrobench_MT_tasks +dataset_path: masakhane/mafand +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: validation +fewshot_split: validation +test_split: test +doc_to_target: !function utils.get_target_reverse +doc_to_text: !function utils.create_reverse_prompt_2 +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-amh.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-amh.yaml new file mode 100644 index 00000000..09c21d21 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-amh +include: mafand +task: mafand_en-amh_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-hau.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-hau.yaml new file mode 100644 index 00000000..e9a91c76 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-hau +include: mafand +task: mafand_en-hau_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-ibo.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-ibo.yaml new file mode 100644 index 00000000..568a845e --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-ibo +include: mafand +task: mafand_en-ibo_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-kin.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-kin.yaml new file mode 100644 index 00000000..09425f64 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-kin +include: mafand +task: mafand_en-kin_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-lug.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-lug.yaml new file mode 100644 index 00000000..13c91d36 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-lug +include: mafand +task: mafand_en-lug_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-luo.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-luo.yaml new file mode 100644 index 00000000..41bb0936 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-luo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-luo +include: mafand +task: mafand_en-luo_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-nya.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-nya.yaml new file mode 100644 index 00000000..90a72810 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-nya.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-nya +include: mafand +task: mafand_en-nya_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-pcm.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-pcm.yaml new file mode 100644 index 00000000..73229c4f --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-pcm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-pcm +include: mafand +task: mafand_en-pcm_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-sna.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-sna.yaml new file mode 100644 index 00000000..ac371871 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-sna +include: mafand +task: mafand_en-sna_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-swa.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-swa.yaml new file mode 100644 index 00000000..21d9fc0e --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-swa +include: mafand +task: mafand_en-swa_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-tsn.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-tsn.yaml new file mode 100644 index 00000000..b3dd4362 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-tsn.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-tsn +include: mafand +task: mafand_en-tsn_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-twi.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-twi.yaml new file mode 100644 index 00000000..5502ffa4 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-twi +include: mafand +task: mafand_en-twi_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-xho.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-xho.yaml new file mode 100644 index 00000000..c8c1ffee --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-xho +include: mafand +task: mafand_en-xho_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-yor.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-yor.yaml new file mode 100644 index 00000000..89c070c7 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-yor.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-yor +include: mafand +task: mafand_en-yor_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-zul.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-zul.yaml new file mode 100644 index 00000000..e5472540 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_en-zul.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-zul +include: mafand +task: mafand_en-zul_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_fr-bam.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_fr-bam.yaml new file mode 100644 index 00000000..15c6e981 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_fr-bam.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-bam +include: mafand +task: mafand_fr-bam_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_fr-bbj.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_fr-bbj.yaml new file mode 100644 index 00000000..4f5101a7 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_fr-bbj.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-bbj +include: mafand +task: mafand_fr-bbj_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_fr-ewe.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_fr-ewe.yaml new file mode 100644 index 00000000..29d4214c --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_fr-ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-ewe +include: mafand +task: mafand_fr-ewe_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_fr-fon.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_fr-fon.yaml new file mode 100644 index 00000000..9710db5b --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_fr-fon.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-fon +include: mafand +task: mafand_fr-fon_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_fr-mos.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_fr-mos.yaml new file mode 100644 index 00000000..682fb19c --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_fr-mos.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-mos +include: mafand +task: mafand_fr-mos_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_fr-wol.yaml b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_fr-wol.yaml new file mode 100644 index 00000000..3740ca9b --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/mafand_fr-wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-wol +include: mafand +task: mafand_fr-wol_prompt_2 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/utils.py b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/utils.py new file mode 100644 index 00000000..0df3a329 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_2/english-african/utils.py @@ -0,0 +1,121 @@ +languages = { + "amh": "Amharic", + "bam": "Bambara", + "bbj": "Gbomala", + "ewe": "Ewe", + "fon": "Fon", + "hau": "Hausa", + "ibo": "Igbo", + "kin": "Kinyarwanda", + "lug": "Luganda", + "luo": "Luo", + "mos": "Mossi", + "nya": "Chichewa", + "pcm": "Nigerian Pidgin", + "sna": "Shona", + "swa": "Swahili", + "tsn": "Setswana", + "twi": "Twi", + "wol": "Wolof", + "xho": "Xhosa", + "yor": "Yoruba", + "zul": "Zulu", +} + + +def get_target(doc): + target = ( + doc["translation"]["en"] + if "en" in doc["translation"].keys() + else doc["translation"]["fr"] + ) + return target + + +def get_target_reverse(doc): + target_key = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + target = doc["translation"][target_key] + return target + + +def create_text_prompt_1(doc): + source_key = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_sentence = doc["translation"][source_key] + source_lang = "English" if "en" in doc["translation"].keys() else "French" + prompt = ( + "You are an advanced Translator, a specialized assistant designed to translate documents from " + f"{languages[source_key]} into {source_lang}. \nYour main goal is to ensure translations are grammatically " + f"correct and human-oriented. \n{languages[source_key]}: {source_sentence} \n{source_lang}: " + ) + return prompt + + +def create_reverse_prompt_1(doc): + target_lang = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_key = "en" if "en" in doc["translation"].keys() else "fr" + source_lang = "English" if source_key == "en" else "French" + source_sentence = doc["translation"][source_key] + prompt = ( + "You are an advanced Translator, a specialized assistant designed to translate documents from " + f"{source_lang} into {languages[target_lang]}. \nYour main goal is to ensure translations are " + f"grammatically correct and human-oriented. \n{source_lang}: {source_sentence} \n{languages[target_lang]}: " + ) + return prompt + + +def create_text_prompt_2(doc): + source_key = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_sentence = doc["translation"][source_key] + source_lang = "English" if "en" in doc["translation"].keys() else "French" + prompt = ( + f"{languages[source_key]} sentence: {source_sentence} \n{source_lang} sentence: ", + ) + return prompt + + +def create_reverse_prompt_2(doc): + target_lang = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_key = "en" if "en" in doc["translation"].keys() else "fr" + source_lang = "English" if source_key == "en" else "French" + source_sentence = doc["translation"][source_key] + prompt = ( + f"{source_lang} sentence: {source_sentence} \n{languages[target_lang]} sentence: \n", + ) + return prompt + + +def create_text_prompt_3(doc): + source_key = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_sentence = doc["translation"][source_key] + source_lang = "English" if "en" in doc["translation"].keys() else "French" + prompt = ( + f"You are a translation expert. Translate the following {languages[source_key]} sentences " + f"to {source_lang}. \n{languages[source_key]} sentence: {source_sentence}\n{source_lang} sentence: " + ) + return prompt + + +def create_reverse_prompt_3(doc): + target_lang = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_key = "en" if "en" in doc["translation"].keys() else "fr" + source_lang = "English" if source_key == "en" else "French" + source_sentence = doc["translation"][source_key] + prompt = ( + f"You are a translation expert. Translate the following {source_lang} sentence into {languages[target_lang]}\n" + f"{source_lang} sentence: {source_sentence}\n{languages[target_lang]} sentence: " + ) + return prompt diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand new file mode 100644 index 00000000..eb7ad988 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand @@ -0,0 +1,28 @@ +tag: +- mafand_tasks +- mafand_afr-eng +- mafand_afr-eng_prompt_3 +- afrobench_MT_tasks +dataset_path: masakhane/mafand +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: validation +fewshot_split: validation +test_split: test +doc_to_target: !function utils.get_target +doc_to_text: !function utils.create_text_prompt_2 +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_amh-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_amh-en.yaml new file mode 100644 index 00000000..856318b5 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_amh-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-amh +include: mafand +task: mafand_amh-en_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_bam-fr.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_bam-fr.yaml new file mode 100644 index 00000000..bed42523 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_bam-fr.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-bam +include: mafand +task: mafand_bam-fr_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_bbj-fr.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_bbj-fr.yaml new file mode 100644 index 00000000..1170c266 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_bbj-fr.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-bbj +include: mafand +task: mafand_bbj-fr_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_ewe-fr.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_ewe-fr.yaml new file mode 100644 index 00000000..39a345cb --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_ewe-fr.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-ewe +include: mafand +task: mafand_ewe-fr_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_fon-fr.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_fon-fr.yaml new file mode 100644 index 00000000..1b464bb9 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_fon-fr.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-fon +include: mafand +task: mafand_fon-fr_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_hau-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_hau-en.yaml new file mode 100644 index 00000000..9c0b0f15 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_hau-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-hau +include: mafand +task: mafand_hau-en_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_ibo-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_ibo-en.yaml new file mode 100644 index 00000000..2f78f55a --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_ibo-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-ibo +include: mafand +task: mafand_ibo-en_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_kin-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_kin-en.yaml new file mode 100644 index 00000000..254b22be --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_kin-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-kin +include: mafand +task: mafand_kin-en_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_lug-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_lug-en.yaml new file mode 100644 index 00000000..ad19b3c8 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_lug-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-lug +include: mafand +task: mafand_lug-en_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_luo-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_luo-en.yaml new file mode 100644 index 00000000..a3a36749 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_luo-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-luo +include: mafand +task: mafand_luo-en_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_mos-fr.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_mos-fr.yaml new file mode 100644 index 00000000..4ea41931 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_mos-fr.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-mos +include: mafand +task: mafand_mos-fr_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_nya-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_nya-en.yaml new file mode 100644 index 00000000..de9ec930 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_nya-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-nya +include: mafand +task: mafand_nya-en_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_pcm-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_pcm-en.yaml new file mode 100644 index 00000000..95ad3380 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_pcm-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-pcm +include: mafand +task: mafand_pcm-en_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_sna-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_sna-en.yaml new file mode 100644 index 00000000..2d86ccc3 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_sna-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-sna +include: mafand +task: mafand_sna-en_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_swa-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_swa-en.yaml new file mode 100644 index 00000000..3c70f2e3 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_swa-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-swa +include: mafand +task: mafand_swa-en_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_tsn-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_tsn-en.yaml new file mode 100644 index 00000000..0ee8f415 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_tsn-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-tsn +include: mafand +task: mafand_tsn-en_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_twi-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_twi-en.yaml new file mode 100644 index 00000000..a37d2395 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_twi-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-twi +include: mafand +task: mafand_twi-en_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_wol-fr.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_wol-fr.yaml new file mode 100644 index 00000000..ed778cbe --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_wol-fr.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-wol +include: mafand +task: mafand_wol-fr_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_xho-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_xho-en.yaml new file mode 100644 index 00000000..93e9e2fe --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_xho-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-xho +include: mafand +task: mafand_xho-en_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_yor-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_yor-en.yaml new file mode 100644 index 00000000..78301f7e --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_yor-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-yor +include: mafand +task: mafand_yor-en_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_zul-en.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_zul-en.yaml new file mode 100644 index 00000000..06177d14 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/mafand_zul-en.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-zul +include: mafand +task: mafand_zul-en_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/utils.py b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/utils.py new file mode 100644 index 00000000..0df3a329 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/african-english/utils.py @@ -0,0 +1,121 @@ +languages = { + "amh": "Amharic", + "bam": "Bambara", + "bbj": "Gbomala", + "ewe": "Ewe", + "fon": "Fon", + "hau": "Hausa", + "ibo": "Igbo", + "kin": "Kinyarwanda", + "lug": "Luganda", + "luo": "Luo", + "mos": "Mossi", + "nya": "Chichewa", + "pcm": "Nigerian Pidgin", + "sna": "Shona", + "swa": "Swahili", + "tsn": "Setswana", + "twi": "Twi", + "wol": "Wolof", + "xho": "Xhosa", + "yor": "Yoruba", + "zul": "Zulu", +} + + +def get_target(doc): + target = ( + doc["translation"]["en"] + if "en" in doc["translation"].keys() + else doc["translation"]["fr"] + ) + return target + + +def get_target_reverse(doc): + target_key = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + target = doc["translation"][target_key] + return target + + +def create_text_prompt_1(doc): + source_key = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_sentence = doc["translation"][source_key] + source_lang = "English" if "en" in doc["translation"].keys() else "French" + prompt = ( + "You are an advanced Translator, a specialized assistant designed to translate documents from " + f"{languages[source_key]} into {source_lang}. \nYour main goal is to ensure translations are grammatically " + f"correct and human-oriented. \n{languages[source_key]}: {source_sentence} \n{source_lang}: " + ) + return prompt + + +def create_reverse_prompt_1(doc): + target_lang = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_key = "en" if "en" in doc["translation"].keys() else "fr" + source_lang = "English" if source_key == "en" else "French" + source_sentence = doc["translation"][source_key] + prompt = ( + "You are an advanced Translator, a specialized assistant designed to translate documents from " + f"{source_lang} into {languages[target_lang]}. \nYour main goal is to ensure translations are " + f"grammatically correct and human-oriented. \n{source_lang}: {source_sentence} \n{languages[target_lang]}: " + ) + return prompt + + +def create_text_prompt_2(doc): + source_key = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_sentence = doc["translation"][source_key] + source_lang = "English" if "en" in doc["translation"].keys() else "French" + prompt = ( + f"{languages[source_key]} sentence: {source_sentence} \n{source_lang} sentence: ", + ) + return prompt + + +def create_reverse_prompt_2(doc): + target_lang = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_key = "en" if "en" in doc["translation"].keys() else "fr" + source_lang = "English" if source_key == "en" else "French" + source_sentence = doc["translation"][source_key] + prompt = ( + f"{source_lang} sentence: {source_sentence} \n{languages[target_lang]} sentence: \n", + ) + return prompt + + +def create_text_prompt_3(doc): + source_key = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_sentence = doc["translation"][source_key] + source_lang = "English" if "en" in doc["translation"].keys() else "French" + prompt = ( + f"You are a translation expert. Translate the following {languages[source_key]} sentences " + f"to {source_lang}. \n{languages[source_key]} sentence: {source_sentence}\n{source_lang} sentence: " + ) + return prompt + + +def create_reverse_prompt_3(doc): + target_lang = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_key = "en" if "en" in doc["translation"].keys() else "fr" + source_lang = "English" if source_key == "en" else "French" + source_sentence = doc["translation"][source_key] + prompt = ( + f"You are a translation expert. Translate the following {source_lang} sentence into {languages[target_lang]}\n" + f"{source_lang} sentence: {source_sentence}\n{languages[target_lang]} sentence: " + ) + return prompt diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand new file mode 100644 index 00000000..9a59654e --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand @@ -0,0 +1,28 @@ +tag: +- mafand_tasks +- mafand_eng-afr +- mafand_eng-afr_prompt_3 +- afrobench_MT_tasks +dataset_path: masakhane/mafand +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: validation +fewshot_split: validation +test_split: test +doc_to_target: !function utils.get_target_reverse +doc_to_text: !function utils.create_reverse_prompt_3 +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-amh.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-amh.yaml new file mode 100644 index 00000000..10872430 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-amh +include: mafand +task: mafand_en-amh_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-hau.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-hau.yaml new file mode 100644 index 00000000..f64e6816 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-hau +include: mafand +task: mafand_en-hau_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-ibo.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-ibo.yaml new file mode 100644 index 00000000..72df05e5 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-ibo +include: mafand +task: mafand_en-ibo_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-kin.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-kin.yaml new file mode 100644 index 00000000..44c48678 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-kin +include: mafand +task: mafand_en-kin_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-lug.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-lug.yaml new file mode 100644 index 00000000..2beae91b --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-lug +include: mafand +task: mafand_en-lug_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-luo.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-luo.yaml new file mode 100644 index 00000000..b4c1aa8b --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-luo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-luo +include: mafand +task: mafand_en-luo_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-nya.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-nya.yaml new file mode 100644 index 00000000..eee7af0c --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-nya.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-nya +include: mafand +task: mafand_en-nya_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-pcm.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-pcm.yaml new file mode 100644 index 00000000..6e606425 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-pcm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-pcm +include: mafand +task: mafand_en-pcm_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-sna.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-sna.yaml new file mode 100644 index 00000000..82abd862 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-sna +include: mafand +task: mafand_en-sna_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-swa.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-swa.yaml new file mode 100644 index 00000000..8a7135ff --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-swa +include: mafand +task: mafand_en-swa_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-tsn.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-tsn.yaml new file mode 100644 index 00000000..b976b5fd --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-tsn.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-tsn +include: mafand +task: mafand_en-tsn_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-twi.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-twi.yaml new file mode 100644 index 00000000..53345a26 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-twi +include: mafand +task: mafand_en-twi_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-xho.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-xho.yaml new file mode 100644 index 00000000..4eba7f69 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-xho +include: mafand +task: mafand_en-xho_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-yor.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-yor.yaml new file mode 100644 index 00000000..9b20e9f9 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-yor.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-yor +include: mafand +task: mafand_en-yor_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-zul.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-zul.yaml new file mode 100644 index 00000000..cb5280b9 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_en-zul.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: en-zul +include: mafand +task: mafand_en-zul_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_fr-bam.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_fr-bam.yaml new file mode 100644 index 00000000..3e94be00 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_fr-bam.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-bam +include: mafand +task: mafand_fr-bam_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_fr-bbj.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_fr-bbj.yaml new file mode 100644 index 00000000..9170a6b5 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_fr-bbj.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-bbj +include: mafand +task: mafand_fr-bbj_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_fr-ewe.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_fr-ewe.yaml new file mode 100644 index 00000000..7139c81f --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_fr-ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-ewe +include: mafand +task: mafand_fr-ewe_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_fr-fon.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_fr-fon.yaml new file mode 100644 index 00000000..b42292ce --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_fr-fon.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-fon +include: mafand +task: mafand_fr-fon_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_fr-mos.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_fr-mos.yaml new file mode 100644 index 00000000..044047c3 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_fr-mos.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-mos +include: mafand +task: mafand_fr-mos_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_fr-wol.yaml b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_fr-wol.yaml new file mode 100644 index 00000000..9fc1bca3 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/mafand_fr-wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fr-wol +include: mafand +task: mafand_fr-wol_prompt_3 diff --git a/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/utils.py b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/utils.py new file mode 100644 index 00000000..0df3a329 --- /dev/null +++ b/lm_eval/tasks/afrobench/mafand/prompt_3/english-african/utils.py @@ -0,0 +1,121 @@ +languages = { + "amh": "Amharic", + "bam": "Bambara", + "bbj": "Gbomala", + "ewe": "Ewe", + "fon": "Fon", + "hau": "Hausa", + "ibo": "Igbo", + "kin": "Kinyarwanda", + "lug": "Luganda", + "luo": "Luo", + "mos": "Mossi", + "nya": "Chichewa", + "pcm": "Nigerian Pidgin", + "sna": "Shona", + "swa": "Swahili", + "tsn": "Setswana", + "twi": "Twi", + "wol": "Wolof", + "xho": "Xhosa", + "yor": "Yoruba", + "zul": "Zulu", +} + + +def get_target(doc): + target = ( + doc["translation"]["en"] + if "en" in doc["translation"].keys() + else doc["translation"]["fr"] + ) + return target + + +def get_target_reverse(doc): + target_key = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + target = doc["translation"][target_key] + return target + + +def create_text_prompt_1(doc): + source_key = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_sentence = doc["translation"][source_key] + source_lang = "English" if "en" in doc["translation"].keys() else "French" + prompt = ( + "You are an advanced Translator, a specialized assistant designed to translate documents from " + f"{languages[source_key]} into {source_lang}. \nYour main goal is to ensure translations are grammatically " + f"correct and human-oriented. \n{languages[source_key]}: {source_sentence} \n{source_lang}: " + ) + return prompt + + +def create_reverse_prompt_1(doc): + target_lang = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_key = "en" if "en" in doc["translation"].keys() else "fr" + source_lang = "English" if source_key == "en" else "French" + source_sentence = doc["translation"][source_key] + prompt = ( + "You are an advanced Translator, a specialized assistant designed to translate documents from " + f"{source_lang} into {languages[target_lang]}. \nYour main goal is to ensure translations are " + f"grammatically correct and human-oriented. \n{source_lang}: {source_sentence} \n{languages[target_lang]}: " + ) + return prompt + + +def create_text_prompt_2(doc): + source_key = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_sentence = doc["translation"][source_key] + source_lang = "English" if "en" in doc["translation"].keys() else "French" + prompt = ( + f"{languages[source_key]} sentence: {source_sentence} \n{source_lang} sentence: ", + ) + return prompt + + +def create_reverse_prompt_2(doc): + target_lang = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_key = "en" if "en" in doc["translation"].keys() else "fr" + source_lang = "English" if source_key == "en" else "French" + source_sentence = doc["translation"][source_key] + prompt = ( + f"{source_lang} sentence: {source_sentence} \n{languages[target_lang]} sentence: \n", + ) + return prompt + + +def create_text_prompt_3(doc): + source_key = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_sentence = doc["translation"][source_key] + source_lang = "English" if "en" in doc["translation"].keys() else "French" + prompt = ( + f"You are a translation expert. Translate the following {languages[source_key]} sentences " + f"to {source_lang}. \n{languages[source_key]} sentence: {source_sentence}\n{source_lang} sentence: " + ) + return prompt + + +def create_reverse_prompt_3(doc): + target_lang = [key for key in doc["translation"].keys() if key not in ["en", "fr"]][ + 0 + ] + source_key = "en" if "en" in doc["translation"].keys() else "fr" + source_lang = "English" if source_key == "en" else "French" + source_sentence = doc["translation"][source_key] + prompt = ( + f"You are a translation expert. Translate the following {source_lang} sentence into {languages[target_lang]}\n" + f"{source_lang} sentence: {source_sentence}\n{languages[target_lang]} sentence: " + ) + return prompt diff --git a/lm_eval/tasks/afrobench/masakhaner/README.md b/lm_eval/tasks/afrobench/masakhaner/README.md new file mode 100644 index 00000000..ca96648e --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/README.md @@ -0,0 +1,76 @@ +# + +## Paper +Title: `MasakhaNER 2.0: Africa-centric Transfer Learning for Named Entity Recognition` + +Paper Link: https://aclanthology.org/2022.emnlp-main.298/ + +## Abstract +>African languages are spoken by over a billion people, but they are under-represented in NLP research and development. Multiple challenges exist, including the limited availability of annotated training and evaluation datasets as well as the lack of understanding of which settings, languages, and recently proposed methods like cross-lingual transfer will be effective. In this paper, we aim to move towards solutions for these challenges, focusing on the task of named entity recognition (NER). We present the creation of the largest to-date human-annotated NER dataset for 20 African languages. We study the behaviour of state-of-the-art cross-lingual transfer methods in an Africa-centric setting, empirically demonstrating that the choice of source transfer language significantly affects performance. While much previous work defaults to using English as the source language, our results show that choosing the best transfer language improves zero-shot F1 scores by an average of 14% over 20 languages as compared to using English. + +HomePage: https://github.com/masakhane-io/masakhane-ner + +### Citation + +``` +@inproceedings{adelani-etal-2022-masakhaner, + title = "{M}asakha{NER} 2.0: {A}frica-centric Transfer Learning for Named Entity Recognition", + author = "Adelani, David Ifeoluwa and + Neubig, Graham and + Ruder, Sebastian and + Rijhwani, Shruti and + Beukman, Michael and + Palen-Michel, Chester and + Lignos, Constantine and + Alabi, Jesujoba O. and + Muhammad, Shamsuddeen H. and + Nabende, Peter and + Dione, Cheikh M. Bamba and + Bukula, Andiswa and + Mabuya, Rooweither and + Dossou, Bonaventure F. P. and + Sibanda, Blessing and + Buzaaba, Happy and + Mukiibi, Jonathan and + Kalipe, Godson and + Mbaye, Derguene and + Taylor, Amelia and + Kabore, Fatoumata and + Emezue, Chris Chinenye and + Aremu, Anuoluwapo and + Ogayo, Perez and + Gitau, Catherine and + Munkoh-Buabeng, Edwin and + Memdjokam Koagne, Victoire and + Tapo, Allahsera Auguste and + Macucwa, Tebogo and + Marivate, Vukosi and + Mboning, Elvis and + Gwadabe, Tajuddeen and + Adewumi, Tosin and + Ahia, Orevaoghene and + Nakatumba-Nabende, Joyce and + Mokono, Neo L. and + Ezeani, Ignatius and + Chukwuneke, Chiamaka and + Adeyemi, Mofetoluwa and + Hacheme, Gilles Q. and + Abdulmumim, Idris and + Ogundepo, Odunayo and + Yousuf, Oreen and + Moteu Ngoli, Tatiana and + Klakow, Dietrich", + editor = "Goldberg, Yoav and + Kozareva, Zornitsa and + Zhang, Yue", + booktitle = "Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing", + month = dec, + year = "2022", + address = "Abu Dhabi, United Arab Emirates", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2022.emnlp-main.298/", + doi = "10.18653/v1/2022.emnlp-main.298", + pages = "4488--4508", + abstract = "African languages are spoken by over a billion people, but they are under-represented in NLP research and development. Multiple challenges exist, including the limited availability of annotated training and evaluation datasets as well as the lack of understanding of which settings, languages, and recently proposed methods like cross-lingual transfer will be effective. In this paper, we aim to move towards solutions for these challenges, focusing on the task of named entity recognition (NER). We present the creation of the largest to-date human-annotated NER dataset for 20 African languages. We study the behaviour of state-of-the-art cross-lingual transfer methods in an Africa-centric setting, empirically demonstrating that the choice of source transfer language significantly affects performance. While much previous work defaults to using English as the source language, our results show that choosing the best transfer language improves zero-shot F1 scores by an average of 14{\%} over 20 languages as compared to using English." +} +``` diff --git a/lm_eval/tasks/afrobench/masakhaner/gen_utils.py b/lm_eval/tasks/afrobench/masakhaner/gen_utils.py new file mode 100644 index 00000000..4d101202 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/gen_utils.py @@ -0,0 +1,138 @@ +import argparse +import os + +import yaml + + +class FunctionTag: + def __init__(self, value): + self.value = value + + +def prompt_func(mode, lang): + prompt_map = { + "prompt_1": "Named entities refers to names of location, organisation and personal name. \n For example, " + "'David is an employee of Amazon and he is visiting New York next week to see Esther' will be \n" + "PERSON: David $ ORGANIZATION: Amazon $ LOCATION: New York $ PERSON: Esther \n\n" + "Ensure the output strictly follows the format: label: entity $ label: entity, with each unique " + "entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity, entity) or " + "irrelevant entries like none. \n\nText: {{text}} \n" + "Return only the output", + "prompt_2": "You are working as a named entity recognition expert and your task is to label a given text " + "with named entity labels. Your task is to identify and label any named entities present in the " + "text. The named entity labels that you will be using are PER (person), LOC (location), " + "ORG (organization) and DATE (date). Label multi-word entities as a single named entity. " + "For words which are not part of any named entity, do not return any value for it. \n" + "Ensure the output strictly follows the format: label: entity $$ label: entity, with each unique " + "entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity, entity) or " + "irrelevant entries like none. Return only the output \n\nText: {{text}}", + "prompt_3": f"You are a Named Entity Recognition expert in {lang} language. \nExtract all named entities from " + f"the following {lang} text and categorize them into PERSON, LOCATION, ORGANIZATION, or DATE. " + f"Ensure the output strictly follows the format: label: entity $$ label: entity, with each unique " + "entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity, entity) or " + "irrelevant entries like none. Return only the output \n\nText: {{text}}", + "prompt_4": f"As a {lang} linguist, label all named entities in the {lang} text below with the categories: " + "PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output strictly follows the format: label: " + "entity $$ label: entity, with each unique entity on a separate label line, avoiding grouped " + "entities (e.g., avoid LOC: entity, entity) or irrelevant entries like none. Return only the " + "output. \n\nText: {{text}}", + "prompt_5": "Provide a concise list of named entities in the text below. Use the following labels: " + "PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output strictly follows the format: label: " + "entity $$ label: entity, with each unique entity on a separate label line, avoiding grouped " + "entities (e.g., avoid LOC: entity, entity) or irrelevant entries like none. Return only the " + "output. \n\nText: {{text}}", + } + return prompt_map[mode] + + +def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None: + """ + Generate a yaml file for each language. + + :param output_dir: The directory to output the files to. + :param overwrite: Whether to overwrite files if they already exist. + """ + err = [] + languages = { + "am": "Amharic", + "bm": "Bambara", + "bbj": "Ghomala", + "ee": "Ewe", + "ha": "Hausa", + "ig": "Igbo", + "rw": "Kinyarwanda", + "lg": "Luganda", + "luo": "Luo", + "mos": "Mossi", + "ny": "Chichewa", + "pcm": "Nigerian Pidgin", + "sn": "chiShona", + "sw": "Kiswahili", + "tn": "Setswana", + "tw": "Twi", + "wo": "Wolof", + "xh": "isiXhosa", + "yo": "Yoruba", + "zu": "isiZulu", + } + + for lang in languages.keys(): + try: + file_name = f"masakhaner_{lang}.yaml" + task_name = f"masakhaner_{lang}_{mode}" + yaml_template = "masakhaner" + yaml_details = { + "include": yaml_template, + "task": task_name, + "dataset_name": lang, + "doc_to_text": prompt_func(mode, languages[lang]), + } + os.makedirs(f"{output_dir}/{mode}", exist_ok=True) + with open( + f"{output_dir}/{mode}/{file_name}", + "w" if overwrite else "x", + encoding="utf8", + ) as f: + f.write("# Generated by utils.py\n") + yaml.dump( + yaml_details, + f, + allow_unicode=True, + ) + except FileExistsError: + err.append(file_name) + + if len(err) > 0: + raise FileExistsError( + "Files were not created because they already exist (use --overwrite flag):" + f" {', '.join(err)}" + ) + + +def main() -> None: + """Parse CLI args and generate language-specific yaml files.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + default=True, + action="store_true", + help="Overwrite files if they already exist", + ) + parser.add_argument( + "--output-dir", + default="./", + help="Directory to write yaml files to", + ) + parser.add_argument( + "--mode", + default="prompt_1", + choices=["prompt_1", "prompt_2", "prompt_3", "prompt_4", "prompt_5"], + help="Prompt number", + ) + args = parser.parse_args() + + gen_lang_yamls(output_dir=args.output_dir, overwrite=args.overwrite, mode=args.mode) + + +if __name__ == "__main__": + main() diff --git a/lm_eval/tasks/afrobench/masakhaner/masakhaner.yaml b/lm_eval/tasks/afrobench/masakhaner/masakhaner.yaml new file mode 100644 index 00000000..b0d374e8 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/masakhaner.yaml @@ -0,0 +1,13 @@ +group: masakhaner +task: + - masakhaner_prompt_1 + - masakhaner_prompt_2 + - masakhaner_prompt_3 + - masakhaner_prompt_4 + - masakhaner_prompt_5 +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 1 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner new file mode 100644 index 00000000..706eb366 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner @@ -0,0 +1,26 @@ +tag: +- masakhaner_tasks +- masakhaner_prompt_1 +dataset_path: masakhane/masakhaner-x +dataset_name: null +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +generation_kwargs: + do_sample: false + until: + - + - <|im_end|> +validation_split: validation +test_split: test +fewshot_split: train +doc_to_target: target +filter_list: + - name: flexible-extract + filter: + - function: format_span +metric_list: + - metric: f1 + aggregation: !function utils.span_f1_agg + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_am.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_am.yaml new file mode 100644 index 00000000..b2128752 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_am.yaml @@ -0,0 +1,11 @@ +# Generated by utils.py +dataset_name: am +doc_to_text: "Named entities refers to names of location, organisation and personal\ + \ name. \n For example, 'David is an employee of Amazon and he is visiting New York\ + \ next week to see Esther' will be \nPERSON: David $ ORGANIZATION: Amazon $ LOCATION:\ + \ New York $ PERSON: Esther \n\nEnsure the output strictly follows the format: label:\ + \ entity $ label: entity, with each unique entity on a separate label line, avoiding\ + \ grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries like\ + \ none. \n\nText: {{text}} \nReturn only the output" +include: masakhaner +task: masakhaner_am_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_bbj.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_bbj.yaml new file mode 100644 index 00000000..3f3a72bd --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_bbj.yaml @@ -0,0 +1,11 @@ +# Generated by utils.py +dataset_name: bbj +doc_to_text: "Named entities refers to names of location, organisation and personal\ + \ name. \n For example, 'David is an employee of Amazon and he is visiting New York\ + \ next week to see Esther' will be \nPERSON: David $ ORGANIZATION: Amazon $ LOCATION:\ + \ New York $ PERSON: Esther \n\nEnsure the output strictly follows the format: label:\ + \ entity $ label: entity, with each unique entity on a separate label line, avoiding\ + \ grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries like\ + \ none. \n\nText: {{text}} \nReturn only the output" +include: masakhaner +task: masakhaner_bbj_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_bm.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_bm.yaml new file mode 100644 index 00000000..c38bdee9 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_bm.yaml @@ -0,0 +1,11 @@ +# Generated by utils.py +dataset_name: bm +doc_to_text: "Named entities refers to names of location, organisation and personal\ + \ name. \n For example, 'David is an employee of Amazon and he is visiting New York\ + \ next week to see Esther' will be \nPERSON: David $ ORGANIZATION: Amazon $ LOCATION:\ + \ New York $ PERSON: Esther \n\nEnsure the output strictly follows the format: label:\ + \ entity $ label: entity, with each unique entity on a separate label line, avoiding\ + \ grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries like\ + \ none. \n\nText: {{text}} \nReturn only the output" +include: masakhaner +task: masakhaner_bm_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_ee.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_ee.yaml new file mode 100644 index 00000000..97903908 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_ee.yaml @@ -0,0 +1,11 @@ +# Generated by utils.py +dataset_name: ee +doc_to_text: "Named entities refers to names of location, organisation and personal\ + \ name. \n For example, 'David is an employee of Amazon and he is visiting New York\ + \ next week to see Esther' will be \nPERSON: David $ ORGANIZATION: Amazon $ LOCATION:\ + \ New York $ PERSON: Esther \n\nEnsure the output strictly follows the format: label:\ + \ entity $ label: entity, with each unique entity on a separate label line, avoiding\ + \ grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries like\ + \ none. \n\nText: {{text}} \nReturn only the output" +include: masakhaner +task: masakhaner_ee_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_ha.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_ha.yaml new file mode 100644 index 00000000..ad117104 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_ha.yaml @@ -0,0 +1,11 @@ +# Generated by utils.py +dataset_name: ha +doc_to_text: "Named entities refers to names of location, organisation and personal\ + \ name. \n For example, 'David is an employee of Amazon and he is visiting New York\ + \ next week to see Esther' will be \nPERSON: David $ ORGANIZATION: Amazon $ LOCATION:\ + \ New York $ PERSON: Esther \n\nEnsure the output strictly follows the format: label:\ + \ entity $ label: entity, with each unique entity on a separate label line, avoiding\ + \ grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries like\ + \ none. \n\nText: {{text}} \nReturn only the output" +include: masakhaner +task: masakhaner_ha_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_ig.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_ig.yaml new file mode 100644 index 00000000..0f06c065 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_ig.yaml @@ -0,0 +1,11 @@ +# Generated by utils.py +dataset_name: ig +doc_to_text: "Named entities refers to names of location, organisation and personal\ + \ name. \n For example, 'David is an employee of Amazon and he is visiting New York\ + \ next week to see Esther' will be \nPERSON: David $ ORGANIZATION: Amazon $ LOCATION:\ + \ New York $ PERSON: Esther \n\nEnsure the output strictly follows the format: label:\ + \ entity $ label: entity, with each unique entity on a separate label line, avoiding\ + \ grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries like\ + \ none. \n\nText: {{text}} \nReturn only the output" +include: masakhaner +task: masakhaner_ig_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_lg.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_lg.yaml new file mode 100644 index 00000000..1823b20f --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_lg.yaml @@ -0,0 +1,11 @@ +# Generated by utils.py +dataset_name: lg +doc_to_text: "Named entities refers to names of location, organisation and personal\ + \ name. \n For example, 'David is an employee of Amazon and he is visiting New York\ + \ next week to see Esther' will be \nPERSON: David $ ORGANIZATION: Amazon $ LOCATION:\ + \ New York $ PERSON: Esther \n\nEnsure the output strictly follows the format: label:\ + \ entity $ label: entity, with each unique entity on a separate label line, avoiding\ + \ grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries like\ + \ none. \n\nText: {{text}} \nReturn only the output" +include: masakhaner +task: masakhaner_lg_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_luo.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_luo.yaml new file mode 100644 index 00000000..55b6d829 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_luo.yaml @@ -0,0 +1,11 @@ +# Generated by utils.py +dataset_name: luo +doc_to_text: "Named entities refers to names of location, organisation and personal\ + \ name. \n For example, 'David is an employee of Amazon and he is visiting New York\ + \ next week to see Esther' will be \nPERSON: David $ ORGANIZATION: Amazon $ LOCATION:\ + \ New York $ PERSON: Esther \n\nEnsure the output strictly follows the format: label:\ + \ entity $ label: entity, with each unique entity on a separate label line, avoiding\ + \ grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries like\ + \ none. \n\nText: {{text}} \nReturn only the output" +include: masakhaner +task: masakhaner_luo_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_mos.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_mos.yaml new file mode 100644 index 00000000..ac5ddf43 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_mos.yaml @@ -0,0 +1,11 @@ +# Generated by utils.py +dataset_name: mos +doc_to_text: "Named entities refers to names of location, organisation and personal\ + \ name. \n For example, 'David is an employee of Amazon and he is visiting New York\ + \ next week to see Esther' will be \nPERSON: David $ ORGANIZATION: Amazon $ LOCATION:\ + \ New York $ PERSON: Esther \n\nEnsure the output strictly follows the format: label:\ + \ entity $ label: entity, with each unique entity on a separate label line, avoiding\ + \ grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries like\ + \ none. \n\nText: {{text}} \nReturn only the output" +include: masakhaner +task: masakhaner_mos_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_ny.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_ny.yaml new file mode 100644 index 00000000..36d12ad2 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_ny.yaml @@ -0,0 +1,11 @@ +# Generated by utils.py +dataset_name: ny +doc_to_text: "Named entities refers to names of location, organisation and personal\ + \ name. \n For example, 'David is an employee of Amazon and he is visiting New York\ + \ next week to see Esther' will be \nPERSON: David $ ORGANIZATION: Amazon $ LOCATION:\ + \ New York $ PERSON: Esther \n\nEnsure the output strictly follows the format: label:\ + \ entity $ label: entity, with each unique entity on a separate label line, avoiding\ + \ grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries like\ + \ none. \n\nText: {{text}} \nReturn only the output" +include: masakhaner +task: masakhaner_ny_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_pcm.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_pcm.yaml new file mode 100644 index 00000000..8c09bf44 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_pcm.yaml @@ -0,0 +1,11 @@ +# Generated by utils.py +dataset_name: pcm +doc_to_text: "Named entities refers to names of location, organisation and personal\ + \ name. \n For example, 'David is an employee of Amazon and he is visiting New York\ + \ next week to see Esther' will be \nPERSON: David $ ORGANIZATION: Amazon $ LOCATION:\ + \ New York $ PERSON: Esther \n\nEnsure the output strictly follows the format: label:\ + \ entity $ label: entity, with each unique entity on a separate label line, avoiding\ + \ grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries like\ + \ none. \n\nText: {{text}} \nReturn only the output" +include: masakhaner +task: masakhaner_pcm_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_rw.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_rw.yaml new file mode 100644 index 00000000..7398e5fb --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_rw.yaml @@ -0,0 +1,11 @@ +# Generated by utils.py +dataset_name: rw +doc_to_text: "Named entities refers to names of location, organisation and personal\ + \ name. \n For example, 'David is an employee of Amazon and he is visiting New York\ + \ next week to see Esther' will be \nPERSON: David $ ORGANIZATION: Amazon $ LOCATION:\ + \ New York $ PERSON: Esther \n\nEnsure the output strictly follows the format: label:\ + \ entity $ label: entity, with each unique entity on a separate label line, avoiding\ + \ grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries like\ + \ none. \n\nText: {{text}} \nReturn only the output" +include: masakhaner +task: masakhaner_rw_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_sn.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_sn.yaml new file mode 100644 index 00000000..ecdd3260 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_sn.yaml @@ -0,0 +1,11 @@ +# Generated by utils.py +dataset_name: sn +doc_to_text: "Named entities refers to names of location, organisation and personal\ + \ name. \n For example, 'David is an employee of Amazon and he is visiting New York\ + \ next week to see Esther' will be \nPERSON: David $ ORGANIZATION: Amazon $ LOCATION:\ + \ New York $ PERSON: Esther \n\nEnsure the output strictly follows the format: label:\ + \ entity $ label: entity, with each unique entity on a separate label line, avoiding\ + \ grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries like\ + \ none. \n\nText: {{text}} \nReturn only the output" +include: masakhaner +task: masakhaner_sn_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_sw.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_sw.yaml new file mode 100644 index 00000000..f2bd3379 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_sw.yaml @@ -0,0 +1,11 @@ +# Generated by utils.py +dataset_name: sw +doc_to_text: "Named entities refers to names of location, organisation and personal\ + \ name. \n For example, 'David is an employee of Amazon and he is visiting New York\ + \ next week to see Esther' will be \nPERSON: David $ ORGANIZATION: Amazon $ LOCATION:\ + \ New York $ PERSON: Esther \n\nEnsure the output strictly follows the format: label:\ + \ entity $ label: entity, with each unique entity on a separate label line, avoiding\ + \ grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries like\ + \ none. \n\nText: {{text}} \nReturn only the output" +include: masakhaner +task: masakhaner_sw_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_tn.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_tn.yaml new file mode 100644 index 00000000..50d80dcb --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_tn.yaml @@ -0,0 +1,11 @@ +# Generated by utils.py +dataset_name: tn +doc_to_text: "Named entities refers to names of location, organisation and personal\ + \ name. \n For example, 'David is an employee of Amazon and he is visiting New York\ + \ next week to see Esther' will be \nPERSON: David $ ORGANIZATION: Amazon $ LOCATION:\ + \ New York $ PERSON: Esther \n\nEnsure the output strictly follows the format: label:\ + \ entity $ label: entity, with each unique entity on a separate label line, avoiding\ + \ grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries like\ + \ none. \n\nText: {{text}} \nReturn only the output" +include: masakhaner +task: masakhaner_tn_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_tw.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_tw.yaml new file mode 100644 index 00000000..6c8a8d40 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_tw.yaml @@ -0,0 +1,11 @@ +# Generated by utils.py +dataset_name: tw +doc_to_text: "Named entities refers to names of location, organisation and personal\ + \ name. \n For example, 'David is an employee of Amazon and he is visiting New York\ + \ next week to see Esther' will be \nPERSON: David $ ORGANIZATION: Amazon $ LOCATION:\ + \ New York $ PERSON: Esther \n\nEnsure the output strictly follows the format: label:\ + \ entity $ label: entity, with each unique entity on a separate label line, avoiding\ + \ grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries like\ + \ none. \n\nText: {{text}} \nReturn only the output" +include: masakhaner +task: masakhaner_tw_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_wo.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_wo.yaml new file mode 100644 index 00000000..5e5f6eea --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_wo.yaml @@ -0,0 +1,11 @@ +# Generated by utils.py +dataset_name: wo +doc_to_text: "Named entities refers to names of location, organisation and personal\ + \ name. \n For example, 'David is an employee of Amazon and he is visiting New York\ + \ next week to see Esther' will be \nPERSON: David $ ORGANIZATION: Amazon $ LOCATION:\ + \ New York $ PERSON: Esther \n\nEnsure the output strictly follows the format: label:\ + \ entity $ label: entity, with each unique entity on a separate label line, avoiding\ + \ grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries like\ + \ none. \n\nText: {{text}} \nReturn only the output" +include: masakhaner +task: masakhaner_wo_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_xh.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_xh.yaml new file mode 100644 index 00000000..8b27051f --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_xh.yaml @@ -0,0 +1,11 @@ +# Generated by utils.py +dataset_name: xh +doc_to_text: "Named entities refers to names of location, organisation and personal\ + \ name. \n For example, 'David is an employee of Amazon and he is visiting New York\ + \ next week to see Esther' will be \nPERSON: David $ ORGANIZATION: Amazon $ LOCATION:\ + \ New York $ PERSON: Esther \n\nEnsure the output strictly follows the format: label:\ + \ entity $ label: entity, with each unique entity on a separate label line, avoiding\ + \ grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries like\ + \ none. \n\nText: {{text}} \nReturn only the output" +include: masakhaner +task: masakhaner_xh_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_yo.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_yo.yaml new file mode 100644 index 00000000..2fdb71aa --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_yo.yaml @@ -0,0 +1,11 @@ +# Generated by utils.py +dataset_name: yo +doc_to_text: "Named entities refers to names of location, organisation and personal\ + \ name. \n For example, 'David is an employee of Amazon and he is visiting New York\ + \ next week to see Esther' will be \nPERSON: David $ ORGANIZATION: Amazon $ LOCATION:\ + \ New York $ PERSON: Esther \n\nEnsure the output strictly follows the format: label:\ + \ entity $ label: entity, with each unique entity on a separate label line, avoiding\ + \ grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries like\ + \ none. \n\nText: {{text}} \nReturn only the output" +include: masakhaner +task: masakhaner_yo_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_zu.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_zu.yaml new file mode 100644 index 00000000..83b9d4b0 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_1/masakhaner_zu.yaml @@ -0,0 +1,11 @@ +# Generated by utils.py +dataset_name: zu +doc_to_text: "Named entities refers to names of location, organisation and personal\ + \ name. \n For example, 'David is an employee of Amazon and he is visiting New York\ + \ next week to see Esther' will be \nPERSON: David $ ORGANIZATION: Amazon $ LOCATION:\ + \ New York $ PERSON: Esther \n\nEnsure the output strictly follows the format: label:\ + \ entity $ label: entity, with each unique entity on a separate label line, avoiding\ + \ grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries like\ + \ none. \n\nText: {{text}} \nReturn only the output" +include: masakhaner +task: masakhaner_zu_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_1/utils.py b/lm_eval/tasks/afrobench/masakhaner/prompt_1/utils.py new file mode 100644 index 00000000..76909044 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_1/utils.py @@ -0,0 +1,146 @@ +import collections +import re + +from lm_eval.utils import weighted_f1_score + + +def doc_to_target(doc): + return transform_text(doc["ner_tags"]) + + +def transform_text(text): + entities = [] + current_entity = "" + current_tag = "" + + for pair in text.split("\n"): + if pair: # Check if the line is not empty + word, tag = pair.strip().split(": ") + tag = tag.upper() + word = word.lower() + word = word.strip(",.").strip() + + if tag.startswith("B-"): + if current_entity: + entities.append(f"{current_tag}: {current_entity}") + current_tag = tag.split("-")[1] + current_entity = word + elif tag.startswith("I-") and tag.split("-")[1] == current_tag: + current_entity += word + else: + if current_entity: + entities.append(f"{current_tag}: {current_entity}") + current_entity = "" + current_tag = "" + if current_entity: + entities.append(f"{current_tag}: {current_entity}") + + # Join all the transformed output lines with $$ as separator + return " $$ ".join(entities) + + +def span_f1_agg(items): + """Computes Span based F1 score. + + This function is copied from + https://github.com/google-research/multilingual-t5/blob/master/multilingual_t5/evaluation/metrics.py + + Args: + targets: list of strings or list of list of strings if multiple references + are present. + predictions: list of strings + + Returns: + span f1 across all targets and predictions (Based on CoNLL script) + """ + unzipped_list = list(zip(*items)) + targets = unzipped_list[0] + predictions = unzipped_list[1] + + true_positives = collections.defaultdict(int) + false_positives = collections.defaultdict(int) + false_negatives = collections.defaultdict(int) + + def normalize_text(strings): + def get_blank_spaces_pattern(): + return re.compile(r"\s{3,}|\t") + + def remove_blank_spaces(text): + text = re.sub(pattern=get_blank_spaces_pattern(), repl="", string=text) + text = re.sub("\s+", " ", text) + return text + + def remove_punctuation(text): + my_punctuation = '!"$%&\'()*+,-./:;<=>?[\\]^_`{|}~•@.""-,`' + text = re.sub( + "[" + my_punctuation + "]+", " ", str(text) + ) # strip punctuation + return text + + def remove_articles(text): + regex = re.compile(r"\b(a|an|the)\b", re.UNICODE) + return re.sub(regex, " ", text) + + def lowercase(text): + text = text.lower() + return text + + strings = remove_punctuation(strings) + strings = remove_articles(strings) + strings = remove_blank_spaces(strings) + strings = lowercase(strings) + + return strings + + def tags_to_spans(tag_sequence, delimiter="$$"): + """Extract spans from IOB1 or BIO tags.""" + if isinstance(tag_sequence, list): + tag_sequence = " ".join(i.strip() for i in tag_sequence) + tag_sequence_split = [ + item.strip() + for sub in tag_sequence.strip().split(delimiter) + for item in sub.split("$") + if item + ] + tag_sequence_split = [ + item.strip() + for value in tag_sequence_split + for sub in value.split(". ") + for item in sub.split(", ") + ] + tags_entities = [] + for tag_entity in tag_sequence_split: + tag_entity_split = tag_entity.split(": ") + if len(tag_entity_split) != 2: + continue + tag = normalize_text(tag_entity_split[0].strip()) + entity = normalize_text(tag_entity_split[1].rstrip().lstrip()) + tags_entities.append((tag, entity)) + return tags_entities + + def compute_f1_metrics(true_positive, false_positive, false_negative): + precision = float(true_positive) / float(true_positive + false_positive + 1e-13) + recall = float(true_positive) / float(true_positive + false_negative + 1e-13) + f1_measures = 2.0 * ((precision * recall) / (precision + recall + 1e-13)) + return precision, recall, f1_measures + + for target, pred in zip(targets, predictions): + gold_spans = tags_to_spans(target) + predicted_spans = tags_to_spans(pred) + + for span in predicted_spans: + if span in gold_spans: + true_positives[span[0]] += 1 + gold_spans.remove(span) + else: + false_positives[span[0]] += 1 + # These spans weren't predicted. + for span in gold_spans: + false_negatives[span[0]] += 1 + + _, _, f1_measure = compute_f1_metrics( + sum(true_positives.values()), + sum(false_positives.values()), + sum(false_negatives.values()), + ) + return f1_measure diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner new file mode 100644 index 00000000..2fd5eb82 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner @@ -0,0 +1,26 @@ +tag: +- masakhaner_tasks +- masakhaner_prompt_2 +dataset_path: masakhane/masakhaner-x +dataset_name: null +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +generation_kwargs: + do_sample: false + until: + - + - <|im_end|> +validation_split: validation +test_split: test +fewshot_split: train +doc_to_target: target +filter_list: + - name: flexible-extract + filter: + - function: format_span +metric_list: + - metric: f1 + aggregation: !function utils.span_f1_agg + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_am.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_am.yaml new file mode 100644 index 00000000..bd1bd335 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_am.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: am +doc_to_text: "You are working as a named entity recognition expert and your task is\ + \ to label a given text with named entity labels. Your task is to identify and label\ + \ any named entities present in the text. The named entity labels that you will\ + \ be using are PER (person), LOC (location), ORG (organization) and DATE (date).\ + \ Label multi-word entities as a single named entity. For words which are not part\ + \ of any named entity, do not return any value for it. \nEnsure the output strictly\ + \ follows the format: label: entity $$ label: entity, with each unique entity on\ + \ a separate label line, avoiding grouped entities (e.g., avoid LOC: entity, entity)\ + \ or irrelevant entries like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_am_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_bbj.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_bbj.yaml new file mode 100644 index 00000000..5d817ecb --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_bbj.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: bbj +doc_to_text: "You are working as a named entity recognition expert and your task is\ + \ to label a given text with named entity labels. Your task is to identify and label\ + \ any named entities present in the text. The named entity labels that you will\ + \ be using are PER (person), LOC (location), ORG (organization) and DATE (date).\ + \ Label multi-word entities as a single named entity. For words which are not part\ + \ of any named entity, do not return any value for it. \nEnsure the output strictly\ + \ follows the format: label: entity $$ label: entity, with each unique entity on\ + \ a separate label line, avoiding grouped entities (e.g., avoid LOC: entity, entity)\ + \ or irrelevant entries like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_bbj_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_bm.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_bm.yaml new file mode 100644 index 00000000..f99a03c7 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_bm.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: bm +doc_to_text: "You are working as a named entity recognition expert and your task is\ + \ to label a given text with named entity labels. Your task is to identify and label\ + \ any named entities present in the text. The named entity labels that you will\ + \ be using are PER (person), LOC (location), ORG (organization) and DATE (date).\ + \ Label multi-word entities as a single named entity. For words which are not part\ + \ of any named entity, do not return any value for it. \nEnsure the output strictly\ + \ follows the format: label: entity $$ label: entity, with each unique entity on\ + \ a separate label line, avoiding grouped entities (e.g., avoid LOC: entity, entity)\ + \ or irrelevant entries like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_bm_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_ee.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_ee.yaml new file mode 100644 index 00000000..da31685e --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_ee.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: ee +doc_to_text: "You are working as a named entity recognition expert and your task is\ + \ to label a given text with named entity labels. Your task is to identify and label\ + \ any named entities present in the text. The named entity labels that you will\ + \ be using are PER (person), LOC (location), ORG (organization) and DATE (date).\ + \ Label multi-word entities as a single named entity. For words which are not part\ + \ of any named entity, do not return any value for it. \nEnsure the output strictly\ + \ follows the format: label: entity $$ label: entity, with each unique entity on\ + \ a separate label line, avoiding grouped entities (e.g., avoid LOC: entity, entity)\ + \ or irrelevant entries like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_ee_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_ha.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_ha.yaml new file mode 100644 index 00000000..8075046a --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_ha.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: ha +doc_to_text: "You are working as a named entity recognition expert and your task is\ + \ to label a given text with named entity labels. Your task is to identify and label\ + \ any named entities present in the text. The named entity labels that you will\ + \ be using are PER (person), LOC (location), ORG (organization) and DATE (date).\ + \ Label multi-word entities as a single named entity. For words which are not part\ + \ of any named entity, do not return any value for it. \nEnsure the output strictly\ + \ follows the format: label: entity $$ label: entity, with each unique entity on\ + \ a separate label line, avoiding grouped entities (e.g., avoid LOC: entity, entity)\ + \ or irrelevant entries like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_ha_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_ig.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_ig.yaml new file mode 100644 index 00000000..c8771f51 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_ig.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: ig +doc_to_text: "You are working as a named entity recognition expert and your task is\ + \ to label a given text with named entity labels. Your task is to identify and label\ + \ any named entities present in the text. The named entity labels that you will\ + \ be using are PER (person), LOC (location), ORG (organization) and DATE (date).\ + \ Label multi-word entities as a single named entity. For words which are not part\ + \ of any named entity, do not return any value for it. \nEnsure the output strictly\ + \ follows the format: label: entity $$ label: entity, with each unique entity on\ + \ a separate label line, avoiding grouped entities (e.g., avoid LOC: entity, entity)\ + \ or irrelevant entries like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_ig_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_lg.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_lg.yaml new file mode 100644 index 00000000..5c6729e3 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_lg.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: lg +doc_to_text: "You are working as a named entity recognition expert and your task is\ + \ to label a given text with named entity labels. Your task is to identify and label\ + \ any named entities present in the text. The named entity labels that you will\ + \ be using are PER (person), LOC (location), ORG (organization) and DATE (date).\ + \ Label multi-word entities as a single named entity. For words which are not part\ + \ of any named entity, do not return any value for it. \nEnsure the output strictly\ + \ follows the format: label: entity $$ label: entity, with each unique entity on\ + \ a separate label line, avoiding grouped entities (e.g., avoid LOC: entity, entity)\ + \ or irrelevant entries like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_lg_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_luo.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_luo.yaml new file mode 100644 index 00000000..a458235f --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_luo.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: luo +doc_to_text: "You are working as a named entity recognition expert and your task is\ + \ to label a given text with named entity labels. Your task is to identify and label\ + \ any named entities present in the text. The named entity labels that you will\ + \ be using are PER (person), LOC (location), ORG (organization) and DATE (date).\ + \ Label multi-word entities as a single named entity. For words which are not part\ + \ of any named entity, do not return any value for it. \nEnsure the output strictly\ + \ follows the format: label: entity $$ label: entity, with each unique entity on\ + \ a separate label line, avoiding grouped entities (e.g., avoid LOC: entity, entity)\ + \ or irrelevant entries like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_luo_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_mos.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_mos.yaml new file mode 100644 index 00000000..816b9bde --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_mos.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: mos +doc_to_text: "You are working as a named entity recognition expert and your task is\ + \ to label a given text with named entity labels. Your task is to identify and label\ + \ any named entities present in the text. The named entity labels that you will\ + \ be using are PER (person), LOC (location), ORG (organization) and DATE (date).\ + \ Label multi-word entities as a single named entity. For words which are not part\ + \ of any named entity, do not return any value for it. \nEnsure the output strictly\ + \ follows the format: label: entity $$ label: entity, with each unique entity on\ + \ a separate label line, avoiding grouped entities (e.g., avoid LOC: entity, entity)\ + \ or irrelevant entries like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_mos_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_ny.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_ny.yaml new file mode 100644 index 00000000..2f8c4c13 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_ny.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: ny +doc_to_text: "You are working as a named entity recognition expert and your task is\ + \ to label a given text with named entity labels. Your task is to identify and label\ + \ any named entities present in the text. The named entity labels that you will\ + \ be using are PER (person), LOC (location), ORG (organization) and DATE (date).\ + \ Label multi-word entities as a single named entity. For words which are not part\ + \ of any named entity, do not return any value for it. \nEnsure the output strictly\ + \ follows the format: label: entity $$ label: entity, with each unique entity on\ + \ a separate label line, avoiding grouped entities (e.g., avoid LOC: entity, entity)\ + \ or irrelevant entries like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_ny_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_pcm.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_pcm.yaml new file mode 100644 index 00000000..75dc6ec0 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_pcm.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: pcm +doc_to_text: "You are working as a named entity recognition expert and your task is\ + \ to label a given text with named entity labels. Your task is to identify and label\ + \ any named entities present in the text. The named entity labels that you will\ + \ be using are PER (person), LOC (location), ORG (organization) and DATE (date).\ + \ Label multi-word entities as a single named entity. For words which are not part\ + \ of any named entity, do not return any value for it. \nEnsure the output strictly\ + \ follows the format: label: entity $$ label: entity, with each unique entity on\ + \ a separate label line, avoiding grouped entities (e.g., avoid LOC: entity, entity)\ + \ or irrelevant entries like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_pcm_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_rw.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_rw.yaml new file mode 100644 index 00000000..fb93e2d4 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_rw.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: rw +doc_to_text: "You are working as a named entity recognition expert and your task is\ + \ to label a given text with named entity labels. Your task is to identify and label\ + \ any named entities present in the text. The named entity labels that you will\ + \ be using are PER (person), LOC (location), ORG (organization) and DATE (date).\ + \ Label multi-word entities as a single named entity. For words which are not part\ + \ of any named entity, do not return any value for it. \nEnsure the output strictly\ + \ follows the format: label: entity $$ label: entity, with each unique entity on\ + \ a separate label line, avoiding grouped entities (e.g., avoid LOC: entity, entity)\ + \ or irrelevant entries like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_rw_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_sn.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_sn.yaml new file mode 100644 index 00000000..60380a51 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_sn.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: sn +doc_to_text: "You are working as a named entity recognition expert and your task is\ + \ to label a given text with named entity labels. Your task is to identify and label\ + \ any named entities present in the text. The named entity labels that you will\ + \ be using are PER (person), LOC (location), ORG (organization) and DATE (date).\ + \ Label multi-word entities as a single named entity. For words which are not part\ + \ of any named entity, do not return any value for it. \nEnsure the output strictly\ + \ follows the format: label: entity $$ label: entity, with each unique entity on\ + \ a separate label line, avoiding grouped entities (e.g., avoid LOC: entity, entity)\ + \ or irrelevant entries like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_sn_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_sw.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_sw.yaml new file mode 100644 index 00000000..82cf74ae --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_sw.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: sw +doc_to_text: "You are working as a named entity recognition expert and your task is\ + \ to label a given text with named entity labels. Your task is to identify and label\ + \ any named entities present in the text. The named entity labels that you will\ + \ be using are PER (person), LOC (location), ORG (organization) and DATE (date).\ + \ Label multi-word entities as a single named entity. For words which are not part\ + \ of any named entity, do not return any value for it. \nEnsure the output strictly\ + \ follows the format: label: entity $$ label: entity, with each unique entity on\ + \ a separate label line, avoiding grouped entities (e.g., avoid LOC: entity, entity)\ + \ or irrelevant entries like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_sw_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_tn.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_tn.yaml new file mode 100644 index 00000000..1852ebe9 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_tn.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: tn +doc_to_text: "You are working as a named entity recognition expert and your task is\ + \ to label a given text with named entity labels. Your task is to identify and label\ + \ any named entities present in the text. The named entity labels that you will\ + \ be using are PER (person), LOC (location), ORG (organization) and DATE (date).\ + \ Label multi-word entities as a single named entity. For words which are not part\ + \ of any named entity, do not return any value for it. \nEnsure the output strictly\ + \ follows the format: label: entity $$ label: entity, with each unique entity on\ + \ a separate label line, avoiding grouped entities (e.g., avoid LOC: entity, entity)\ + \ or irrelevant entries like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_tn_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_tw.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_tw.yaml new file mode 100644 index 00000000..ea354958 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_tw.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: tw +doc_to_text: "You are working as a named entity recognition expert and your task is\ + \ to label a given text with named entity labels. Your task is to identify and label\ + \ any named entities present in the text. The named entity labels that you will\ + \ be using are PER (person), LOC (location), ORG (organization) and DATE (date).\ + \ Label multi-word entities as a single named entity. For words which are not part\ + \ of any named entity, do not return any value for it. \nEnsure the output strictly\ + \ follows the format: label: entity $$ label: entity, with each unique entity on\ + \ a separate label line, avoiding grouped entities (e.g., avoid LOC: entity, entity)\ + \ or irrelevant entries like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_tw_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_wo.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_wo.yaml new file mode 100644 index 00000000..e7cd0d75 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_wo.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: wo +doc_to_text: "You are working as a named entity recognition expert and your task is\ + \ to label a given text with named entity labels. Your task is to identify and label\ + \ any named entities present in the text. The named entity labels that you will\ + \ be using are PER (person), LOC (location), ORG (organization) and DATE (date).\ + \ Label multi-word entities as a single named entity. For words which are not part\ + \ of any named entity, do not return any value for it. \nEnsure the output strictly\ + \ follows the format: label: entity $$ label: entity, with each unique entity on\ + \ a separate label line, avoiding grouped entities (e.g., avoid LOC: entity, entity)\ + \ or irrelevant entries like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_wo_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_xh.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_xh.yaml new file mode 100644 index 00000000..9451f0ed --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_xh.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: xh +doc_to_text: "You are working as a named entity recognition expert and your task is\ + \ to label a given text with named entity labels. Your task is to identify and label\ + \ any named entities present in the text. The named entity labels that you will\ + \ be using are PER (person), LOC (location), ORG (organization) and DATE (date).\ + \ Label multi-word entities as a single named entity. For words which are not part\ + \ of any named entity, do not return any value for it. \nEnsure the output strictly\ + \ follows the format: label: entity $$ label: entity, with each unique entity on\ + \ a separate label line, avoiding grouped entities (e.g., avoid LOC: entity, entity)\ + \ or irrelevant entries like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_xh_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_yo.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_yo.yaml new file mode 100644 index 00000000..fc0d92c5 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_yo.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: yo +doc_to_text: "You are working as a named entity recognition expert and your task is\ + \ to label a given text with named entity labels. Your task is to identify and label\ + \ any named entities present in the text. The named entity labels that you will\ + \ be using are PER (person), LOC (location), ORG (organization) and DATE (date).\ + \ Label multi-word entities as a single named entity. For words which are not part\ + \ of any named entity, do not return any value for it. \nEnsure the output strictly\ + \ follows the format: label: entity $$ label: entity, with each unique entity on\ + \ a separate label line, avoiding grouped entities (e.g., avoid LOC: entity, entity)\ + \ or irrelevant entries like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_yo_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_zu.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_zu.yaml new file mode 100644 index 00000000..e06bf3ce --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_2/masakhaner_zu.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: zu +doc_to_text: "You are working as a named entity recognition expert and your task is\ + \ to label a given text with named entity labels. Your task is to identify and label\ + \ any named entities present in the text. The named entity labels that you will\ + \ be using are PER (person), LOC (location), ORG (organization) and DATE (date).\ + \ Label multi-word entities as a single named entity. For words which are not part\ + \ of any named entity, do not return any value for it. \nEnsure the output strictly\ + \ follows the format: label: entity $$ label: entity, with each unique entity on\ + \ a separate label line, avoiding grouped entities (e.g., avoid LOC: entity, entity)\ + \ or irrelevant entries like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_zu_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_2/utils.py b/lm_eval/tasks/afrobench/masakhaner/prompt_2/utils.py new file mode 100644 index 00000000..76909044 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_2/utils.py @@ -0,0 +1,146 @@ +import collections +import re + +from lm_eval.utils import weighted_f1_score + + +def doc_to_target(doc): + return transform_text(doc["ner_tags"]) + + +def transform_text(text): + entities = [] + current_entity = "" + current_tag = "" + + for pair in text.split("\n"): + if pair: # Check if the line is not empty + word, tag = pair.strip().split(": ") + tag = tag.upper() + word = word.lower() + word = word.strip(",.").strip() + + if tag.startswith("B-"): + if current_entity: + entities.append(f"{current_tag}: {current_entity}") + current_tag = tag.split("-")[1] + current_entity = word + elif tag.startswith("I-") and tag.split("-")[1] == current_tag: + current_entity += word + else: + if current_entity: + entities.append(f"{current_tag}: {current_entity}") + current_entity = "" + current_tag = "" + if current_entity: + entities.append(f"{current_tag}: {current_entity}") + + # Join all the transformed output lines with $$ as separator + return " $$ ".join(entities) + + +def span_f1_agg(items): + """Computes Span based F1 score. + + This function is copied from + https://github.com/google-research/multilingual-t5/blob/master/multilingual_t5/evaluation/metrics.py + + Args: + targets: list of strings or list of list of strings if multiple references + are present. + predictions: list of strings + + Returns: + span f1 across all targets and predictions (Based on CoNLL script) + """ + unzipped_list = list(zip(*items)) + targets = unzipped_list[0] + predictions = unzipped_list[1] + + true_positives = collections.defaultdict(int) + false_positives = collections.defaultdict(int) + false_negatives = collections.defaultdict(int) + + def normalize_text(strings): + def get_blank_spaces_pattern(): + return re.compile(r"\s{3,}|\t") + + def remove_blank_spaces(text): + text = re.sub(pattern=get_blank_spaces_pattern(), repl="", string=text) + text = re.sub("\s+", " ", text) + return text + + def remove_punctuation(text): + my_punctuation = '!"$%&\'()*+,-./:;<=>?[\\]^_`{|}~•@.""-,`' + text = re.sub( + "[" + my_punctuation + "]+", " ", str(text) + ) # strip punctuation + return text + + def remove_articles(text): + regex = re.compile(r"\b(a|an|the)\b", re.UNICODE) + return re.sub(regex, " ", text) + + def lowercase(text): + text = text.lower() + return text + + strings = remove_punctuation(strings) + strings = remove_articles(strings) + strings = remove_blank_spaces(strings) + strings = lowercase(strings) + + return strings + + def tags_to_spans(tag_sequence, delimiter="$$"): + """Extract spans from IOB1 or BIO tags.""" + if isinstance(tag_sequence, list): + tag_sequence = " ".join(i.strip() for i in tag_sequence) + tag_sequence_split = [ + item.strip() + for sub in tag_sequence.strip().split(delimiter) + for item in sub.split("$") + if item + ] + tag_sequence_split = [ + item.strip() + for value in tag_sequence_split + for sub in value.split(". ") + for item in sub.split(", ") + ] + tags_entities = [] + for tag_entity in tag_sequence_split: + tag_entity_split = tag_entity.split(": ") + if len(tag_entity_split) != 2: + continue + tag = normalize_text(tag_entity_split[0].strip()) + entity = normalize_text(tag_entity_split[1].rstrip().lstrip()) + tags_entities.append((tag, entity)) + return tags_entities + + def compute_f1_metrics(true_positive, false_positive, false_negative): + precision = float(true_positive) / float(true_positive + false_positive + 1e-13) + recall = float(true_positive) / float(true_positive + false_negative + 1e-13) + f1_measures = 2.0 * ((precision * recall) / (precision + recall + 1e-13)) + return precision, recall, f1_measures + + for target, pred in zip(targets, predictions): + gold_spans = tags_to_spans(target) + predicted_spans = tags_to_spans(pred) + + for span in predicted_spans: + if span in gold_spans: + true_positives[span[0]] += 1 + gold_spans.remove(span) + else: + false_positives[span[0]] += 1 + # These spans weren't predicted. + for span in gold_spans: + false_negatives[span[0]] += 1 + + _, _, f1_measure = compute_f1_metrics( + sum(true_positives.values()), + sum(false_positives.values()), + sum(false_negatives.values()), + ) + return f1_measure diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner new file mode 100644 index 00000000..7f32f86b --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner @@ -0,0 +1,26 @@ +tag: +- masakhaner_tasks +- masakhaner_prompt_3 +dataset_path: masakhane/masakhaner-x +dataset_name: null +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +generation_kwargs: + do_sample: false + until: + - + - <|im_end|> +validation_split: validation +test_split: test +fewshot_split: train +doc_to_target: target +filter_list: + - name: flexible-extract + filter: + - function: format_span +metric_list: + - metric: f1 + aggregation: !function utils.span_f1_agg + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_am.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_am.yaml new file mode 100644 index 00000000..54ad8b54 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_am.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: am +doc_to_text: "You are a Named Entity Recognition expert in Amharic language. \nExtract\ + \ all named entities from the following Amharic text and categorize them into PERSON,\ + \ LOCATION, ORGANIZATION, or DATE. Ensure the output strictly follows the format:\ + \ label: entity $$ label: entity, with each unique entity on a separate label line,\ + \ avoiding grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries\ + \ like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_am_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_bbj.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_bbj.yaml new file mode 100644 index 00000000..23e724f4 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_bbj.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: bbj +doc_to_text: "You are a Named Entity Recognition expert in Ghomala language. \nExtract\ + \ all named entities from the following Ghomala text and categorize them into PERSON,\ + \ LOCATION, ORGANIZATION, or DATE. Ensure the output strictly follows the format:\ + \ label: entity $$ label: entity, with each unique entity on a separate label line,\ + \ avoiding grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries\ + \ like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_bbj_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_bm.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_bm.yaml new file mode 100644 index 00000000..62b5b80e --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_bm.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: bm +doc_to_text: "You are a Named Entity Recognition expert in Bambara language. \nExtract\ + \ all named entities from the following Bambara text and categorize them into PERSON,\ + \ LOCATION, ORGANIZATION, or DATE. Ensure the output strictly follows the format:\ + \ label: entity $$ label: entity, with each unique entity on a separate label line,\ + \ avoiding grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries\ + \ like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_bm_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_ee.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_ee.yaml new file mode 100644 index 00000000..2cdadd27 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_ee.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: ee +doc_to_text: "You are a Named Entity Recognition expert in Ewe language. \nExtract\ + \ all named entities from the following Ewe text and categorize them into PERSON,\ + \ LOCATION, ORGANIZATION, or DATE. Ensure the output strictly follows the format:\ + \ label: entity $$ label: entity, with each unique entity on a separate label line,\ + \ avoiding grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries\ + \ like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_ee_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_ha.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_ha.yaml new file mode 100644 index 00000000..9d19d26f --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_ha.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: ha +doc_to_text: "You are a Named Entity Recognition expert in Hausa language. \nExtract\ + \ all named entities from the following Hausa text and categorize them into PERSON,\ + \ LOCATION, ORGANIZATION, or DATE. Ensure the output strictly follows the format:\ + \ label: entity $$ label: entity, with each unique entity on a separate label line,\ + \ avoiding grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries\ + \ like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_ha_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_ig.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_ig.yaml new file mode 100644 index 00000000..edf61196 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_ig.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: ig +doc_to_text: "You are a Named Entity Recognition expert in Igbo language. \nExtract\ + \ all named entities from the following Igbo text and categorize them into PERSON,\ + \ LOCATION, ORGANIZATION, or DATE. Ensure the output strictly follows the format:\ + \ label: entity $$ label: entity, with each unique entity on a separate label line,\ + \ avoiding grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries\ + \ like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_ig_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_lg.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_lg.yaml new file mode 100644 index 00000000..9318a782 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_lg.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: lg +doc_to_text: "You are a Named Entity Recognition expert in Luganda language. \nExtract\ + \ all named entities from the following Luganda text and categorize them into PERSON,\ + \ LOCATION, ORGANIZATION, or DATE. Ensure the output strictly follows the format:\ + \ label: entity $$ label: entity, with each unique entity on a separate label line,\ + \ avoiding grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries\ + \ like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_lg_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_luo.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_luo.yaml new file mode 100644 index 00000000..61254fc3 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_luo.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: luo +doc_to_text: "You are a Named Entity Recognition expert in Luo language. \nExtract\ + \ all named entities from the following Luo text and categorize them into PERSON,\ + \ LOCATION, ORGANIZATION, or DATE. Ensure the output strictly follows the format:\ + \ label: entity $$ label: entity, with each unique entity on a separate label line,\ + \ avoiding grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries\ + \ like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_luo_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_mos.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_mos.yaml new file mode 100644 index 00000000..84ff6b24 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_mos.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: mos +doc_to_text: "You are a Named Entity Recognition expert in Mossi language. \nExtract\ + \ all named entities from the following Mossi text and categorize them into PERSON,\ + \ LOCATION, ORGANIZATION, or DATE. Ensure the output strictly follows the format:\ + \ label: entity $$ label: entity, with each unique entity on a separate label line,\ + \ avoiding grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries\ + \ like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_mos_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_ny.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_ny.yaml new file mode 100644 index 00000000..bd592c5b --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_ny.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: ny +doc_to_text: "You are a Named Entity Recognition expert in Chichewa language. \nExtract\ + \ all named entities from the following Chichewa text and categorize them into PERSON,\ + \ LOCATION, ORGANIZATION, or DATE. Ensure the output strictly follows the format:\ + \ label: entity $$ label: entity, with each unique entity on a separate label line,\ + \ avoiding grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries\ + \ like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_ny_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_pcm.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_pcm.yaml new file mode 100644 index 00000000..b448b244 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_pcm.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: pcm +doc_to_text: "You are a Named Entity Recognition expert in Nigerian Pidgin language.\ + \ \nExtract all named entities from the following Nigerian Pidgin text and categorize\ + \ them into PERSON, LOCATION, ORGANIZATION, or DATE. Ensure the output strictly\ + \ follows the format: label: entity $$ label: entity, with each unique entity on\ + \ a separate label line, avoiding grouped entities (e.g., avoid LOC: entity, entity)\ + \ or irrelevant entries like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_pcm_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_rw.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_rw.yaml new file mode 100644 index 00000000..5356ce8b --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_rw.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: rw +doc_to_text: "You are a Named Entity Recognition expert in Kinyarwanda language. \n\ + Extract all named entities from the following Kinyarwanda text and categorize them\ + \ into PERSON, LOCATION, ORGANIZATION, or DATE. Ensure the output strictly follows\ + \ the format: label: entity $$ label: entity, with each unique entity on a separate\ + \ label line, avoiding grouped entities (e.g., avoid LOC: entity, entity) or irrelevant\ + \ entries like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_rw_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_sn.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_sn.yaml new file mode 100644 index 00000000..ab356ae8 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_sn.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: sn +doc_to_text: "You are a Named Entity Recognition expert in chiShona language. \nExtract\ + \ all named entities from the following chiShona text and categorize them into PERSON,\ + \ LOCATION, ORGANIZATION, or DATE. Ensure the output strictly follows the format:\ + \ label: entity $$ label: entity, with each unique entity on a separate label line,\ + \ avoiding grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries\ + \ like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_sn_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_sw.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_sw.yaml new file mode 100644 index 00000000..bb3d6959 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_sw.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: sw +doc_to_text: "You are a Named Entity Recognition expert in Kiswahili language. \n\ + Extract all named entities from the following Kiswahili text and categorize them\ + \ into PERSON, LOCATION, ORGANIZATION, or DATE. Ensure the output strictly follows\ + \ the format: label: entity $$ label: entity, with each unique entity on a separate\ + \ label line, avoiding grouped entities (e.g., avoid LOC: entity, entity) or irrelevant\ + \ entries like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_sw_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_tn.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_tn.yaml new file mode 100644 index 00000000..d42d164a --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_tn.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: tn +doc_to_text: "You are a Named Entity Recognition expert in Setswana language. \nExtract\ + \ all named entities from the following Setswana text and categorize them into PERSON,\ + \ LOCATION, ORGANIZATION, or DATE. Ensure the output strictly follows the format:\ + \ label: entity $$ label: entity, with each unique entity on a separate label line,\ + \ avoiding grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries\ + \ like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_tn_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_tw.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_tw.yaml new file mode 100644 index 00000000..62b4e2af --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_tw.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: tw +doc_to_text: "You are a Named Entity Recognition expert in Twi language. \nExtract\ + \ all named entities from the following Twi text and categorize them into PERSON,\ + \ LOCATION, ORGANIZATION, or DATE. Ensure the output strictly follows the format:\ + \ label: entity $$ label: entity, with each unique entity on a separate label line,\ + \ avoiding grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries\ + \ like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_tw_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_wo.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_wo.yaml new file mode 100644 index 00000000..6db45e2b --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_wo.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: wo +doc_to_text: "You are a Named Entity Recognition expert in Wolof language. \nExtract\ + \ all named entities from the following Wolof text and categorize them into PERSON,\ + \ LOCATION, ORGANIZATION, or DATE. Ensure the output strictly follows the format:\ + \ label: entity $$ label: entity, with each unique entity on a separate label line,\ + \ avoiding grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries\ + \ like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_wo_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_xh.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_xh.yaml new file mode 100644 index 00000000..6a697b27 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_xh.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: xh +doc_to_text: "You are a Named Entity Recognition expert in isiXhosa language. \nExtract\ + \ all named entities from the following isiXhosa text and categorize them into PERSON,\ + \ LOCATION, ORGANIZATION, or DATE. Ensure the output strictly follows the format:\ + \ label: entity $$ label: entity, with each unique entity on a separate label line,\ + \ avoiding grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries\ + \ like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_xh_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_yo.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_yo.yaml new file mode 100644 index 00000000..589cd5b3 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_yo.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: yo +doc_to_text: "You are a Named Entity Recognition expert in Yoruba language. \nExtract\ + \ all named entities from the following Yoruba text and categorize them into PERSON,\ + \ LOCATION, ORGANIZATION, or DATE. Ensure the output strictly follows the format:\ + \ label: entity $$ label: entity, with each unique entity on a separate label line,\ + \ avoiding grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries\ + \ like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_yo_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_zu.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_zu.yaml new file mode 100644 index 00000000..c25d5a0c --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_3/masakhaner_zu.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: zu +doc_to_text: "You are a Named Entity Recognition expert in isiZulu language. \nExtract\ + \ all named entities from the following isiZulu text and categorize them into PERSON,\ + \ LOCATION, ORGANIZATION, or DATE. Ensure the output strictly follows the format:\ + \ label: entity $$ label: entity, with each unique entity on a separate label line,\ + \ avoiding grouped entities (e.g., avoid LOC: entity, entity) or irrelevant entries\ + \ like none. Return only the output \n\nText: {{text}}" +include: masakhaner +task: masakhaner_zu_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_3/utils.py b/lm_eval/tasks/afrobench/masakhaner/prompt_3/utils.py new file mode 100644 index 00000000..76909044 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_3/utils.py @@ -0,0 +1,146 @@ +import collections +import re + +from lm_eval.utils import weighted_f1_score + + +def doc_to_target(doc): + return transform_text(doc["ner_tags"]) + + +def transform_text(text): + entities = [] + current_entity = "" + current_tag = "" + + for pair in text.split("\n"): + if pair: # Check if the line is not empty + word, tag = pair.strip().split(": ") + tag = tag.upper() + word = word.lower() + word = word.strip(",.").strip() + + if tag.startswith("B-"): + if current_entity: + entities.append(f"{current_tag}: {current_entity}") + current_tag = tag.split("-")[1] + current_entity = word + elif tag.startswith("I-") and tag.split("-")[1] == current_tag: + current_entity += word + else: + if current_entity: + entities.append(f"{current_tag}: {current_entity}") + current_entity = "" + current_tag = "" + if current_entity: + entities.append(f"{current_tag}: {current_entity}") + + # Join all the transformed output lines with $$ as separator + return " $$ ".join(entities) + + +def span_f1_agg(items): + """Computes Span based F1 score. + + This function is copied from + https://github.com/google-research/multilingual-t5/blob/master/multilingual_t5/evaluation/metrics.py + + Args: + targets: list of strings or list of list of strings if multiple references + are present. + predictions: list of strings + + Returns: + span f1 across all targets and predictions (Based on CoNLL script) + """ + unzipped_list = list(zip(*items)) + targets = unzipped_list[0] + predictions = unzipped_list[1] + + true_positives = collections.defaultdict(int) + false_positives = collections.defaultdict(int) + false_negatives = collections.defaultdict(int) + + def normalize_text(strings): + def get_blank_spaces_pattern(): + return re.compile(r"\s{3,}|\t") + + def remove_blank_spaces(text): + text = re.sub(pattern=get_blank_spaces_pattern(), repl="", string=text) + text = re.sub("\s+", " ", text) + return text + + def remove_punctuation(text): + my_punctuation = '!"$%&\'()*+,-./:;<=>?[\\]^_`{|}~•@.""-,`' + text = re.sub( + "[" + my_punctuation + "]+", " ", str(text) + ) # strip punctuation + return text + + def remove_articles(text): + regex = re.compile(r"\b(a|an|the)\b", re.UNICODE) + return re.sub(regex, " ", text) + + def lowercase(text): + text = text.lower() + return text + + strings = remove_punctuation(strings) + strings = remove_articles(strings) + strings = remove_blank_spaces(strings) + strings = lowercase(strings) + + return strings + + def tags_to_spans(tag_sequence, delimiter="$$"): + """Extract spans from IOB1 or BIO tags.""" + if isinstance(tag_sequence, list): + tag_sequence = " ".join(i.strip() for i in tag_sequence) + tag_sequence_split = [ + item.strip() + for sub in tag_sequence.strip().split(delimiter) + for item in sub.split("$") + if item + ] + tag_sequence_split = [ + item.strip() + for value in tag_sequence_split + for sub in value.split(". ") + for item in sub.split(", ") + ] + tags_entities = [] + for tag_entity in tag_sequence_split: + tag_entity_split = tag_entity.split(": ") + if len(tag_entity_split) != 2: + continue + tag = normalize_text(tag_entity_split[0].strip()) + entity = normalize_text(tag_entity_split[1].rstrip().lstrip()) + tags_entities.append((tag, entity)) + return tags_entities + + def compute_f1_metrics(true_positive, false_positive, false_negative): + precision = float(true_positive) / float(true_positive + false_positive + 1e-13) + recall = float(true_positive) / float(true_positive + false_negative + 1e-13) + f1_measures = 2.0 * ((precision * recall) / (precision + recall + 1e-13)) + return precision, recall, f1_measures + + for target, pred in zip(targets, predictions): + gold_spans = tags_to_spans(target) + predicted_spans = tags_to_spans(pred) + + for span in predicted_spans: + if span in gold_spans: + true_positives[span[0]] += 1 + gold_spans.remove(span) + else: + false_positives[span[0]] += 1 + # These spans weren't predicted. + for span in gold_spans: + false_negatives[span[0]] += 1 + + _, _, f1_measure = compute_f1_metrics( + sum(true_positives.values()), + sum(false_positives.values()), + sum(false_negatives.values()), + ) + return f1_measure diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner new file mode 100644 index 00000000..5c0ae52e --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner @@ -0,0 +1,26 @@ +tag: +- masakhaner_tasks +- masakhaner_prompt_4 +dataset_path: masakhane/masakhaner-x +dataset_name: null +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +generation_kwargs: + do_sample: false + until: + - + - <|im_end|> +validation_split: validation +test_split: test +fewshot_split: train +doc_to_target: target +filter_list: + - name: flexible-extract + filter: + - function: format_span +metric_list: + - metric: f1 + aggregation: !function utils.span_f1_agg + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_am.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_am.yaml new file mode 100644 index 00000000..19b06221 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_am.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: am +doc_to_text: "As a Amharic linguist, label all named entities in the Amharic text\ + \ below with the categories: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the\ + \ output strictly follows the format: label: entity $$ label: entity, with each\ + \ unique entity on a separate label line, avoiding grouped entities (e.g., avoid\ + \ LOC: entity, entity) or irrelevant entries like none. Return only the output.\ + \ \n\nText: {{text}}" +include: masakhaner +task: masakhaner_am_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_bbj.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_bbj.yaml new file mode 100644 index 00000000..03ed5210 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_bbj.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: bbj +doc_to_text: "As a Ghomala linguist, label all named entities in the Ghomala text\ + \ below with the categories: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the\ + \ output strictly follows the format: label: entity $$ label: entity, with each\ + \ unique entity on a separate label line, avoiding grouped entities (e.g., avoid\ + \ LOC: entity, entity) or irrelevant entries like none. Return only the output.\ + \ \n\nText: {{text}}" +include: masakhaner +task: masakhaner_bbj_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_bm.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_bm.yaml new file mode 100644 index 00000000..e719db9a --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_bm.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: bm +doc_to_text: "As a Bambara linguist, label all named entities in the Bambara text\ + \ below with the categories: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the\ + \ output strictly follows the format: label: entity $$ label: entity, with each\ + \ unique entity on a separate label line, avoiding grouped entities (e.g., avoid\ + \ LOC: entity, entity) or irrelevant entries like none. Return only the output.\ + \ \n\nText: {{text}}" +include: masakhaner +task: masakhaner_bm_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_ee.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_ee.yaml new file mode 100644 index 00000000..fe5fc75d --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_ee.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: ee +doc_to_text: "As a Ewe linguist, label all named entities in the Ewe text below with\ + \ the categories: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output strictly\ + \ follows the format: label: entity $$ label: entity, with each unique entity on\ + \ a separate label line, avoiding grouped entities (e.g., avoid LOC: entity, entity)\ + \ or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_ee_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_ha.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_ha.yaml new file mode 100644 index 00000000..9f88b9d1 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_ha.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: ha +doc_to_text: "As a Hausa linguist, label all named entities in the Hausa text below\ + \ with the categories: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_ha_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_ig.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_ig.yaml new file mode 100644 index 00000000..d4712d7e --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_ig.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: ig +doc_to_text: "As a Igbo linguist, label all named entities in the Igbo text below\ + \ with the categories: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_ig_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_lg.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_lg.yaml new file mode 100644 index 00000000..cd7bde4a --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_lg.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: lg +doc_to_text: "As a Luganda linguist, label all named entities in the Luganda text\ + \ below with the categories: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the\ + \ output strictly follows the format: label: entity $$ label: entity, with each\ + \ unique entity on a separate label line, avoiding grouped entities (e.g., avoid\ + \ LOC: entity, entity) or irrelevant entries like none. Return only the output.\ + \ \n\nText: {{text}}" +include: masakhaner +task: masakhaner_lg_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_luo.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_luo.yaml new file mode 100644 index 00000000..92c0ddfa --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_luo.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: luo +doc_to_text: "As a Luo linguist, label all named entities in the Luo text below with\ + \ the categories: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output strictly\ + \ follows the format: label: entity $$ label: entity, with each unique entity on\ + \ a separate label line, avoiding grouped entities (e.g., avoid LOC: entity, entity)\ + \ or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_luo_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_mos.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_mos.yaml new file mode 100644 index 00000000..2eb75d8e --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_mos.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: mos +doc_to_text: "As a Mossi linguist, label all named entities in the Mossi text below\ + \ with the categories: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_mos_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_ny.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_ny.yaml new file mode 100644 index 00000000..e8cb8218 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_ny.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: ny +doc_to_text: "As a Chichewa linguist, label all named entities in the Chichewa text\ + \ below with the categories: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the\ + \ output strictly follows the format: label: entity $$ label: entity, with each\ + \ unique entity on a separate label line, avoiding grouped entities (e.g., avoid\ + \ LOC: entity, entity) or irrelevant entries like none. Return only the output.\ + \ \n\nText: {{text}}" +include: masakhaner +task: masakhaner_ny_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_pcm.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_pcm.yaml new file mode 100644 index 00000000..93f8ae3a --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_pcm.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: pcm +doc_to_text: "As a Nigerian Pidgin linguist, label all named entities in the Nigerian\ + \ Pidgin text below with the categories: PERSON, LOCATION, ORGANIZATION, and DATE.\ + \ Ensure the output strictly follows the format: label: entity $$ label: entity,\ + \ with each unique entity on a separate label line, avoiding grouped entities (e.g.,\ + \ avoid LOC: entity, entity) or irrelevant entries like none. Return only the output.\ + \ \n\nText: {{text}}" +include: masakhaner +task: masakhaner_pcm_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_rw.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_rw.yaml new file mode 100644 index 00000000..d64d4992 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_rw.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: rw +doc_to_text: "As a Kinyarwanda linguist, label all named entities in the Kinyarwanda\ + \ text below with the categories: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure\ + \ the output strictly follows the format: label: entity $$ label: entity, with each\ + \ unique entity on a separate label line, avoiding grouped entities (e.g., avoid\ + \ LOC: entity, entity) or irrelevant entries like none. Return only the output.\ + \ \n\nText: {{text}}" +include: masakhaner +task: masakhaner_rw_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_sn.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_sn.yaml new file mode 100644 index 00000000..40230fb1 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_sn.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: sn +doc_to_text: "As a chiShona linguist, label all named entities in the chiShona text\ + \ below with the categories: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the\ + \ output strictly follows the format: label: entity $$ label: entity, with each\ + \ unique entity on a separate label line, avoiding grouped entities (e.g., avoid\ + \ LOC: entity, entity) or irrelevant entries like none. Return only the output.\ + \ \n\nText: {{text}}" +include: masakhaner +task: masakhaner_sn_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_sw.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_sw.yaml new file mode 100644 index 00000000..7b27554d --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_sw.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: sw +doc_to_text: "As a Kiswahili linguist, label all named entities in the Kiswahili text\ + \ below with the categories: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the\ + \ output strictly follows the format: label: entity $$ label: entity, with each\ + \ unique entity on a separate label line, avoiding grouped entities (e.g., avoid\ + \ LOC: entity, entity) or irrelevant entries like none. Return only the output.\ + \ \n\nText: {{text}}" +include: masakhaner +task: masakhaner_sw_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_tn.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_tn.yaml new file mode 100644 index 00000000..88080456 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_tn.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: tn +doc_to_text: "As a Setswana linguist, label all named entities in the Setswana text\ + \ below with the categories: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the\ + \ output strictly follows the format: label: entity $$ label: entity, with each\ + \ unique entity on a separate label line, avoiding grouped entities (e.g., avoid\ + \ LOC: entity, entity) or irrelevant entries like none. Return only the output.\ + \ \n\nText: {{text}}" +include: masakhaner +task: masakhaner_tn_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_tw.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_tw.yaml new file mode 100644 index 00000000..8d2eec6b --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_tw.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: tw +doc_to_text: "As a Twi linguist, label all named entities in the Twi text below with\ + \ the categories: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output strictly\ + \ follows the format: label: entity $$ label: entity, with each unique entity on\ + \ a separate label line, avoiding grouped entities (e.g., avoid LOC: entity, entity)\ + \ or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_tw_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_wo.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_wo.yaml new file mode 100644 index 00000000..41501cb3 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_wo.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: wo +doc_to_text: "As a Wolof linguist, label all named entities in the Wolof text below\ + \ with the categories: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_wo_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_xh.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_xh.yaml new file mode 100644 index 00000000..4b29fda3 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_xh.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: xh +doc_to_text: "As a isiXhosa linguist, label all named entities in the isiXhosa text\ + \ below with the categories: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the\ + \ output strictly follows the format: label: entity $$ label: entity, with each\ + \ unique entity on a separate label line, avoiding grouped entities (e.g., avoid\ + \ LOC: entity, entity) or irrelevant entries like none. Return only the output.\ + \ \n\nText: {{text}}" +include: masakhaner +task: masakhaner_xh_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_yo.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_yo.yaml new file mode 100644 index 00000000..a0c327bd --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_yo.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: yo +doc_to_text: "As a Yoruba linguist, label all named entities in the Yoruba text below\ + \ with the categories: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_yo_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_zu.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_zu.yaml new file mode 100644 index 00000000..24961ec7 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_4/masakhaner_zu.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: zu +doc_to_text: "As a isiZulu linguist, label all named entities in the isiZulu text\ + \ below with the categories: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the\ + \ output strictly follows the format: label: entity $$ label: entity, with each\ + \ unique entity on a separate label line, avoiding grouped entities (e.g., avoid\ + \ LOC: entity, entity) or irrelevant entries like none. Return only the output.\ + \ \n\nText: {{text}}" +include: masakhaner +task: masakhaner_zu_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_4/utils.py b/lm_eval/tasks/afrobench/masakhaner/prompt_4/utils.py new file mode 100644 index 00000000..76909044 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_4/utils.py @@ -0,0 +1,146 @@ +import collections +import re + +from lm_eval.utils import weighted_f1_score + + +def doc_to_target(doc): + return transform_text(doc["ner_tags"]) + + +def transform_text(text): + entities = [] + current_entity = "" + current_tag = "" + + for pair in text.split("\n"): + if pair: # Check if the line is not empty + word, tag = pair.strip().split(": ") + tag = tag.upper() + word = word.lower() + word = word.strip(",.").strip() + + if tag.startswith("B-"): + if current_entity: + entities.append(f"{current_tag}: {current_entity}") + current_tag = tag.split("-")[1] + current_entity = word + elif tag.startswith("I-") and tag.split("-")[1] == current_tag: + current_entity += word + else: + if current_entity: + entities.append(f"{current_tag}: {current_entity}") + current_entity = "" + current_tag = "" + if current_entity: + entities.append(f"{current_tag}: {current_entity}") + + # Join all the transformed output lines with $$ as separator + return " $$ ".join(entities) + + +def span_f1_agg(items): + """Computes Span based F1 score. + + This function is copied from + https://github.com/google-research/multilingual-t5/blob/master/multilingual_t5/evaluation/metrics.py + + Args: + targets: list of strings or list of list of strings if multiple references + are present. + predictions: list of strings + + Returns: + span f1 across all targets and predictions (Based on CoNLL script) + """ + unzipped_list = list(zip(*items)) + targets = unzipped_list[0] + predictions = unzipped_list[1] + + true_positives = collections.defaultdict(int) + false_positives = collections.defaultdict(int) + false_negatives = collections.defaultdict(int) + + def normalize_text(strings): + def get_blank_spaces_pattern(): + return re.compile(r"\s{3,}|\t") + + def remove_blank_spaces(text): + text = re.sub(pattern=get_blank_spaces_pattern(), repl="", string=text) + text = re.sub("\s+", " ", text) + return text + + def remove_punctuation(text): + my_punctuation = '!"$%&\'()*+,-./:;<=>?[\\]^_`{|}~•@.""-,`' + text = re.sub( + "[" + my_punctuation + "]+", " ", str(text) + ) # strip punctuation + return text + + def remove_articles(text): + regex = re.compile(r"\b(a|an|the)\b", re.UNICODE) + return re.sub(regex, " ", text) + + def lowercase(text): + text = text.lower() + return text + + strings = remove_punctuation(strings) + strings = remove_articles(strings) + strings = remove_blank_spaces(strings) + strings = lowercase(strings) + + return strings + + def tags_to_spans(tag_sequence, delimiter="$$"): + """Extract spans from IOB1 or BIO tags.""" + if isinstance(tag_sequence, list): + tag_sequence = " ".join(i.strip() for i in tag_sequence) + tag_sequence_split = [ + item.strip() + for sub in tag_sequence.strip().split(delimiter) + for item in sub.split("$") + if item + ] + tag_sequence_split = [ + item.strip() + for value in tag_sequence_split + for sub in value.split(". ") + for item in sub.split(", ") + ] + tags_entities = [] + for tag_entity in tag_sequence_split: + tag_entity_split = tag_entity.split(": ") + if len(tag_entity_split) != 2: + continue + tag = normalize_text(tag_entity_split[0].strip()) + entity = normalize_text(tag_entity_split[1].rstrip().lstrip()) + tags_entities.append((tag, entity)) + return tags_entities + + def compute_f1_metrics(true_positive, false_positive, false_negative): + precision = float(true_positive) / float(true_positive + false_positive + 1e-13) + recall = float(true_positive) / float(true_positive + false_negative + 1e-13) + f1_measures = 2.0 * ((precision * recall) / (precision + recall + 1e-13)) + return precision, recall, f1_measures + + for target, pred in zip(targets, predictions): + gold_spans = tags_to_spans(target) + predicted_spans = tags_to_spans(pred) + + for span in predicted_spans: + if span in gold_spans: + true_positives[span[0]] += 1 + gold_spans.remove(span) + else: + false_positives[span[0]] += 1 + # These spans weren't predicted. + for span in gold_spans: + false_negatives[span[0]] += 1 + + _, _, f1_measure = compute_f1_metrics( + sum(true_positives.values()), + sum(false_positives.values()), + sum(false_negatives.values()), + ) + return f1_measure diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner new file mode 100644 index 00000000..09cd77e1 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner @@ -0,0 +1,26 @@ +tag: +- masakhaner_tasks +- masakhaner_prompt_5 +dataset_path: masakhane/masakhaner-x +dataset_name: null +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +generation_kwargs: + do_sample: false + until: + - + - <|im_end|> +validation_split: validation +test_split: test +fewshot_split: train +doc_to_target: target +filter_list: + - name: flexible-extract + filter: + - function: format_span +metric_list: + - metric: f1 + aggregation: !function utils.span_f1_agg + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_am.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_am.yaml new file mode 100644 index 00000000..90c48574 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_am.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: am +doc_to_text: "Provide a concise list of named entities in the text below. Use the\ + \ following labels: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_am_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_bbj.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_bbj.yaml new file mode 100644 index 00000000..74726694 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_bbj.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: bbj +doc_to_text: "Provide a concise list of named entities in the text below. Use the\ + \ following labels: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_bbj_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_bm.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_bm.yaml new file mode 100644 index 00000000..c97e0c22 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_bm.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: bm +doc_to_text: "Provide a concise list of named entities in the text below. Use the\ + \ following labels: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_bm_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_ee.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_ee.yaml new file mode 100644 index 00000000..6371649d --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_ee.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: ee +doc_to_text: "Provide a concise list of named entities in the text below. Use the\ + \ following labels: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_ee_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_ha.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_ha.yaml new file mode 100644 index 00000000..1d68c7ee --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_ha.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: ha +doc_to_text: "Provide a concise list of named entities in the text below. Use the\ + \ following labels: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_ha_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_ig.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_ig.yaml new file mode 100644 index 00000000..3b8a4295 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_ig.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: ig +doc_to_text: "Provide a concise list of named entities in the text below. Use the\ + \ following labels: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_ig_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_lg.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_lg.yaml new file mode 100644 index 00000000..84bdc8b9 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_lg.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: lg +doc_to_text: "Provide a concise list of named entities in the text below. Use the\ + \ following labels: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_lg_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_luo.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_luo.yaml new file mode 100644 index 00000000..55a0b574 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_luo.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: luo +doc_to_text: "Provide a concise list of named entities in the text below. Use the\ + \ following labels: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_luo_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_mos.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_mos.yaml new file mode 100644 index 00000000..06bcc446 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_mos.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: mos +doc_to_text: "Provide a concise list of named entities in the text below. Use the\ + \ following labels: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_mos_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_ny.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_ny.yaml new file mode 100644 index 00000000..e400f10e --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_ny.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: ny +doc_to_text: "Provide a concise list of named entities in the text below. Use the\ + \ following labels: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_ny_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_pcm.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_pcm.yaml new file mode 100644 index 00000000..b9d897bc --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_pcm.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: pcm +doc_to_text: "Provide a concise list of named entities in the text below. Use the\ + \ following labels: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_pcm_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_rw.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_rw.yaml new file mode 100644 index 00000000..0742bc4d --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_rw.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: rw +doc_to_text: "Provide a concise list of named entities in the text below. Use the\ + \ following labels: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_rw_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_sn.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_sn.yaml new file mode 100644 index 00000000..56711335 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_sn.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: sn +doc_to_text: "Provide a concise list of named entities in the text below. Use the\ + \ following labels: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_sn_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_sw.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_sw.yaml new file mode 100644 index 00000000..c418beb4 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_sw.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: sw +doc_to_text: "Provide a concise list of named entities in the text below. Use the\ + \ following labels: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_sw_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_tn.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_tn.yaml new file mode 100644 index 00000000..bf94a108 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_tn.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: tn +doc_to_text: "Provide a concise list of named entities in the text below. Use the\ + \ following labels: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_tn_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_tw.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_tw.yaml new file mode 100644 index 00000000..cad2e2e3 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_tw.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: tw +doc_to_text: "Provide a concise list of named entities in the text below. Use the\ + \ following labels: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_tw_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_wo.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_wo.yaml new file mode 100644 index 00000000..ec7af039 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_wo.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: wo +doc_to_text: "Provide a concise list of named entities in the text below. Use the\ + \ following labels: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_wo_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_xh.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_xh.yaml new file mode 100644 index 00000000..debb164a --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_xh.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: xh +doc_to_text: "Provide a concise list of named entities in the text below. Use the\ + \ following labels: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_xh_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_yo.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_yo.yaml new file mode 100644 index 00000000..9abe1acb --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_yo.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: yo +doc_to_text: "Provide a concise list of named entities in the text below. Use the\ + \ following labels: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_yo_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_zu.yaml b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_zu.yaml new file mode 100644 index 00000000..c5af591a --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_5/masakhaner_zu.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: zu +doc_to_text: "Provide a concise list of named entities in the text below. Use the\ + \ following labels: PERSON, LOCATION, ORGANIZATION, and DATE. Ensure the output\ + \ strictly follows the format: label: entity $$ label: entity, with each unique\ + \ entity on a separate label line, avoiding grouped entities (e.g., avoid LOC: entity,\ + \ entity) or irrelevant entries like none. Return only the output. \n\nText: {{text}}" +include: masakhaner +task: masakhaner_zu_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhaner/prompt_5/utils.py b/lm_eval/tasks/afrobench/masakhaner/prompt_5/utils.py new file mode 100644 index 00000000..76909044 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhaner/prompt_5/utils.py @@ -0,0 +1,146 @@ +import collections +import re + +from lm_eval.utils import weighted_f1_score + + +def doc_to_target(doc): + return transform_text(doc["ner_tags"]) + + +def transform_text(text): + entities = [] + current_entity = "" + current_tag = "" + + for pair in text.split("\n"): + if pair: # Check if the line is not empty + word, tag = pair.strip().split(": ") + tag = tag.upper() + word = word.lower() + word = word.strip(",.").strip() + + if tag.startswith("B-"): + if current_entity: + entities.append(f"{current_tag}: {current_entity}") + current_tag = tag.split("-")[1] + current_entity = word + elif tag.startswith("I-") and tag.split("-")[1] == current_tag: + current_entity += word + else: + if current_entity: + entities.append(f"{current_tag}: {current_entity}") + current_entity = "" + current_tag = "" + if current_entity: + entities.append(f"{current_tag}: {current_entity}") + + # Join all the transformed output lines with $$ as separator + return " $$ ".join(entities) + + +def span_f1_agg(items): + """Computes Span based F1 score. + + This function is copied from + https://github.com/google-research/multilingual-t5/blob/master/multilingual_t5/evaluation/metrics.py + + Args: + targets: list of strings or list of list of strings if multiple references + are present. + predictions: list of strings + + Returns: + span f1 across all targets and predictions (Based on CoNLL script) + """ + unzipped_list = list(zip(*items)) + targets = unzipped_list[0] + predictions = unzipped_list[1] + + true_positives = collections.defaultdict(int) + false_positives = collections.defaultdict(int) + false_negatives = collections.defaultdict(int) + + def normalize_text(strings): + def get_blank_spaces_pattern(): + return re.compile(r"\s{3,}|\t") + + def remove_blank_spaces(text): + text = re.sub(pattern=get_blank_spaces_pattern(), repl="", string=text) + text = re.sub("\s+", " ", text) + return text + + def remove_punctuation(text): + my_punctuation = '!"$%&\'()*+,-./:;<=>?[\\]^_`{|}~•@.""-,`' + text = re.sub( + "[" + my_punctuation + "]+", " ", str(text) + ) # strip punctuation + return text + + def remove_articles(text): + regex = re.compile(r"\b(a|an|the)\b", re.UNICODE) + return re.sub(regex, " ", text) + + def lowercase(text): + text = text.lower() + return text + + strings = remove_punctuation(strings) + strings = remove_articles(strings) + strings = remove_blank_spaces(strings) + strings = lowercase(strings) + + return strings + + def tags_to_spans(tag_sequence, delimiter="$$"): + """Extract spans from IOB1 or BIO tags.""" + if isinstance(tag_sequence, list): + tag_sequence = " ".join(i.strip() for i in tag_sequence) + tag_sequence_split = [ + item.strip() + for sub in tag_sequence.strip().split(delimiter) + for item in sub.split("$") + if item + ] + tag_sequence_split = [ + item.strip() + for value in tag_sequence_split + for sub in value.split(". ") + for item in sub.split(", ") + ] + tags_entities = [] + for tag_entity in tag_sequence_split: + tag_entity_split = tag_entity.split(": ") + if len(tag_entity_split) != 2: + continue + tag = normalize_text(tag_entity_split[0].strip()) + entity = normalize_text(tag_entity_split[1].rstrip().lstrip()) + tags_entities.append((tag, entity)) + return tags_entities + + def compute_f1_metrics(true_positive, false_positive, false_negative): + precision = float(true_positive) / float(true_positive + false_positive + 1e-13) + recall = float(true_positive) / float(true_positive + false_negative + 1e-13) + f1_measures = 2.0 * ((precision * recall) / (precision + recall + 1e-13)) + return precision, recall, f1_measures + + for target, pred in zip(targets, predictions): + gold_spans = tags_to_spans(target) + predicted_spans = tags_to_spans(pred) + + for span in predicted_spans: + if span in gold_spans: + true_positives[span[0]] += 1 + gold_spans.remove(span) + else: + false_positives[span[0]] += 1 + # These spans weren't predicted. + for span in gold_spans: + false_negatives[span[0]] += 1 + + _, _, f1_measure = compute_f1_metrics( + sum(true_positives.values()), + sum(false_positives.values()), + sum(false_negatives.values()), + ) + return f1_measure diff --git a/lm_eval/tasks/afrobench/masakhanews/README.md b/lm_eval/tasks/afrobench/masakhanews/README.md new file mode 100644 index 00000000..16df2df1 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/README.md @@ -0,0 +1,99 @@ +# + +## Paper +Title: `MasakhaNEWS: News Topic Classification for African languages` + +Paper Link: https://aclanthology.org/2023.ijcnlp-main.10/ + +## Abstract +>African languages are severely under-represented in NLP research due to lack of datasets covering several NLP tasks. While there are individual language specific datasets that are being expanded to different tasks, only a handful of NLP tasks (e.g. named entity recognition and machine translation) have standardized benchmark datasets covering several geographical and typologically-diverse African languages. In this paper, we develop MasakhaNEWS -- a new benchmark dataset for news topic classification covering 16 languages widely spoken in Africa. We provide an evaluation of baseline models by training classical machine learning models and fine-tuning several language models. Furthermore, we explore several alternatives to full fine-tuning of language models that are better suited for zero-shot and few-shot learning such as cross-lingual parameter-efficient fine-tuning (like MAD-X), pattern exploiting training (PET), prompting language models (like ChatGPT), and prompt-free sentence transformer fine-tuning (SetFit and Cohere Embedding API). Our evaluation in zero-shot setting shows the potential of prompting ChatGPT for news topic classification in low-resource African languages, achieving an average performance of 70 F1 points without leveraging additional supervision like MAD-X. In few-shot setting, we show that with as little as 10 examples per label, we achieved more than 90% (i.e. 86.0 F1 points) of the performance of full supervised training (92.6 F1 points) leveraging the PET approach. + +HomePage: https://github.com/masakhane-io/masakhane-news + +### Citation + +``` +@inproceedings{adelani-etal-2023-masakhanews, + title = "{M}asakha{NEWS}: News Topic Classification for {A}frican languages", + author = "Adelani, David Ifeoluwa and + Masiak, Marek and + Azime, Israel Abebe and + Alabi, Jesujoba and + Tonja, Atnafu Lambebo and + Mwase, Christine and + Ogundepo, Odunayo and + Dossou, Bonaventure F. P. and + Oladipo, Akintunde and + Nixdorf, Doreen and + Emezue, Chris Chinenye and + Al-azzawi, Sana and + Sibanda, Blessing and + David, Davis and + Ndolela, Lolwethu and + Mukiibi, Jonathan and + Ajayi, Tunde and + Moteu, Tatiana and + Odhiambo, Brian and + Owodunni, Abraham and + Obiefuna, Nnaemeka and + Mohamed, Muhidin and + Muhammad, Shamsuddeen Hassan and + Ababu, Teshome Mulugeta and + Salahudeen, Saheed Abdullahi and + Yigezu, Mesay Gemeda and + Gwadabe, Tajuddeen and + Abdulmumin, Idris and + Taye, Mahlet and + Awoyomi, Oluwabusayo and + Shode, Iyanuoluwa and + Adelani, Tolulope and + Abdulganiyu, Habiba and + Omotayo, Abdul-Hakeem and + Adeeko, Adetola and + Afolabi, Abeeb and + Aremu, Anuoluwapo and + Samuel, Olanrewaju and + Siro, Clemencia and + Kimotho, Wangari and + Ogbu, Onyekachi and + Mbonu, Chinedu and + Chukwuneke, Chiamaka and + Fanijo, Samuel and + Ojo, Jessica and + Awosan, Oyinkansola and + Kebede, Tadesse and + Sakayo, Toadoum Sari and + Nyatsine, Pamela and + Sidume, Freedmore and + Yousuf, Oreen and + Oduwole, Mardiyyah and + Tshinu, Kanda and + Kimanuka, Ussen and + Diko, Thina and + Nxakama, Siyanda and + Nigusse, Sinodos and + Johar, Abdulmejid and + Mohamed, Shafie and + Hassan, Fuad Mire and + Mehamed, Moges Ahmed and + Ngabire, Evrard and + Jules, Jules and + Ssenkungu, Ivan and + Stenetorp, Pontus", + editor = "Park, Jong C. and + Arase, Yuki and + Hu, Baotian and + Lu, Wei and + Wijaya, Derry and + Purwarianti, Ayu and + Krisnadhi, Adila Alfa", + booktitle = "Proceedings of the 13th International Joint Conference on Natural Language Processing and the 3rd Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)", + month = nov, + year = "2023", + address = "Nusa Dua, Bali", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2023.ijcnlp-main.10/", + doi = "10.18653/v1/2023.ijcnlp-main.10", + pages = "144--159" +} +``` diff --git a/lm_eval/tasks/afrobench/masakhanews/masakhanews.yaml b/lm_eval/tasks/afrobench/masakhanews/masakhanews.yaml new file mode 100644 index 00000000..93b6f29d --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/masakhanews.yaml @@ -0,0 +1,13 @@ +group: masakhanews +task: + - masakhanews_prompt_1 + - masakhanews_prompt_2 + - masakhanews_prompt_3 + - masakhanews_prompt_4 + - masakhanews_prompt_5 +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 1 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews b/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews new file mode 100644 index 00000000..282a3842 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews @@ -0,0 +1,43 @@ +tag: +- masakhanews_tasks +- masakhanews_prompt_1 +- afrobench_TC_tasks +dataset_path: masakhane/masakhanews +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: label +doc_to_choice: + - "business" + - "entertainment" + - "health" + - "politics" + - "religion" + - "sports" + - "technology" +should_decontaminate: true +doc_to_decontamination_query: headline_text +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_amh.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_amh.yaml new file mode 100644 index 00000000..d45b784f --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_amh.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: "Given the categories technology, business, politics, sports, health,\ + \ entertainment, or religion; what category does the text: '{{headline_text}}' belong\ + \ to: \n\n" +include: masakhanews +task: masakhanews_amh_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_eng.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_eng.yaml new file mode 100644 index 00000000..40685c17 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng +doc_to_text: "Given the categories technology, business, politics, sports, health,\ + \ entertainment, or religion; what category does the text: '{{headline_text}}' belong\ + \ to: \n\n" +include: masakhanews +task: masakhanews_eng_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_fra.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_fra.yaml new file mode 100644 index 00000000..23711721 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_fra.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fra +doc_to_text: "Given the categories technology, business, politics, sports, health,\ + \ entertainment, or religion; what category does the text: '{{headline_text}}' belong\ + \ to: \n\n" +include: masakhanews +task: masakhanews_fra_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_hau.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_hau.yaml new file mode 100644 index 00000000..c7288982 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_hau.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: "Given the categories technology, business, politics, sports, health,\ + \ entertainment, or religion; what category does the text: '{{headline_text}}' belong\ + \ to: \n\n" +include: masakhanews +task: masakhanews_hau_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_ibo.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_ibo.yaml new file mode 100644 index 00000000..4bf65cca --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_ibo.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "Given the categories technology, business, politics, sports, health,\ + \ entertainment, or religion; what category does the text: '{{headline_text}}' belong\ + \ to: \n\n" +include: masakhanews +task: masakhanews_ibo_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_lin.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_lin.yaml new file mode 100644 index 00000000..c6cdbe8d --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_lin.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: "Given the categories technology, business, politics, sports, health,\ + \ entertainment, or religion; what category does the text: '{{headline_text}}' belong\ + \ to: \n\n" +include: masakhanews +task: masakhanews_lin_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_lug.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_lug.yaml new file mode 100644 index 00000000..e2f0ec1b --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_lug.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: "Given the categories technology, business, politics, sports, health,\ + \ entertainment, or religion; what category does the text: '{{headline_text}}' belong\ + \ to: \n\n" +include: masakhanews +task: masakhanews_lug_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_orm.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_orm.yaml new file mode 100644 index 00000000..a9bff1ac --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_orm.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: "Given the categories technology, business, politics, sports, health,\ + \ entertainment, or religion; what category does the text: '{{headline_text}}' belong\ + \ to: \n\n" +include: masakhanews +task: masakhanews_orm_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_pcm.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_pcm.yaml new file mode 100644 index 00000000..119b01bb --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_pcm.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: pcm +doc_to_text: "Given the categories technology, business, politics, sports, health,\ + \ entertainment, or religion; what category does the text: '{{headline_text}}' belong\ + \ to: \n\n" +include: masakhanews +task: masakhanews_pcm_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_run.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_run.yaml new file mode 100644 index 00000000..d8bc2923 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_run.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: run +doc_to_text: "Given the categories technology, business, politics, sports, health,\ + \ entertainment, or religion; what category does the text: '{{headline_text}}' belong\ + \ to: \n\n" +include: masakhanews +task: masakhanews_run_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_sna.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_sna.yaml new file mode 100644 index 00000000..ee4fabdc --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_sna.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: "Given the categories technology, business, politics, sports, health,\ + \ entertainment, or religion; what category does the text: '{{headline_text}}' belong\ + \ to: \n\n" +include: masakhanews +task: masakhanews_sna_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_som.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_som.yaml new file mode 100644 index 00000000..88d7774c --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_som.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: som +doc_to_text: "Given the categories technology, business, politics, sports, health,\ + \ entertainment, or religion; what category does the text: '{{headline_text}}' belong\ + \ to: \n\n" +include: masakhanews +task: masakhanews_som_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_swa.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_swa.yaml new file mode 100644 index 00000000..c4e02aae --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_swa.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: "Given the categories technology, business, politics, sports, health,\ + \ entertainment, or religion; what category does the text: '{{headline_text}}' belong\ + \ to: \n\n" +include: masakhanews +task: masakhanews_swa_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_tir.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_tir.yaml new file mode 100644 index 00000000..72fa30ae --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_tir.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tir +doc_to_text: "Given the categories technology, business, politics, sports, health,\ + \ entertainment, or religion; what category does the text: '{{headline_text}}' belong\ + \ to: \n\n" +include: masakhanews +task: masakhanews_tir_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_xho.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_xho.yaml new file mode 100644 index 00000000..1d98b3b6 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_xho.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: "Given the categories technology, business, politics, sports, health,\ + \ entertainment, or religion; what category does the text: '{{headline_text}}' belong\ + \ to: \n\n" +include: masakhanews +task: masakhanews_xho_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_yor.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_yor.yaml new file mode 100644 index 00000000..3ef4eec0 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_1/masakhanews_yor.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: "Given the categories technology, business, politics, sports, health,\ + \ entertainment, or religion; what category does the text: '{{headline_text}}' belong\ + \ to: \n\n" +include: masakhanews +task: masakhanews_yor_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_1/utils.py b/lm_eval/tasks/afrobench/masakhanews/prompt_1/utils.py new file mode 100644 index 00000000..3e735e2d --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_1/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews b/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews new file mode 100644 index 00000000..c174d2c7 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews @@ -0,0 +1,43 @@ +tag: +- masakhanews_tasks +- masakhanews_prompt_2 +- afrobench_TC_tasks +dataset_path: masakhane/masakhanews +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: label +doc_to_choice: + - "business" + - "entertainment" + - "health" + - "politics" + - "religion" + - "sports" + - "technology" +should_decontaminate: true +doc_to_decontamination_query: headline_text +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_amh.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_amh.yaml new file mode 100644 index 00000000..cee7619c --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_amh.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: 'Does this Amharic topic; ''{{headline_text}}'' belong to one of the + following categories: technology, business, politics, sports, health, entertainment, + or religion? category only + + + ' +include: masakhanews +task: masakhanews_amh_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_eng.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_eng.yaml new file mode 100644 index 00000000..d3d6dd16 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_eng.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: eng +doc_to_text: 'Does this English topic; ''{{headline_text}}'' belong to one of the + following categories: technology, business, politics, sports, health, entertainment, + or religion? category only + + + ' +include: masakhanews +task: masakhanews_eng_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_fra.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_fra.yaml new file mode 100644 index 00000000..c35a6a1d --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_fra.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: fra +doc_to_text: 'Does this French topic; ''{{headline_text}}'' belong to one of the following + categories: technology, business, politics, sports, health, entertainment, or religion? + category only + + + ' +include: masakhanews +task: masakhanews_fra_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_hau.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_hau.yaml new file mode 100644 index 00000000..93e9cc5a --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_hau.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: 'Does this Hausa topic; ''{{headline_text}}'' belong to one of the following + categories: technology, business, politics, sports, health, entertainment, or religion? + category only + + + ' +include: masakhanews +task: masakhanews_hau_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_ibo.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_ibo.yaml new file mode 100644 index 00000000..1638e435 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_ibo.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: 'Does this Igbo topic; ''{{headline_text}}'' belong to one of the following + categories: technology, business, politics, sports, health, entertainment, or religion? + category only + + + ' +include: masakhanews +task: masakhanews_ibo_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_lin.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_lin.yaml new file mode 100644 index 00000000..0010d0e1 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_lin.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: 'Does this Lingala topic; ''{{headline_text}}'' belong to one of the + following categories: technology, business, politics, sports, health, entertainment, + or religion? category only + + + ' +include: masakhanews +task: masakhanews_lin_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_lug.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_lug.yaml new file mode 100644 index 00000000..d5260672 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_lug.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: 'Does this Luganda topic; ''{{headline_text}}'' belong to one of the + following categories: technology, business, politics, sports, health, entertainment, + or religion? category only + + + ' +include: masakhanews +task: masakhanews_lug_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_orm.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_orm.yaml new file mode 100644 index 00000000..cd04c845 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_orm.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: 'Does this Afaan Oromoo topic; ''{{headline_text}}'' belong to one of + the following categories: technology, business, politics, sports, health, entertainment, + or religion? category only + + + ' +include: masakhanews +task: masakhanews_orm_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_pcm.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_pcm.yaml new file mode 100644 index 00000000..de685e3a --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_pcm.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: pcm +doc_to_text: 'Does this Nigerian Pidgin topic; ''{{headline_text}}'' belong to one + of the following categories: technology, business, politics, sports, health, entertainment, + or religion? category only + + + ' +include: masakhanews +task: masakhanews_pcm_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_run.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_run.yaml new file mode 100644 index 00000000..62236d59 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_run.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: run +doc_to_text: 'Does this Kirundi topic; ''{{headline_text}}'' belong to one of the + following categories: technology, business, politics, sports, health, entertainment, + or religion? category only + + + ' +include: masakhanews +task: masakhanews_run_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_sna.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_sna.yaml new file mode 100644 index 00000000..2a97e176 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_sna.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: 'Does this Shona topic; ''{{headline_text}}'' belong to one of the following + categories: technology, business, politics, sports, health, entertainment, or religion? + category only + + + ' +include: masakhanews +task: masakhanews_sna_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_som.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_som.yaml new file mode 100644 index 00000000..318b9b87 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_som.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: som +doc_to_text: 'Does this Somali topic; ''{{headline_text}}'' belong to one of the following + categories: technology, business, politics, sports, health, entertainment, or religion? + category only + + + ' +include: masakhanews +task: masakhanews_som_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_swa.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_swa.yaml new file mode 100644 index 00000000..75b9345f --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_swa.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: 'Does this Swahili topic; ''{{headline_text}}'' belong to one of the + following categories: technology, business, politics, sports, health, entertainment, + or religion? category only + + + ' +include: masakhanews +task: masakhanews_swa_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_tir.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_tir.yaml new file mode 100644 index 00000000..258a2bd3 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_tir.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: tir +doc_to_text: 'Does this Tigrinya topic; ''{{headline_text}}'' belong to one of the + following categories: technology, business, politics, sports, health, entertainment, + or religion? category only + + + ' +include: masakhanews +task: masakhanews_tir_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_xho.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_xho.yaml new file mode 100644 index 00000000..30c4c3ac --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_xho.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: 'Does this Xhosa topic; ''{{headline_text}}'' belong to one of the following + categories: technology, business, politics, sports, health, entertainment, or religion? + category only + + + ' +include: masakhanews +task: masakhanews_xho_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_yor.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_yor.yaml new file mode 100644 index 00000000..067cf106 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_2/masakhanews_yor.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: 'Does this Yoruba topic; ''{{headline_text}}'' belong to one of the following + categories: technology, business, politics, sports, health, entertainment, or religion? + category only + + + ' +include: masakhanews +task: masakhanews_yor_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_2/utils.py b/lm_eval/tasks/afrobench/masakhanews/prompt_2/utils.py new file mode 100644 index 00000000..3e735e2d --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_2/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews b/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews new file mode 100644 index 00000000..ecc21089 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews @@ -0,0 +1,43 @@ +tag: +- masakhanews_tasks +- masakhanews_prompt_3 +- afrobench_TC_tasks +dataset_path: masakhane/masakhanews +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: label +doc_to_choice: + - "business" + - "entertainment" + - "health" + - "politics" + - "religion" + - "sports" + - "technology" +should_decontaminate: true +doc_to_decontamination_query: headline_text +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_amh.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_amh.yaml new file mode 100644 index 00000000..dec10d29 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_amh.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories technology, religion, politics, sports, health, entertainment, or business;\ + \ what is the topic of the Amharic statement below? Return only the category. \n\ + \ntext: {{headline_text}} \\category:\n\n" +include: masakhanews +task: masakhanews_amh_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_eng.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_eng.yaml new file mode 100644 index 00000000..a8b7159e --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_eng.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: eng +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories technology, religion, politics, sports, health, entertainment, or business;\ + \ what is the topic of the English statement below? Return only the category. \n\ + \ntext: {{headline_text}} \\category:\n\n" +include: masakhanews +task: masakhanews_eng_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_fra.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_fra.yaml new file mode 100644 index 00000000..328316a8 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_fra.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: fra +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories technology, religion, politics, sports, health, entertainment, or business;\ + \ what is the topic of the French statement below? Return only the category. \n\n\ + text: {{headline_text}} \\category:\n\n" +include: masakhanews +task: masakhanews_fra_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_hau.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_hau.yaml new file mode 100644 index 00000000..c18ff277 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_hau.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories technology, religion, politics, sports, health, entertainment, or business;\ + \ what is the topic of the Hausa statement below? Return only the category. \n\n\ + text: {{headline_text}} \\category:\n\n" +include: masakhanews +task: masakhanews_hau_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_ibo.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_ibo.yaml new file mode 100644 index 00000000..3a91db84 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_ibo.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories technology, religion, politics, sports, health, entertainment, or business;\ + \ what is the topic of the Igbo statement below? Return only the category. \n\n\ + text: {{headline_text}} \\category:\n\n" +include: masakhanews +task: masakhanews_ibo_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_lin.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_lin.yaml new file mode 100644 index 00000000..19c4cca2 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_lin.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories technology, religion, politics, sports, health, entertainment, or business;\ + \ what is the topic of the Lingala statement below? Return only the category. \n\ + \ntext: {{headline_text}} \\category:\n\n" +include: masakhanews +task: masakhanews_lin_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_lug.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_lug.yaml new file mode 100644 index 00000000..9e3d4319 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_lug.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories technology, religion, politics, sports, health, entertainment, or business;\ + \ what is the topic of the Luganda statement below? Return only the category. \n\ + \ntext: {{headline_text}} \\category:\n\n" +include: masakhanews +task: masakhanews_lug_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_orm.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_orm.yaml new file mode 100644 index 00000000..9bacf042 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_orm.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories technology, religion, politics, sports, health, entertainment, or business;\ + \ what is the topic of the Afaan Oromoo statement below? Return only the category.\ + \ \n\ntext: {{headline_text}} \\category:\n\n" +include: masakhanews +task: masakhanews_orm_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_pcm.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_pcm.yaml new file mode 100644 index 00000000..e873becd --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_pcm.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: pcm +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories technology, religion, politics, sports, health, entertainment, or business;\ + \ what is the topic of the Nigerian Pidgin statement below? Return only the category.\ + \ \n\ntext: {{headline_text}} \\category:\n\n" +include: masakhanews +task: masakhanews_pcm_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_run.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_run.yaml new file mode 100644 index 00000000..307e1371 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_run.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: run +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories technology, religion, politics, sports, health, entertainment, or business;\ + \ what is the topic of the Kirundi statement below? Return only the category. \n\ + \ntext: {{headline_text}} \\category:\n\n" +include: masakhanews +task: masakhanews_run_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_sna.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_sna.yaml new file mode 100644 index 00000000..ee69be3d --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_sna.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories technology, religion, politics, sports, health, entertainment, or business;\ + \ what is the topic of the Shona statement below? Return only the category. \n\n\ + text: {{headline_text}} \\category:\n\n" +include: masakhanews +task: masakhanews_sna_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_som.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_som.yaml new file mode 100644 index 00000000..8c181fdd --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_som.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: som +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories technology, religion, politics, sports, health, entertainment, or business;\ + \ what is the topic of the Somali statement below? Return only the category. \n\n\ + text: {{headline_text}} \\category:\n\n" +include: masakhanews +task: masakhanews_som_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_swa.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_swa.yaml new file mode 100644 index 00000000..fbe1c420 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_swa.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories technology, religion, politics, sports, health, entertainment, or business;\ + \ what is the topic of the Swahili statement below? Return only the category. \n\ + \ntext: {{headline_text}} \\category:\n\n" +include: masakhanews +task: masakhanews_swa_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_tir.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_tir.yaml new file mode 100644 index 00000000..b6055da2 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_tir.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: tir +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories technology, religion, politics, sports, health, entertainment, or business;\ + \ what is the topic of the Tigrinya statement below? Return only the category. \n\ + \ntext: {{headline_text}} \\category:\n\n" +include: masakhanews +task: masakhanews_tir_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_xho.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_xho.yaml new file mode 100644 index 00000000..110fc087 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_xho.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories technology, religion, politics, sports, health, entertainment, or business;\ + \ what is the topic of the Xhosa statement below? Return only the category. \n\n\ + text: {{headline_text}} \\category:\n\n" +include: masakhanews +task: masakhanews_xho_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_yor.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_yor.yaml new file mode 100644 index 00000000..d31e9b23 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_3/masakhanews_yor.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories technology, religion, politics, sports, health, entertainment, or business;\ + \ what is the topic of the Yoruba statement below? Return only the category. \n\n\ + text: {{headline_text}} \\category:\n\n" +include: masakhanews +task: masakhanews_yor_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_3/utils.py b/lm_eval/tasks/afrobench/masakhanews/prompt_3/utils.py new file mode 100644 index 00000000..3e735e2d --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_3/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews b/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews new file mode 100644 index 00000000..a1801f4e --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews @@ -0,0 +1,43 @@ +tag: +- masakhanews_tasks +- masakhanews_prompt_4 +- afrobench_TC_tasks +dataset_path: masakhane/masakhanews +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: label +doc_to_choice: + - "business" + - "entertainment" + - "health" + - "politics" + - "religion" + - "sports" + - "technology" +should_decontaminate: true +doc_to_decontamination_query: headline_text +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_amh.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_amh.yaml new file mode 100644 index 00000000..a7630585 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_amh.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: "Label the following text as technology, religion, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_amh_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_eng.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_eng.yaml new file mode 100644 index 00000000..85671137 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng +doc_to_text: "Label the following text as technology, religion, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_eng_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_fra.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_fra.yaml new file mode 100644 index 00000000..3f86635f --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_fra.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fra +doc_to_text: "Label the following text as technology, religion, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_fra_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_hau.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_hau.yaml new file mode 100644 index 00000000..c1b7ce56 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_hau.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: "Label the following text as technology, religion, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_hau_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_ibo.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_ibo.yaml new file mode 100644 index 00000000..d76a905d --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_ibo.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "Label the following text as technology, religion, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_ibo_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_lin.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_lin.yaml new file mode 100644 index 00000000..e0247529 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_lin.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: "Label the following text as technology, religion, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_lin_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_lug.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_lug.yaml new file mode 100644 index 00000000..ca02c0a5 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_lug.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: "Label the following text as technology, religion, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_lug_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_orm.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_orm.yaml new file mode 100644 index 00000000..781eb4cc --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_orm.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: "Label the following text as technology, religion, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_orm_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_pcm.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_pcm.yaml new file mode 100644 index 00000000..93ad9f48 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_pcm.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: pcm +doc_to_text: "Label the following text as technology, religion, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_pcm_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_run.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_run.yaml new file mode 100644 index 00000000..f5d98548 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_run.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: run +doc_to_text: "Label the following text as technology, religion, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_run_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_sna.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_sna.yaml new file mode 100644 index 00000000..2676db85 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_sna.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: "Label the following text as technology, religion, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_sna_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_som.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_som.yaml new file mode 100644 index 00000000..6562da41 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_som.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: som +doc_to_text: "Label the following text as technology, religion, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_som_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_swa.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_swa.yaml new file mode 100644 index 00000000..3bb9764a --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_swa.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: "Label the following text as technology, religion, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_swa_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_tir.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_tir.yaml new file mode 100644 index 00000000..3dfb1d4e --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_tir.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tir +doc_to_text: "Label the following text as technology, religion, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_tir_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_xho.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_xho.yaml new file mode 100644 index 00000000..9c1b51c2 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_xho.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: "Label the following text as technology, religion, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_xho_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_yor.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_yor.yaml new file mode 100644 index 00000000..9d22d1c7 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_4/masakhanews_yor.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: "Label the following text as technology, religion, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_yor_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_4/utils.py b/lm_eval/tasks/afrobench/masakhanews/prompt_4/utils.py new file mode 100644 index 00000000..3e735e2d --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_4/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews b/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews new file mode 100644 index 00000000..8d76af03 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews @@ -0,0 +1,43 @@ +tag: +- masakhanews_tasks +- masakhanews_prompt_5 +- afrobench_TC_tasks +dataset_path: masakhane/masakhanews +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: label +doc_to_choice: + - "business" + - "entertainment" + - "health" + - "politics" + - "religion" + - "sports" + - "technology" +should_decontaminate: true +doc_to_decontamination_query: headline_text +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_amh.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_amh.yaml new file mode 100644 index 00000000..759ce913 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_amh.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: amh +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Amharic text. For each input, classify the topic as technology, business, politics,\ + \ sports, health, entertainment, or religion. Use the following guidelines: \n\n\ + \ technology: The text discusses scientific discoveries, technological advancements,\ + \ or related topics. \npolitics: The text covers political events, policies, or\ + \ related topics. \nsports: The text talks about sports events, athletes, or related\ + \ topics. \nhealth: The text addresses health issues, medical advancements, or related\ + \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\ + \ topics. \nreligion: The text talks about relgions, religious institutions and\ + \ beliefs or related topics. \n\nbusiness: The text covers economy, business, or\ + \ related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_amh_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_eng.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_eng.yaml new file mode 100644 index 00000000..2c03032b --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_eng.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: eng +doc_to_text: "You are tasked with performing topic classification on the following\ + \ English text. For each input, classify the topic as technology, business, politics,\ + \ sports, health, entertainment, or religion. Use the following guidelines: \n\n\ + \ technology: The text discusses scientific discoveries, technological advancements,\ + \ or related topics. \npolitics: The text covers political events, policies, or\ + \ related topics. \nsports: The text talks about sports events, athletes, or related\ + \ topics. \nhealth: The text addresses health issues, medical advancements, or related\ + \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\ + \ topics. \nreligion: The text talks about relgions, religious institutions and\ + \ beliefs or related topics. \n\nbusiness: The text covers economy, business, or\ + \ related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_eng_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_fra.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_fra.yaml new file mode 100644 index 00000000..603d149d --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_fra.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: fra +doc_to_text: "You are tasked with performing topic classification on the following\ + \ French text. For each input, classify the topic as technology, business, politics,\ + \ sports, health, entertainment, or religion. Use the following guidelines: \n\n\ + \ technology: The text discusses scientific discoveries, technological advancements,\ + \ or related topics. \npolitics: The text covers political events, policies, or\ + \ related topics. \nsports: The text talks about sports events, athletes, or related\ + \ topics. \nhealth: The text addresses health issues, medical advancements, or related\ + \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\ + \ topics. \nreligion: The text talks about relgions, religious institutions and\ + \ beliefs or related topics. \n\nbusiness: The text covers economy, business, or\ + \ related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_fra_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_hau.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_hau.yaml new file mode 100644 index 00000000..04a478cf --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_hau.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Hausa text. For each input, classify the topic as technology, business, politics,\ + \ sports, health, entertainment, or religion. Use the following guidelines: \n\n\ + \ technology: The text discusses scientific discoveries, technological advancements,\ + \ or related topics. \npolitics: The text covers political events, policies, or\ + \ related topics. \nsports: The text talks about sports events, athletes, or related\ + \ topics. \nhealth: The text addresses health issues, medical advancements, or related\ + \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\ + \ topics. \nreligion: The text talks about relgions, religious institutions and\ + \ beliefs or related topics. \n\nbusiness: The text covers economy, business, or\ + \ related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_hau_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_ibo.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_ibo.yaml new file mode 100644 index 00000000..ce3cc15b --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_ibo.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Igbo text. For each input, classify the topic as technology, business, politics,\ + \ sports, health, entertainment, or religion. Use the following guidelines: \n\n\ + \ technology: The text discusses scientific discoveries, technological advancements,\ + \ or related topics. \npolitics: The text covers political events, policies, or\ + \ related topics. \nsports: The text talks about sports events, athletes, or related\ + \ topics. \nhealth: The text addresses health issues, medical advancements, or related\ + \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\ + \ topics. \nreligion: The text talks about relgions, religious institutions and\ + \ beliefs or related topics. \n\nbusiness: The text covers economy, business, or\ + \ related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_ibo_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_lin.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_lin.yaml new file mode 100644 index 00000000..e22303fe --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_lin.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: lin +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Lingala text. For each input, classify the topic as technology, business, politics,\ + \ sports, health, entertainment, or religion. Use the following guidelines: \n\n\ + \ technology: The text discusses scientific discoveries, technological advancements,\ + \ or related topics. \npolitics: The text covers political events, policies, or\ + \ related topics. \nsports: The text talks about sports events, athletes, or related\ + \ topics. \nhealth: The text addresses health issues, medical advancements, or related\ + \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\ + \ topics. \nreligion: The text talks about relgions, religious institutions and\ + \ beliefs or related topics. \n\nbusiness: The text covers economy, business, or\ + \ related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_lin_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_lug.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_lug.yaml new file mode 100644 index 00000000..fe949b6f --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_lug.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Luganda text. For each input, classify the topic as technology, business, politics,\ + \ sports, health, entertainment, or religion. Use the following guidelines: \n\n\ + \ technology: The text discusses scientific discoveries, technological advancements,\ + \ or related topics. \npolitics: The text covers political events, policies, or\ + \ related topics. \nsports: The text talks about sports events, athletes, or related\ + \ topics. \nhealth: The text addresses health issues, medical advancements, or related\ + \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\ + \ topics. \nreligion: The text talks about relgions, religious institutions and\ + \ beliefs or related topics. \n\nbusiness: The text covers economy, business, or\ + \ related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_lug_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_orm.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_orm.yaml new file mode 100644 index 00000000..413e8812 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_orm.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: orm +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Afaan Oromoo text. For each input, classify the topic as technology, business,\ + \ politics, sports, health, entertainment, or religion. Use the following guidelines:\ + \ \n\n technology: The text discusses scientific discoveries, technological advancements,\ + \ or related topics. \npolitics: The text covers political events, policies, or\ + \ related topics. \nsports: The text talks about sports events, athletes, or related\ + \ topics. \nhealth: The text addresses health issues, medical advancements, or related\ + \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\ + \ topics. \nreligion: The text talks about relgions, religious institutions and\ + \ beliefs or related topics. \n\nbusiness: The text covers economy, business, or\ + \ related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_orm_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_pcm.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_pcm.yaml new file mode 100644 index 00000000..b9322857 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_pcm.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: pcm +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Nigerian Pidgin text. For each input, classify the topic as technology, business,\ + \ politics, sports, health, entertainment, or religion. Use the following guidelines:\ + \ \n\n technology: The text discusses scientific discoveries, technological advancements,\ + \ or related topics. \npolitics: The text covers political events, policies, or\ + \ related topics. \nsports: The text talks about sports events, athletes, or related\ + \ topics. \nhealth: The text addresses health issues, medical advancements, or related\ + \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\ + \ topics. \nreligion: The text talks about relgions, religious institutions and\ + \ beliefs or related topics. \n\nbusiness: The text covers economy, business, or\ + \ related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_pcm_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_run.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_run.yaml new file mode 100644 index 00000000..6f207fb7 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_run.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: run +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Kirundi text. For each input, classify the topic as technology, business, politics,\ + \ sports, health, entertainment, or religion. Use the following guidelines: \n\n\ + \ technology: The text discusses scientific discoveries, technological advancements,\ + \ or related topics. \npolitics: The text covers political events, policies, or\ + \ related topics. \nsports: The text talks about sports events, athletes, or related\ + \ topics. \nhealth: The text addresses health issues, medical advancements, or related\ + \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\ + \ topics. \nreligion: The text talks about relgions, religious institutions and\ + \ beliefs or related topics. \n\nbusiness: The text covers economy, business, or\ + \ related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_run_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_sna.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_sna.yaml new file mode 100644 index 00000000..737d335e --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_sna.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Shona text. For each input, classify the topic as technology, business, politics,\ + \ sports, health, entertainment, or religion. Use the following guidelines: \n\n\ + \ technology: The text discusses scientific discoveries, technological advancements,\ + \ or related topics. \npolitics: The text covers political events, policies, or\ + \ related topics. \nsports: The text talks about sports events, athletes, or related\ + \ topics. \nhealth: The text addresses health issues, medical advancements, or related\ + \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\ + \ topics. \nreligion: The text talks about relgions, religious institutions and\ + \ beliefs or related topics. \n\nbusiness: The text covers economy, business, or\ + \ related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_sna_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_som.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_som.yaml new file mode 100644 index 00000000..39bb80c4 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_som.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: som +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Somali text. For each input, classify the topic as technology, business, politics,\ + \ sports, health, entertainment, or religion. Use the following guidelines: \n\n\ + \ technology: The text discusses scientific discoveries, technological advancements,\ + \ or related topics. \npolitics: The text covers political events, policies, or\ + \ related topics. \nsports: The text talks about sports events, athletes, or related\ + \ topics. \nhealth: The text addresses health issues, medical advancements, or related\ + \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\ + \ topics. \nreligion: The text talks about relgions, religious institutions and\ + \ beliefs or related topics. \n\nbusiness: The text covers economy, business, or\ + \ related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_som_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_swa.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_swa.yaml new file mode 100644 index 00000000..c59e359c --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_swa.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Swahili text. For each input, classify the topic as technology, business, politics,\ + \ sports, health, entertainment, or religion. Use the following guidelines: \n\n\ + \ technology: The text discusses scientific discoveries, technological advancements,\ + \ or related topics. \npolitics: The text covers political events, policies, or\ + \ related topics. \nsports: The text talks about sports events, athletes, or related\ + \ topics. \nhealth: The text addresses health issues, medical advancements, or related\ + \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\ + \ topics. \nreligion: The text talks about relgions, religious institutions and\ + \ beliefs or related topics. \n\nbusiness: The text covers economy, business, or\ + \ related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_swa_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_tir.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_tir.yaml new file mode 100644 index 00000000..959de7a8 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_tir.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: tir +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Tigrinya text. For each input, classify the topic as technology, business, politics,\ + \ sports, health, entertainment, or religion. Use the following guidelines: \n\n\ + \ technology: The text discusses scientific discoveries, technological advancements,\ + \ or related topics. \npolitics: The text covers political events, policies, or\ + \ related topics. \nsports: The text talks about sports events, athletes, or related\ + \ topics. \nhealth: The text addresses health issues, medical advancements, or related\ + \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\ + \ topics. \nreligion: The text talks about relgions, religious institutions and\ + \ beliefs or related topics. \n\nbusiness: The text covers economy, business, or\ + \ related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_tir_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_xho.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_xho.yaml new file mode 100644 index 00000000..35cad729 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_xho.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Xhosa text. For each input, classify the topic as technology, business, politics,\ + \ sports, health, entertainment, or religion. Use the following guidelines: \n\n\ + \ technology: The text discusses scientific discoveries, technological advancements,\ + \ or related topics. \npolitics: The text covers political events, policies, or\ + \ related topics. \nsports: The text talks about sports events, athletes, or related\ + \ topics. \nhealth: The text addresses health issues, medical advancements, or related\ + \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\ + \ topics. \nreligion: The text talks about relgions, religious institutions and\ + \ beliefs or related topics. \n\nbusiness: The text covers economy, business, or\ + \ related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_xho_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_yor.yaml b/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_yor.yaml new file mode 100644 index 00000000..e83c7045 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_5/masakhanews_yor.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Yoruba text. For each input, classify the topic as technology, business, politics,\ + \ sports, health, entertainment, or religion. Use the following guidelines: \n\n\ + \ technology: The text discusses scientific discoveries, technological advancements,\ + \ or related topics. \npolitics: The text covers political events, policies, or\ + \ related topics. \nsports: The text talks about sports events, athletes, or related\ + \ topics. \nhealth: The text addresses health issues, medical advancements, or related\ + \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\ + \ topics. \nreligion: The text talks about relgions, religious institutions and\ + \ beliefs or related topics. \n\nbusiness: The text covers economy, business, or\ + \ related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{headline_text}} \\category: \n\n" +include: masakhanews +task: masakhanews_yor_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhanews/prompt_5/utils.py b/lm_eval/tasks/afrobench/masakhanews/prompt_5/utils.py new file mode 100644 index 00000000..3e735e2d --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/prompt_5/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm_eval/tasks/afrobench/masakhanews/utils.py b/lm_eval/tasks/afrobench/masakhanews/utils.py new file mode 100644 index 00000000..310a7aeb --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhanews/utils.py @@ -0,0 +1,127 @@ +import argparse +import os + +import yaml + + +def prompt_func(mode, lang): + prompt_map = { + "prompt_1": "Given the categories technology, business, politics, sports, health, entertainment, or religion; what category does the text: '{{headline}}' belong to: \n\n", + "prompt_2": f"Does this {lang} topic; " + "'{{headline}}' belong to one of the following categories: technology, business, politics, sports, health, entertainment, or religion? category only\n\n", + "prompt_3": f"You are an assistant able to classify topics in texts. \n\n" + f"Given the categories technology, religion, politics, sports, health, entertainment, or business; what is " + f"the topic of the {lang} statement below? Return only the category. " + "\n\ntext: {{headline}} \category:\n\n", + "prompt_4": "Label the following text as technology, religion, politics, sports, health, entertainment, or geography. Provide only the category as your " + "response. \n\ntext: {{headline}} \category: \n\n", + "prompt_5": f"You are tasked with performing topic classification on the following {lang} text. " + f"For each input, classify the topic as technology, business, politics, sports, health, entertainment, or religion. " + f"Use the following guidelines: \n\n " + f"technology: The text discusses scientific discoveries, technological advancements, or related topics. \n" + f"politics: The text covers political events, policies, or related topics. \n" + f"sports: The text talks about sports events, athletes, or related topics. \n" + f"health: The text addresses health issues, medical advancements, or related topics. \n" + f"entertainment: The text pertains to movies, music, celebrities, or related topics. \n" + f"religion: The text talks about relgions, religious institutions and beliefs or related topics. \n\n" + f"business: The text covers economy, business, or related topics. \n\n" + f"If the text contains multiple topics, choose the dominant topic. " + f"For ambiguous or unclear topics, select the category that best reflects the overall content. " + "Please provide a single classification for each input.\n\ntext: {{headline}} \category: \n\n", + } + return prompt_map[mode] + + +def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None: + """ + Generate a yaml file for each language. + + :param output_dir: The directory to output the files to. + :param overwrite: Whether to overwrite files if they already exist. + """ + err = [] + languages = { + "amh": "Amharic", + "eng": "English", + "fra": "French", + "hau": "Hausa", + "ibo": "Igbo", + "lin": "Lingala", + "lug": "Luganda", + "orm": "Afaan Oromoo", + "pcm": "Nigerian Pidgin", + "run": "Kirundi", + "sna": "Shona", + "som": "Somali", + "swa": "Swahili", + "tir": "Tigrinya", + "xho": "Xhosa", + "yor": "Yoruba", + } + + for lang in languages.keys(): + try: + file_name = f"masakhanews_{lang}.yaml" + task_name = f"masakhanews_{lang}_{mode}" + yaml_template = "masakhanews" + yaml_details = { + "include": yaml_template, + "task": task_name, + "dataset_name": lang, + "doc_to_text": prompt_func(mode, languages[lang]), + } + file_path = os.path.join(output_dir, mode) + os.makedirs(file_path, exist_ok=True) + + with open( + f"{output_dir}/{mode}/{file_name}", + "w" if overwrite else "x", + encoding="utf8", + ) as f: + f.write("# Generated by utils.py\n") + yaml.dump( + yaml_details, + f, + allow_unicode=True, + ) + except FileExistsError: + err.append(file_name) + + if len(err) > 0: + raise FileExistsError( + "Files were not created because they already exist (use --overwrite flag):" + f" {', '.join(err)}" + ) + + +def main() -> None: + """Parse CLI args and generate language-specific yaml files.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + default=True, + action="store_true", + help="Overwrite files if they already exist", + ) + parser.add_argument( + "--output-dir", + default="./", + help="Directory to write yaml files to", + ) + + PROMPT_CHOICES = ["prompt_1", "prompt_2", "prompt_3", "prompt_4", "prompt_5"] + parser.add_argument( + "--mode", + nargs="*", + default=PROMPT_CHOICES, + choices=PROMPT_CHOICES, + help="Prompt number(s)", + ) + args = parser.parse_args() + + for mode in args.mode: + gen_lang_yamls(output_dir=args.output_dir, overwrite=args.overwrite, mode=mode) + + +if __name__ == "__main__": + main() diff --git a/lm_eval/tasks/afrobench/masakhapos/README.md b/lm_eval/tasks/afrobench/masakhapos/README.md new file mode 100644 index 00000000..1fcf11c7 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/README.md @@ -0,0 +1,75 @@ +# + +## Paper +Title: `MasakhaPOS: Part-of-Speech Tagging for Typologically Diverse African languages` + +Paper Link: https://aclanthology.org/2023.acl-long.609/ + +## Abstract +>In this paper, we present AfricaPOS, the largest part-of-speech (POS) dataset for 20 typologically diverse African languages. We discuss the challenges in annotating POS for these languages using the universal dependencies (UD) guidelines. We conducted extensive POS baseline experiments using both conditional random field and several multilingual pre-trained language models. We applied various cross-lingual transfer models trained with data available in the UD. Evaluating on the AfricaPOS dataset, we show that choosing the best transfer language(s) in both single-source and multi-source setups greatly improves the POS tagging performance of the target languages, in particular when combined with parameter-fine-tuning methods. Crucially, transferring knowledge from a language that matches the language family and morphosyntactic properties seems to be more effective for POS tagging in unseen languages. + +HomePage: https://github.com/masakhane-io/masakhane-pos + +### Citation + +``` +@inproceedings{dione-etal-2023-masakhapos, + title = "{M}asakha{POS}: Part-of-Speech Tagging for Typologically Diverse {A}frican languages", + author = "Dione, Cheikh M. Bamba and + Adelani, David Ifeoluwa and + Nabende, Peter and + Alabi, Jesujoba and + Sindane, Thapelo and + Buzaaba, Happy and + Muhammad, Shamsuddeen Hassan and + Emezue, Chris Chinenye and + Ogayo, Perez and + Aremu, Anuoluwapo and + Gitau, Catherine and + Mbaye, Derguene and + Mukiibi, Jonathan and + Sibanda, Blessing and + Dossou, Bonaventure F. P. and + Bukula, Andiswa and + Mabuya, Rooweither and + Tapo, Allahsera Auguste and + Munkoh-Buabeng, Edwin and + Memdjokam Koagne, Victoire and + Ouoba Kabore, Fatoumata and + Taylor, Amelia and + Kalipe, Godson and + Macucwa, Tebogo and + Marivate, Vukosi and + Gwadabe, Tajuddeen and + Elvis, Mboning Tchiaze and + Onyenwe, Ikechukwu and + Atindogbe, Gratien and + Adelani, Tolulope and + Akinade, Idris and + Samuel, Olanrewaju and + Nahimana, Marien and + Musabeyezu, Th{\'e}og{\`e}ne and + Niyomutabazi, Emile and + Chimhenga, Ester and + Gotosa, Kudzai and + Mizha, Patrick and + Agbolo, Apelete and + Traore, Seydou and + Uchechukwu, Chinedu and + Yusuf, Aliyu and + Abdullahi, Muhammad and + Klakow, Dietrich", + editor = "Rogers, Anna and + Boyd-Graber, Jordan and + Okazaki, Naoaki", + booktitle = "Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)", + month = jul, + year = "2023", + address = "Toronto, Canada", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2023.acl-long.609/", + doi = "10.18653/v1/2023.acl-long.609", + pages = "10883--10900", + abstract = "In this paper, we present AfricaPOS, the largest part-of-speech (POS) dataset for 20 typologically diverse African languages. We discuss the challenges in annotating POS for these languages using the universal dependencies (UD) guidelines. We conducted extensive POS baseline experiments using both conditional random field and several multilingual pre-trained language models. We applied various cross-lingual transfer models trained with data available in the UD. Evaluating on the AfricaPOS dataset, we show that choosing the best transfer language(s) in both single-source and multi-source setups greatly improves the POS tagging performance of the target languages, in particular when combined with parameter-fine-tuning methods. Crucially, transferring knowledge from a language that matches the language family and morphosyntactic properties seems to be more effective for POS tagging in unseen languages." +} +``` diff --git a/lm_eval/tasks/afrobench/masakhapos/gen_utils.py b/lm_eval/tasks/afrobench/masakhapos/gen_utils.py new file mode 100644 index 00000000..52b9dafb --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/gen_utils.py @@ -0,0 +1,151 @@ +import argparse +import os + +import yaml + + +class FunctionTag: + def __init__(self, value): + self.value = value + + +def prompt_func(mode, lang): + prompt_map = { + "prompt_1": "Please provide the POS tags for each word in the input sentence. The input will be a list of " + "words in the sentence. The output format should be a list of tuples, where each tuple consists of " + "a word from the input text and its corresponding POS tag label from the tag label set: ['ADJ', " + "'ADP', 'ADV', 'AUX', 'CCONJ, 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', " + "'SCONJ', 'SYM', 'VERB', 'X']. \nYour response should include only a list of tuples, in the order " + "that the words appear in the input sentence, including punctuations, with each tuple containing the corresponding POS tag " + "label for a word. \n\nSentence: {{tokens}} \nOutput: ", + "prompt_2": f"You are an expert in tagging words and sentences in {lang} with the right POS tag. " + f"\n\nPlease provide the POS tags for each word in the {lang} sentence. The input is a list of words in" + " the sentence. POS tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET', 'INTJ', 'NOUN', " + "'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB', 'X']. The output format should " + "be a list of tuples, where each tuple consists of a word from the input text and its corresponding" + " POS tag label from the POS tag label set provided\nYour response should include only a list of " + "tuples, in the order that the words appear in the input sentence, including punctuations, with each tuple containing the " + "corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: ", + "prompt_3": f"Acting as a {lang} linguist and without making any corrections or changes to the text, perform a part of " + "speech (POS) analysis of the sentences using the following POS tag label annotation ['ADJ', " + "'ADP', 'ADV', 'AUX', 'CCONJ, 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', " + "'SCONJ', 'SYM', 'VERB', 'X']. The input will be a list of words in the sentence. The output format should " + "be a list of tuples, where each tuple consists of a word from the input text and its corresponding" + " POS tag label from the POS tag label set provided\nYour response should include only a list of " + "tuples, in the order that the words appear in the input sentence, including punctuations, with each tuple containing the " + "corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: ", + "prompt_4": "Annotate each word in the provided sentence with the appropriate POS tag. The annotation " + "list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', " + "'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB', 'X']. The input sentence will be a list of words" + " in the sentence. The output format should " + "be a list of tuples, where each tuple consists of a word from the input text and its corresponding" + " POS tag label from the POS tag label set provided\nYour response should include only a list of " + "tuples, in the order that the words appear in the input sentence, including punctuations, with each tuple containing the " + "corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: ", + "prompt_5": "Given the following sentence, identify the part of speech (POS) for each word. Use the following " + "POS tag set: \nNOUN: Noun (person, place, thing), \nVERB: Verb (action, state), " + "\nADJ: Adjective (describes a noun), \nADV: Adverb (modifies a verb, adjective, or adverb), " + "\nPRON: Pronoun (replaces a noun), \nDET: Determiner (introduces a noun), " + "\nADP: Adposition (preposition or postposition), \nCCONJ: Conjunction (connects words, phrases, clauses)" + "\nPUNCT: Punctuation, \nPROPN: Proper Noun, \nAUX: Auxiliary verb (helper verb), " + "\nSCONJ: Subordinating conjunction \nPART: Particle, \nSYM: Symbol, \nINTJ: Interjection, " + "\nNUM: Numeral, \nX: others. The output format should " + "be a list of tuples, where each tuple consists of a word from the input text and its corresponding" + " POS tag label key only from the POS tag set provided\nYour response should include only a list of " + "tuples, in the order that the words appear in the input sentence, including punctuations, with each tuple containing the " + "corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: ", + } + return prompt_map[mode] + + +def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None: + """ + Generate a yaml file for each language. + + :param output_dir: The directory to output the files to. + :param overwrite: Whether to overwrite files if they already exist. + """ + err = [] + languages = { + "bam": "Bambara", + "bbj": "Ghomala", + "ewe": "Ewe", + "fon": "Fon", + "hau": "Hausa", + "ibo": "Igbo", + "kin": "Kinyarwanda", + "lug": "Luganda", + "luo": "Dholuo", + "mos": "Mossi", + "nya": "Chichewa", + "pcm": "Nigerian Pidgin", + "sna": "chiShona", + "swa": "Kiswahili", + "tsn": "Setswana", + "twi": "Twi", + "wol": "Wolof", + "xho": "isiXhosa", + "yor": "Yoruba", + "zul": "isiZulu", + } + + for lang in languages.keys(): + try: + file_name = f"masakhapos_{lang}.yaml" + task_name = f"masakhapos_{lang}_{mode}" + yaml_template = "masakhapos_yaml" + yaml_details = { + "include": yaml_template, + "task": task_name, + "dataset_name": lang, + "doc_to_text": prompt_func(mode, languages[lang]), + } + os.makedirs(f"{output_dir}/{mode}", exist_ok=True) + with open( + f"{output_dir}/{mode}/{file_name}", + "w" if overwrite else "x", + encoding="utf8", + ) as f: + f.write("# Generated by utils.py\n") + yaml.dump( + yaml_details, + f, + allow_unicode=True, + ) + except FileExistsError: + err.append(file_name) + + if len(err) > 0: + raise FileExistsError( + "Files were not created because they already exist (use --overwrite flag):" + f" {', '.join(err)}" + ) + + +def main() -> None: + """Parse CLI args and generate language-specific yaml files.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + default=True, + action="store_true", + help="Overwrite files if they already exist", + ) + parser.add_argument( + "--output-dir", + default="./", + help="Directory to write yaml files to", + ) + parser.add_argument( + "--mode", + default="prompt_1", + choices=["prompt_1", "prompt_2", "prompt_3", "prompt_4", "prompt_5"], + help="Prompt number", + ) + args = parser.parse_args() + + gen_lang_yamls(output_dir=args.output_dir, overwrite=args.overwrite, mode=args.mode) + + +if __name__ == "__main__": + main() diff --git a/lm_eval/tasks/afrobench/masakhapos/masakhapos.yaml b/lm_eval/tasks/afrobench/masakhapos/masakhapos.yaml new file mode 100644 index 00000000..3fb1574e --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/masakhapos.yaml @@ -0,0 +1,13 @@ +group: masakhapos +task: + - masakhapos_prompt_1 + - masakhapos_prompt_2 + - masakhapos_prompt_3 + - masakhapos_prompt_4 + - masakhapos_prompt_5 +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 1 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_bam.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_bam.yaml new file mode 100644 index 00000000..b1c64e38 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_bam.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: bam +doc_to_text: "Please provide the POS tags for each word in the input sentence. The\ + \ input will be a list of words in the sentence. The output format should be a list\ + \ of tuples, where each tuple consists of a word from the input text and its corresponding\ + \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. \nYour response should include only a list of tuples, in the order that\ + \ the words appear in the input sentence, including punctuations, with each tuple\ + \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\ + \ \nOutput: " +include: masakhapos_yaml +task: masakhapos_bam_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_bbj.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_bbj.yaml new file mode 100644 index 00000000..418c8e0c --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_bbj.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: bbj +doc_to_text: "Please provide the POS tags for each word in the input sentence. The\ + \ input will be a list of words in the sentence. The output format should be a list\ + \ of tuples, where each tuple consists of a word from the input text and its corresponding\ + \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. \nYour response should include only a list of tuples, in the order that\ + \ the words appear in the input sentence, including punctuations, with each tuple\ + \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\ + \ \nOutput: " +include: masakhapos_yaml +task: masakhapos_bbj_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_ewe.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_ewe.yaml new file mode 100644 index 00000000..1eeb2497 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_ewe.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: ewe +doc_to_text: "Please provide the POS tags for each word in the input sentence. The\ + \ input will be a list of words in the sentence. The output format should be a list\ + \ of tuples, where each tuple consists of a word from the input text and its corresponding\ + \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. \nYour response should include only a list of tuples, in the order that\ + \ the words appear in the input sentence, including punctuations, with each tuple\ + \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\ + \ \nOutput: " +include: masakhapos_yaml +task: masakhapos_ewe_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_fon.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_fon.yaml new file mode 100644 index 00000000..da7eb7ae --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_fon.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: fon +doc_to_text: "Please provide the POS tags for each word in the input sentence. The\ + \ input will be a list of words in the sentence. The output format should be a list\ + \ of tuples, where each tuple consists of a word from the input text and its corresponding\ + \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. \nYour response should include only a list of tuples, in the order that\ + \ the words appear in the input sentence, including punctuations, with each tuple\ + \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\ + \ \nOutput: " +include: masakhapos_yaml +task: masakhapos_fon_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_hau.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_hau.yaml new file mode 100644 index 00000000..431ed8f1 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_hau.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: "Please provide the POS tags for each word in the input sentence. The\ + \ input will be a list of words in the sentence. The output format should be a list\ + \ of tuples, where each tuple consists of a word from the input text and its corresponding\ + \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. \nYour response should include only a list of tuples, in the order that\ + \ the words appear in the input sentence, including punctuations, with each tuple\ + \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\ + \ \nOutput: " +include: masakhapos_yaml +task: masakhapos_hau_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_ibo.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_ibo.yaml new file mode 100644 index 00000000..0cb171fe --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_ibo.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "Please provide the POS tags for each word in the input sentence. The\ + \ input will be a list of words in the sentence. The output format should be a list\ + \ of tuples, where each tuple consists of a word from the input text and its corresponding\ + \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. \nYour response should include only a list of tuples, in the order that\ + \ the words appear in the input sentence, including punctuations, with each tuple\ + \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\ + \ \nOutput: " +include: masakhapos_yaml +task: masakhapos_ibo_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_kin.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_kin.yaml new file mode 100644 index 00000000..dced04f2 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_kin.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: "Please provide the POS tags for each word in the input sentence. The\ + \ input will be a list of words in the sentence. The output format should be a list\ + \ of tuples, where each tuple consists of a word from the input text and its corresponding\ + \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. \nYour response should include only a list of tuples, in the order that\ + \ the words appear in the input sentence, including punctuations, with each tuple\ + \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\ + \ \nOutput: " +include: masakhapos_yaml +task: masakhapos_kin_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_lug.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_lug.yaml new file mode 100644 index 00000000..e773f643 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_lug.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: "Please provide the POS tags for each word in the input sentence. The\ + \ input will be a list of words in the sentence. The output format should be a list\ + \ of tuples, where each tuple consists of a word from the input text and its corresponding\ + \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. \nYour response should include only a list of tuples, in the order that\ + \ the words appear in the input sentence, including punctuations, with each tuple\ + \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\ + \ \nOutput: " +include: masakhapos_yaml +task: masakhapos_lug_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_luo.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_luo.yaml new file mode 100644 index 00000000..4544e2b1 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_luo.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: luo +doc_to_text: "Please provide the POS tags for each word in the input sentence. The\ + \ input will be a list of words in the sentence. The output format should be a list\ + \ of tuples, where each tuple consists of a word from the input text and its corresponding\ + \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. \nYour response should include only a list of tuples, in the order that\ + \ the words appear in the input sentence, including punctuations, with each tuple\ + \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\ + \ \nOutput: " +include: masakhapos_yaml +task: masakhapos_luo_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_mos.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_mos.yaml new file mode 100644 index 00000000..b0c7d3f6 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_mos.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: mos +doc_to_text: "Please provide the POS tags for each word in the input sentence. The\ + \ input will be a list of words in the sentence. The output format should be a list\ + \ of tuples, where each tuple consists of a word from the input text and its corresponding\ + \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. \nYour response should include only a list of tuples, in the order that\ + \ the words appear in the input sentence, including punctuations, with each tuple\ + \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\ + \ \nOutput: " +include: masakhapos_yaml +task: masakhapos_mos_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_nya.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_nya.yaml new file mode 100644 index 00000000..f8d4fcbf --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_nya.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: nya +doc_to_text: "Please provide the POS tags for each word in the input sentence. The\ + \ input will be a list of words in the sentence. The output format should be a list\ + \ of tuples, where each tuple consists of a word from the input text and its corresponding\ + \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. \nYour response should include only a list of tuples, in the order that\ + \ the words appear in the input sentence, including punctuations, with each tuple\ + \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\ + \ \nOutput: " +include: masakhapos_yaml +task: masakhapos_nya_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_pcm.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_pcm.yaml new file mode 100644 index 00000000..2d05924e --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_pcm.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: pcm +doc_to_text: "Please provide the POS tags for each word in the input sentence. The\ + \ input will be a list of words in the sentence. The output format should be a list\ + \ of tuples, where each tuple consists of a word from the input text and its corresponding\ + \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. \nYour response should include only a list of tuples, in the order that\ + \ the words appear in the input sentence, including punctuations, with each tuple\ + \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\ + \ \nOutput: " +include: masakhapos_yaml +task: masakhapos_pcm_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_sna.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_sna.yaml new file mode 100644 index 00000000..7afa02f4 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_sna.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: "Please provide the POS tags for each word in the input sentence. The\ + \ input will be a list of words in the sentence. The output format should be a list\ + \ of tuples, where each tuple consists of a word from the input text and its corresponding\ + \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. \nYour response should include only a list of tuples, in the order that\ + \ the words appear in the input sentence, including punctuations, with each tuple\ + \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\ + \ \nOutput: " +include: masakhapos_yaml +task: masakhapos_sna_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_swa.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_swa.yaml new file mode 100644 index 00000000..ab2f123e --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_swa.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: "Please provide the POS tags for each word in the input sentence. The\ + \ input will be a list of words in the sentence. The output format should be a list\ + \ of tuples, where each tuple consists of a word from the input text and its corresponding\ + \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. \nYour response should include only a list of tuples, in the order that\ + \ the words appear in the input sentence, including punctuations, with each tuple\ + \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\ + \ \nOutput: " +include: masakhapos_yaml +task: masakhapos_swa_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_tsn.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_tsn.yaml new file mode 100644 index 00000000..ca02f064 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_tsn.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: tsn +doc_to_text: "Please provide the POS tags for each word in the input sentence. The\ + \ input will be a list of words in the sentence. The output format should be a list\ + \ of tuples, where each tuple consists of a word from the input text and its corresponding\ + \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. \nYour response should include only a list of tuples, in the order that\ + \ the words appear in the input sentence, including punctuations, with each tuple\ + \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\ + \ \nOutput: " +include: masakhapos_yaml +task: masakhapos_tsn_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_twi.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_twi.yaml new file mode 100644 index 00000000..f22c0936 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_twi.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: "Please provide the POS tags for each word in the input sentence. The\ + \ input will be a list of words in the sentence. The output format should be a list\ + \ of tuples, where each tuple consists of a word from the input text and its corresponding\ + \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. \nYour response should include only a list of tuples, in the order that\ + \ the words appear in the input sentence, including punctuations, with each tuple\ + \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\ + \ \nOutput: " +include: masakhapos_yaml +task: masakhapos_twi_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_wol.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_wol.yaml new file mode 100644 index 00000000..e0bdd23a --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_wol.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: "Please provide the POS tags for each word in the input sentence. The\ + \ input will be a list of words in the sentence. The output format should be a list\ + \ of tuples, where each tuple consists of a word from the input text and its corresponding\ + \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. \nYour response should include only a list of tuples, in the order that\ + \ the words appear in the input sentence, including punctuations, with each tuple\ + \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\ + \ \nOutput: " +include: masakhapos_yaml +task: masakhapos_wol_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_xho.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_xho.yaml new file mode 100644 index 00000000..f712a874 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_xho.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: "Please provide the POS tags for each word in the input sentence. The\ + \ input will be a list of words in the sentence. The output format should be a list\ + \ of tuples, where each tuple consists of a word from the input text and its corresponding\ + \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. \nYour response should include only a list of tuples, in the order that\ + \ the words appear in the input sentence, including punctuations, with each tuple\ + \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\ + \ \nOutput: " +include: masakhapos_yaml +task: masakhapos_xho_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_yaml new file mode 100644 index 00000000..bdca7a85 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_yaml @@ -0,0 +1,32 @@ +tag: +- masakhapos_tasks +- masakhapos_prompt_1 +dataset_path: masakhane/masakhapos +dataset_name: null +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +generation_kwargs: + do_sample: false + until: + - + - <|im_end|> +validation_split: validation +test_split: test +fewshot_split: train +doc_to_target: !function utils.doc_to_target +should_decontaminate: true +doc_to_decontamination_query: "Sentence: {{token}}\nOutput:" +filter_list: + - filter: + - function: regex_pos + name: flexible-extract +metric_list: + - metric: acc + aggregation: !function utils.acc_score + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_yor.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_yor.yaml new file mode 100644 index 00000000..efa8750a --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_yor.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: "Please provide the POS tags for each word in the input sentence. The\ + \ input will be a list of words in the sentence. The output format should be a list\ + \ of tuples, where each tuple consists of a word from the input text and its corresponding\ + \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. \nYour response should include only a list of tuples, in the order that\ + \ the words appear in the input sentence, including punctuations, with each tuple\ + \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\ + \ \nOutput: " +include: masakhapos_yaml +task: masakhapos_yor_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_zul.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_zul.yaml new file mode 100644 index 00000000..362c9934 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_1/masakhapos_zul.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: "Please provide the POS tags for each word in the input sentence. The\ + \ input will be a list of words in the sentence. The output format should be a list\ + \ of tuples, where each tuple consists of a word from the input text and its corresponding\ + \ POS tag label from the tag label set: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. \nYour response should include only a list of tuples, in the order that\ + \ the words appear in the input sentence, including punctuations, with each tuple\ + \ containing the corresponding POS tag label for a word. \n\nSentence: {{tokens}}\ + \ \nOutput: " +include: masakhapos_yaml +task: masakhapos_zul_prompt_1 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_1/utils.py b/lm_eval/tasks/afrobench/masakhapos/prompt_1/utils.py new file mode 100644 index 00000000..4ccc66d9 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_1/utils.py @@ -0,0 +1,55 @@ +from itertools import chain + +from sklearn.metrics import accuracy_score + +from lm_eval.utils import weighted_f1_score + + +def doc_to_target(doc): + pos_tag_map = { + 0: "NOUN", + 1: "PUNCT", + 2: "ADP", + 3: "NUM", + 4: "SYM", + 5: "SCONJ", + 6: "ADJ", + 7: "PART", + 8: "DET", + 9: "CCONJ", + 10: "PROPN", + 11: "PRON", + 12: "X", + 13: "_", + 14: "ADV", + 15: "INTJ", + 16: "VERB", + 17: "AUX", + } + return [pos_tag_map[tag] for tag in doc["upos"]] + + +def acc_score(items): + unzipped_list = list(zip(*items)) + + golds, preds = unzipped_list[0], unzipped_list[1] + + # Flatten preds' inner lists + flattened_preds = [list(chain.from_iterable(p)) for p in preds] + + # Calculate the accuracy for each gold-pred pair + accuracy_scores = [] + for gold, pred in zip(golds, flattened_preds): + # Ensure both lists are of the same length, otherwise truncate to match + min_length = min(len(gold), len(pred)) + gold = gold[:min_length] + pred = pred[:min_length] + + # Calculate accuracy for the current pair and add to the list + accuracy = accuracy_score(gold, pred) + accuracy_scores.append(accuracy) + + mean_accuracy = ( + sum(accuracy_scores) / len(accuracy_scores) if accuracy_scores else 0 + ) + return mean_accuracy diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_bam.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_bam.yaml new file mode 100644 index 00000000..bde25d7e --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_bam.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: bam +doc_to_text: "You are an expert in tagging words and sentences in Bambara with the\ + \ right POS tag. \n\nPlease provide the POS tags for each word in the Bambara sentence.\ + \ The input is a list of words in the sentence. POS tag label set: ['ADJ', 'ADP',\ + \ 'ADV', 'AUX', 'CCONJ, 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT',\ + \ 'SCONJ', 'SYM', 'VERB', 'X']. The output format should be a list of tuples, where\ + \ each tuple consists of a word from the input text and its corresponding POS tag\ + \ label from the POS tag label set provided\nYour response should include only a\ + \ list of tuples, in the order that the words appear in the input sentence, including\ + \ punctuations, with each tuple containing the corresponding POS tag label for a\ + \ word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_bam_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_bbj.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_bbj.yaml new file mode 100644 index 00000000..8439e6b0 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_bbj.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: bbj +doc_to_text: "You are an expert in tagging words and sentences in Ghomala with the\ + \ right POS tag. \n\nPlease provide the POS tags for each word in the Ghomala sentence.\ + \ The input is a list of words in the sentence. POS tag label set: ['ADJ', 'ADP',\ + \ 'ADV', 'AUX', 'CCONJ, 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT',\ + \ 'SCONJ', 'SYM', 'VERB', 'X']. The output format should be a list of tuples, where\ + \ each tuple consists of a word from the input text and its corresponding POS tag\ + \ label from the POS tag label set provided\nYour response should include only a\ + \ list of tuples, in the order that the words appear in the input sentence, including\ + \ punctuations, with each tuple containing the corresponding POS tag label for a\ + \ word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_bbj_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_ewe.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_ewe.yaml new file mode 100644 index 00000000..5ffa2ba9 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_ewe.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: ewe +doc_to_text: "You are an expert in tagging words and sentences in Ewe with the right\ + \ POS tag. \n\nPlease provide the POS tags for each word in the Ewe sentence. The\ + \ input is a list of words in the sentence. POS tag label set: ['ADJ', 'ADP', 'ADV',\ + \ 'AUX', 'CCONJ, 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT',\ + \ 'SCONJ', 'SYM', 'VERB', 'X']. The output format should be a list of tuples, where\ + \ each tuple consists of a word from the input text and its corresponding POS tag\ + \ label from the POS tag label set provided\nYour response should include only a\ + \ list of tuples, in the order that the words appear in the input sentence, including\ + \ punctuations, with each tuple containing the corresponding POS tag label for a\ + \ word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_ewe_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_fon.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_fon.yaml new file mode 100644 index 00000000..548f2de4 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_fon.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: fon +doc_to_text: "You are an expert in tagging words and sentences in Fon with the right\ + \ POS tag. \n\nPlease provide the POS tags for each word in the Fon sentence. The\ + \ input is a list of words in the sentence. POS tag label set: ['ADJ', 'ADP', 'ADV',\ + \ 'AUX', 'CCONJ, 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT',\ + \ 'SCONJ', 'SYM', 'VERB', 'X']. The output format should be a list of tuples, where\ + \ each tuple consists of a word from the input text and its corresponding POS tag\ + \ label from the POS tag label set provided\nYour response should include only a\ + \ list of tuples, in the order that the words appear in the input sentence, including\ + \ punctuations, with each tuple containing the corresponding POS tag label for a\ + \ word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_fon_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_hau.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_hau.yaml new file mode 100644 index 00000000..4bc03457 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_hau.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: "You are an expert in tagging words and sentences in Hausa with the right\ + \ POS tag. \n\nPlease provide the POS tags for each word in the Hausa sentence.\ + \ The input is a list of words in the sentence. POS tag label set: ['ADJ', 'ADP',\ + \ 'ADV', 'AUX', 'CCONJ, 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT',\ + \ 'SCONJ', 'SYM', 'VERB', 'X']. The output format should be a list of tuples, where\ + \ each tuple consists of a word from the input text and its corresponding POS tag\ + \ label from the POS tag label set provided\nYour response should include only a\ + \ list of tuples, in the order that the words appear in the input sentence, including\ + \ punctuations, with each tuple containing the corresponding POS tag label for a\ + \ word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_hau_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_ibo.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_ibo.yaml new file mode 100644 index 00000000..d0f5d357 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_ibo.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "You are an expert in tagging words and sentences in Igbo with the right\ + \ POS tag. \n\nPlease provide the POS tags for each word in the Igbo sentence. The\ + \ input is a list of words in the sentence. POS tag label set: ['ADJ', 'ADP', 'ADV',\ + \ 'AUX', 'CCONJ, 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT',\ + \ 'SCONJ', 'SYM', 'VERB', 'X']. The output format should be a list of tuples, where\ + \ each tuple consists of a word from the input text and its corresponding POS tag\ + \ label from the POS tag label set provided\nYour response should include only a\ + \ list of tuples, in the order that the words appear in the input sentence, including\ + \ punctuations, with each tuple containing the corresponding POS tag label for a\ + \ word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_ibo_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_kin.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_kin.yaml new file mode 100644 index 00000000..95fd232a --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_kin.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: "You are an expert in tagging words and sentences in Kinyarwanda with\ + \ the right POS tag. \n\nPlease provide the POS tags for each word in the Kinyarwanda\ + \ sentence. The input is a list of words in the sentence. POS tag label set: ['ADJ',\ + \ 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN',\ + \ 'PUNCT', 'SCONJ', 'SYM', 'VERB', 'X']. The output format should be a list of tuples,\ + \ where each tuple consists of a word from the input text and its corresponding\ + \ POS tag label from the POS tag label set provided\nYour response should include\ + \ only a list of tuples, in the order that the words appear in the input sentence,\ + \ including punctuations, with each tuple containing the corresponding POS tag label\ + \ for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_kin_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_lug.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_lug.yaml new file mode 100644 index 00000000..21b02b10 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_lug.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: "You are an expert in tagging words and sentences in Luganda with the\ + \ right POS tag. \n\nPlease provide the POS tags for each word in the Luganda sentence.\ + \ The input is a list of words in the sentence. POS tag label set: ['ADJ', 'ADP',\ + \ 'ADV', 'AUX', 'CCONJ, 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT',\ + \ 'SCONJ', 'SYM', 'VERB', 'X']. The output format should be a list of tuples, where\ + \ each tuple consists of a word from the input text and its corresponding POS tag\ + \ label from the POS tag label set provided\nYour response should include only a\ + \ list of tuples, in the order that the words appear in the input sentence, including\ + \ punctuations, with each tuple containing the corresponding POS tag label for a\ + \ word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_lug_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_luo.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_luo.yaml new file mode 100644 index 00000000..42ccb34f --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_luo.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: luo +doc_to_text: "You are an expert in tagging words and sentences in Dholuo with the\ + \ right POS tag. \n\nPlease provide the POS tags for each word in the Dholuo sentence.\ + \ The input is a list of words in the sentence. POS tag label set: ['ADJ', 'ADP',\ + \ 'ADV', 'AUX', 'CCONJ, 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT',\ + \ 'SCONJ', 'SYM', 'VERB', 'X']. The output format should be a list of tuples, where\ + \ each tuple consists of a word from the input text and its corresponding POS tag\ + \ label from the POS tag label set provided\nYour response should include only a\ + \ list of tuples, in the order that the words appear in the input sentence, including\ + \ punctuations, with each tuple containing the corresponding POS tag label for a\ + \ word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_luo_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_mos.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_mos.yaml new file mode 100644 index 00000000..cfa74aef --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_mos.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: mos +doc_to_text: "You are an expert in tagging words and sentences in Mossi with the right\ + \ POS tag. \n\nPlease provide the POS tags for each word in the Mossi sentence.\ + \ The input is a list of words in the sentence. POS tag label set: ['ADJ', 'ADP',\ + \ 'ADV', 'AUX', 'CCONJ, 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT',\ + \ 'SCONJ', 'SYM', 'VERB', 'X']. The output format should be a list of tuples, where\ + \ each tuple consists of a word from the input text and its corresponding POS tag\ + \ label from the POS tag label set provided\nYour response should include only a\ + \ list of tuples, in the order that the words appear in the input sentence, including\ + \ punctuations, with each tuple containing the corresponding POS tag label for a\ + \ word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_mos_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_nya.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_nya.yaml new file mode 100644 index 00000000..27de8386 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_nya.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: nya +doc_to_text: "You are an expert in tagging words and sentences in Chichewa with the\ + \ right POS tag. \n\nPlease provide the POS tags for each word in the Chichewa sentence.\ + \ The input is a list of words in the sentence. POS tag label set: ['ADJ', 'ADP',\ + \ 'ADV', 'AUX', 'CCONJ, 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT',\ + \ 'SCONJ', 'SYM', 'VERB', 'X']. The output format should be a list of tuples, where\ + \ each tuple consists of a word from the input text and its corresponding POS tag\ + \ label from the POS tag label set provided\nYour response should include only a\ + \ list of tuples, in the order that the words appear in the input sentence, including\ + \ punctuations, with each tuple containing the corresponding POS tag label for a\ + \ word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_nya_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_pcm.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_pcm.yaml new file mode 100644 index 00000000..0c532569 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_pcm.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: pcm +doc_to_text: "You are an expert in tagging words and sentences in Nigerian Pidgin\ + \ with the right POS tag. \n\nPlease provide the POS tags for each word in the Nigerian\ + \ Pidgin sentence. The input is a list of words in the sentence. POS tag label set:\ + \ ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON',\ + \ 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB', 'X']. The output format should be a\ + \ list of tuples, where each tuple consists of a word from the input text and its\ + \ corresponding POS tag label from the POS tag label set provided\nYour response\ + \ should include only a list of tuples, in the order that the words appear in the\ + \ input sentence, including punctuations, with each tuple containing the corresponding\ + \ POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_pcm_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_sna.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_sna.yaml new file mode 100644 index 00000000..c6c6467d --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_sna.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: "You are an expert in tagging words and sentences in chiShona with the\ + \ right POS tag. \n\nPlease provide the POS tags for each word in the chiShona sentence.\ + \ The input is a list of words in the sentence. POS tag label set: ['ADJ', 'ADP',\ + \ 'ADV', 'AUX', 'CCONJ, 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT',\ + \ 'SCONJ', 'SYM', 'VERB', 'X']. The output format should be a list of tuples, where\ + \ each tuple consists of a word from the input text and its corresponding POS tag\ + \ label from the POS tag label set provided\nYour response should include only a\ + \ list of tuples, in the order that the words appear in the input sentence, including\ + \ punctuations, with each tuple containing the corresponding POS tag label for a\ + \ word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_sna_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_swa.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_swa.yaml new file mode 100644 index 00000000..b1ca8780 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_swa.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: "You are an expert in tagging words and sentences in Kiswahili with the\ + \ right POS tag. \n\nPlease provide the POS tags for each word in the Kiswahili\ + \ sentence. The input is a list of words in the sentence. POS tag label set: ['ADJ',\ + \ 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN',\ + \ 'PUNCT', 'SCONJ', 'SYM', 'VERB', 'X']. The output format should be a list of tuples,\ + \ where each tuple consists of a word from the input text and its corresponding\ + \ POS tag label from the POS tag label set provided\nYour response should include\ + \ only a list of tuples, in the order that the words appear in the input sentence,\ + \ including punctuations, with each tuple containing the corresponding POS tag label\ + \ for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_swa_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_tsn.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_tsn.yaml new file mode 100644 index 00000000..a6988664 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_tsn.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: tsn +doc_to_text: "You are an expert in tagging words and sentences in Setswana with the\ + \ right POS tag. \n\nPlease provide the POS tags for each word in the Setswana sentence.\ + \ The input is a list of words in the sentence. POS tag label set: ['ADJ', 'ADP',\ + \ 'ADV', 'AUX', 'CCONJ, 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT',\ + \ 'SCONJ', 'SYM', 'VERB', 'X']. The output format should be a list of tuples, where\ + \ each tuple consists of a word from the input text and its corresponding POS tag\ + \ label from the POS tag label set provided\nYour response should include only a\ + \ list of tuples, in the order that the words appear in the input sentence, including\ + \ punctuations, with each tuple containing the corresponding POS tag label for a\ + \ word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_tsn_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_twi.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_twi.yaml new file mode 100644 index 00000000..22a6f414 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_twi.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: "You are an expert in tagging words and sentences in Twi with the right\ + \ POS tag. \n\nPlease provide the POS tags for each word in the Twi sentence. The\ + \ input is a list of words in the sentence. POS tag label set: ['ADJ', 'ADP', 'ADV',\ + \ 'AUX', 'CCONJ, 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT',\ + \ 'SCONJ', 'SYM', 'VERB', 'X']. The output format should be a list of tuples, where\ + \ each tuple consists of a word from the input text and its corresponding POS tag\ + \ label from the POS tag label set provided\nYour response should include only a\ + \ list of tuples, in the order that the words appear in the input sentence, including\ + \ punctuations, with each tuple containing the corresponding POS tag label for a\ + \ word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_twi_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_wol.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_wol.yaml new file mode 100644 index 00000000..e64fcc3d --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_wol.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: "You are an expert in tagging words and sentences in Wolof with the right\ + \ POS tag. \n\nPlease provide the POS tags for each word in the Wolof sentence.\ + \ The input is a list of words in the sentence. POS tag label set: ['ADJ', 'ADP',\ + \ 'ADV', 'AUX', 'CCONJ, 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT',\ + \ 'SCONJ', 'SYM', 'VERB', 'X']. The output format should be a list of tuples, where\ + \ each tuple consists of a word from the input text and its corresponding POS tag\ + \ label from the POS tag label set provided\nYour response should include only a\ + \ list of tuples, in the order that the words appear in the input sentence, including\ + \ punctuations, with each tuple containing the corresponding POS tag label for a\ + \ word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_wol_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_xho.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_xho.yaml new file mode 100644 index 00000000..b0d8d8de --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_xho.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: "You are an expert in tagging words and sentences in isiXhosa with the\ + \ right POS tag. \n\nPlease provide the POS tags for each word in the isiXhosa sentence.\ + \ The input is a list of words in the sentence. POS tag label set: ['ADJ', 'ADP',\ + \ 'ADV', 'AUX', 'CCONJ, 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT',\ + \ 'SCONJ', 'SYM', 'VERB', 'X']. The output format should be a list of tuples, where\ + \ each tuple consists of a word from the input text and its corresponding POS tag\ + \ label from the POS tag label set provided\nYour response should include only a\ + \ list of tuples, in the order that the words appear in the input sentence, including\ + \ punctuations, with each tuple containing the corresponding POS tag label for a\ + \ word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_xho_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_yaml new file mode 100644 index 00000000..044fffdb --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_yaml @@ -0,0 +1,32 @@ +tag: +- masakhapos_tasks +- masakhapos_prompt_2 +dataset_path: masakhane/masakhapos +dataset_name: null +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +generation_kwargs: + do_sample: false + until: + - + - <|im_end|> +validation_split: validation +test_split: test +fewshot_split: train +doc_to_target: !function utils.doc_to_target +should_decontaminate: true +doc_to_decontamination_query: "Sentence: {{token}}\nOutput:" +filter_list: + - filter: + - function: regex_pos + name: flexible-extract +metric_list: + - metric: acc + aggregation: !function utils.acc_score + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_yor.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_yor.yaml new file mode 100644 index 00000000..1a9d1b78 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_yor.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: "You are an expert in tagging words and sentences in Yoruba with the\ + \ right POS tag. \n\nPlease provide the POS tags for each word in the Yoruba sentence.\ + \ The input is a list of words in the sentence. POS tag label set: ['ADJ', 'ADP',\ + \ 'ADV', 'AUX', 'CCONJ, 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT',\ + \ 'SCONJ', 'SYM', 'VERB', 'X']. The output format should be a list of tuples, where\ + \ each tuple consists of a word from the input text and its corresponding POS tag\ + \ label from the POS tag label set provided\nYour response should include only a\ + \ list of tuples, in the order that the words appear in the input sentence, including\ + \ punctuations, with each tuple containing the corresponding POS tag label for a\ + \ word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_yor_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_zul.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_zul.yaml new file mode 100644 index 00000000..1aa1ca4c --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_2/masakhapos_zul.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: "You are an expert in tagging words and sentences in isiZulu with the\ + \ right POS tag. \n\nPlease provide the POS tags for each word in the isiZulu sentence.\ + \ The input is a list of words in the sentence. POS tag label set: ['ADJ', 'ADP',\ + \ 'ADV', 'AUX', 'CCONJ, 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT',\ + \ 'SCONJ', 'SYM', 'VERB', 'X']. The output format should be a list of tuples, where\ + \ each tuple consists of a word from the input text and its corresponding POS tag\ + \ label from the POS tag label set provided\nYour response should include only a\ + \ list of tuples, in the order that the words appear in the input sentence, including\ + \ punctuations, with each tuple containing the corresponding POS tag label for a\ + \ word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_zul_prompt_2 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_2/utils.py b/lm_eval/tasks/afrobench/masakhapos/prompt_2/utils.py new file mode 100644 index 00000000..4ccc66d9 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_2/utils.py @@ -0,0 +1,55 @@ +from itertools import chain + +from sklearn.metrics import accuracy_score + +from lm_eval.utils import weighted_f1_score + + +def doc_to_target(doc): + pos_tag_map = { + 0: "NOUN", + 1: "PUNCT", + 2: "ADP", + 3: "NUM", + 4: "SYM", + 5: "SCONJ", + 6: "ADJ", + 7: "PART", + 8: "DET", + 9: "CCONJ", + 10: "PROPN", + 11: "PRON", + 12: "X", + 13: "_", + 14: "ADV", + 15: "INTJ", + 16: "VERB", + 17: "AUX", + } + return [pos_tag_map[tag] for tag in doc["upos"]] + + +def acc_score(items): + unzipped_list = list(zip(*items)) + + golds, preds = unzipped_list[0], unzipped_list[1] + + # Flatten preds' inner lists + flattened_preds = [list(chain.from_iterable(p)) for p in preds] + + # Calculate the accuracy for each gold-pred pair + accuracy_scores = [] + for gold, pred in zip(golds, flattened_preds): + # Ensure both lists are of the same length, otherwise truncate to match + min_length = min(len(gold), len(pred)) + gold = gold[:min_length] + pred = pred[:min_length] + + # Calculate accuracy for the current pair and add to the list + accuracy = accuracy_score(gold, pred) + accuracy_scores.append(accuracy) + + mean_accuracy = ( + sum(accuracy_scores) / len(accuracy_scores) if accuracy_scores else 0 + ) + return mean_accuracy diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_bam.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_bam.yaml new file mode 100644 index 00000000..64bf664f --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_bam.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: bam +doc_to_text: "Acting as a Bambara linguist and without making any corrections or changes\ + \ to the text, perform a part of speech (POS) analysis of the sentences using the\ + \ following POS tag label annotation ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. The input will be a list of words in the sentence. The output format should\ + \ be a list of tuples, where each tuple consists of a word from the input text and\ + \ its corresponding POS tag label from the POS tag label set provided\nYour response\ + \ should include only a list of tuples, in the order that the words appear in the\ + \ input sentence, including punctuations, with each tuple containing the corresponding\ + \ POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_bam_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_bbj.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_bbj.yaml new file mode 100644 index 00000000..50d00b6d --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_bbj.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: bbj +doc_to_text: "Acting as a Ghomala linguist and without making any corrections or changes\ + \ to the text, perform a part of speech (POS) analysis of the sentences using the\ + \ following POS tag label annotation ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. The input will be a list of words in the sentence. The output format should\ + \ be a list of tuples, where each tuple consists of a word from the input text and\ + \ its corresponding POS tag label from the POS tag label set provided\nYour response\ + \ should include only a list of tuples, in the order that the words appear in the\ + \ input sentence, including punctuations, with each tuple containing the corresponding\ + \ POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_bbj_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_ewe.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_ewe.yaml new file mode 100644 index 00000000..c83ad4ba --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_ewe.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: ewe +doc_to_text: "Acting as a Ewe linguist and without making any corrections or changes\ + \ to the text, perform a part of speech (POS) analysis of the sentences using the\ + \ following POS tag label annotation ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. The input will be a list of words in the sentence. The output format should\ + \ be a list of tuples, where each tuple consists of a word from the input text and\ + \ its corresponding POS tag label from the POS tag label set provided\nYour response\ + \ should include only a list of tuples, in the order that the words appear in the\ + \ input sentence, including punctuations, with each tuple containing the corresponding\ + \ POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_ewe_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_fon.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_fon.yaml new file mode 100644 index 00000000..b12efe16 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_fon.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: fon +doc_to_text: "Acting as a Fon linguist and without making any corrections or changes\ + \ to the text, perform a part of speech (POS) analysis of the sentences using the\ + \ following POS tag label annotation ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. The input will be a list of words in the sentence. The output format should\ + \ be a list of tuples, where each tuple consists of a word from the input text and\ + \ its corresponding POS tag label from the POS tag label set provided\nYour response\ + \ should include only a list of tuples, in the order that the words appear in the\ + \ input sentence, including punctuations, with each tuple containing the corresponding\ + \ POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_fon_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_hau.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_hau.yaml new file mode 100644 index 00000000..613384cf --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_hau.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: "Acting as a Hausa linguist and without making any corrections or changes\ + \ to the text, perform a part of speech (POS) analysis of the sentences using the\ + \ following POS tag label annotation ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. The input will be a list of words in the sentence. The output format should\ + \ be a list of tuples, where each tuple consists of a word from the input text and\ + \ its corresponding POS tag label from the POS tag label set provided\nYour response\ + \ should include only a list of tuples, in the order that the words appear in the\ + \ input sentence, including punctuations, with each tuple containing the corresponding\ + \ POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_hau_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_ibo.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_ibo.yaml new file mode 100644 index 00000000..d7af7e36 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_ibo.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "Acting as a Igbo linguist and without making any corrections or changes\ + \ to the text, perform a part of speech (POS) analysis of the sentences using the\ + \ following POS tag label annotation ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. The input will be a list of words in the sentence. The output format should\ + \ be a list of tuples, where each tuple consists of a word from the input text and\ + \ its corresponding POS tag label from the POS tag label set provided\nYour response\ + \ should include only a list of tuples, in the order that the words appear in the\ + \ input sentence, including punctuations, with each tuple containing the corresponding\ + \ POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_ibo_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_kin.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_kin.yaml new file mode 100644 index 00000000..1255d99f --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_kin.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: "Acting as a Kinyarwanda linguist and without making any corrections\ + \ or changes to the text, perform a part of speech (POS) analysis of the sentences\ + \ using the following POS tag label annotation ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ,\ + \ 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM',\ + \ 'VERB', 'X']. The input will be a list of words in the sentence. The output format\ + \ should be a list of tuples, where each tuple consists of a word from the input\ + \ text and its corresponding POS tag label from the POS tag label set provided\n\ + Your response should include only a list of tuples, in the order that the words\ + \ appear in the input sentence, including punctuations, with each tuple containing\ + \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_kin_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_lug.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_lug.yaml new file mode 100644 index 00000000..0eb3fad6 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_lug.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: "Acting as a Luganda linguist and without making any corrections or changes\ + \ to the text, perform a part of speech (POS) analysis of the sentences using the\ + \ following POS tag label annotation ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. The input will be a list of words in the sentence. The output format should\ + \ be a list of tuples, where each tuple consists of a word from the input text and\ + \ its corresponding POS tag label from the POS tag label set provided\nYour response\ + \ should include only a list of tuples, in the order that the words appear in the\ + \ input sentence, including punctuations, with each tuple containing the corresponding\ + \ POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_lug_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_luo.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_luo.yaml new file mode 100644 index 00000000..6d9ceb84 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_luo.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: luo +doc_to_text: "Acting as a Dholuo linguist and without making any corrections or changes\ + \ to the text, perform a part of speech (POS) analysis of the sentences using the\ + \ following POS tag label annotation ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. The input will be a list of words in the sentence. The output format should\ + \ be a list of tuples, where each tuple consists of a word from the input text and\ + \ its corresponding POS tag label from the POS tag label set provided\nYour response\ + \ should include only a list of tuples, in the order that the words appear in the\ + \ input sentence, including punctuations, with each tuple containing the corresponding\ + \ POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_luo_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_mos.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_mos.yaml new file mode 100644 index 00000000..705e4d51 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_mos.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: mos +doc_to_text: "Acting as a Mossi linguist and without making any corrections or changes\ + \ to the text, perform a part of speech (POS) analysis of the sentences using the\ + \ following POS tag label annotation ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. The input will be a list of words in the sentence. The output format should\ + \ be a list of tuples, where each tuple consists of a word from the input text and\ + \ its corresponding POS tag label from the POS tag label set provided\nYour response\ + \ should include only a list of tuples, in the order that the words appear in the\ + \ input sentence, including punctuations, with each tuple containing the corresponding\ + \ POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_mos_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_nya.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_nya.yaml new file mode 100644 index 00000000..fecb644d --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_nya.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: nya +doc_to_text: "Acting as a Chichewa linguist and without making any corrections or\ + \ changes to the text, perform a part of speech (POS) analysis of the sentences\ + \ using the following POS tag label annotation ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ,\ + \ 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM',\ + \ 'VERB', 'X']. The input will be a list of words in the sentence. The output format\ + \ should be a list of tuples, where each tuple consists of a word from the input\ + \ text and its corresponding POS tag label from the POS tag label set provided\n\ + Your response should include only a list of tuples, in the order that the words\ + \ appear in the input sentence, including punctuations, with each tuple containing\ + \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_nya_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_pcm.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_pcm.yaml new file mode 100644 index 00000000..9cfc76c5 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_pcm.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: pcm +doc_to_text: "Acting as a Nigerian Pidgin linguist and without making any corrections\ + \ or changes to the text, perform a part of speech (POS) analysis of the sentences\ + \ using the following POS tag label annotation ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ,\ + \ 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM',\ + \ 'VERB', 'X']. The input will be a list of words in the sentence. The output format\ + \ should be a list of tuples, where each tuple consists of a word from the input\ + \ text and its corresponding POS tag label from the POS tag label set provided\n\ + Your response should include only a list of tuples, in the order that the words\ + \ appear in the input sentence, including punctuations, with each tuple containing\ + \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_pcm_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_sna.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_sna.yaml new file mode 100644 index 00000000..947b68fe --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_sna.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: "Acting as a chiShona linguist and without making any corrections or\ + \ changes to the text, perform a part of speech (POS) analysis of the sentences\ + \ using the following POS tag label annotation ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ,\ + \ 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM',\ + \ 'VERB', 'X']. The input will be a list of words in the sentence. The output format\ + \ should be a list of tuples, where each tuple consists of a word from the input\ + \ text and its corresponding POS tag label from the POS tag label set provided\n\ + Your response should include only a list of tuples, in the order that the words\ + \ appear in the input sentence, including punctuations, with each tuple containing\ + \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_sna_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_swa.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_swa.yaml new file mode 100644 index 00000000..0cc2e6ef --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_swa.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: "Acting as a Kiswahili linguist and without making any corrections or\ + \ changes to the text, perform a part of speech (POS) analysis of the sentences\ + \ using the following POS tag label annotation ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ,\ + \ 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM',\ + \ 'VERB', 'X']. The input will be a list of words in the sentence. The output format\ + \ should be a list of tuples, where each tuple consists of a word from the input\ + \ text and its corresponding POS tag label from the POS tag label set provided\n\ + Your response should include only a list of tuples, in the order that the words\ + \ appear in the input sentence, including punctuations, with each tuple containing\ + \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_swa_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_tsn.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_tsn.yaml new file mode 100644 index 00000000..a37aa2e6 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_tsn.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: tsn +doc_to_text: "Acting as a Setswana linguist and without making any corrections or\ + \ changes to the text, perform a part of speech (POS) analysis of the sentences\ + \ using the following POS tag label annotation ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ,\ + \ 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM',\ + \ 'VERB', 'X']. The input will be a list of words in the sentence. The output format\ + \ should be a list of tuples, where each tuple consists of a word from the input\ + \ text and its corresponding POS tag label from the POS tag label set provided\n\ + Your response should include only a list of tuples, in the order that the words\ + \ appear in the input sentence, including punctuations, with each tuple containing\ + \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_tsn_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_twi.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_twi.yaml new file mode 100644 index 00000000..40bf3c17 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_twi.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: "Acting as a Twi linguist and without making any corrections or changes\ + \ to the text, perform a part of speech (POS) analysis of the sentences using the\ + \ following POS tag label annotation ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. The input will be a list of words in the sentence. The output format should\ + \ be a list of tuples, where each tuple consists of a word from the input text and\ + \ its corresponding POS tag label from the POS tag label set provided\nYour response\ + \ should include only a list of tuples, in the order that the words appear in the\ + \ input sentence, including punctuations, with each tuple containing the corresponding\ + \ POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_twi_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_wol.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_wol.yaml new file mode 100644 index 00000000..97e98aa7 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_wol.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: "Acting as a Wolof linguist and without making any corrections or changes\ + \ to the text, perform a part of speech (POS) analysis of the sentences using the\ + \ following POS tag label annotation ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. The input will be a list of words in the sentence. The output format should\ + \ be a list of tuples, where each tuple consists of a word from the input text and\ + \ its corresponding POS tag label from the POS tag label set provided\nYour response\ + \ should include only a list of tuples, in the order that the words appear in the\ + \ input sentence, including punctuations, with each tuple containing the corresponding\ + \ POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_wol_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_xho.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_xho.yaml new file mode 100644 index 00000000..72dafcfa --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_xho.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: "Acting as a isiXhosa linguist and without making any corrections or\ + \ changes to the text, perform a part of speech (POS) analysis of the sentences\ + \ using the following POS tag label annotation ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ,\ + \ 'DET', 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM',\ + \ 'VERB', 'X']. The input will be a list of words in the sentence. The output format\ + \ should be a list of tuples, where each tuple consists of a word from the input\ + \ text and its corresponding POS tag label from the POS tag label set provided\n\ + Your response should include only a list of tuples, in the order that the words\ + \ appear in the input sentence, including punctuations, with each tuple containing\ + \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_xho_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_yaml new file mode 100644 index 00000000..681b6216 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_yaml @@ -0,0 +1,32 @@ +tag: +- masakhapos_tasks +- masakhapos_prompt_3 +dataset_path: masakhane/masakhapos +dataset_name: null +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +generation_kwargs: + do_sample: false + until: + - + - <|im_end|> +validation_split: validation +test_split: test +fewshot_split: train +doc_to_target: !function utils.doc_to_target +should_decontaminate: true +doc_to_decontamination_query: "Sentence: {{token}}\nOutput:" +filter_list: + - filter: + - function: regex_pos + name: flexible-extract +metric_list: + - metric: acc + aggregation: !function utils.acc_score + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_yor.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_yor.yaml new file mode 100644 index 00000000..c11f48aa --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_yor.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: "Acting as a Yoruba linguist and without making any corrections or changes\ + \ to the text, perform a part of speech (POS) analysis of the sentences using the\ + \ following POS tag label annotation ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. The input will be a list of words in the sentence. The output format should\ + \ be a list of tuples, where each tuple consists of a word from the input text and\ + \ its corresponding POS tag label from the POS tag label set provided\nYour response\ + \ should include only a list of tuples, in the order that the words appear in the\ + \ input sentence, including punctuations, with each tuple containing the corresponding\ + \ POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_yor_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_zul.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_zul.yaml new file mode 100644 index 00000000..d89dcf41 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_3/masakhapos_zul.yaml @@ -0,0 +1,14 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: "Acting as a isiZulu linguist and without making any corrections or changes\ + \ to the text, perform a part of speech (POS) analysis of the sentences using the\ + \ following POS tag label annotation ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. The input will be a list of words in the sentence. The output format should\ + \ be a list of tuples, where each tuple consists of a word from the input text and\ + \ its corresponding POS tag label from the POS tag label set provided\nYour response\ + \ should include only a list of tuples, in the order that the words appear in the\ + \ input sentence, including punctuations, with each tuple containing the corresponding\ + \ POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_zul_prompt_3 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_3/utils.py b/lm_eval/tasks/afrobench/masakhapos/prompt_3/utils.py new file mode 100644 index 00000000..4ccc66d9 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_3/utils.py @@ -0,0 +1,55 @@ +from itertools import chain + +from sklearn.metrics import accuracy_score + +from lm_eval.utils import weighted_f1_score + + +def doc_to_target(doc): + pos_tag_map = { + 0: "NOUN", + 1: "PUNCT", + 2: "ADP", + 3: "NUM", + 4: "SYM", + 5: "SCONJ", + 6: "ADJ", + 7: "PART", + 8: "DET", + 9: "CCONJ", + 10: "PROPN", + 11: "PRON", + 12: "X", + 13: "_", + 14: "ADV", + 15: "INTJ", + 16: "VERB", + 17: "AUX", + } + return [pos_tag_map[tag] for tag in doc["upos"]] + + +def acc_score(items): + unzipped_list = list(zip(*items)) + + golds, preds = unzipped_list[0], unzipped_list[1] + + # Flatten preds' inner lists + flattened_preds = [list(chain.from_iterable(p)) for p in preds] + + # Calculate the accuracy for each gold-pred pair + accuracy_scores = [] + for gold, pred in zip(golds, flattened_preds): + # Ensure both lists are of the same length, otherwise truncate to match + min_length = min(len(gold), len(pred)) + gold = gold[:min_length] + pred = pred[:min_length] + + # Calculate accuracy for the current pair and add to the list + accuracy = accuracy_score(gold, pred) + accuracy_scores.append(accuracy) + + mean_accuracy = ( + sum(accuracy_scores) / len(accuracy_scores) if accuracy_scores else 0 + ) + return mean_accuracy diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_bam.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_bam.yaml new file mode 100644 index 00000000..318a1507 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_bam.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: bam +doc_to_text: "Annotate each word in the provided sentence with the appropriate POS\ + \ tag. The annotation list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. The input sentence will be a list of words in the sentence. The output format\ + \ should be a list of tuples, where each tuple consists of a word from the input\ + \ text and its corresponding POS tag label from the POS tag label set provided\n\ + Your response should include only a list of tuples, in the order that the words\ + \ appear in the input sentence, including punctuations, with each tuple containing\ + \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_bam_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_bbj.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_bbj.yaml new file mode 100644 index 00000000..24680e2d --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_bbj.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: bbj +doc_to_text: "Annotate each word in the provided sentence with the appropriate POS\ + \ tag. The annotation list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. The input sentence will be a list of words in the sentence. The output format\ + \ should be a list of tuples, where each tuple consists of a word from the input\ + \ text and its corresponding POS tag label from the POS tag label set provided\n\ + Your response should include only a list of tuples, in the order that the words\ + \ appear in the input sentence, including punctuations, with each tuple containing\ + \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_bbj_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_ewe.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_ewe.yaml new file mode 100644 index 00000000..74823221 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_ewe.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: ewe +doc_to_text: "Annotate each word in the provided sentence with the appropriate POS\ + \ tag. The annotation list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. The input sentence will be a list of words in the sentence. The output format\ + \ should be a list of tuples, where each tuple consists of a word from the input\ + \ text and its corresponding POS tag label from the POS tag label set provided\n\ + Your response should include only a list of tuples, in the order that the words\ + \ appear in the input sentence, including punctuations, with each tuple containing\ + \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_ewe_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_fon.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_fon.yaml new file mode 100644 index 00000000..2deca67e --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_fon.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: fon +doc_to_text: "Annotate each word in the provided sentence with the appropriate POS\ + \ tag. The annotation list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. The input sentence will be a list of words in the sentence. The output format\ + \ should be a list of tuples, where each tuple consists of a word from the input\ + \ text and its corresponding POS tag label from the POS tag label set provided\n\ + Your response should include only a list of tuples, in the order that the words\ + \ appear in the input sentence, including punctuations, with each tuple containing\ + \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_fon_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_hau.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_hau.yaml new file mode 100644 index 00000000..8a1f5b77 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_hau.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: "Annotate each word in the provided sentence with the appropriate POS\ + \ tag. The annotation list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. The input sentence will be a list of words in the sentence. The output format\ + \ should be a list of tuples, where each tuple consists of a word from the input\ + \ text and its corresponding POS tag label from the POS tag label set provided\n\ + Your response should include only a list of tuples, in the order that the words\ + \ appear in the input sentence, including punctuations, with each tuple containing\ + \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_hau_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_ibo.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_ibo.yaml new file mode 100644 index 00000000..789b0897 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_ibo.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "Annotate each word in the provided sentence with the appropriate POS\ + \ tag. The annotation list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. The input sentence will be a list of words in the sentence. The output format\ + \ should be a list of tuples, where each tuple consists of a word from the input\ + \ text and its corresponding POS tag label from the POS tag label set provided\n\ + Your response should include only a list of tuples, in the order that the words\ + \ appear in the input sentence, including punctuations, with each tuple containing\ + \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_ibo_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_kin.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_kin.yaml new file mode 100644 index 00000000..1486b4fa --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_kin.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: "Annotate each word in the provided sentence with the appropriate POS\ + \ tag. The annotation list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. The input sentence will be a list of words in the sentence. The output format\ + \ should be a list of tuples, where each tuple consists of a word from the input\ + \ text and its corresponding POS tag label from the POS tag label set provided\n\ + Your response should include only a list of tuples, in the order that the words\ + \ appear in the input sentence, including punctuations, with each tuple containing\ + \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_kin_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_lug.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_lug.yaml new file mode 100644 index 00000000..a80c5602 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_lug.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: "Annotate each word in the provided sentence with the appropriate POS\ + \ tag. The annotation list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. The input sentence will be a list of words in the sentence. The output format\ + \ should be a list of tuples, where each tuple consists of a word from the input\ + \ text and its corresponding POS tag label from the POS tag label set provided\n\ + Your response should include only a list of tuples, in the order that the words\ + \ appear in the input sentence, including punctuations, with each tuple containing\ + \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_lug_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_luo.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_luo.yaml new file mode 100644 index 00000000..3136f885 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_luo.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: luo +doc_to_text: "Annotate each word in the provided sentence with the appropriate POS\ + \ tag. The annotation list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. The input sentence will be a list of words in the sentence. The output format\ + \ should be a list of tuples, where each tuple consists of a word from the input\ + \ text and its corresponding POS tag label from the POS tag label set provided\n\ + Your response should include only a list of tuples, in the order that the words\ + \ appear in the input sentence, including punctuations, with each tuple containing\ + \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_luo_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_mos.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_mos.yaml new file mode 100644 index 00000000..24ae470c --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_mos.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: mos +doc_to_text: "Annotate each word in the provided sentence with the appropriate POS\ + \ tag. The annotation list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. The input sentence will be a list of words in the sentence. The output format\ + \ should be a list of tuples, where each tuple consists of a word from the input\ + \ text and its corresponding POS tag label from the POS tag label set provided\n\ + Your response should include only a list of tuples, in the order that the words\ + \ appear in the input sentence, including punctuations, with each tuple containing\ + \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_mos_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_nya.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_nya.yaml new file mode 100644 index 00000000..616c003d --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_nya.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: nya +doc_to_text: "Annotate each word in the provided sentence with the appropriate POS\ + \ tag. The annotation list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. The input sentence will be a list of words in the sentence. The output format\ + \ should be a list of tuples, where each tuple consists of a word from the input\ + \ text and its corresponding POS tag label from the POS tag label set provided\n\ + Your response should include only a list of tuples, in the order that the words\ + \ appear in the input sentence, including punctuations, with each tuple containing\ + \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_nya_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_pcm.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_pcm.yaml new file mode 100644 index 00000000..dcaae118 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_pcm.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: pcm +doc_to_text: "Annotate each word in the provided sentence with the appropriate POS\ + \ tag. The annotation list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. The input sentence will be a list of words in the sentence. The output format\ + \ should be a list of tuples, where each tuple consists of a word from the input\ + \ text and its corresponding POS tag label from the POS tag label set provided\n\ + Your response should include only a list of tuples, in the order that the words\ + \ appear in the input sentence, including punctuations, with each tuple containing\ + \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_pcm_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_sna.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_sna.yaml new file mode 100644 index 00000000..07237cee --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_sna.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: "Annotate each word in the provided sentence with the appropriate POS\ + \ tag. The annotation list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. The input sentence will be a list of words in the sentence. The output format\ + \ should be a list of tuples, where each tuple consists of a word from the input\ + \ text and its corresponding POS tag label from the POS tag label set provided\n\ + Your response should include only a list of tuples, in the order that the words\ + \ appear in the input sentence, including punctuations, with each tuple containing\ + \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_sna_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_swa.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_swa.yaml new file mode 100644 index 00000000..c937299b --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_swa.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: "Annotate each word in the provided sentence with the appropriate POS\ + \ tag. The annotation list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. The input sentence will be a list of words in the sentence. The output format\ + \ should be a list of tuples, where each tuple consists of a word from the input\ + \ text and its corresponding POS tag label from the POS tag label set provided\n\ + Your response should include only a list of tuples, in the order that the words\ + \ appear in the input sentence, including punctuations, with each tuple containing\ + \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_swa_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_tsn.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_tsn.yaml new file mode 100644 index 00000000..e1bc5ad5 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_tsn.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: tsn +doc_to_text: "Annotate each word in the provided sentence with the appropriate POS\ + \ tag. The annotation list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. The input sentence will be a list of words in the sentence. The output format\ + \ should be a list of tuples, where each tuple consists of a word from the input\ + \ text and its corresponding POS tag label from the POS tag label set provided\n\ + Your response should include only a list of tuples, in the order that the words\ + \ appear in the input sentence, including punctuations, with each tuple containing\ + \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_tsn_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_twi.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_twi.yaml new file mode 100644 index 00000000..bf3a523b --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_twi.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: "Annotate each word in the provided sentence with the appropriate POS\ + \ tag. The annotation list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. The input sentence will be a list of words in the sentence. The output format\ + \ should be a list of tuples, where each tuple consists of a word from the input\ + \ text and its corresponding POS tag label from the POS tag label set provided\n\ + Your response should include only a list of tuples, in the order that the words\ + \ appear in the input sentence, including punctuations, with each tuple containing\ + \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_twi_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_wol.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_wol.yaml new file mode 100644 index 00000000..d427cee3 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_wol.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: "Annotate each word in the provided sentence with the appropriate POS\ + \ tag. The annotation list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. The input sentence will be a list of words in the sentence. The output format\ + \ should be a list of tuples, where each tuple consists of a word from the input\ + \ text and its corresponding POS tag label from the POS tag label set provided\n\ + Your response should include only a list of tuples, in the order that the words\ + \ appear in the input sentence, including punctuations, with each tuple containing\ + \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_wol_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_xho.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_xho.yaml new file mode 100644 index 00000000..4b6525f9 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_xho.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: "Annotate each word in the provided sentence with the appropriate POS\ + \ tag. The annotation list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. The input sentence will be a list of words in the sentence. The output format\ + \ should be a list of tuples, where each tuple consists of a word from the input\ + \ text and its corresponding POS tag label from the POS tag label set provided\n\ + Your response should include only a list of tuples, in the order that the words\ + \ appear in the input sentence, including punctuations, with each tuple containing\ + \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_xho_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_yaml new file mode 100644 index 00000000..ba629386 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_yaml @@ -0,0 +1,32 @@ +tag: +- masakhapos_tasks +- masakhapos_prompt_4 +dataset_path: masakhane/masakhapos +dataset_name: null +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +generation_kwargs: + do_sample: false + until: + - + - <|im_end|> +validation_split: validation +test_split: test +fewshot_split: train +doc_to_target: !function utils.doc_to_target +should_decontaminate: true +doc_to_decontamination_query: "Sentence: {{token}}\nOutput:" +filter_list: + - filter: + - function: regex_pos + name: flexible-extract +metric_list: + - metric: acc + aggregation: !function utils.acc_score + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_yor.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_yor.yaml new file mode 100644 index 00000000..a7d70f67 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_yor.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: "Annotate each word in the provided sentence with the appropriate POS\ + \ tag. The annotation list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. The input sentence will be a list of words in the sentence. The output format\ + \ should be a list of tuples, where each tuple consists of a word from the input\ + \ text and its corresponding POS tag label from the POS tag label set provided\n\ + Your response should include only a list of tuples, in the order that the words\ + \ appear in the input sentence, including punctuations, with each tuple containing\ + \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_yor_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_zul.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_zul.yaml new file mode 100644 index 00000000..2a03cc5d --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_4/masakhapos_zul.yaml @@ -0,0 +1,13 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: "Annotate each word in the provided sentence with the appropriate POS\ + \ tag. The annotation list is given as: ['ADJ', 'ADP', 'ADV', 'AUX', 'CCONJ, 'DET',\ + \ 'INTJ', 'NOUN', 'NUM', 'PART', 'PRON', 'PROPN', 'PUNCT', 'SCONJ', 'SYM', 'VERB',\ + \ 'X']. The input sentence will be a list of words in the sentence. The output format\ + \ should be a list of tuples, where each tuple consists of a word from the input\ + \ text and its corresponding POS tag label from the POS tag label set provided\n\ + Your response should include only a list of tuples, in the order that the words\ + \ appear in the input sentence, including punctuations, with each tuple containing\ + \ the corresponding POS tag label for a word. \n\nSentence: {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_zul_prompt_4 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_4/utils.py b/lm_eval/tasks/afrobench/masakhapos/prompt_4/utils.py new file mode 100644 index 00000000..4ccc66d9 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_4/utils.py @@ -0,0 +1,55 @@ +from itertools import chain + +from sklearn.metrics import accuracy_score + +from lm_eval.utils import weighted_f1_score + + +def doc_to_target(doc): + pos_tag_map = { + 0: "NOUN", + 1: "PUNCT", + 2: "ADP", + 3: "NUM", + 4: "SYM", + 5: "SCONJ", + 6: "ADJ", + 7: "PART", + 8: "DET", + 9: "CCONJ", + 10: "PROPN", + 11: "PRON", + 12: "X", + 13: "_", + 14: "ADV", + 15: "INTJ", + 16: "VERB", + 17: "AUX", + } + return [pos_tag_map[tag] for tag in doc["upos"]] + + +def acc_score(items): + unzipped_list = list(zip(*items)) + + golds, preds = unzipped_list[0], unzipped_list[1] + + # Flatten preds' inner lists + flattened_preds = [list(chain.from_iterable(p)) for p in preds] + + # Calculate the accuracy for each gold-pred pair + accuracy_scores = [] + for gold, pred in zip(golds, flattened_preds): + # Ensure both lists are of the same length, otherwise truncate to match + min_length = min(len(gold), len(pred)) + gold = gold[:min_length] + pred = pred[:min_length] + + # Calculate accuracy for the current pair and add to the list + accuracy = accuracy_score(gold, pred) + accuracy_scores.append(accuracy) + + mean_accuracy = ( + sum(accuracy_scores) / len(accuracy_scores) if accuracy_scores else 0 + ) + return mean_accuracy diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_bam.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_bam.yaml new file mode 100644 index 00000000..4cd65c90 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_bam.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: bam +doc_to_text: "Given the following sentence, identify the part of speech (POS) for\ + \ each word. Use the following POS tag set: \nNOUN: Noun (person, place, thing),\ + \ \nVERB: Verb (action, state), \nADJ: Adjective (describes a noun), \nADV: Adverb\ + \ (modifies a verb, adjective, or adverb), \nPRON: Pronoun (replaces a noun), \n\ + DET: Determiner (introduces a noun), \nADP: Adposition (preposition or postposition),\ + \ \nCCONJ: Conjunction (connects words, phrases, clauses)\nPUNCT: Punctuation, \n\ + PROPN: Proper Noun, \nAUX: Auxiliary verb (helper verb), \nSCONJ: Subordinating\ + \ conjunction \nPART: Particle, \nSYM: Symbol, \nINTJ: Interjection, \nNUM: Numeral,\ + \ \nX: others. The output format should be a list of tuples, where each tuple consists\ + \ of a word from the input text and its corresponding POS tag label key only from\ + \ the POS tag set provided\nYour response should include only a list of tuples,\ + \ in the order that the words appear in the input sentence, including punctuations,\ + \ with each tuple containing the corresponding POS tag label for a word. \n\nSentence:\ + \ {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_bam_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_bbj.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_bbj.yaml new file mode 100644 index 00000000..969406dc --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_bbj.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: bbj +doc_to_text: "Given the following sentence, identify the part of speech (POS) for\ + \ each word. Use the following POS tag set: \nNOUN: Noun (person, place, thing),\ + \ \nVERB: Verb (action, state), \nADJ: Adjective (describes a noun), \nADV: Adverb\ + \ (modifies a verb, adjective, or adverb), \nPRON: Pronoun (replaces a noun), \n\ + DET: Determiner (introduces a noun), \nADP: Adposition (preposition or postposition),\ + \ \nCCONJ: Conjunction (connects words, phrases, clauses)\nPUNCT: Punctuation, \n\ + PROPN: Proper Noun, \nAUX: Auxiliary verb (helper verb), \nSCONJ: Subordinating\ + \ conjunction \nPART: Particle, \nSYM: Symbol, \nINTJ: Interjection, \nNUM: Numeral,\ + \ \nX: others. The output format should be a list of tuples, where each tuple consists\ + \ of a word from the input text and its corresponding POS tag label key only from\ + \ the POS tag set provided\nYour response should include only a list of tuples,\ + \ in the order that the words appear in the input sentence, including punctuations,\ + \ with each tuple containing the corresponding POS tag label for a word. \n\nSentence:\ + \ {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_bbj_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_ewe.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_ewe.yaml new file mode 100644 index 00000000..aacc83ee --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_ewe.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: ewe +doc_to_text: "Given the following sentence, identify the part of speech (POS) for\ + \ each word. Use the following POS tag set: \nNOUN: Noun (person, place, thing),\ + \ \nVERB: Verb (action, state), \nADJ: Adjective (describes a noun), \nADV: Adverb\ + \ (modifies a verb, adjective, or adverb), \nPRON: Pronoun (replaces a noun), \n\ + DET: Determiner (introduces a noun), \nADP: Adposition (preposition or postposition),\ + \ \nCCONJ: Conjunction (connects words, phrases, clauses)\nPUNCT: Punctuation, \n\ + PROPN: Proper Noun, \nAUX: Auxiliary verb (helper verb), \nSCONJ: Subordinating\ + \ conjunction \nPART: Particle, \nSYM: Symbol, \nINTJ: Interjection, \nNUM: Numeral,\ + \ \nX: others. The output format should be a list of tuples, where each tuple consists\ + \ of a word from the input text and its corresponding POS tag label key only from\ + \ the POS tag set provided\nYour response should include only a list of tuples,\ + \ in the order that the words appear in the input sentence, including punctuations,\ + \ with each tuple containing the corresponding POS tag label for a word. \n\nSentence:\ + \ {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_ewe_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_fon.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_fon.yaml new file mode 100644 index 00000000..642d1d0a --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_fon.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: fon +doc_to_text: "Given the following sentence, identify the part of speech (POS) for\ + \ each word. Use the following POS tag set: \nNOUN: Noun (person, place, thing),\ + \ \nVERB: Verb (action, state), \nADJ: Adjective (describes a noun), \nADV: Adverb\ + \ (modifies a verb, adjective, or adverb), \nPRON: Pronoun (replaces a noun), \n\ + DET: Determiner (introduces a noun), \nADP: Adposition (preposition or postposition),\ + \ \nCCONJ: Conjunction (connects words, phrases, clauses)\nPUNCT: Punctuation, \n\ + PROPN: Proper Noun, \nAUX: Auxiliary verb (helper verb), \nSCONJ: Subordinating\ + \ conjunction \nPART: Particle, \nSYM: Symbol, \nINTJ: Interjection, \nNUM: Numeral,\ + \ \nX: others. The output format should be a list of tuples, where each tuple consists\ + \ of a word from the input text and its corresponding POS tag label key only from\ + \ the POS tag set provided\nYour response should include only a list of tuples,\ + \ in the order that the words appear in the input sentence, including punctuations,\ + \ with each tuple containing the corresponding POS tag label for a word. \n\nSentence:\ + \ {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_fon_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_hau.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_hau.yaml new file mode 100644 index 00000000..b2c07ce7 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_hau.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: "Given the following sentence, identify the part of speech (POS) for\ + \ each word. Use the following POS tag set: \nNOUN: Noun (person, place, thing),\ + \ \nVERB: Verb (action, state), \nADJ: Adjective (describes a noun), \nADV: Adverb\ + \ (modifies a verb, adjective, or adverb), \nPRON: Pronoun (replaces a noun), \n\ + DET: Determiner (introduces a noun), \nADP: Adposition (preposition or postposition),\ + \ \nCCONJ: Conjunction (connects words, phrases, clauses)\nPUNCT: Punctuation, \n\ + PROPN: Proper Noun, \nAUX: Auxiliary verb (helper verb), \nSCONJ: Subordinating\ + \ conjunction \nPART: Particle, \nSYM: Symbol, \nINTJ: Interjection, \nNUM: Numeral,\ + \ \nX: others. The output format should be a list of tuples, where each tuple consists\ + \ of a word from the input text and its corresponding POS tag label key only from\ + \ the POS tag set provided\nYour response should include only a list of tuples,\ + \ in the order that the words appear in the input sentence, including punctuations,\ + \ with each tuple containing the corresponding POS tag label for a word. \n\nSentence:\ + \ {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_hau_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_ibo.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_ibo.yaml new file mode 100644 index 00000000..bef4b994 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_ibo.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: "Given the following sentence, identify the part of speech (POS) for\ + \ each word. Use the following POS tag set: \nNOUN: Noun (person, place, thing),\ + \ \nVERB: Verb (action, state), \nADJ: Adjective (describes a noun), \nADV: Adverb\ + \ (modifies a verb, adjective, or adverb), \nPRON: Pronoun (replaces a noun), \n\ + DET: Determiner (introduces a noun), \nADP: Adposition (preposition or postposition),\ + \ \nCCONJ: Conjunction (connects words, phrases, clauses)\nPUNCT: Punctuation, \n\ + PROPN: Proper Noun, \nAUX: Auxiliary verb (helper verb), \nSCONJ: Subordinating\ + \ conjunction \nPART: Particle, \nSYM: Symbol, \nINTJ: Interjection, \nNUM: Numeral,\ + \ \nX: others. The output format should be a list of tuples, where each tuple consists\ + \ of a word from the input text and its corresponding POS tag label key only from\ + \ the POS tag set provided\nYour response should include only a list of tuples,\ + \ in the order that the words appear in the input sentence, including punctuations,\ + \ with each tuple containing the corresponding POS tag label for a word. \n\nSentence:\ + \ {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_ibo_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_kin.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_kin.yaml new file mode 100644 index 00000000..1983540b --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_kin.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: kin +doc_to_text: "Given the following sentence, identify the part of speech (POS) for\ + \ each word. Use the following POS tag set: \nNOUN: Noun (person, place, thing),\ + \ \nVERB: Verb (action, state), \nADJ: Adjective (describes a noun), \nADV: Adverb\ + \ (modifies a verb, adjective, or adverb), \nPRON: Pronoun (replaces a noun), \n\ + DET: Determiner (introduces a noun), \nADP: Adposition (preposition or postposition),\ + \ \nCCONJ: Conjunction (connects words, phrases, clauses)\nPUNCT: Punctuation, \n\ + PROPN: Proper Noun, \nAUX: Auxiliary verb (helper verb), \nSCONJ: Subordinating\ + \ conjunction \nPART: Particle, \nSYM: Symbol, \nINTJ: Interjection, \nNUM: Numeral,\ + \ \nX: others. The output format should be a list of tuples, where each tuple consists\ + \ of a word from the input text and its corresponding POS tag label key only from\ + \ the POS tag set provided\nYour response should include only a list of tuples,\ + \ in the order that the words appear in the input sentence, including punctuations,\ + \ with each tuple containing the corresponding POS tag label for a word. \n\nSentence:\ + \ {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_kin_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_lug.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_lug.yaml new file mode 100644 index 00000000..55b9210a --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_lug.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: lug +doc_to_text: "Given the following sentence, identify the part of speech (POS) for\ + \ each word. Use the following POS tag set: \nNOUN: Noun (person, place, thing),\ + \ \nVERB: Verb (action, state), \nADJ: Adjective (describes a noun), \nADV: Adverb\ + \ (modifies a verb, adjective, or adverb), \nPRON: Pronoun (replaces a noun), \n\ + DET: Determiner (introduces a noun), \nADP: Adposition (preposition or postposition),\ + \ \nCCONJ: Conjunction (connects words, phrases, clauses)\nPUNCT: Punctuation, \n\ + PROPN: Proper Noun, \nAUX: Auxiliary verb (helper verb), \nSCONJ: Subordinating\ + \ conjunction \nPART: Particle, \nSYM: Symbol, \nINTJ: Interjection, \nNUM: Numeral,\ + \ \nX: others. The output format should be a list of tuples, where each tuple consists\ + \ of a word from the input text and its corresponding POS tag label key only from\ + \ the POS tag set provided\nYour response should include only a list of tuples,\ + \ in the order that the words appear in the input sentence, including punctuations,\ + \ with each tuple containing the corresponding POS tag label for a word. \n\nSentence:\ + \ {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_lug_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_luo.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_luo.yaml new file mode 100644 index 00000000..5a17e407 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_luo.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: luo +doc_to_text: "Given the following sentence, identify the part of speech (POS) for\ + \ each word. Use the following POS tag set: \nNOUN: Noun (person, place, thing),\ + \ \nVERB: Verb (action, state), \nADJ: Adjective (describes a noun), \nADV: Adverb\ + \ (modifies a verb, adjective, or adverb), \nPRON: Pronoun (replaces a noun), \n\ + DET: Determiner (introduces a noun), \nADP: Adposition (preposition or postposition),\ + \ \nCCONJ: Conjunction (connects words, phrases, clauses)\nPUNCT: Punctuation, \n\ + PROPN: Proper Noun, \nAUX: Auxiliary verb (helper verb), \nSCONJ: Subordinating\ + \ conjunction \nPART: Particle, \nSYM: Symbol, \nINTJ: Interjection, \nNUM: Numeral,\ + \ \nX: others. The output format should be a list of tuples, where each tuple consists\ + \ of a word from the input text and its corresponding POS tag label key only from\ + \ the POS tag set provided\nYour response should include only a list of tuples,\ + \ in the order that the words appear in the input sentence, including punctuations,\ + \ with each tuple containing the corresponding POS tag label for a word. \n\nSentence:\ + \ {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_luo_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_mos.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_mos.yaml new file mode 100644 index 00000000..43479749 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_mos.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: mos +doc_to_text: "Given the following sentence, identify the part of speech (POS) for\ + \ each word. Use the following POS tag set: \nNOUN: Noun (person, place, thing),\ + \ \nVERB: Verb (action, state), \nADJ: Adjective (describes a noun), \nADV: Adverb\ + \ (modifies a verb, adjective, or adverb), \nPRON: Pronoun (replaces a noun), \n\ + DET: Determiner (introduces a noun), \nADP: Adposition (preposition or postposition),\ + \ \nCCONJ: Conjunction (connects words, phrases, clauses)\nPUNCT: Punctuation, \n\ + PROPN: Proper Noun, \nAUX: Auxiliary verb (helper verb), \nSCONJ: Subordinating\ + \ conjunction \nPART: Particle, \nSYM: Symbol, \nINTJ: Interjection, \nNUM: Numeral,\ + \ \nX: others. The output format should be a list of tuples, where each tuple consists\ + \ of a word from the input text and its corresponding POS tag label key only from\ + \ the POS tag set provided\nYour response should include only a list of tuples,\ + \ in the order that the words appear in the input sentence, including punctuations,\ + \ with each tuple containing the corresponding POS tag label for a word. \n\nSentence:\ + \ {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_mos_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_nya.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_nya.yaml new file mode 100644 index 00000000..7d2d0ec1 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_nya.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: nya +doc_to_text: "Given the following sentence, identify the part of speech (POS) for\ + \ each word. Use the following POS tag set: \nNOUN: Noun (person, place, thing),\ + \ \nVERB: Verb (action, state), \nADJ: Adjective (describes a noun), \nADV: Adverb\ + \ (modifies a verb, adjective, or adverb), \nPRON: Pronoun (replaces a noun), \n\ + DET: Determiner (introduces a noun), \nADP: Adposition (preposition or postposition),\ + \ \nCCONJ: Conjunction (connects words, phrases, clauses)\nPUNCT: Punctuation, \n\ + PROPN: Proper Noun, \nAUX: Auxiliary verb (helper verb), \nSCONJ: Subordinating\ + \ conjunction \nPART: Particle, \nSYM: Symbol, \nINTJ: Interjection, \nNUM: Numeral,\ + \ \nX: others. The output format should be a list of tuples, where each tuple consists\ + \ of a word from the input text and its corresponding POS tag label key only from\ + \ the POS tag set provided\nYour response should include only a list of tuples,\ + \ in the order that the words appear in the input sentence, including punctuations,\ + \ with each tuple containing the corresponding POS tag label for a word. \n\nSentence:\ + \ {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_nya_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_pcm.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_pcm.yaml new file mode 100644 index 00000000..cd5ea927 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_pcm.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: pcm +doc_to_text: "Given the following sentence, identify the part of speech (POS) for\ + \ each word. Use the following POS tag set: \nNOUN: Noun (person, place, thing),\ + \ \nVERB: Verb (action, state), \nADJ: Adjective (describes a noun), \nADV: Adverb\ + \ (modifies a verb, adjective, or adverb), \nPRON: Pronoun (replaces a noun), \n\ + DET: Determiner (introduces a noun), \nADP: Adposition (preposition or postposition),\ + \ \nCCONJ: Conjunction (connects words, phrases, clauses)\nPUNCT: Punctuation, \n\ + PROPN: Proper Noun, \nAUX: Auxiliary verb (helper verb), \nSCONJ: Subordinating\ + \ conjunction \nPART: Particle, \nSYM: Symbol, \nINTJ: Interjection, \nNUM: Numeral,\ + \ \nX: others. The output format should be a list of tuples, where each tuple consists\ + \ of a word from the input text and its corresponding POS tag label key only from\ + \ the POS tag set provided\nYour response should include only a list of tuples,\ + \ in the order that the words appear in the input sentence, including punctuations,\ + \ with each tuple containing the corresponding POS tag label for a word. \n\nSentence:\ + \ {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_pcm_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_sna.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_sna.yaml new file mode 100644 index 00000000..3cc21f0c --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_sna.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: sna +doc_to_text: "Given the following sentence, identify the part of speech (POS) for\ + \ each word. Use the following POS tag set: \nNOUN: Noun (person, place, thing),\ + \ \nVERB: Verb (action, state), \nADJ: Adjective (describes a noun), \nADV: Adverb\ + \ (modifies a verb, adjective, or adverb), \nPRON: Pronoun (replaces a noun), \n\ + DET: Determiner (introduces a noun), \nADP: Adposition (preposition or postposition),\ + \ \nCCONJ: Conjunction (connects words, phrases, clauses)\nPUNCT: Punctuation, \n\ + PROPN: Proper Noun, \nAUX: Auxiliary verb (helper verb), \nSCONJ: Subordinating\ + \ conjunction \nPART: Particle, \nSYM: Symbol, \nINTJ: Interjection, \nNUM: Numeral,\ + \ \nX: others. The output format should be a list of tuples, where each tuple consists\ + \ of a word from the input text and its corresponding POS tag label key only from\ + \ the POS tag set provided\nYour response should include only a list of tuples,\ + \ in the order that the words appear in the input sentence, including punctuations,\ + \ with each tuple containing the corresponding POS tag label for a word. \n\nSentence:\ + \ {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_sna_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_swa.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_swa.yaml new file mode 100644 index 00000000..b08dacde --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_swa.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: swa +doc_to_text: "Given the following sentence, identify the part of speech (POS) for\ + \ each word. Use the following POS tag set: \nNOUN: Noun (person, place, thing),\ + \ \nVERB: Verb (action, state), \nADJ: Adjective (describes a noun), \nADV: Adverb\ + \ (modifies a verb, adjective, or adverb), \nPRON: Pronoun (replaces a noun), \n\ + DET: Determiner (introduces a noun), \nADP: Adposition (preposition or postposition),\ + \ \nCCONJ: Conjunction (connects words, phrases, clauses)\nPUNCT: Punctuation, \n\ + PROPN: Proper Noun, \nAUX: Auxiliary verb (helper verb), \nSCONJ: Subordinating\ + \ conjunction \nPART: Particle, \nSYM: Symbol, \nINTJ: Interjection, \nNUM: Numeral,\ + \ \nX: others. The output format should be a list of tuples, where each tuple consists\ + \ of a word from the input text and its corresponding POS tag label key only from\ + \ the POS tag set provided\nYour response should include only a list of tuples,\ + \ in the order that the words appear in the input sentence, including punctuations,\ + \ with each tuple containing the corresponding POS tag label for a word. \n\nSentence:\ + \ {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_swa_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_tsn.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_tsn.yaml new file mode 100644 index 00000000..bbc20d6e --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_tsn.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: tsn +doc_to_text: "Given the following sentence, identify the part of speech (POS) for\ + \ each word. Use the following POS tag set: \nNOUN: Noun (person, place, thing),\ + \ \nVERB: Verb (action, state), \nADJ: Adjective (describes a noun), \nADV: Adverb\ + \ (modifies a verb, adjective, or adverb), \nPRON: Pronoun (replaces a noun), \n\ + DET: Determiner (introduces a noun), \nADP: Adposition (preposition or postposition),\ + \ \nCCONJ: Conjunction (connects words, phrases, clauses)\nPUNCT: Punctuation, \n\ + PROPN: Proper Noun, \nAUX: Auxiliary verb (helper verb), \nSCONJ: Subordinating\ + \ conjunction \nPART: Particle, \nSYM: Symbol, \nINTJ: Interjection, \nNUM: Numeral,\ + \ \nX: others. The output format should be a list of tuples, where each tuple consists\ + \ of a word from the input text and its corresponding POS tag label key only from\ + \ the POS tag set provided\nYour response should include only a list of tuples,\ + \ in the order that the words appear in the input sentence, including punctuations,\ + \ with each tuple containing the corresponding POS tag label for a word. \n\nSentence:\ + \ {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_tsn_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_twi.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_twi.yaml new file mode 100644 index 00000000..11af3b87 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_twi.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: twi +doc_to_text: "Given the following sentence, identify the part of speech (POS) for\ + \ each word. Use the following POS tag set: \nNOUN: Noun (person, place, thing),\ + \ \nVERB: Verb (action, state), \nADJ: Adjective (describes a noun), \nADV: Adverb\ + \ (modifies a verb, adjective, or adverb), \nPRON: Pronoun (replaces a noun), \n\ + DET: Determiner (introduces a noun), \nADP: Adposition (preposition or postposition),\ + \ \nCCONJ: Conjunction (connects words, phrases, clauses)\nPUNCT: Punctuation, \n\ + PROPN: Proper Noun, \nAUX: Auxiliary verb (helper verb), \nSCONJ: Subordinating\ + \ conjunction \nPART: Particle, \nSYM: Symbol, \nINTJ: Interjection, \nNUM: Numeral,\ + \ \nX: others. The output format should be a list of tuples, where each tuple consists\ + \ of a word from the input text and its corresponding POS tag label key only from\ + \ the POS tag set provided\nYour response should include only a list of tuples,\ + \ in the order that the words appear in the input sentence, including punctuations,\ + \ with each tuple containing the corresponding POS tag label for a word. \n\nSentence:\ + \ {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_twi_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_wol.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_wol.yaml new file mode 100644 index 00000000..ca294724 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_wol.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: wol +doc_to_text: "Given the following sentence, identify the part of speech (POS) for\ + \ each word. Use the following POS tag set: \nNOUN: Noun (person, place, thing),\ + \ \nVERB: Verb (action, state), \nADJ: Adjective (describes a noun), \nADV: Adverb\ + \ (modifies a verb, adjective, or adverb), \nPRON: Pronoun (replaces a noun), \n\ + DET: Determiner (introduces a noun), \nADP: Adposition (preposition or postposition),\ + \ \nCCONJ: Conjunction (connects words, phrases, clauses)\nPUNCT: Punctuation, \n\ + PROPN: Proper Noun, \nAUX: Auxiliary verb (helper verb), \nSCONJ: Subordinating\ + \ conjunction \nPART: Particle, \nSYM: Symbol, \nINTJ: Interjection, \nNUM: Numeral,\ + \ \nX: others. The output format should be a list of tuples, where each tuple consists\ + \ of a word from the input text and its corresponding POS tag label key only from\ + \ the POS tag set provided\nYour response should include only a list of tuples,\ + \ in the order that the words appear in the input sentence, including punctuations,\ + \ with each tuple containing the corresponding POS tag label for a word. \n\nSentence:\ + \ {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_wol_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_xho.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_xho.yaml new file mode 100644 index 00000000..345354c3 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_xho.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: xho +doc_to_text: "Given the following sentence, identify the part of speech (POS) for\ + \ each word. Use the following POS tag set: \nNOUN: Noun (person, place, thing),\ + \ \nVERB: Verb (action, state), \nADJ: Adjective (describes a noun), \nADV: Adverb\ + \ (modifies a verb, adjective, or adverb), \nPRON: Pronoun (replaces a noun), \n\ + DET: Determiner (introduces a noun), \nADP: Adposition (preposition or postposition),\ + \ \nCCONJ: Conjunction (connects words, phrases, clauses)\nPUNCT: Punctuation, \n\ + PROPN: Proper Noun, \nAUX: Auxiliary verb (helper verb), \nSCONJ: Subordinating\ + \ conjunction \nPART: Particle, \nSYM: Symbol, \nINTJ: Interjection, \nNUM: Numeral,\ + \ \nX: others. The output format should be a list of tuples, where each tuple consists\ + \ of a word from the input text and its corresponding POS tag label key only from\ + \ the POS tag set provided\nYour response should include only a list of tuples,\ + \ in the order that the words appear in the input sentence, including punctuations,\ + \ with each tuple containing the corresponding POS tag label for a word. \n\nSentence:\ + \ {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_xho_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_yaml new file mode 100644 index 00000000..df148e8a --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_yaml @@ -0,0 +1,32 @@ +tag: +- masakhapos_tasks +- masakhapos_prompt_5 +dataset_path: masakhane/masakhapos +dataset_name: null +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +generation_kwargs: + do_sample: false + until: + - + - <|im_end|> +validation_split: validation +test_split: test +fewshot_split: train +doc_to_target: !function utils.doc_to_target +should_decontaminate: true +doc_to_decontamination_query: "Sentence: {{token}}\nOutput:" +filter_list: + - filter: + - function: regex_pos + name: flexible-extract +metric_list: + - metric: acc + aggregation: !function utils.acc_score + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_yor.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_yor.yaml new file mode 100644 index 00000000..84bb266a --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_yor.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: "Given the following sentence, identify the part of speech (POS) for\ + \ each word. Use the following POS tag set: \nNOUN: Noun (person, place, thing),\ + \ \nVERB: Verb (action, state), \nADJ: Adjective (describes a noun), \nADV: Adverb\ + \ (modifies a verb, adjective, or adverb), \nPRON: Pronoun (replaces a noun), \n\ + DET: Determiner (introduces a noun), \nADP: Adposition (preposition or postposition),\ + \ \nCCONJ: Conjunction (connects words, phrases, clauses)\nPUNCT: Punctuation, \n\ + PROPN: Proper Noun, \nAUX: Auxiliary verb (helper verb), \nSCONJ: Subordinating\ + \ conjunction \nPART: Particle, \nSYM: Symbol, \nINTJ: Interjection, \nNUM: Numeral,\ + \ \nX: others. The output format should be a list of tuples, where each tuple consists\ + \ of a word from the input text and its corresponding POS tag label key only from\ + \ the POS tag set provided\nYour response should include only a list of tuples,\ + \ in the order that the words appear in the input sentence, including punctuations,\ + \ with each tuple containing the corresponding POS tag label for a word. \n\nSentence:\ + \ {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_yor_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_zul.yaml b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_zul.yaml new file mode 100644 index 00000000..4e400bfe --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_5/masakhapos_zul.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: zul +doc_to_text: "Given the following sentence, identify the part of speech (POS) for\ + \ each word. Use the following POS tag set: \nNOUN: Noun (person, place, thing),\ + \ \nVERB: Verb (action, state), \nADJ: Adjective (describes a noun), \nADV: Adverb\ + \ (modifies a verb, adjective, or adverb), \nPRON: Pronoun (replaces a noun), \n\ + DET: Determiner (introduces a noun), \nADP: Adposition (preposition or postposition),\ + \ \nCCONJ: Conjunction (connects words, phrases, clauses)\nPUNCT: Punctuation, \n\ + PROPN: Proper Noun, \nAUX: Auxiliary verb (helper verb), \nSCONJ: Subordinating\ + \ conjunction \nPART: Particle, \nSYM: Symbol, \nINTJ: Interjection, \nNUM: Numeral,\ + \ \nX: others. The output format should be a list of tuples, where each tuple consists\ + \ of a word from the input text and its corresponding POS tag label key only from\ + \ the POS tag set provided\nYour response should include only a list of tuples,\ + \ in the order that the words appear in the input sentence, including punctuations,\ + \ with each tuple containing the corresponding POS tag label for a word. \n\nSentence:\ + \ {{tokens}} \nOutput: " +include: masakhapos_yaml +task: masakhapos_zul_prompt_5 diff --git a/lm_eval/tasks/afrobench/masakhapos/prompt_5/utils.py b/lm_eval/tasks/afrobench/masakhapos/prompt_5/utils.py new file mode 100644 index 00000000..4ccc66d9 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/prompt_5/utils.py @@ -0,0 +1,55 @@ +from itertools import chain + +from sklearn.metrics import accuracy_score + +from lm_eval.utils import weighted_f1_score + + +def doc_to_target(doc): + pos_tag_map = { + 0: "NOUN", + 1: "PUNCT", + 2: "ADP", + 3: "NUM", + 4: "SYM", + 5: "SCONJ", + 6: "ADJ", + 7: "PART", + 8: "DET", + 9: "CCONJ", + 10: "PROPN", + 11: "PRON", + 12: "X", + 13: "_", + 14: "ADV", + 15: "INTJ", + 16: "VERB", + 17: "AUX", + } + return [pos_tag_map[tag] for tag in doc["upos"]] + + +def acc_score(items): + unzipped_list = list(zip(*items)) + + golds, preds = unzipped_list[0], unzipped_list[1] + + # Flatten preds' inner lists + flattened_preds = [list(chain.from_iterable(p)) for p in preds] + + # Calculate the accuracy for each gold-pred pair + accuracy_scores = [] + for gold, pred in zip(golds, flattened_preds): + # Ensure both lists are of the same length, otherwise truncate to match + min_length = min(len(gold), len(pred)) + gold = gold[:min_length] + pred = pred[:min_length] + + # Calculate accuracy for the current pair and add to the list + accuracy = accuracy_score(gold, pred) + accuracy_scores.append(accuracy) + + mean_accuracy = ( + sum(accuracy_scores) / len(accuracy_scores) if accuracy_scores else 0 + ) + return mean_accuracy diff --git a/lm_eval/tasks/afrobench/masakhapos/utils.py b/lm_eval/tasks/afrobench/masakhapos/utils.py new file mode 100644 index 00000000..d7976f84 --- /dev/null +++ b/lm_eval/tasks/afrobench/masakhapos/utils.py @@ -0,0 +1,40 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_text(doc): + output = """Please provide the POS tags for each word in the input sentence. The input will be a list of words in + the sentence. The output format should be a list of tuples, where each tuple consists of a word from the input text + and its corresponding POS tag label from the tag label set: ["ADJ", "ADP", "ADV", "AUX", "CCONJ, "DET", "INTJ", + "NOUN", "NUM", "PART", "PRON", "PROPN", "PUNCT" "SCONJ", "SYM", "VERB", "X"]. \nYour response should include only a + list of tuples, in the order that the words appear in the input sentence, with each tuple containing the + corresponding POS tag label for a word. + + Input: {tokens} + Output: """ + + text = output.format(subject=doc["tokens"]) + return text + + +def doc_to_target(doc): + pos_tag_map = { + 0: "NOUN", + 1: "PUNCT", + 2: "ADP", + 3: "NUM", + 4: "SYM", + 5: "SCONJ", + 6: "ADJ", + 7: "PART", + 8: "DET", + 9: "CCONJ", + 10: "PROPN", + 11: "PRON", + 12: "X", + 13: "_", + 14: "ADV", + 15: "INTJ", + 16: "VERB", + 17: "AUX", + } + return [pos_tag_map[tag] for tag in doc["upos"]] diff --git a/lm_eval/tasks/afrobench/naijarc/README.md b/lm_eval/tasks/afrobench/naijarc/README.md new file mode 100644 index 00000000..f6f98178 --- /dev/null +++ b/lm_eval/tasks/afrobench/naijarc/README.md @@ -0,0 +1,25 @@ +# + +## Paper +Title: `NaijaRC: A Multi-choice Reading Comprehension Dataset for Nigerian Languages` + +Paper Link: https://arxiv.org/abs/2308.09768 + +## Abstract +>In this paper, we create NaijaRC: a new multi-choice Reading Comprehension dataset for three native Nigeria languages that is based on high-school reading comprehension examination. We provide baseline results by performing cross-lingual transfer using existing English RACE and Belebele training dataset based on a pre-trained encoder-only model. Additionally, we provide results by prompting large language models (LLMs) like GPT-4. + +HomePage: https://huggingface.co/datasets/aremuadeolajr/NaijaRC + +### Citation + +``` +@misc{aremu2024naijarcmultichoicereadingcomprehension, + title={NaijaRC: A Multi-choice Reading Comprehension Dataset for Nigerian Languages}, + author={Anuoluwapo Aremu and Jesujoba O. Alabi and Daud Abolade and Nkechinyere F. Aguobi and Shamsuddeen Hassan Muhammad and David Ifeoluwa Adelani}, + year={2024}, + eprint={2308.09768}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2308.09768}, +} +``` diff --git a/lm_eval/tasks/afrobench/naijarc/naijarc.yaml b/lm_eval/tasks/afrobench/naijarc/naijarc.yaml new file mode 100644 index 00000000..4230ed64 --- /dev/null +++ b/lm_eval/tasks/afrobench/naijarc/naijarc.yaml @@ -0,0 +1,13 @@ +group: naijarc +task: + - naijarc_prompt_1 + - naijarc_prompt_2 + - naijarc_prompt_3 + - naijarc_prompt_4 + - naijarc_prompt_5 +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 1 diff --git a/lm_eval/tasks/afrobench/naijarc/prompt_1/naijarc b/lm_eval/tasks/afrobench/naijarc/prompt_1/naijarc new file mode 100644 index 00000000..b077e3bb --- /dev/null +++ b/lm_eval/tasks/afrobench/naijarc/prompt_1/naijarc @@ -0,0 +1,24 @@ +tag: + - naijarc_tasks + - naijarc_prompt_1 + - RC_tasks +dataset_path: Davlan/NaijaRC +dataset_name: null +output_type: multiple_choice +test_split: test +fewshot_split: test +fewshot_config: + sampler: first_n +doc_to_target: "{{['A', 'B', 'C', 'D'].index(Answer.strip())}}" +should_decontaminate: true +doc_to_decontamination_query: "{{question}}" +doc_to_choice: ["A", "B", "C", "D"] +metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/naijarc/prompt_1/naijarc_hau.yaml b/lm_eval/tasks/afrobench/naijarc/prompt_1/naijarc_hau.yaml new file mode 100644 index 00000000..1144a9a2 --- /dev/null +++ b/lm_eval/tasks/afrobench/naijarc/prompt_1/naijarc_hau.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: 'P: {{story}} + + Q: {{question.strip()}} + + A: {{options_A}} + + B: {{options_B}} + + C: {{options_C}} + + D: {{options_D}} + + Please choose the correct answer from the options above:' +include: naijarc +task: naijarc_hau_prompt_1 diff --git a/lm_eval/tasks/afrobench/naijarc/prompt_1/naijarc_ibo.yaml b/lm_eval/tasks/afrobench/naijarc/prompt_1/naijarc_ibo.yaml new file mode 100644 index 00000000..1db68523 --- /dev/null +++ b/lm_eval/tasks/afrobench/naijarc/prompt_1/naijarc_ibo.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: 'P: {{story}} + + Q: {{question.strip()}} + + A: {{options_A}} + + B: {{options_B}} + + C: {{options_C}} + + D: {{options_D}} + + Please choose the correct answer from the options above:' +include: naijarc +task: naijarc_ibo_prompt_1 diff --git a/lm_eval/tasks/afrobench/naijarc/prompt_1/naijarc_yor.yaml b/lm_eval/tasks/afrobench/naijarc/prompt_1/naijarc_yor.yaml new file mode 100644 index 00000000..2bb83fea --- /dev/null +++ b/lm_eval/tasks/afrobench/naijarc/prompt_1/naijarc_yor.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: 'P: {{story}} + + Q: {{question.strip()}} + + A: {{options_A}} + + B: {{options_B}} + + C: {{options_C}} + + D: {{options_D}} + + Please choose the correct answer from the options above:' +include: naijarc +task: naijarc_yor_prompt_1 diff --git a/lm_eval/tasks/afrobench/naijarc/prompt_2/naijarc b/lm_eval/tasks/afrobench/naijarc/prompt_2/naijarc new file mode 100644 index 00000000..3a8ec09a --- /dev/null +++ b/lm_eval/tasks/afrobench/naijarc/prompt_2/naijarc @@ -0,0 +1,23 @@ +tag: + - naijarc_tasks + - naijarc_prompt_2 + - RC_tasks +dataset_path: Davlan/NaijaRC +dataset_name: null +output_type: multiple_choice +test_split: test +fewshot_config: + sampler: first_n +doc_to_target: "{{['A', 'B', 'C', 'D'].index(Answer.strip())}}" +should_decontaminate: true +doc_to_decontamination_query: "{{question}}" +doc_to_choice: ["A", "B", "C", "D"] +metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/naijarc/prompt_2/naijarc_hau.yaml b/lm_eval/tasks/afrobench/naijarc/prompt_2/naijarc_hau.yaml new file mode 100644 index 00000000..a1d94db4 --- /dev/null +++ b/lm_eval/tasks/afrobench/naijarc/prompt_2/naijarc_hau.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: 'Passage: {{story}} + + Question: {{question.strip()}} + + 1: {{options_A}} + + 2: {{options_B}} + + 3: {{options_C}} + + 4: {{options_D}} + + Please select the correct answer from the given choices:' +include: naijarc +task: naijarc_hau_prompt_2 diff --git a/lm_eval/tasks/afrobench/naijarc/prompt_2/naijarc_ibo.yaml b/lm_eval/tasks/afrobench/naijarc/prompt_2/naijarc_ibo.yaml new file mode 100644 index 00000000..8384fad1 --- /dev/null +++ b/lm_eval/tasks/afrobench/naijarc/prompt_2/naijarc_ibo.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: 'Passage: {{story}} + + Question: {{question.strip()}} + + 1: {{options_A}} + + 2: {{options_B}} + + 3: {{options_C}} + + 4: {{options_D}} + + Please select the correct answer from the given choices:' +include: naijarc +task: naijarc_ibo_prompt_2 diff --git a/lm_eval/tasks/afrobench/naijarc/prompt_2/naijarc_yor.yaml b/lm_eval/tasks/afrobench/naijarc/prompt_2/naijarc_yor.yaml new file mode 100644 index 00000000..88b1c198 --- /dev/null +++ b/lm_eval/tasks/afrobench/naijarc/prompt_2/naijarc_yor.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: 'Passage: {{story}} + + Question: {{question.strip()}} + + 1: {{options_A}} + + 2: {{options_B}} + + 3: {{options_C}} + + 4: {{options_D}} + + Please select the correct answer from the given choices:' +include: naijarc +task: naijarc_yor_prompt_2 diff --git a/lm_eval/tasks/afrobench/naijarc/prompt_3/naijarc b/lm_eval/tasks/afrobench/naijarc/prompt_3/naijarc new file mode 100644 index 00000000..06746a43 --- /dev/null +++ b/lm_eval/tasks/afrobench/naijarc/prompt_3/naijarc @@ -0,0 +1,23 @@ +tag: + - naijarc_tasks + - naijarc_prompt_3 + - RC_tasks +dataset_path: Davlan/NaijaRC +dataset_name: null +output_type: multiple_choice +test_split: test +fewshot_config: + sampler: first_n +doc_to_target: "{{['A', 'B', 'C', 'D'].index(Answer.strip())}}" +should_decontaminate: true +doc_to_decontamination_query: "{{question}}" +doc_to_choice: ["A", "B", "C", "D"] +metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/naijarc/prompt_3/naijarc_hau.yaml b/lm_eval/tasks/afrobench/naijarc/prompt_3/naijarc_hau.yaml new file mode 100644 index 00000000..fb4b4431 --- /dev/null +++ b/lm_eval/tasks/afrobench/naijarc/prompt_3/naijarc_hau.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: 'Context: {{story}} + + Query: {{question.strip()}} + + Option A: {{options_A}} + + Option B: {{options_B}} + + Option C: {{options_C}} + + Option D: {{options_D}} + + Please indicate the correct option from the list above:' +include: naijarc +task: naijarc_hau_prompt_3 diff --git a/lm_eval/tasks/afrobench/naijarc/prompt_3/naijarc_ibo.yaml b/lm_eval/tasks/afrobench/naijarc/prompt_3/naijarc_ibo.yaml new file mode 100644 index 00000000..dad37fe9 --- /dev/null +++ b/lm_eval/tasks/afrobench/naijarc/prompt_3/naijarc_ibo.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: 'Context: {{story}} + + Query: {{question.strip()}} + + Option A: {{options_A}} + + Option B: {{options_B}} + + Option C: {{options_C}} + + Option D: {{options_D}} + + Please indicate the correct option from the list above:' +include: naijarc +task: naijarc_ibo_prompt_3 diff --git a/lm_eval/tasks/afrobench/naijarc/prompt_3/naijarc_yor.yaml b/lm_eval/tasks/afrobench/naijarc/prompt_3/naijarc_yor.yaml new file mode 100644 index 00000000..5ab84a8b --- /dev/null +++ b/lm_eval/tasks/afrobench/naijarc/prompt_3/naijarc_yor.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: 'Context: {{story}} + + Query: {{question.strip()}} + + Option A: {{options_A}} + + Option B: {{options_B}} + + Option C: {{options_C}} + + Option D: {{options_D}} + + Please indicate the correct option from the list above:' +include: naijarc +task: naijarc_yor_prompt_3 diff --git a/lm_eval/tasks/afrobench/naijarc/prompt_4/naijarc b/lm_eval/tasks/afrobench/naijarc/prompt_4/naijarc new file mode 100644 index 00000000..27bbc8c9 --- /dev/null +++ b/lm_eval/tasks/afrobench/naijarc/prompt_4/naijarc @@ -0,0 +1,23 @@ +tag: + - naijarc_tasks + - naijarc_prompt_4 + - RC_tasks +dataset_path: Davlan/NaijaRC +dataset_name: null +output_type: multiple_choice +test_split: test +fewshot_config: + sampler: first_n +doc_to_target: "{{['A', 'B', 'C', 'D'].index(Answer.strip())}}" +should_decontaminate: true +doc_to_decontamination_query: "{{question}}" +doc_to_choice: ["A", "B", "C", "D"] +metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/naijarc/prompt_4/naijarc_hau.yaml b/lm_eval/tasks/afrobench/naijarc/prompt_4/naijarc_hau.yaml new file mode 100644 index 00000000..4f846a8c --- /dev/null +++ b/lm_eval/tasks/afrobench/naijarc/prompt_4/naijarc_hau.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: '{{story}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{options_A}} + + B) {{options_B}} + + C) {{options_C}} + + D) {{options_D}} + + Please provide the correct answer from the choices given:' +include: naijarc +task: naijarc_hau_prompt_4 diff --git a/lm_eval/tasks/afrobench/naijarc/prompt_4/naijarc_ibo.yaml b/lm_eval/tasks/afrobench/naijarc/prompt_4/naijarc_ibo.yaml new file mode 100644 index 00000000..926d7a8f --- /dev/null +++ b/lm_eval/tasks/afrobench/naijarc/prompt_4/naijarc_ibo.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: '{{story}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{options_A}} + + B) {{options_B}} + + C) {{options_C}} + + D) {{options_D}} + + Please provide the correct answer from the choices given:' +include: naijarc +task: naijarc_ibo_prompt_4 diff --git a/lm_eval/tasks/afrobench/naijarc/prompt_4/naijarc_yor.yaml b/lm_eval/tasks/afrobench/naijarc/prompt_4/naijarc_yor.yaml new file mode 100644 index 00000000..13ad793c --- /dev/null +++ b/lm_eval/tasks/afrobench/naijarc/prompt_4/naijarc_yor.yaml @@ -0,0 +1,21 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: '{{story}} + + Based on the above passage, answer the following question: + + {{question.strip()}} + + Choices: + + A) {{options_A}} + + B) {{options_B}} + + C) {{options_C}} + + D) {{options_D}} + + Please provide the correct answer from the choices given:' +include: naijarc +task: naijarc_yor_prompt_4 diff --git a/lm_eval/tasks/afrobench/naijarc/prompt_5/naijarc b/lm_eval/tasks/afrobench/naijarc/prompt_5/naijarc new file mode 100644 index 00000000..0aa06d34 --- /dev/null +++ b/lm_eval/tasks/afrobench/naijarc/prompt_5/naijarc @@ -0,0 +1,23 @@ +tag: + - naijarc_tasks + - naijarc_prompt_5 + - RC_tasks +dataset_path: Davlan/NaijaRC +dataset_name: null +output_type: multiple_choice +test_split: test +fewshot_config: + sampler: first_n +doc_to_target: "{{['A', 'B', 'C', 'D'].index(Answer.strip())}}" +should_decontaminate: true +doc_to_decontamination_query: "{{question}}" +doc_to_choice: ["A", "B", "C", "D"] +metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/naijarc/prompt_5/naijarc_hau.yaml b/lm_eval/tasks/afrobench/naijarc/prompt_5/naijarc_hau.yaml new file mode 100644 index 00000000..c6ba82f9 --- /dev/null +++ b/lm_eval/tasks/afrobench/naijarc/prompt_5/naijarc_hau.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: hau +doc_to_text: 'Read the passage: {{story}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{options_A}} + + B. {{options_B}} + + C. {{options_C}} + + D. {{options_D}} + + Please choose the correct option from the above list:' +include: naijarc +task: naijarc_hau_prompt_5 diff --git a/lm_eval/tasks/afrobench/naijarc/prompt_5/naijarc_ibo.yaml b/lm_eval/tasks/afrobench/naijarc/prompt_5/naijarc_ibo.yaml new file mode 100644 index 00000000..b527dc1f --- /dev/null +++ b/lm_eval/tasks/afrobench/naijarc/prompt_5/naijarc_ibo.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_text: 'Read the passage: {{story}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{options_A}} + + B. {{options_B}} + + C. {{options_C}} + + D. {{options_D}} + + Please choose the correct option from the above list:' +include: naijarc +task: naijarc_ibo_prompt_5 diff --git a/lm_eval/tasks/afrobench/naijarc/prompt_5/naijarc_yor.yaml b/lm_eval/tasks/afrobench/naijarc/prompt_5/naijarc_yor.yaml new file mode 100644 index 00000000..0959e327 --- /dev/null +++ b/lm_eval/tasks/afrobench/naijarc/prompt_5/naijarc_yor.yaml @@ -0,0 +1,19 @@ +# Generated by utils.py +dataset_name: yor +doc_to_text: 'Read the passage: {{story}} + + Then answer the question: {{question.strip()}} + + Options: + + A. {{options_A}} + + B. {{options_B}} + + C. {{options_C}} + + D. {{options_D}} + + Please choose the correct option from the above list:' +include: naijarc +task: naijarc_yor_prompt_5 diff --git a/lm_eval/tasks/afrobench/naijarc/utils.py b/lm_eval/tasks/afrobench/naijarc/utils.py new file mode 100644 index 00000000..ad636a8e --- /dev/null +++ b/lm_eval/tasks/afrobench/naijarc/utils.py @@ -0,0 +1,93 @@ +import argparse +import os + +import yaml + + +def prompt_func(mode, lang): + prompt_map = { + "prompt_1": "P: {{story}}\nQ: {{question.strip()}}\nA: {{options_A}}\nB: {{options_B}}\nC: {{options_C}}\nD: {{options_D}}\nPlease choose the correct answer from the options above:", + "prompt_2": "Passage: {{story}}\nQuestion: {{question.strip()}}\n1: {{options_A}}\n2: {{options_B}}\n3: {{options_C}}\n4: {{options_D}}\nPlease select the correct answer from the given choices:", + "prompt_3": "Context: {{story}}\nQuery: {{question.strip()}}\nOption A: {{options_A}}\nOption B: {{options_B}}\nOption C: {{options_C}}\nOption D: {{options_D}}\nPlease indicate the correct option from the list above:", + "prompt_4": "{{story}}\nBased on the above passage, answer the following question:\n{{question.strip()}}\nChoices:\nA) {{options_A}}\nB) {{options_B}}\nC) {{options_C}}\nD) {{options_D}}\nPlease provide the correct answer from the choices given:", + "prompt_5": "Read the passage: {{story}}\nThen answer the question: {{question.strip()}}\nOptions:\nA. {{options_A}}\nB. {{options_B}}\nC. {{options_C}}\nD. {{options_D}}\nPlease choose the correct option from the above list:", + } + return prompt_map[mode] + + +def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None: + """ + Generate a yaml file for each language. + + :param output_dir: The directory to output the files to. + :param overwrite: Whether to overwrite files if they already exist. + """ + err = [] + languages = { + "hau": "Hausa", + "ibo": "Igbo", + "yor": "Yoruba", + } + + for lang in languages.keys(): + try: + file_name = f"naijarc_{lang}.yaml" + task_name = f"naijarc_{lang}_{mode}" + yaml_template = "naijarc" + yaml_details = { + "include": yaml_template, + "task": task_name, + "dataset_name": lang, + "doc_to_text": prompt_func(mode, languages[lang]), + } + file_path = os.path.join(output_dir, mode) + os.makedirs(file_path, exist_ok=True) + + with open( + f"{output_dir}/{mode}/{file_name}", + "w" if overwrite else "x", + encoding="utf8", + ) as f: + f.write("# Generated by utils.py\n") + yaml.dump( + yaml_details, + f, + allow_unicode=True, + ) + except FileExistsError: + err.append(file_name) + + if len(err) > 0: + raise FileExistsError( + "Files were not created because they already exist (use --overwrite flag):" + f" {', '.join(err)}" + ) + + +def main() -> None: + """Parse CLI args and generate language-specific yaml files.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + default=True, + action="store_true", + help="Overwrite files if they already exist", + ) + parser.add_argument( + "--output-dir", + default="./", + help="Directory to write yaml files to", + ) + parser.add_argument( + "--mode", + default="prompt_1", + choices=["prompt_1", "prompt_2", "prompt_3", "prompt_4", "prompt_5"], + help="Prompt number", + ) + args = parser.parse_args() + + gen_lang_yamls(output_dir=args.output_dir, overwrite=args.overwrite, mode=args.mode) + + +if __name__ == "__main__": + main() diff --git a/lm_eval/tasks/afrobench/nollysenti/README.md b/lm_eval/tasks/afrobench/nollysenti/README.md new file mode 100644 index 00000000..fa241319 --- /dev/null +++ b/lm_eval/tasks/afrobench/nollysenti/README.md @@ -0,0 +1,35 @@ +# + +## Paper +Title: `NollySenti: Leveraging Transfer Learning and Machine Translation for Nigerian Movie Sentiment Classification` + +Paper Link: https://aclanthology.org/2023.acl-short.85/ + +## Abstract +>Africa has over 2000 indigenous languages but they are under-represented in NLP research due to lack of datasets. In recent years, there have been progress in developing labelled corpora for African languages. However, they are often available in a single domain and may not generalize to other domains. In this paper, we focus on the task of sentiment classification for cross-domain adaptation. We create a new dataset, Nollywood movie reviews for five languages widely spoken in Nigeria (English, Hausa, Igbo, Nigerian Pidgin, and Yoruba). We provide an extensive empirical evaluation using classical machine learning methods and pre-trained language models. By leveraging transfer learning, we compare the performance of cross-domain adaptation from Twitter domain, and cross-lingual adaptation from English language. Our evaluation shows that transfer from English in the same target domain leads to more than 5% improvement in accuracy compared to transfer from Twitter in the same language. To further mitigate the domain difference, we leverage machine translation from English to other Nigerian languages, which leads to a further improvement of 7% over cross-lingual evaluation. While machine translation to low-resource languages are often of low quality, our analysis shows that sentiment related words are often preserved. + +HomePage: https://github.com/IyanuSh/NollySenti + +### Citation + +``` +@inproceedings{shode-etal-2023-nollysenti, + title = "{N}olly{S}enti: Leveraging Transfer Learning and Machine Translation for {N}igerian Movie Sentiment Classification", + author = "Shode, Iyanuoluwa and + Adelani, David Ifeoluwa and + Peng, JIng and + Feldman, Anna", + editor = "Rogers, Anna and + Boyd-Graber, Jordan and + Okazaki, Naoaki", + booktitle = "Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers)", + month = jul, + year = "2023", + address = "Toronto, Canada", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2023.acl-short.85/", + doi = "10.18653/v1/2023.acl-short.85", + pages = "986--998", + abstract = "Africa has over 2000 indigenous languages but they are under-represented in NLP research due to lack of datasets. In recent years, there have been progress in developing labelled corpora for African languages. However, they are often available in a single domain and may not generalize to other domains. In this paper, we focus on the task of sentiment classification for cross-domain adaptation. We create a new dataset, Nollywood movie reviews for five languages widely spoken in Nigeria (English, Hausa, Igbo, Nigerian Pidgin, and Yoruba). We provide an extensive empirical evaluation using classical machine learning methods and pre-trained language models. By leveraging transfer learning, we compare the performance of cross-domain adaptation from Twitter domain, and cross-lingual adaptation from English language. Our evaluation shows that transfer from English in the same target domain leads to more than 5{\%} improvement in accuracy compared to transfer from Twitter in the same language. To further mitigate the domain difference, we leverage machine translation from English to other Nigerian languages, which leads to a further improvement of 7{\%} over cross-lingual evaluation. While machine translation to low-resource languages are often of low quality, our analysis shows that sentiment related words are often preserved." +} +``` diff --git a/lm_eval/tasks/afrobench/nollysenti/nollysenti.yaml b/lm_eval/tasks/afrobench/nollysenti/nollysenti.yaml new file mode 100644 index 00000000..7fb13262 --- /dev/null +++ b/lm_eval/tasks/afrobench/nollysenti/nollysenti.yaml @@ -0,0 +1,13 @@ +group: nollysenti +task: + - nollysenti_prompt_1 + - nollysenti_prompt_2 + - nollysenti_prompt_3 + - nollysenti_prompt_4 + - nollysenti_prompt_5 +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 1 diff --git a/lm_eval/tasks/afrobench/nollysenti/prompt_1/nollysenti b/lm_eval/tasks/afrobench/nollysenti/prompt_1/nollysenti new file mode 100644 index 00000000..0476cdc0 --- /dev/null +++ b/lm_eval/tasks/afrobench/nollysenti/prompt_1/nollysenti @@ -0,0 +1,38 @@ +tag: + - afrobench_sentiment_tasks + - nollysenti_prompt_1 +dataset_path: Davlan/nollysenti +dataset_kwargs: {trust_remote_code: True} +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_text: 'Does this movie description "{{review}}" have a Positive or Negative sentiment? Labels only\n' +doc_to_target: label +doc_to_choice: + - "positive" + - "negative" +should_decontaminate: true +doc_to_decontamination_query: review +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/nollysenti/prompt_1/nollysenti_eng.yaml b/lm_eval/tasks/afrobench/nollysenti/prompt_1/nollysenti_eng.yaml new file mode 100644 index 00000000..5cf3a85f --- /dev/null +++ b/lm_eval/tasks/afrobench/nollysenti/prompt_1/nollysenti_eng.yaml @@ -0,0 +1,3 @@ +dataset_name: en +include: nollysenti +task: nollysenti_eng_prompt_1 diff --git a/lm_eval/tasks/afrobench/nollysenti/prompt_1/nollysenti_hau.yaml b/lm_eval/tasks/afrobench/nollysenti/prompt_1/nollysenti_hau.yaml new file mode 100644 index 00000000..157e97db --- /dev/null +++ b/lm_eval/tasks/afrobench/nollysenti/prompt_1/nollysenti_hau.yaml @@ -0,0 +1,3 @@ +dataset_name: ha +include: nollysenti +task: nollysenti_hau_prompt_1 diff --git a/lm_eval/tasks/afrobench/nollysenti/prompt_1/nollysenti_ibo.yaml b/lm_eval/tasks/afrobench/nollysenti/prompt_1/nollysenti_ibo.yaml new file mode 100644 index 00000000..77c9bfd4 --- /dev/null +++ b/lm_eval/tasks/afrobench/nollysenti/prompt_1/nollysenti_ibo.yaml @@ -0,0 +1,3 @@ +dataset_name: ig +include: nollysenti +task: nollysenti_ibo_prompt_1 diff --git a/lm_eval/tasks/afrobench/nollysenti/prompt_1/nollysenti_pcm.yaml b/lm_eval/tasks/afrobench/nollysenti/prompt_1/nollysenti_pcm.yaml new file mode 100644 index 00000000..53630168 --- /dev/null +++ b/lm_eval/tasks/afrobench/nollysenti/prompt_1/nollysenti_pcm.yaml @@ -0,0 +1,3 @@ +dataset_name: pcm +include: nollysenti +task: nollysenti_pcm_prompt_1 diff --git a/lm_eval/tasks/afrobench/nollysenti/prompt_1/nollysenti_yor.yaml b/lm_eval/tasks/afrobench/nollysenti/prompt_1/nollysenti_yor.yaml new file mode 100644 index 00000000..6dc1cfab --- /dev/null +++ b/lm_eval/tasks/afrobench/nollysenti/prompt_1/nollysenti_yor.yaml @@ -0,0 +1,3 @@ +dataset_name: yo +include: nollysenti +task: nollysenti_yor_prompt_1 diff --git a/lm_eval/tasks/afrobench/nollysenti/prompt_1/utils.py b/lm_eval/tasks/afrobench/nollysenti/prompt_1/utils.py new file mode 100644 index 00000000..3e735e2d --- /dev/null +++ b/lm_eval/tasks/afrobench/nollysenti/prompt_1/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm_eval/tasks/afrobench/nollysenti/prompt_2/nollysenti b/lm_eval/tasks/afrobench/nollysenti/prompt_2/nollysenti new file mode 100644 index 00000000..76f664fe --- /dev/null +++ b/lm_eval/tasks/afrobench/nollysenti/prompt_2/nollysenti @@ -0,0 +1,37 @@ +tag: + - afrobench_sentiment_tasks + - nollysenti_prompt_2 +dataset_path: Davlan/nollysenti +dataset_kwargs: {trust_remote_code: True} +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: label +doc_to_choice: + - "positive" + - "negative" +should_decontaminate: true +doc_to_decontamination_query: review +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/nollysenti/prompt_2/nollysenti_eng.yaml b/lm_eval/tasks/afrobench/nollysenti/prompt_2/nollysenti_eng.yaml new file mode 100644 index 00000000..ac3bb04d --- /dev/null +++ b/lm_eval/tasks/afrobench/nollysenti/prompt_2/nollysenti_eng.yaml @@ -0,0 +1,4 @@ +dataset_name: en +include: nollysenti +doc_to_text: 'Does this English movie description; "{{review}}" have a Positive or Negative sentiment? Labels only\n' +task: nollysenti_eng_prompt_2 diff --git a/lm_eval/tasks/afrobench/nollysenti/prompt_2/nollysenti_hau.yaml b/lm_eval/tasks/afrobench/nollysenti/prompt_2/nollysenti_hau.yaml new file mode 100644 index 00000000..f87bce67 --- /dev/null +++ b/lm_eval/tasks/afrobench/nollysenti/prompt_2/nollysenti_hau.yaml @@ -0,0 +1,4 @@ +dataset_name: ha +include: nollysenti +doc_to_text: 'Does this Hausa movie description; "{{review}}" have a Positive or Negative sentiment? Labels only\n' +task: nollysenti_hau_prompt_2 diff --git a/lm_eval/tasks/afrobench/nollysenti/prompt_2/nollysenti_ibo.yaml b/lm_eval/tasks/afrobench/nollysenti/prompt_2/nollysenti_ibo.yaml new file mode 100644 index 00000000..2f7ae185 --- /dev/null +++ b/lm_eval/tasks/afrobench/nollysenti/prompt_2/nollysenti_ibo.yaml @@ -0,0 +1,4 @@ +dataset_name: ig +include: nollysenti +doc_to_text: 'Does this Igbo movie description; "{{review}}" have a Positive or Negative sentiment? Labels only\n' +task: nollysenti_ibo_prompt_2 diff --git a/lm_eval/tasks/afrobench/nollysenti/prompt_2/nollysenti_pcm.yaml b/lm_eval/tasks/afrobench/nollysenti/prompt_2/nollysenti_pcm.yaml new file mode 100644 index 00000000..b0305c76 --- /dev/null +++ b/lm_eval/tasks/afrobench/nollysenti/prompt_2/nollysenti_pcm.yaml @@ -0,0 +1,4 @@ +dataset_name: pcm +include: nollysenti +doc_to_text: 'Does this Naija Pidgin movie description; "{{review}}" have a Positive or Negative sentiment? Labels only\n' +task: nollysenti_pcm_prompt_2 diff --git a/lm_eval/tasks/afrobench/nollysenti/prompt_2/nollysenti_yor.yaml b/lm_eval/tasks/afrobench/nollysenti/prompt_2/nollysenti_yor.yaml new file mode 100644 index 00000000..03c89d8b --- /dev/null +++ b/lm_eval/tasks/afrobench/nollysenti/prompt_2/nollysenti_yor.yaml @@ -0,0 +1,4 @@ +dataset_name: yo +include: nollysenti +doc_to_text: 'Does this Yoruba movie description; "{{review}}" have a Positive or Negative sentiment? Labels only\n' +task: nollysenti_yor_prompt_2 diff --git a/lm_eval/tasks/afrobench/nollysenti/prompt_2/utils.py b/lm_eval/tasks/afrobench/nollysenti/prompt_2/utils.py new file mode 100644 index 00000000..3e735e2d --- /dev/null +++ b/lm_eval/tasks/afrobench/nollysenti/prompt_2/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti b/lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti new file mode 100644 index 00000000..472928ac --- /dev/null +++ b/lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti @@ -0,0 +1,37 @@ +tag: + - afrobench_sentiment_tasks + - nollysenti_prompt_3 +dataset_path: Davlan/nollysenti +dataset_kwargs: {trust_remote_code: True} +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: label +doc_to_choice: + - "positive" + - "negative" +should_decontaminate: true +doc_to_decontamination_query: review +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti_eng.yaml b/lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti_eng.yaml new file mode 100644 index 00000000..df21a145 --- /dev/null +++ b/lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti_eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: en +doc_to_text: "You are an assistant able to detect sentiment in movie reviews. \n\nGiven\ + \ the sentiment labels Positive or Negative; what is the sentiment of the\ + \ English statement below? Return only the labels\n\nReview: {{review}}\n" +include: nollysenti +task: nollysenti_eng_prompt_3 diff --git a/lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti_hau.yaml b/lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti_hau.yaml new file mode 100644 index 00000000..5d15488d --- /dev/null +++ b/lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti_hau.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ha +doc_to_text: "You are an assistant able to detect sentiment in movie reviews. \n\nGiven\ + \ the sentiment labels Positive or Negative; what is the sentiment of the\ + \ Hausa statement below? Return only the labels\n\nReview: {{review}}\n" +include: nollysenti +task: nollysenti_hau_prompt_3 diff --git a/lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti_ibo.yaml b/lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti_ibo.yaml new file mode 100644 index 00000000..2f6bb7b2 --- /dev/null +++ b/lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti_ibo.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ig +doc_to_text: "You are an assistant able to detect sentiment in movie reviews. \n\nGiven\ + \ the sentiment labels Positive or Negative; what is the sentiment of the\ + \ Igbo statement below? Return only the labels\n\nReview: {{review}}\n" +include: nollysenti +task: nollysenti_ibo_prompt_3 diff --git a/lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti_pcm.yaml b/lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti_pcm.yaml new file mode 100644 index 00000000..f98519f3 --- /dev/null +++ b/lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti_pcm.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: pcm +doc_to_text: "You are an assistant able to detect sentiment in movie reviews. \n\nGiven\ + \ the sentiment labels Positive or Negative; what is the sentiment of the\ + \ Naija Pidgin statement below? Return only the labels\n\nReview: {{review}}\n" +include: nollysenti +task: nollysenti_pcm_prompt_3 diff --git a/lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti_yor.yaml b/lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti_yor.yaml new file mode 100644 index 00000000..fd64d1ed --- /dev/null +++ b/lm_eval/tasks/afrobench/nollysenti/prompt_3/nollysenti_yor.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: yo +doc_to_text: "You are an assistant able to detect sentiment in movie reviews. \n\nGiven\ + \ the sentiment labels Positive or Negative; what is the sentiment of the\ + \ Yoruba statement below? Return only the labels\n\nReview: {{review}}\n" +include: nollysenti +task: nollysenti_yor_prompt_3 diff --git a/lm_eval/tasks/afrobench/nollysenti/prompt_3/utils.py b/lm_eval/tasks/afrobench/nollysenti/prompt_3/utils.py new file mode 100644 index 00000000..3e735e2d --- /dev/null +++ b/lm_eval/tasks/afrobench/nollysenti/prompt_3/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti b/lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti new file mode 100644 index 00000000..de1bb486 --- /dev/null +++ b/lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti @@ -0,0 +1,37 @@ +tag: + - afrobench_sentiment_tasks + - nollysenti_prompt_4 +dataset_path: Davlan/nollysenti +dataset_kwargs: {trust_remote_code: True} +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: label +doc_to_choice: + - "positive" + - "negative" +should_decontaminate: true +doc_to_decontamination_query: review +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti_eng.yaml b/lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti_eng.yaml new file mode 100644 index 00000000..d8e01ab6 --- /dev/null +++ b/lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti_eng.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: en +doc_to_text: "Label the following text as Positive, or Negative. Provide\ + \ only the label as your response. \n\ntext: {{review}} \nlabel: \n" +include: nollysenti +task: nollysenti_eng_prompt_4 diff --git a/lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti_hau.yaml b/lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti_hau.yaml new file mode 100644 index 00000000..abc95704 --- /dev/null +++ b/lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti_hau.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ha +doc_to_text: "Label the following text as Positive, or Negative. Provide\ + \ only the label as your response. \n\ntext: {{review}} \nlabel: \n" +include: nollysenti +task: nollysenti_hau_prompt_4 diff --git a/lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti_ibo.yaml b/lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti_ibo.yaml new file mode 100644 index 00000000..8962cf72 --- /dev/null +++ b/lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti_ibo.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ig +doc_to_text: "Label the following text as Positive, or Negative. Provide\ + \ only the label as your response. \n\ntext: {{review}} \nlabel: \n" +include: nollysenti +task: nollysenti_ibo_prompt_4 diff --git a/lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti_pcm.yaml b/lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti_pcm.yaml new file mode 100644 index 00000000..36d43b79 --- /dev/null +++ b/lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti_pcm.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: pcm +doc_to_text: "Label the following text as Positive, or Negative. Provide\ + \ only the label as your response. \n\ntext: {{review}} \nlabel: \n" +include: nollysenti +task: nollysenti_pcm_prompt_4 diff --git a/lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti_yor.yaml b/lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti_yor.yaml new file mode 100644 index 00000000..2c100c4d --- /dev/null +++ b/lm_eval/tasks/afrobench/nollysenti/prompt_4/nollysenti_yor.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: yo +doc_to_text: "Label the following text as Positive, or Negative. Provide\ + \ only the label as your response. \n\ntext: {{review}} \nlabel: \n" +include: nollysenti +task: nollysenti_yor_prompt_4 diff --git a/lm_eval/tasks/afrobench/nollysenti/prompt_4/utils.py b/lm_eval/tasks/afrobench/nollysenti/prompt_4/utils.py new file mode 100644 index 00000000..3e735e2d --- /dev/null +++ b/lm_eval/tasks/afrobench/nollysenti/prompt_4/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti b/lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti new file mode 100644 index 00000000..2e25f2f0 --- /dev/null +++ b/lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti @@ -0,0 +1,37 @@ +tag: + - afrobench_sentiment_tasks + - nollysenti_prompt_5 +dataset_path: Davlan/nollysenti +dataset_kwargs: {trust_remote_code: True} +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: label +doc_to_choice: + - "positive" + - "negative" +should_decontaminate: true +doc_to_decontamination_query: review +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti_eng.yaml b/lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti_eng.yaml new file mode 100644 index 00000000..d485ffe1 --- /dev/null +++ b/lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti_eng.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: en +doc_to_text: "You are tasked with performing sentiment classification on the following\ + \ English text. For each input, classify the sentiment as positive, negative.\ + \ Use the following guidelines: \n\n Positive: The text expresses happiness,\ + \ satisfaction, or optimism. \nNegative: The text conveys disappointment, dissatisfaction,\ + \ or pessimism. \n\nIf the text contains both positive and negative sentiments, choose\ + \ the dominant sentiment. For ambiguous or unclear sentiments, select the label\ + \ that best reflects the overall tone. Please provide a single classification for\ + \ each input.\n\ntext: {{review}} \nlabel: \n" +include: nollysenti +task: nollysenti_eng_prompt_5 diff --git a/lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti_hau.yaml b/lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti_hau.yaml new file mode 100644 index 00000000..7ed16af7 --- /dev/null +++ b/lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti_hau.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: ha +doc_to_text: "You are tasked with performing sentiment classification on the following\ + \ Hausa text. For each input, classify the sentiment as positive, negative.\ + \ Use the following guidelines: \n\n Positive: The text expresses happiness,\ + \ satisfaction, or optimism. \nNegative: The text conveys disappointment, dissatisfaction,\ + \ or pessimism. \n\nIf the text contains both positive and negative sentiments, choose\ + \ the dominant sentiment. For ambiguous or unclear sentiments, select the label\ + \ that best reflects the overall tone. Please provide a single classification for\ + \ each input.\n\ntext: {{review}} \nlabel: \n" +include: nollysenti +task: nollysenti_hau_prompt_5 diff --git a/lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti_ibo.yaml b/lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti_ibo.yaml new file mode 100644 index 00000000..c75f2690 --- /dev/null +++ b/lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti_ibo.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: ig +doc_to_text: "You are tasked with performing sentiment classification on the following\ + \ Igbo text. For each input, classify the sentiment as positive, negative.\ + \ Use the following guidelines: \n\n Positive: The text expresses happiness,\ + \ satisfaction, or optimism. \nNegative: The text conveys disappointment, dissatisfaction,\ + \ or pessimism. \n\nIf the text contains both positive and negative sentiments, choose\ + \ the dominant sentiment. For ambiguous or unclear sentiments, select the label\ + \ that best reflects the overall tone. Please provide a single classification for\ + \ each input. \n\ntext: {{review}} \nlabel: \n" +include: nollysenti +task: nollysenti_ibo_prompt_5 diff --git a/lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti_pcm.yaml b/lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti_pcm.yaml new file mode 100644 index 00000000..29b5cda0 --- /dev/null +++ b/lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti_pcm.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: pcm +doc_to_text: "You are tasked with performing sentiment classification on the following\ + \ Naija Pidgin text. For each input, classify the sentiment as positive, negative.\ + \ Use the following guidelines: \n\n Positive: The text expresses happiness,\ + \ satisfaction, or optimism. \nNegative: The text conveys disappointment, dissatisfaction,\ + \ or pessimism. \n\nIf the text contains both positive and negative sentiments, choose\ + \ the dominant sentiment. For ambiguous or unclear sentiments, select the label\ + \ that best reflects the overall tone. Please provide a single classification for\ + \ each input. \n\ntext: {{review}} \nlabel: \n" +include: nollysenti +task: nollysenti_pcm_prompt_5 diff --git a/lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti_yor.yaml b/lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti_yor.yaml new file mode 100644 index 00000000..f1aea028 --- /dev/null +++ b/lm_eval/tasks/afrobench/nollysenti/prompt_5/nollysenti_yor.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: yo +doc_to_text: "You are tasked with performing sentiment classification on the following\ + \ Yoruba text. For each input, classify the sentiment as positive, negative.\ + \ Use the following guidelines: \n\n Positive: The text expresses happiness,\ + \ satisfaction, or optimism. \nNegative: The text conveys disappointment, dissatisfaction,\ + \ or pessimism. \n\nIf the text contains both positive and negative sentiments, choose\ + \ the dominant sentiment. For ambiguous or unclear sentiments, select the label\ + \ that best reflects the overall tone. Please provide a single classification for\ + \ each input. \n\ntext: {{review}} \nlabel: \n" +include: nollysenti +task: nollysenti_yor_prompt_5 diff --git a/lm_eval/tasks/afrobench/nollysenti/prompt_5/utils.py b/lm_eval/tasks/afrobench/nollysenti/prompt_5/utils.py new file mode 100644 index 00000000..3e735e2d --- /dev/null +++ b/lm_eval/tasks/afrobench/nollysenti/prompt_5/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm_eval/tasks/afrobench/ntrex/README.md b/lm_eval/tasks/afrobench/ntrex/README.md new file mode 100644 index 00000000..d68cf8c9 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/README.md @@ -0,0 +1,38 @@ +# + +## Paper +Title: `NTREX-128 – News Test References for MT Evaluation of 128 Languages` + +Paper Link: https://aclanthology.org/2022.sumeval-1.4/ + +## Abstract +>We release NTREX-128, a data set for machine translation (MT) evaluation from English into a total of 128 target languages. The paper describes the data creation process and proposes a quality filtering method based on human evaluation. We show experimental results which confirm that the directionality of test sets translation indeed plays an important role wrt. the usefulness of the corresponding metrics’ scores. Thus, we recommend that the NTREX-128 data set should be used for evaluation of Englishsourced translation models but not in reverse direction. The test set release introduces another benchmark for the evaluation of massively multilingual machine translation research. + +HomePage: https://github.com/MicrosoftTranslator/NTREX + +### Citation + +``` +@inproceedings{federmann-etal-2022-ntrex, + title = "{NTREX}-128 {--} News Test References for {MT} Evaluation of 128 Languages", + author = "Federmann, Christian and + Kocmi, Tom and + Xin, Ying", + editor = "Ahuja, Kabir and + Anastasopoulos, Antonios and + Patra, Barun and + Neubig, Graham and + Choudhury, Monojit and + Dandapat, Sandipan and + Sitaram, Sunayana and + Chaudhary, Vishrav", + booktitle = "Proceedings of the First Workshop on Scaling Up Multilingual Evaluation", + month = nov, + year = "2022", + address = "Online", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2022.sumeval-1.4/", + doi = "10.18653/v1/2022.sumeval-1.4", + pages = "21--24" +} +``` diff --git a/lm_eval/tasks/afrobench/ntrex/gen_utils.py b/lm_eval/tasks/afrobench/ntrex/gen_utils.py new file mode 100644 index 00000000..ba549de2 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/gen_utils.py @@ -0,0 +1,171 @@ +import argparse +import os + +import yaml + + +class FunctionTag: + def __init__(self, value): + self.value = value + + +def prompt_func(mode, lang, lang_dict): + language_column_name = f"sentence_{lang}" + prompt_map = { + "prompt_1": f"{lang_dict[lang]}: {{{{{language_column_name}}}}} \nEnglish: ", + "prompt_1_reverse": f"English: {{{{sentence_eng_Latn}}}} \n{lang_dict[lang]}: ", + "prompt_2": f"You are a translation expert. Translate the following {lang_dict[lang]} sentences to English \n" + f"{lang_dict[lang]}: {{{{{language_column_name}}}}}\nEnglish: ", + "prompt_2_reverse": f"You are a translation expert. Translate the following English sentences to " + f"{lang_dict[lang]} " + "\nEnglish: {{sentence_eng_Latn}} " + f"\n{lang_dict[lang]}: ", + "prompt_3": f"As a {lang_dict[lang]} and English linguist, translate the following {lang_dict[lang]} sentences " + f"to English \n{lang_dict[lang]}: {{{{{language_column_name}}}}}\nEnglish: ", + "prompt_3_reverse": f"As a {lang_dict[lang]} and English linguist, translate the following English sentences to " + f"{lang_dict[lang]} " + "\nEnglish: {{sentence_eng_Latn}} " + f"\n{lang_dict[lang]}: ", + } + return prompt_map[mode] + + +def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str, reverse: bool) -> None: + """ + Generate a yaml file for each language. + + :param output_dir: The directory to output the files to. + :param overwrite: Whether to overwrite files if they already exist. + """ + err = [] + languages = { + "afr_Latn": "Afrikaans", + "amh_Ethi": "Amharic", + "arb_Arab": "Arabic", + "bem_Latn": "Bemba", + "ewe_Latn": "Ewe", + "fra_Latn": "French", + "hau_Latn": "Hausa", + "ibo_Latn": "Igbo", + "kin_Latn": "Kinyarwanda", + "mey_Arab": "Hassaniya Arabic", + "mlg_Latn": "Malagasy", + "msa_Latn": "Malay", + "nde_Latn": "North Ndebele", + "nso_Latn": "Northern Sotho", + "nya_Latn": "Chichewa", + "orm_Ethi": "Oromo", + "shi_Arab": "Tachelhit", + "sna_Latn": "Shona (Latin)", + "som_Latn": "Somali", + "ssw_Latn": "Swati", + "swa_Latn": "Swahili", + "tam_Taml": "Tamil", + "tel_Telu": "Telugu", + "tir_Ethi": "Tigrinya", + "ton_Latn": "Tongan", + "tsn_Latn": "Tswana", + "urd_Arab": "Urdu", + "ven_Latn": "Venda", + "wol_Latn": "Wolof", + "xho_Latn": "Xhosa", + "yor_Latn": "Yoruba", + "zul_Latn": "Zulu", + } + + for lang in languages.keys(): + try: + if not reverse: + file_name = f"ntrex_{lang}-eng_Latn.yaml" + task_name = f"ntrex_{lang}-eng_Latn_{mode}" + yaml_template = "ntrex" + yaml_details = { + "include": yaml_template, + "dataset_name": f"{lang}", + "task": task_name, + "doc_to_target": "sentence_eng_Latn", + "doc_to_text": prompt_func(mode, lang, languages), + } + os.makedirs(f"{output_dir}/{mode}/african-english", exist_ok=True) + with open( + f"{output_dir}/{mode}/african-english/{file_name}", + "w" if overwrite else "x", + encoding="utf8", + ) as f: + f.write("# Generated by utils.py\n") + yaml.dump( + yaml_details, + f, + allow_unicode=True, + ) + else: + file_name = f"ntrex_eng_Latn-{lang}.yaml" + task_name = f"ntrex_eng_Latn-{lang}_{mode}" + yaml_template = "ntrex" + yaml_details = { + "include": yaml_template, + "dataset_name": f"{lang}", + "task": task_name, + "doc_to_target": f"sentence_{lang}", + "doc_to_text": prompt_func(f"{mode}_reverse", lang, languages), + } + os.makedirs(f"{output_dir}/{mode}/english-african", exist_ok=True) + with open( + f"{output_dir}/{mode}/english-african/{file_name}", + "w" if overwrite else "x", + encoding="utf8", + ) as f: + f.write("# Generated by utils.py\n") + yaml.dump( + yaml_details, + f, + allow_unicode=True, + ) + except FileExistsError: + err.append(file_name) + + if len(err) > 0: + raise FileExistsError( + "Files were not created because they already exist (use --overwrite flag):" + f" {', '.join(err)}" + ) + + +def main() -> None: + """Parse CLI args and generate language-specific yaml files.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + default=True, + action="store_true", + help="Overwrite files if they already exist", + ) + parser.add_argument( + "--output-dir", + default="./", + help="Directory to write yaml files to", + ) + parser.add_argument( + "--mode", + default="prompt_1", + choices=["prompt_1", "prompt_2", "prompt_3"], + help="Prompt number", + ) + parser.add_argument( + "--reverse", + default=False, + choices=[True, False], + help="Reverse the translation direction", + ) + args = parser.parse_args() + + gen_lang_yamls( + output_dir=args.output_dir, + overwrite=args.overwrite, + mode=args.mode, + reverse=args.reverse, + ) + + +if __name__ == "__main__": + main() diff --git a/lm_eval/tasks/afrobench/ntrex/ntrex.yaml b/lm_eval/tasks/afrobench/ntrex/ntrex.yaml new file mode 100644 index 00000000..c30b08ce --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/ntrex.yaml @@ -0,0 +1,14 @@ +group: african_ntrex +task: + - ntrex_eng-afr_prompt_1 + - ntrex_eng-afr_prompt_2 + - ntrex_eng-afr_prompt_3 + - ntrex_afr-eng_prompt_1 + - ntrex_afr-eng_prompt_2 + - ntrex_afr-eng_prompt_3 +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex new file mode 100644 index 00000000..3c2659d7 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex @@ -0,0 +1,26 @@ +tag: +- ntrex_tasks +- ntrex_afr-eng +- ntrex_afr-eng_prompt_1 +- afrobench_MT_tasks +dataset_path: masakhane/ntrex_african +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: test +fewshot_split: test +test_split: test +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_afr_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_afr_Latn-eng_Latn.yaml new file mode 100644 index 00000000..eb119043 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_afr_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: afr_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Afrikaans: {{sentence_afr_Latn}} \nEnglish: " +include: ntrex +task: ntrex_afr_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_amh_Ethi-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_amh_Ethi-eng_Latn.yaml new file mode 100644 index 00000000..0114a212 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_amh_Ethi-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: amh_Ethi +doc_to_target: sentence_eng_Latn +doc_to_text: "Amharic: {{sentence_amh_Ethi}} \nEnglish: " +include: ntrex +task: ntrex_amh_Ethi-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_arb_Arab-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_arb_Arab-eng_Latn.yaml new file mode 100644 index 00000000..4ddc8c4b --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_arb_Arab-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: arb_Arab +doc_to_target: sentence_eng_Latn +doc_to_text: "Arabic: {{sentence_arb_Arab}} \nEnglish: " +include: ntrex +task: ntrex_arb_Arab-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_bem_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_bem_Latn-eng_Latn.yaml new file mode 100644 index 00000000..c33ab35a --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_bem_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: bem_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Bemba: {{sentence_bem_Latn}} \nEnglish: " +include: ntrex +task: ntrex_bem_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_ewe_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_ewe_Latn-eng_Latn.yaml new file mode 100644 index 00000000..c5f69c00 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_ewe_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ewe_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Ewe: {{sentence_ewe_Latn}} \nEnglish: " +include: ntrex +task: ntrex_ewe_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_fra_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_fra_Latn-eng_Latn.yaml new file mode 100644 index 00000000..fa3fad61 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_fra_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: fra_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "French: {{sentence_fra_Latn}} \nEnglish: " +include: ntrex +task: ntrex_fra_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_hau_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_hau_Latn-eng_Latn.yaml new file mode 100644 index 00000000..8b6d0f28 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_hau_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: hau_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Hausa: {{sentence_hau_Latn}} \nEnglish: " +include: ntrex +task: ntrex_hau_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_ibo_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_ibo_Latn-eng_Latn.yaml new file mode 100644 index 00000000..99259861 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_ibo_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ibo_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Igbo: {{sentence_ibo_Latn}} \nEnglish: " +include: ntrex +task: ntrex_ibo_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_kin_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_kin_Latn-eng_Latn.yaml new file mode 100644 index 00000000..eee96a62 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_kin_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: kin_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Kinyarwanda: {{sentence_kin_Latn}} \nEnglish: " +include: ntrex +task: ntrex_kin_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_mey_Arab-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_mey_Arab-eng_Latn.yaml new file mode 100644 index 00000000..6650e644 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_mey_Arab-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: mey_Arab +doc_to_target: sentence_eng_Latn +doc_to_text: "Hassaniya Arabic: {{sentence_mey_Arab}} \nEnglish: " +include: ntrex +task: ntrex_mey_Arab-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_mlg_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_mlg_Latn-eng_Latn.yaml new file mode 100644 index 00000000..375522c5 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_mlg_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: mlg_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Malagasy: {{sentence_mlg_Latn}} \nEnglish: " +include: ntrex +task: ntrex_mlg_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_msa_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_msa_Latn-eng_Latn.yaml new file mode 100644 index 00000000..65aaaa80 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_msa_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: msa_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Malay: {{sentence_msa_Latn}} \nEnglish: " +include: ntrex +task: ntrex_msa_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_nde_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_nde_Latn-eng_Latn.yaml new file mode 100644 index 00000000..d63548fb --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_nde_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: nde_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "North Ndebele: {{sentence_nde_Latn}} \nEnglish: " +include: ntrex +task: ntrex_nde_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_nso_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_nso_Latn-eng_Latn.yaml new file mode 100644 index 00000000..4cf1cccf --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_nso_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: nso_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Northern Sotho: {{sentence_nso_Latn}} \nEnglish: " +include: ntrex +task: ntrex_nso_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_nya_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_nya_Latn-eng_Latn.yaml new file mode 100644 index 00000000..ee4ac6d7 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_nya_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: nya_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Chichewa: {{sentence_nya_Latn}} \nEnglish: " +include: ntrex +task: ntrex_nya_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_orm_Ethi-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_orm_Ethi-eng_Latn.yaml new file mode 100644 index 00000000..44687306 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_orm_Ethi-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: orm_Ethi +doc_to_target: sentence_eng_Latn +doc_to_text: "Oromo: {{sentence_orm_Ethi}} \nEnglish: " +include: ntrex +task: ntrex_orm_Ethi-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_shi_Arab-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_shi_Arab-eng_Latn.yaml new file mode 100644 index 00000000..10972893 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_shi_Arab-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: shi_Arab +doc_to_target: sentence_eng_Latn +doc_to_text: "Tachelhit: {{sentence_shi_Arab}} \nEnglish: " +include: ntrex +task: ntrex_shi_Arab-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_sna_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_sna_Latn-eng_Latn.yaml new file mode 100644 index 00000000..63d83528 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_sna_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: sna_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Shona (Latin): {{sentence_sna_Latn}} \nEnglish: " +include: ntrex +task: ntrex_sna_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_som_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_som_Latn-eng_Latn.yaml new file mode 100644 index 00000000..d6eb91e0 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_som_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: som_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Somali: {{sentence_som_Latn}} \nEnglish: " +include: ntrex +task: ntrex_som_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_ssw_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_ssw_Latn-eng_Latn.yaml new file mode 100644 index 00000000..48c5c109 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_ssw_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ssw_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Swati: {{sentence_ssw_Latn}} \nEnglish: " +include: ntrex +task: ntrex_ssw_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_swa_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_swa_Latn-eng_Latn.yaml new file mode 100644 index 00000000..863222f7 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_swa_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: swa_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Swahili: {{sentence_swa_Latn}} \nEnglish: " +include: ntrex +task: ntrex_swa_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_tam_Taml-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_tam_Taml-eng_Latn.yaml new file mode 100644 index 00000000..993b480f --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_tam_Taml-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: tam_Taml +doc_to_target: sentence_eng_Latn +doc_to_text: "Tamil: {{sentence_tam_Taml}} \nEnglish: " +include: ntrex +task: ntrex_tam_Taml-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_tel_Telu-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_tel_Telu-eng_Latn.yaml new file mode 100644 index 00000000..d91e9a1f --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_tel_Telu-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: tel_Telu +doc_to_target: sentence_eng_Latn +doc_to_text: "Telugu: {{sentence_tel_Telu}} \nEnglish: " +include: ntrex +task: ntrex_tel_Telu-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_tir_Ethi-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_tir_Ethi-eng_Latn.yaml new file mode 100644 index 00000000..f23f332c --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_tir_Ethi-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: tir_Ethi +doc_to_target: sentence_eng_Latn +doc_to_text: "Tigrinya: {{sentence_tir_Ethi}} \nEnglish: " +include: ntrex +task: ntrex_tir_Ethi-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_ton_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_ton_Latn-eng_Latn.yaml new file mode 100644 index 00000000..5676a1a9 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_ton_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ton_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Tongan: {{sentence_ton_Latn}} \nEnglish: " +include: ntrex +task: ntrex_ton_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_tsn_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_tsn_Latn-eng_Latn.yaml new file mode 100644 index 00000000..caa0f9e5 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_tsn_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: tsn_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Tswana: {{sentence_tsn_Latn}} \nEnglish: " +include: ntrex +task: ntrex_tsn_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_urd_Arab-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_urd_Arab-eng_Latn.yaml new file mode 100644 index 00000000..4e07e678 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_urd_Arab-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: urd_Arab +doc_to_target: sentence_eng_Latn +doc_to_text: "Urdu: {{sentence_urd_Arab}} \nEnglish: " +include: ntrex +task: ntrex_urd_Arab-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_ven_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_ven_Latn-eng_Latn.yaml new file mode 100644 index 00000000..7ba8ceaf --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_ven_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ven_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Venda: {{sentence_ven_Latn}} \nEnglish: " +include: ntrex +task: ntrex_ven_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_wol_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_wol_Latn-eng_Latn.yaml new file mode 100644 index 00000000..8dcacb69 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_wol_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: wol_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Wolof: {{sentence_wol_Latn}} \nEnglish: " +include: ntrex +task: ntrex_wol_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_xho_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_xho_Latn-eng_Latn.yaml new file mode 100644 index 00000000..1b6abc9d --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_xho_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: xho_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Xhosa: {{sentence_xho_Latn}} \nEnglish: " +include: ntrex +task: ntrex_xho_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_yor_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_yor_Latn-eng_Latn.yaml new file mode 100644 index 00000000..e98aecd5 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_yor_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: yor_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Yoruba: {{sentence_yor_Latn}} \nEnglish: " +include: ntrex +task: ntrex_yor_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_zul_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_zul_Latn-eng_Latn.yaml new file mode 100644 index 00000000..a38abee1 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/african-english/ntrex_zul_Latn-eng_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: zul_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "Zulu: {{sentence_zul_Latn}} \nEnglish: " +include: ntrex +task: ntrex_zul_Latn-eng_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex new file mode 100644 index 00000000..2b5aa84f --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex @@ -0,0 +1,26 @@ +tag: +- ntrex_tasks +- ntrex_eng-afr +- ntrex_eng-afr_prompt_1 +- afrobench_MT_tasks +dataset_path: masakhane/ntrex_african +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: test +fewshot_split: test +test_split: test +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-afr_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-afr_Latn.yaml new file mode 100644 index 00000000..40471f80 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-afr_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: afr_Latn +doc_to_target: sentence_afr_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nAfrikaans: " +include: ntrex +task: ntrex_eng_Latn-afr_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-amh_Ethi.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-amh_Ethi.yaml new file mode 100644 index 00000000..6e4dfba5 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-amh_Ethi.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: amh_Ethi +doc_to_target: sentence_amh_Ethi +doc_to_text: "English: {{sentence_eng_Latn}} \nAmharic: " +include: ntrex +task: ntrex_eng_Latn-amh_Ethi_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-arb_Arab.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-arb_Arab.yaml new file mode 100644 index 00000000..1a248a9a --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-arb_Arab.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: arb_Arab +doc_to_target: sentence_arb_Arab +doc_to_text: "English: {{sentence_eng_Latn}} \nArabic: " +include: ntrex +task: ntrex_eng_Latn-arb_Arab_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-bem_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-bem_Latn.yaml new file mode 100644 index 00000000..035c6822 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-bem_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: bem_Latn +doc_to_target: sentence_bem_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nBemba: " +include: ntrex +task: ntrex_eng_Latn-bem_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-ewe_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-ewe_Latn.yaml new file mode 100644 index 00000000..f5deae5c --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-ewe_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ewe_Latn +doc_to_target: sentence_ewe_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nEwe: " +include: ntrex +task: ntrex_eng_Latn-ewe_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-fra_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-fra_Latn.yaml new file mode 100644 index 00000000..cf079cf4 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-fra_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: fra_Latn +doc_to_target: sentence_fra_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nFrench: " +include: ntrex +task: ntrex_eng_Latn-fra_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-hau_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-hau_Latn.yaml new file mode 100644 index 00000000..377acbfb --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-hau_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: hau_Latn +doc_to_target: sentence_hau_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nHausa: " +include: ntrex +task: ntrex_eng_Latn-hau_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-ibo_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-ibo_Latn.yaml new file mode 100644 index 00000000..1c3a14df --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-ibo_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ibo_Latn +doc_to_target: sentence_ibo_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nIgbo: " +include: ntrex +task: ntrex_eng_Latn-ibo_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-kin_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-kin_Latn.yaml new file mode 100644 index 00000000..ec14399e --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-kin_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: kin_Latn +doc_to_target: sentence_kin_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nKinyarwanda: " +include: ntrex +task: ntrex_eng_Latn-kin_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-mey_Arab.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-mey_Arab.yaml new file mode 100644 index 00000000..fb696cc5 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-mey_Arab.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: mey_Arab +doc_to_target: sentence_mey_Arab +doc_to_text: "English: {{sentence_eng_Latn}} \nHassaniya Arabic: " +include: ntrex +task: ntrex_eng_Latn-mey_Arab_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-mlg_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-mlg_Latn.yaml new file mode 100644 index 00000000..035c98c3 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-mlg_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: mlg_Latn +doc_to_target: sentence_mlg_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nMalagasy: " +include: ntrex +task: ntrex_eng_Latn-mlg_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-msa_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-msa_Latn.yaml new file mode 100644 index 00000000..c4c6b7d7 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-msa_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: msa_Latn +doc_to_target: sentence_msa_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nMalay: " +include: ntrex +task: ntrex_eng_Latn-msa_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-nde_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-nde_Latn.yaml new file mode 100644 index 00000000..c66b44be --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-nde_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: nde_Latn +doc_to_target: sentence_nde_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nNorth Ndebele: " +include: ntrex +task: ntrex_eng_Latn-nde_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-nso_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-nso_Latn.yaml new file mode 100644 index 00000000..ab6cf296 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-nso_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: nso_Latn +doc_to_target: sentence_nso_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nNorthern Sotho: " +include: ntrex +task: ntrex_eng_Latn-nso_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-nya_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-nya_Latn.yaml new file mode 100644 index 00000000..74cbd1ff --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-nya_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: nya_Latn +doc_to_target: sentence_nya_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nChichewa: " +include: ntrex +task: ntrex_eng_Latn-nya_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-orm_Ethi.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-orm_Ethi.yaml new file mode 100644 index 00000000..ad875cab --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-orm_Ethi.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: orm_Ethi +doc_to_target: sentence_orm_Ethi +doc_to_text: "English: {{sentence_eng_Latn}} \nOromo: " +include: ntrex +task: ntrex_eng_Latn-orm_Ethi_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-shi_Arab.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-shi_Arab.yaml new file mode 100644 index 00000000..5441bbdb --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-shi_Arab.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: shi_Arab +doc_to_target: sentence_shi_Arab +doc_to_text: "English: {{sentence_eng_Latn}} \nTachelhit: " +include: ntrex +task: ntrex_eng_Latn-shi_Arab_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-sna_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-sna_Latn.yaml new file mode 100644 index 00000000..0bed0f6c --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-sna_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: sna_Latn +doc_to_target: sentence_sna_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nShona (Latin): " +include: ntrex +task: ntrex_eng_Latn-sna_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-som_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-som_Latn.yaml new file mode 100644 index 00000000..5e4aafdf --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-som_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: som_Latn +doc_to_target: sentence_som_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nSomali: " +include: ntrex +task: ntrex_eng_Latn-som_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-ssw_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-ssw_Latn.yaml new file mode 100644 index 00000000..fa18ebf2 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-ssw_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ssw_Latn +doc_to_target: sentence_ssw_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nSwati: " +include: ntrex +task: ntrex_eng_Latn-ssw_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-swa_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-swa_Latn.yaml new file mode 100644 index 00000000..a7079ec0 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-swa_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: swa_Latn +doc_to_target: sentence_swa_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nSwahili: " +include: ntrex +task: ntrex_eng_Latn-swa_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-tam_Taml.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-tam_Taml.yaml new file mode 100644 index 00000000..b7e42a36 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-tam_Taml.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: tam_Taml +doc_to_target: sentence_tam_Taml +doc_to_text: "English: {{sentence_eng_Latn}} \nTamil: " +include: ntrex +task: ntrex_eng_Latn-tam_Taml_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-tel_Telu.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-tel_Telu.yaml new file mode 100644 index 00000000..db8eb6b2 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-tel_Telu.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: tel_Telu +doc_to_target: sentence_tel_Telu +doc_to_text: "English: {{sentence_eng_Latn}} \nTelugu: " +include: ntrex +task: ntrex_eng_Latn-tel_Telu_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-tir_Ethi.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-tir_Ethi.yaml new file mode 100644 index 00000000..45c6ae84 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-tir_Ethi.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: tir_Ethi +doc_to_target: sentence_tir_Ethi +doc_to_text: "English: {{sentence_eng_Latn}} \nTigrinya: " +include: ntrex +task: ntrex_eng_Latn-tir_Ethi_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-ton_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-ton_Latn.yaml new file mode 100644 index 00000000..0a680a2c --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-ton_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ton_Latn +doc_to_target: sentence_ton_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nTongan: " +include: ntrex +task: ntrex_eng_Latn-ton_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-tsn_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-tsn_Latn.yaml new file mode 100644 index 00000000..d5a7a4ca --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-tsn_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: tsn_Latn +doc_to_target: sentence_tsn_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nTswana: " +include: ntrex +task: ntrex_eng_Latn-tsn_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-urd_Arab.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-urd_Arab.yaml new file mode 100644 index 00000000..4ee69ded --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-urd_Arab.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: urd_Arab +doc_to_target: sentence_urd_Arab +doc_to_text: "English: {{sentence_eng_Latn}} \nUrdu: " +include: ntrex +task: ntrex_eng_Latn-urd_Arab_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-ven_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-ven_Latn.yaml new file mode 100644 index 00000000..4277ce08 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-ven_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ven_Latn +doc_to_target: sentence_ven_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nVenda: " +include: ntrex +task: ntrex_eng_Latn-ven_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-wol_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-wol_Latn.yaml new file mode 100644 index 00000000..dea533ee --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-wol_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: wol_Latn +doc_to_target: sentence_wol_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nWolof: " +include: ntrex +task: ntrex_eng_Latn-wol_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-xho_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-xho_Latn.yaml new file mode 100644 index 00000000..62ab64bf --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-xho_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: xho_Latn +doc_to_target: sentence_xho_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nXhosa: " +include: ntrex +task: ntrex_eng_Latn-xho_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-yor_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-yor_Latn.yaml new file mode 100644 index 00000000..9d96624a --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-yor_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: yor_Latn +doc_to_target: sentence_yor_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nYoruba: " +include: ntrex +task: ntrex_eng_Latn-yor_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-zul_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-zul_Latn.yaml new file mode 100644 index 00000000..db60fb59 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_1/english-african/ntrex_eng_Latn-zul_Latn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: zul_Latn +doc_to_target: sentence_zul_Latn +doc_to_text: "English: {{sentence_eng_Latn}} \nZulu: " +include: ntrex +task: ntrex_eng_Latn-zul_Latn_prompt_1 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex new file mode 100644 index 00000000..3dc29226 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex @@ -0,0 +1,25 @@ +tag: +- ntrex_afr-eng +- ntrex_afr-eng_prompt_2 +- afrobench_MT_tasks +dataset_path: masakhane/ntrex_african +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: test +fewshot_split: test +test_split: test +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_afr_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_afr_Latn-eng_Latn.yaml new file mode 100644 index 00000000..16cfc7d5 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_afr_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: afr_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Afrikaans sentences\ + \ to English \nAfrikaans: {{sentence_afr_Latn}}\nEnglish: " +include: ntrex +task: ntrex_afr_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_amh_Ethi-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_amh_Ethi-eng_Latn.yaml new file mode 100644 index 00000000..20e88c36 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_amh_Ethi-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: amh_Ethi +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Amharic sentences\ + \ to English \nAmharic: {{sentence_amh_Ethi}}\nEnglish: " +include: ntrex +task: ntrex_amh_Ethi-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_arb_Arab-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_arb_Arab-eng_Latn.yaml new file mode 100644 index 00000000..a88a478a --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_arb_Arab-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: arb_Arab +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Arabic sentences\ + \ to English \nArabic: {{sentence_arb_Arab}}\nEnglish: " +include: ntrex +task: ntrex_arb_Arab-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_bem_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_bem_Latn-eng_Latn.yaml new file mode 100644 index 00000000..3e114a34 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_bem_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: bem_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Bemba sentences\ + \ to English \nBemba: {{sentence_bem_Latn}}\nEnglish: " +include: ntrex +task: ntrex_bem_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_ewe_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_ewe_Latn-eng_Latn.yaml new file mode 100644 index 00000000..5e4facd5 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_ewe_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ewe_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Ewe sentences\ + \ to English \nEwe: {{sentence_ewe_Latn}}\nEnglish: " +include: ntrex +task: ntrex_ewe_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_fra_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_fra_Latn-eng_Latn.yaml new file mode 100644 index 00000000..ad46aedf --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_fra_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fra_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following French sentences\ + \ to English \nFrench: {{sentence_fra_Latn}}\nEnglish: " +include: ntrex +task: ntrex_fra_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_hau_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_hau_Latn-eng_Latn.yaml new file mode 100644 index 00000000..018a6396 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_hau_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: hau_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Hausa sentences\ + \ to English \nHausa: {{sentence_hau_Latn}}\nEnglish: " +include: ntrex +task: ntrex_hau_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_ibo_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_ibo_Latn-eng_Latn.yaml new file mode 100644 index 00000000..0b93d2d8 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_ibo_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ibo_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Igbo sentences\ + \ to English \nIgbo: {{sentence_ibo_Latn}}\nEnglish: " +include: ntrex +task: ntrex_ibo_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_kin_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_kin_Latn-eng_Latn.yaml new file mode 100644 index 00000000..45b18a64 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_kin_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kin_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Kinyarwanda sentences\ + \ to English \nKinyarwanda: {{sentence_kin_Latn}}\nEnglish: " +include: ntrex +task: ntrex_kin_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_mey_Arab-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_mey_Arab-eng_Latn.yaml new file mode 100644 index 00000000..d155b62c --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_mey_Arab-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: mey_Arab +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Hassaniya Arabic\ + \ sentences to English \nHassaniya Arabic: {{sentence_mey_Arab}}\nEnglish: " +include: ntrex +task: ntrex_mey_Arab-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_mlg_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_mlg_Latn-eng_Latn.yaml new file mode 100644 index 00000000..10a7507b --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_mlg_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: mlg_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Malagasy sentences\ + \ to English \nMalagasy: {{sentence_mlg_Latn}}\nEnglish: " +include: ntrex +task: ntrex_mlg_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_msa_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_msa_Latn-eng_Latn.yaml new file mode 100644 index 00000000..be65a0ff --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_msa_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: msa_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Malay sentences\ + \ to English \nMalay: {{sentence_msa_Latn}}\nEnglish: " +include: ntrex +task: ntrex_msa_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_nde_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_nde_Latn-eng_Latn.yaml new file mode 100644 index 00000000..c4a39fc2 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_nde_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nde_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following North Ndebele\ + \ sentences to English \nNorth Ndebele: {{sentence_nde_Latn}}\nEnglish: " +include: ntrex +task: ntrex_nde_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_nso_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_nso_Latn-eng_Latn.yaml new file mode 100644 index 00000000..290122fa --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_nso_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nso_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Northern Sotho\ + \ sentences to English \nNorthern Sotho: {{sentence_nso_Latn}}\nEnglish: " +include: ntrex +task: ntrex_nso_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_nya_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_nya_Latn-eng_Latn.yaml new file mode 100644 index 00000000..de365e01 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_nya_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nya_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Chichewa sentences\ + \ to English \nChichewa: {{sentence_nya_Latn}}\nEnglish: " +include: ntrex +task: ntrex_nya_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_orm_Ethi-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_orm_Ethi-eng_Latn.yaml new file mode 100644 index 00000000..ebe353d1 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_orm_Ethi-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: orm_Ethi +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Oromo sentences\ + \ to English \nOromo: {{sentence_orm_Ethi}}\nEnglish: " +include: ntrex +task: ntrex_orm_Ethi-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_shi_Arab-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_shi_Arab-eng_Latn.yaml new file mode 100644 index 00000000..b2db11ae --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_shi_Arab-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: shi_Arab +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Tachelhit sentences\ + \ to English \nTachelhit: {{sentence_shi_Arab}}\nEnglish: " +include: ntrex +task: ntrex_shi_Arab-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_sna_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_sna_Latn-eng_Latn.yaml new file mode 100644 index 00000000..25600d63 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_sna_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sna_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Shona (Latin)\ + \ sentences to English \nShona (Latin): {{sentence_sna_Latn}}\nEnglish: " +include: ntrex +task: ntrex_sna_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_som_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_som_Latn-eng_Latn.yaml new file mode 100644 index 00000000..0ea6a71d --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_som_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: som_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Somali sentences\ + \ to English \nSomali: {{sentence_som_Latn}}\nEnglish: " +include: ntrex +task: ntrex_som_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_ssw_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_ssw_Latn-eng_Latn.yaml new file mode 100644 index 00000000..b2e690a6 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_ssw_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ssw_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Swati sentences\ + \ to English \nSwati: {{sentence_ssw_Latn}}\nEnglish: " +include: ntrex +task: ntrex_ssw_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_swa_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_swa_Latn-eng_Latn.yaml new file mode 100644 index 00000000..2e609435 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_swa_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: swa_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Swahili sentences\ + \ to English \nSwahili: {{sentence_swa_Latn}}\nEnglish: " +include: ntrex +task: ntrex_swa_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_tam_Taml-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_tam_Taml-eng_Latn.yaml new file mode 100644 index 00000000..e2c9f278 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_tam_Taml-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tam_Taml +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Tamil sentences\ + \ to English \nTamil: {{sentence_tam_Taml}}\nEnglish: " +include: ntrex +task: ntrex_tam_Taml-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_tel_Telu-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_tel_Telu-eng_Latn.yaml new file mode 100644 index 00000000..15dc3598 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_tel_Telu-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tel_Telu +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Telugu sentences\ + \ to English \nTelugu: {{sentence_tel_Telu}}\nEnglish: " +include: ntrex +task: ntrex_tel_Telu-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_tir_Ethi-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_tir_Ethi-eng_Latn.yaml new file mode 100644 index 00000000..5f0bb2b8 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_tir_Ethi-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tir_Ethi +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Tigrinya sentences\ + \ to English \nTigrinya: {{sentence_tir_Ethi}}\nEnglish: " +include: ntrex +task: ntrex_tir_Ethi-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_ton_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_ton_Latn-eng_Latn.yaml new file mode 100644 index 00000000..84f7d281 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_ton_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ton_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Tongan sentences\ + \ to English \nTongan: {{sentence_ton_Latn}}\nEnglish: " +include: ntrex +task: ntrex_ton_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_tsn_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_tsn_Latn-eng_Latn.yaml new file mode 100644 index 00000000..a5699641 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_tsn_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tsn_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Tswana sentences\ + \ to English \nTswana: {{sentence_tsn_Latn}}\nEnglish: " +include: ntrex +task: ntrex_tsn_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_urd_Arab-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_urd_Arab-eng_Latn.yaml new file mode 100644 index 00000000..47a47875 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_urd_Arab-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: urd_Arab +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Urdu sentences\ + \ to English \nUrdu: {{sentence_urd_Arab}}\nEnglish: " +include: ntrex +task: ntrex_urd_Arab-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_ven_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_ven_Latn-eng_Latn.yaml new file mode 100644 index 00000000..5f27b185 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_ven_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ven_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Venda sentences\ + \ to English \nVenda: {{sentence_ven_Latn}}\nEnglish: " +include: ntrex +task: ntrex_ven_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_wol_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_wol_Latn-eng_Latn.yaml new file mode 100644 index 00000000..fa2da55c --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_wol_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: wol_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Wolof sentences\ + \ to English \nWolof: {{sentence_wol_Latn}}\nEnglish: " +include: ntrex +task: ntrex_wol_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_xho_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_xho_Latn-eng_Latn.yaml new file mode 100644 index 00000000..b504cd31 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_xho_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: xho_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Xhosa sentences\ + \ to English \nXhosa: {{sentence_xho_Latn}}\nEnglish: " +include: ntrex +task: ntrex_xho_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_yor_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_yor_Latn-eng_Latn.yaml new file mode 100644 index 00000000..03c4cbac --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_yor_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: yor_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Yoruba sentences\ + \ to English \nYoruba: {{sentence_yor_Latn}}\nEnglish: " +include: ntrex +task: ntrex_yor_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_zul_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_zul_Latn-eng_Latn.yaml new file mode 100644 index 00000000..760abb6f --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/african-english/ntrex_zul_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: zul_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "You are a translation expert. Translate the following Zulu sentences\ + \ to English \nZulu: {{sentence_zul_Latn}}\nEnglish: " +include: ntrex +task: ntrex_zul_Latn-eng_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex new file mode 100644 index 00000000..8dd411c3 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex @@ -0,0 +1,25 @@ +tag: +- ntrex_eng-afr +- ntrex_eng-afr_prompt_2 +- afrobench_MT_tasks +dataset_path: masakhane/ntrex_african +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: test +fewshot_split: test +test_split: test +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-afr_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-afr_Latn.yaml new file mode 100644 index 00000000..678e5b21 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-afr_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: afr_Latn +doc_to_target: sentence_afr_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Afrikaans \nEnglish: {{sentence_eng_Latn}} \nAfrikaans: " +include: ntrex +task: ntrex_eng_Latn-afr_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-amh_Ethi.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-amh_Ethi.yaml new file mode 100644 index 00000000..a5ae3dd1 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-amh_Ethi.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: amh_Ethi +doc_to_target: sentence_amh_Ethi +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Amharic \nEnglish: {{sentence_eng_Latn}} \nAmharic: " +include: ntrex +task: ntrex_eng_Latn-amh_Ethi_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-arb_Arab.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-arb_Arab.yaml new file mode 100644 index 00000000..303ccf47 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-arb_Arab.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: arb_Arab +doc_to_target: sentence_arb_Arab +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Arabic \nEnglish: {{sentence_eng_Latn}} \nArabic: " +include: ntrex +task: ntrex_eng_Latn-arb_Arab_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-bem_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-bem_Latn.yaml new file mode 100644 index 00000000..7992529a --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-bem_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: bem_Latn +doc_to_target: sentence_bem_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Bemba \nEnglish: {{sentence_eng_Latn}} \nBemba: " +include: ntrex +task: ntrex_eng_Latn-bem_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-ewe_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-ewe_Latn.yaml new file mode 100644 index 00000000..3de8e8eb --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-ewe_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ewe_Latn +doc_to_target: sentence_ewe_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Ewe \nEnglish: {{sentence_eng_Latn}} \nEwe: " +include: ntrex +task: ntrex_eng_Latn-ewe_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-fra_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-fra_Latn.yaml new file mode 100644 index 00000000..fc230efe --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-fra_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fra_Latn +doc_to_target: sentence_fra_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to French \nEnglish: {{sentence_eng_Latn}} \nFrench: " +include: ntrex +task: ntrex_eng_Latn-fra_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-hau_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-hau_Latn.yaml new file mode 100644 index 00000000..009d1a51 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-hau_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: hau_Latn +doc_to_target: sentence_hau_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Hausa \nEnglish: {{sentence_eng_Latn}} \nHausa: " +include: ntrex +task: ntrex_eng_Latn-hau_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-ibo_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-ibo_Latn.yaml new file mode 100644 index 00000000..a2b27ab5 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-ibo_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ibo_Latn +doc_to_target: sentence_ibo_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Igbo \nEnglish: {{sentence_eng_Latn}} \nIgbo: " +include: ntrex +task: ntrex_eng_Latn-ibo_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-kin_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-kin_Latn.yaml new file mode 100644 index 00000000..f76077d3 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-kin_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kin_Latn +doc_to_target: sentence_kin_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Kinyarwanda \nEnglish: {{sentence_eng_Latn}} \nKinyarwanda: " +include: ntrex +task: ntrex_eng_Latn-kin_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-mey_Arab.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-mey_Arab.yaml new file mode 100644 index 00000000..2c5b2aba --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-mey_Arab.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: mey_Arab +doc_to_target: sentence_mey_Arab +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Hassaniya Arabic \nEnglish: {{sentence_eng_Latn}} \nHassaniya Arabic: " +include: ntrex +task: ntrex_eng_Latn-mey_Arab_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-mlg_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-mlg_Latn.yaml new file mode 100644 index 00000000..1d25afa1 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-mlg_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: mlg_Latn +doc_to_target: sentence_mlg_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Malagasy \nEnglish: {{sentence_eng_Latn}} \nMalagasy: " +include: ntrex +task: ntrex_eng_Latn-mlg_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-msa_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-msa_Latn.yaml new file mode 100644 index 00000000..c7b7972b --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-msa_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: msa_Latn +doc_to_target: sentence_msa_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Malay \nEnglish: {{sentence_eng_Latn}} \nMalay: " +include: ntrex +task: ntrex_eng_Latn-msa_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-nde_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-nde_Latn.yaml new file mode 100644 index 00000000..31252c02 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-nde_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nde_Latn +doc_to_target: sentence_nde_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to North Ndebele \nEnglish: {{sentence_eng_Latn}} \nNorth Ndebele: " +include: ntrex +task: ntrex_eng_Latn-nde_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-nso_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-nso_Latn.yaml new file mode 100644 index 00000000..b8daba4d --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-nso_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nso_Latn +doc_to_target: sentence_nso_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Northern Sotho \nEnglish: {{sentence_eng_Latn}} \nNorthern Sotho: " +include: ntrex +task: ntrex_eng_Latn-nso_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-nya_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-nya_Latn.yaml new file mode 100644 index 00000000..fe01ef87 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-nya_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nya_Latn +doc_to_target: sentence_nya_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Chichewa \nEnglish: {{sentence_eng_Latn}} \nChichewa: " +include: ntrex +task: ntrex_eng_Latn-nya_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-orm_Ethi.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-orm_Ethi.yaml new file mode 100644 index 00000000..f78e4db6 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-orm_Ethi.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: orm_Ethi +doc_to_target: sentence_orm_Ethi +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Oromo \nEnglish: {{sentence_eng_Latn}} \nOromo: " +include: ntrex +task: ntrex_eng_Latn-orm_Ethi_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-shi_Arab.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-shi_Arab.yaml new file mode 100644 index 00000000..57ea6c04 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-shi_Arab.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: shi_Arab +doc_to_target: sentence_shi_Arab +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Tachelhit \nEnglish: {{sentence_eng_Latn}} \nTachelhit: " +include: ntrex +task: ntrex_eng_Latn-shi_Arab_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-sna_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-sna_Latn.yaml new file mode 100644 index 00000000..399668d3 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-sna_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sna_Latn +doc_to_target: sentence_sna_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Shona (Latin) \nEnglish: {{sentence_eng_Latn}} \nShona (Latin): " +include: ntrex +task: ntrex_eng_Latn-sna_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-som_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-som_Latn.yaml new file mode 100644 index 00000000..8a29749a --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-som_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: som_Latn +doc_to_target: sentence_som_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Somali \nEnglish: {{sentence_eng_Latn}} \nSomali: " +include: ntrex +task: ntrex_eng_Latn-som_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-ssw_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-ssw_Latn.yaml new file mode 100644 index 00000000..a67ffdee --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-ssw_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ssw_Latn +doc_to_target: sentence_ssw_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Swati \nEnglish: {{sentence_eng_Latn}} \nSwati: " +include: ntrex +task: ntrex_eng_Latn-ssw_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-swa_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-swa_Latn.yaml new file mode 100644 index 00000000..0be54151 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-swa_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: swa_Latn +doc_to_target: sentence_swa_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Swahili \nEnglish: {{sentence_eng_Latn}} \nSwahili: " +include: ntrex +task: ntrex_eng_Latn-swa_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-tam_Taml.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-tam_Taml.yaml new file mode 100644 index 00000000..387ac60d --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-tam_Taml.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tam_Taml +doc_to_target: sentence_tam_Taml +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Tamil \nEnglish: {{sentence_eng_Latn}} \nTamil: " +include: ntrex +task: ntrex_eng_Latn-tam_Taml_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-tel_Telu.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-tel_Telu.yaml new file mode 100644 index 00000000..7b1de396 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-tel_Telu.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tel_Telu +doc_to_target: sentence_tel_Telu +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Telugu \nEnglish: {{sentence_eng_Latn}} \nTelugu: " +include: ntrex +task: ntrex_eng_Latn-tel_Telu_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-tir_Ethi.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-tir_Ethi.yaml new file mode 100644 index 00000000..da402211 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-tir_Ethi.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tir_Ethi +doc_to_target: sentence_tir_Ethi +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Tigrinya \nEnglish: {{sentence_eng_Latn}} \nTigrinya: " +include: ntrex +task: ntrex_eng_Latn-tir_Ethi_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-ton_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-ton_Latn.yaml new file mode 100644 index 00000000..f8c46692 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-ton_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ton_Latn +doc_to_target: sentence_ton_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Tongan \nEnglish: {{sentence_eng_Latn}} \nTongan: " +include: ntrex +task: ntrex_eng_Latn-ton_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-tsn_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-tsn_Latn.yaml new file mode 100644 index 00000000..ca918e1d --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-tsn_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tsn_Latn +doc_to_target: sentence_tsn_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Tswana \nEnglish: {{sentence_eng_Latn}} \nTswana: " +include: ntrex +task: ntrex_eng_Latn-tsn_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-urd_Arab.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-urd_Arab.yaml new file mode 100644 index 00000000..8539df76 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-urd_Arab.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: urd_Arab +doc_to_target: sentence_urd_Arab +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Urdu \nEnglish: {{sentence_eng_Latn}} \nUrdu: " +include: ntrex +task: ntrex_eng_Latn-urd_Arab_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-ven_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-ven_Latn.yaml new file mode 100644 index 00000000..e923b12c --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-ven_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ven_Latn +doc_to_target: sentence_ven_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Venda \nEnglish: {{sentence_eng_Latn}} \nVenda: " +include: ntrex +task: ntrex_eng_Latn-ven_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-wol_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-wol_Latn.yaml new file mode 100644 index 00000000..707b76a5 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-wol_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: wol_Latn +doc_to_target: sentence_wol_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Wolof \nEnglish: {{sentence_eng_Latn}} \nWolof: " +include: ntrex +task: ntrex_eng_Latn-wol_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-xho_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-xho_Latn.yaml new file mode 100644 index 00000000..e7f51491 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-xho_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: xho_Latn +doc_to_target: sentence_xho_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Xhosa \nEnglish: {{sentence_eng_Latn}} \nXhosa: " +include: ntrex +task: ntrex_eng_Latn-xho_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-yor_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-yor_Latn.yaml new file mode 100644 index 00000000..6f3e4be5 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-yor_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: yor_Latn +doc_to_target: sentence_yor_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Yoruba \nEnglish: {{sentence_eng_Latn}} \nYoruba: " +include: ntrex +task: ntrex_eng_Latn-yor_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-zul_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-zul_Latn.yaml new file mode 100644 index 00000000..946d0020 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_2/english-african/ntrex_eng_Latn-zul_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: zul_Latn +doc_to_target: sentence_zul_Latn +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Zulu \nEnglish: {{sentence_eng_Latn}} \nZulu: " +include: ntrex +task: ntrex_eng_Latn-zul_Latn_prompt_2 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex new file mode 100644 index 00000000..3bab54d8 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex @@ -0,0 +1,25 @@ +tag: +- ntrex_afr-eng +- ntrex_afr-eng_prompt_3 +- afrobench_MT_tasks +dataset_path: masakhane/ntrex_african +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: test +fewshot_split: test +test_split: test +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_afr_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_afr_Latn-eng_Latn.yaml new file mode 100644 index 00000000..09cbbfc5 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_afr_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: afr_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Afrikaans and English linguist, translate the following Afrikaans\ + \ sentences to English \nAfrikaans: {{sentence_afr_Latn}}\nEnglish: " +include: ntrex +task: ntrex_afr_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_amh_Ethi-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_amh_Ethi-eng_Latn.yaml new file mode 100644 index 00000000..33530440 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_amh_Ethi-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: amh_Ethi +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Amharic and English linguist, translate the following Amharic sentences\ + \ to English \nAmharic: {{sentence_amh_Ethi}}\nEnglish: " +include: ntrex +task: ntrex_amh_Ethi-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_arb_Arab-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_arb_Arab-eng_Latn.yaml new file mode 100644 index 00000000..858c0605 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_arb_Arab-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: arb_Arab +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Arabic and English linguist, translate the following Arabic sentences\ + \ to English \nArabic: {{sentence_arb_Arab}}\nEnglish: " +include: ntrex +task: ntrex_arb_Arab-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_bem_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_bem_Latn-eng_Latn.yaml new file mode 100644 index 00000000..3be00855 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_bem_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: bem_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Bemba and English linguist, translate the following Bemba sentences\ + \ to English \nBemba: {{sentence_bem_Latn}}\nEnglish: " +include: ntrex +task: ntrex_bem_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_ewe_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_ewe_Latn-eng_Latn.yaml new file mode 100644 index 00000000..493176a7 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_ewe_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ewe_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Ewe and English linguist, translate the following Ewe sentences\ + \ to English \nEwe: {{sentence_ewe_Latn}}\nEnglish: " +include: ntrex +task: ntrex_ewe_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_fra_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_fra_Latn-eng_Latn.yaml new file mode 100644 index 00000000..b009a37b --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_fra_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fra_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a French and English linguist, translate the following French sentences\ + \ to English \nFrench: {{sentence_fra_Latn}}\nEnglish: " +include: ntrex +task: ntrex_fra_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_hau_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_hau_Latn-eng_Latn.yaml new file mode 100644 index 00000000..a3c6f721 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_hau_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: hau_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Hausa and English linguist, translate the following Hausa sentences\ + \ to English \nHausa: {{sentence_hau_Latn}}\nEnglish: " +include: ntrex +task: ntrex_hau_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_ibo_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_ibo_Latn-eng_Latn.yaml new file mode 100644 index 00000000..d4b7e768 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_ibo_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ibo_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Igbo and English linguist, translate the following Igbo sentences\ + \ to English \nIgbo: {{sentence_ibo_Latn}}\nEnglish: " +include: ntrex +task: ntrex_ibo_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_kin_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_kin_Latn-eng_Latn.yaml new file mode 100644 index 00000000..bba2d323 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_kin_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kin_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Kinyarwanda and English linguist, translate the following Kinyarwanda\ + \ sentences to English \nKinyarwanda: {{sentence_kin_Latn}}\nEnglish: " +include: ntrex +task: ntrex_kin_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_mey_Arab-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_mey_Arab-eng_Latn.yaml new file mode 100644 index 00000000..9a567548 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_mey_Arab-eng_Latn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: mey_Arab +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Hassaniya Arabic and English linguist, translate the following\ + \ Hassaniya Arabic sentences to English \nHassaniya Arabic: {{sentence_mey_Arab}}\n\ + English: " +include: ntrex +task: ntrex_mey_Arab-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_mlg_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_mlg_Latn-eng_Latn.yaml new file mode 100644 index 00000000..23d55c5a --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_mlg_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: mlg_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Malagasy and English linguist, translate the following Malagasy\ + \ sentences to English \nMalagasy: {{sentence_mlg_Latn}}\nEnglish: " +include: ntrex +task: ntrex_mlg_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_msa_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_msa_Latn-eng_Latn.yaml new file mode 100644 index 00000000..fa1a9618 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_msa_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: msa_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Malay and English linguist, translate the following Malay sentences\ + \ to English \nMalay: {{sentence_msa_Latn}}\nEnglish: " +include: ntrex +task: ntrex_msa_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_nde_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_nde_Latn-eng_Latn.yaml new file mode 100644 index 00000000..855defd0 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_nde_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nde_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a North Ndebele and English linguist, translate the following North\ + \ Ndebele sentences to English \nNorth Ndebele: {{sentence_nde_Latn}}\nEnglish: " +include: ntrex +task: ntrex_nde_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_nso_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_nso_Latn-eng_Latn.yaml new file mode 100644 index 00000000..29a7452c --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_nso_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nso_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Northern Sotho and English linguist, translate the following Northern\ + \ Sotho sentences to English \nNorthern Sotho: {{sentence_nso_Latn}}\nEnglish: " +include: ntrex +task: ntrex_nso_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_nya_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_nya_Latn-eng_Latn.yaml new file mode 100644 index 00000000..50558651 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_nya_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nya_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Chichewa and English linguist, translate the following Chichewa\ + \ sentences to English \nChichewa: {{sentence_nya_Latn}}\nEnglish: " +include: ntrex +task: ntrex_nya_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_orm_Ethi-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_orm_Ethi-eng_Latn.yaml new file mode 100644 index 00000000..9a38e931 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_orm_Ethi-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: orm_Ethi +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Oromo and English linguist, translate the following Oromo sentences\ + \ to English \nOromo: {{sentence_orm_Ethi}}\nEnglish: " +include: ntrex +task: ntrex_orm_Ethi-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_shi_Arab-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_shi_Arab-eng_Latn.yaml new file mode 100644 index 00000000..19f363ef --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_shi_Arab-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: shi_Arab +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Tachelhit and English linguist, translate the following Tachelhit\ + \ sentences to English \nTachelhit: {{sentence_shi_Arab}}\nEnglish: " +include: ntrex +task: ntrex_shi_Arab-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_sna_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_sna_Latn-eng_Latn.yaml new file mode 100644 index 00000000..1c7a6315 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_sna_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sna_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Shona (Latin) and English linguist, translate the following Shona\ + \ (Latin) sentences to English \nShona (Latin): {{sentence_sna_Latn}}\nEnglish: " +include: ntrex +task: ntrex_sna_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_som_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_som_Latn-eng_Latn.yaml new file mode 100644 index 00000000..685f3823 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_som_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: som_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Somali and English linguist, translate the following Somali sentences\ + \ to English \nSomali: {{sentence_som_Latn}}\nEnglish: " +include: ntrex +task: ntrex_som_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_ssw_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_ssw_Latn-eng_Latn.yaml new file mode 100644 index 00000000..dd95665f --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_ssw_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ssw_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Swati and English linguist, translate the following Swati sentences\ + \ to English \nSwati: {{sentence_ssw_Latn}}\nEnglish: " +include: ntrex +task: ntrex_ssw_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_swa_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_swa_Latn-eng_Latn.yaml new file mode 100644 index 00000000..d0731d37 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_swa_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: swa_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Swahili and English linguist, translate the following Swahili sentences\ + \ to English \nSwahili: {{sentence_swa_Latn}}\nEnglish: " +include: ntrex +task: ntrex_swa_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_tam_Taml-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_tam_Taml-eng_Latn.yaml new file mode 100644 index 00000000..834320d8 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_tam_Taml-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tam_Taml +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Tamil and English linguist, translate the following Tamil sentences\ + \ to English \nTamil: {{sentence_tam_Taml}}\nEnglish: " +include: ntrex +task: ntrex_tam_Taml-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_tel_Telu-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_tel_Telu-eng_Latn.yaml new file mode 100644 index 00000000..7fde743d --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_tel_Telu-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tel_Telu +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Telugu and English linguist, translate the following Telugu sentences\ + \ to English \nTelugu: {{sentence_tel_Telu}}\nEnglish: " +include: ntrex +task: ntrex_tel_Telu-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_tir_Ethi-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_tir_Ethi-eng_Latn.yaml new file mode 100644 index 00000000..60189ee7 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_tir_Ethi-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tir_Ethi +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Tigrinya and English linguist, translate the following Tigrinya\ + \ sentences to English \nTigrinya: {{sentence_tir_Ethi}}\nEnglish: " +include: ntrex +task: ntrex_tir_Ethi-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_ton_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_ton_Latn-eng_Latn.yaml new file mode 100644 index 00000000..ec2b5ba9 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_ton_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ton_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Tongan and English linguist, translate the following Tongan sentences\ + \ to English \nTongan: {{sentence_ton_Latn}}\nEnglish: " +include: ntrex +task: ntrex_ton_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_tsn_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_tsn_Latn-eng_Latn.yaml new file mode 100644 index 00000000..fa63ca4b --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_tsn_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tsn_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Tswana and English linguist, translate the following Tswana sentences\ + \ to English \nTswana: {{sentence_tsn_Latn}}\nEnglish: " +include: ntrex +task: ntrex_tsn_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_urd_Arab-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_urd_Arab-eng_Latn.yaml new file mode 100644 index 00000000..2b520795 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_urd_Arab-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: urd_Arab +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Urdu and English linguist, translate the following Urdu sentences\ + \ to English \nUrdu: {{sentence_urd_Arab}}\nEnglish: " +include: ntrex +task: ntrex_urd_Arab-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_ven_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_ven_Latn-eng_Latn.yaml new file mode 100644 index 00000000..82372de2 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_ven_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ven_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Venda and English linguist, translate the following Venda sentences\ + \ to English \nVenda: {{sentence_ven_Latn}}\nEnglish: " +include: ntrex +task: ntrex_ven_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_wol_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_wol_Latn-eng_Latn.yaml new file mode 100644 index 00000000..ae0124f2 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_wol_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: wol_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Wolof and English linguist, translate the following Wolof sentences\ + \ to English \nWolof: {{sentence_wol_Latn}}\nEnglish: " +include: ntrex +task: ntrex_wol_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_xho_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_xho_Latn-eng_Latn.yaml new file mode 100644 index 00000000..7f0528af --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_xho_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: xho_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Xhosa and English linguist, translate the following Xhosa sentences\ + \ to English \nXhosa: {{sentence_xho_Latn}}\nEnglish: " +include: ntrex +task: ntrex_xho_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_yor_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_yor_Latn-eng_Latn.yaml new file mode 100644 index 00000000..99d7cf49 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_yor_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: yor_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Yoruba and English linguist, translate the following Yoruba sentences\ + \ to English \nYoruba: {{sentence_yor_Latn}}\nEnglish: " +include: ntrex +task: ntrex_yor_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_zul_Latn-eng_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_zul_Latn-eng_Latn.yaml new file mode 100644 index 00000000..30f3b307 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/african-english/ntrex_zul_Latn-eng_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: zul_Latn +doc_to_target: sentence_eng_Latn +doc_to_text: "As a Zulu and English linguist, translate the following Zulu sentences\ + \ to English \nZulu: {{sentence_zul_Latn}}\nEnglish: " +include: ntrex +task: ntrex_zul_Latn-eng_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex new file mode 100644 index 00000000..d001e1f6 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex @@ -0,0 +1,25 @@ +tag: +- ntrex_eng-afr +- ntrex_eng-afr_prompt_3 +- afrobench_MT_tasks +dataset_path: masakhane/ntrex_african +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: test +fewshot_split: test +test_split: test +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +repeats: 1 +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-afr_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-afr_Latn.yaml new file mode 100644 index 00000000..4aaa928b --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-afr_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: afr_Latn +doc_to_target: sentence_afr_Latn +doc_to_text: "As a Afrikaans and English linguist, translate the following English\ + \ sentences to Afrikaans \nEnglish: {{sentence_eng_Latn}} \nAfrikaans: " +include: ntrex +task: ntrex_eng_Latn-afr_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-amh_Ethi.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-amh_Ethi.yaml new file mode 100644 index 00000000..008f7302 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-amh_Ethi.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: amh_Ethi +doc_to_target: sentence_amh_Ethi +doc_to_text: "As a Amharic and English linguist, translate the following English sentences\ + \ to Amharic \nEnglish: {{sentence_eng_Latn}} \nAmharic: " +include: ntrex +task: ntrex_eng_Latn-amh_Ethi_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-arb_Arab.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-arb_Arab.yaml new file mode 100644 index 00000000..d0c9e813 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-arb_Arab.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: arb_Arab +doc_to_target: sentence_arb_Arab +doc_to_text: "As a Arabic and English linguist, translate the following English sentences\ + \ to Arabic \nEnglish: {{sentence_eng_Latn}} \nArabic: " +include: ntrex +task: ntrex_eng_Latn-arb_Arab_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-bem_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-bem_Latn.yaml new file mode 100644 index 00000000..e4ab2af3 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-bem_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: bem_Latn +doc_to_target: sentence_bem_Latn +doc_to_text: "As a Bemba and English linguist, translate the following English sentences\ + \ to Bemba \nEnglish: {{sentence_eng_Latn}} \nBemba: " +include: ntrex +task: ntrex_eng_Latn-bem_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-ewe_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-ewe_Latn.yaml new file mode 100644 index 00000000..e1c99ad0 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-ewe_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ewe_Latn +doc_to_target: sentence_ewe_Latn +doc_to_text: "As a Ewe and English linguist, translate the following English sentences\ + \ to Ewe \nEnglish: {{sentence_eng_Latn}} \nEwe: " +include: ntrex +task: ntrex_eng_Latn-ewe_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-fra_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-fra_Latn.yaml new file mode 100644 index 00000000..3668db57 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-fra_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fra_Latn +doc_to_target: sentence_fra_Latn +doc_to_text: "As a French and English linguist, translate the following English sentences\ + \ to French \nEnglish: {{sentence_eng_Latn}} \nFrench: " +include: ntrex +task: ntrex_eng_Latn-fra_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-hau_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-hau_Latn.yaml new file mode 100644 index 00000000..6bca042c --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-hau_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: hau_Latn +doc_to_target: sentence_hau_Latn +doc_to_text: "As a Hausa and English linguist, translate the following English sentences\ + \ to Hausa \nEnglish: {{sentence_eng_Latn}} \nHausa: " +include: ntrex +task: ntrex_eng_Latn-hau_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-ibo_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-ibo_Latn.yaml new file mode 100644 index 00000000..c23fcce8 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-ibo_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ibo_Latn +doc_to_target: sentence_ibo_Latn +doc_to_text: "As a Igbo and English linguist, translate the following English sentences\ + \ to Igbo \nEnglish: {{sentence_eng_Latn}} \nIgbo: " +include: ntrex +task: ntrex_eng_Latn-ibo_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-kin_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-kin_Latn.yaml new file mode 100644 index 00000000..b0041bfb --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-kin_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kin_Latn +doc_to_target: sentence_kin_Latn +doc_to_text: "As a Kinyarwanda and English linguist, translate the following English\ + \ sentences to Kinyarwanda \nEnglish: {{sentence_eng_Latn}} \nKinyarwanda: " +include: ntrex +task: ntrex_eng_Latn-kin_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-mey_Arab.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-mey_Arab.yaml new file mode 100644 index 00000000..435df83d --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-mey_Arab.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: mey_Arab +doc_to_target: sentence_mey_Arab +doc_to_text: "As a Hassaniya Arabic and English linguist, translate the following\ + \ English sentences to Hassaniya Arabic \nEnglish: {{sentence_eng_Latn}} \nHassaniya\ + \ Arabic: " +include: ntrex +task: ntrex_eng_Latn-mey_Arab_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-mlg_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-mlg_Latn.yaml new file mode 100644 index 00000000..74f92d92 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-mlg_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: mlg_Latn +doc_to_target: sentence_mlg_Latn +doc_to_text: "As a Malagasy and English linguist, translate the following English\ + \ sentences to Malagasy \nEnglish: {{sentence_eng_Latn}} \nMalagasy: " +include: ntrex +task: ntrex_eng_Latn-mlg_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-msa_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-msa_Latn.yaml new file mode 100644 index 00000000..bc9a3245 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-msa_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: msa_Latn +doc_to_target: sentence_msa_Latn +doc_to_text: "As a Malay and English linguist, translate the following English sentences\ + \ to Malay \nEnglish: {{sentence_eng_Latn}} \nMalay: " +include: ntrex +task: ntrex_eng_Latn-msa_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-nde_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-nde_Latn.yaml new file mode 100644 index 00000000..f7cf0924 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-nde_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nde_Latn +doc_to_target: sentence_nde_Latn +doc_to_text: "As a North Ndebele and English linguist, translate the following English\ + \ sentences to North Ndebele \nEnglish: {{sentence_eng_Latn}} \nNorth Ndebele: " +include: ntrex +task: ntrex_eng_Latn-nde_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-nso_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-nso_Latn.yaml new file mode 100644 index 00000000..d52c1ef1 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-nso_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nso_Latn +doc_to_target: sentence_nso_Latn +doc_to_text: "As a Northern Sotho and English linguist, translate the following English\ + \ sentences to Northern Sotho \nEnglish: {{sentence_eng_Latn}} \nNorthern Sotho: " +include: ntrex +task: ntrex_eng_Latn-nso_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-nya_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-nya_Latn.yaml new file mode 100644 index 00000000..5a3d3955 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-nya_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nya_Latn +doc_to_target: sentence_nya_Latn +doc_to_text: "As a Chichewa and English linguist, translate the following English\ + \ sentences to Chichewa \nEnglish: {{sentence_eng_Latn}} \nChichewa: " +include: ntrex +task: ntrex_eng_Latn-nya_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-orm_Ethi.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-orm_Ethi.yaml new file mode 100644 index 00000000..d3de07b0 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-orm_Ethi.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: orm_Ethi +doc_to_target: sentence_orm_Ethi +doc_to_text: "As a Oromo and English linguist, translate the following English sentences\ + \ to Oromo \nEnglish: {{sentence_eng_Latn}} \nOromo: " +include: ntrex +task: ntrex_eng_Latn-orm_Ethi_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-shi_Arab.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-shi_Arab.yaml new file mode 100644 index 00000000..e193c7a3 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-shi_Arab.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: shi_Arab +doc_to_target: sentence_shi_Arab +doc_to_text: "As a Tachelhit and English linguist, translate the following English\ + \ sentences to Tachelhit \nEnglish: {{sentence_eng_Latn}} \nTachelhit: " +include: ntrex +task: ntrex_eng_Latn-shi_Arab_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-sna_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-sna_Latn.yaml new file mode 100644 index 00000000..ce8c50f5 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-sna_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sna_Latn +doc_to_target: sentence_sna_Latn +doc_to_text: "As a Shona (Latin) and English linguist, translate the following English\ + \ sentences to Shona (Latin) \nEnglish: {{sentence_eng_Latn}} \nShona (Latin): " +include: ntrex +task: ntrex_eng_Latn-sna_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-som_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-som_Latn.yaml new file mode 100644 index 00000000..4b7f4632 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-som_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: som_Latn +doc_to_target: sentence_som_Latn +doc_to_text: "As a Somali and English linguist, translate the following English sentences\ + \ to Somali \nEnglish: {{sentence_eng_Latn}} \nSomali: " +include: ntrex +task: ntrex_eng_Latn-som_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-ssw_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-ssw_Latn.yaml new file mode 100644 index 00000000..6f02e88c --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-ssw_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ssw_Latn +doc_to_target: sentence_ssw_Latn +doc_to_text: "As a Swati and English linguist, translate the following English sentences\ + \ to Swati \nEnglish: {{sentence_eng_Latn}} \nSwati: " +include: ntrex +task: ntrex_eng_Latn-ssw_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-swa_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-swa_Latn.yaml new file mode 100644 index 00000000..47090821 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-swa_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: swa_Latn +doc_to_target: sentence_swa_Latn +doc_to_text: "As a Swahili and English linguist, translate the following English sentences\ + \ to Swahili \nEnglish: {{sentence_eng_Latn}} \nSwahili: " +include: ntrex +task: ntrex_eng_Latn-swa_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-tam_Taml.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-tam_Taml.yaml new file mode 100644 index 00000000..78d61866 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-tam_Taml.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tam_Taml +doc_to_target: sentence_tam_Taml +doc_to_text: "As a Tamil and English linguist, translate the following English sentences\ + \ to Tamil \nEnglish: {{sentence_eng_Latn}} \nTamil: " +include: ntrex +task: ntrex_eng_Latn-tam_Taml_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-tel_Telu.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-tel_Telu.yaml new file mode 100644 index 00000000..82963531 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-tel_Telu.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tel_Telu +doc_to_target: sentence_tel_Telu +doc_to_text: "As a Telugu and English linguist, translate the following English sentences\ + \ to Telugu \nEnglish: {{sentence_eng_Latn}} \nTelugu: " +include: ntrex +task: ntrex_eng_Latn-tel_Telu_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-tir_Ethi.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-tir_Ethi.yaml new file mode 100644 index 00000000..1f27f438 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-tir_Ethi.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tir_Ethi +doc_to_target: sentence_tir_Ethi +doc_to_text: "As a Tigrinya and English linguist, translate the following English\ + \ sentences to Tigrinya \nEnglish: {{sentence_eng_Latn}} \nTigrinya: " +include: ntrex +task: ntrex_eng_Latn-tir_Ethi_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-ton_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-ton_Latn.yaml new file mode 100644 index 00000000..3ffeb74f --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-ton_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ton_Latn +doc_to_target: sentence_ton_Latn +doc_to_text: "As a Tongan and English linguist, translate the following English sentences\ + \ to Tongan \nEnglish: {{sentence_eng_Latn}} \nTongan: " +include: ntrex +task: ntrex_eng_Latn-ton_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-tsn_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-tsn_Latn.yaml new file mode 100644 index 00000000..ed11f2cb --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-tsn_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tsn_Latn +doc_to_target: sentence_tsn_Latn +doc_to_text: "As a Tswana and English linguist, translate the following English sentences\ + \ to Tswana \nEnglish: {{sentence_eng_Latn}} \nTswana: " +include: ntrex +task: ntrex_eng_Latn-tsn_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-urd_Arab.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-urd_Arab.yaml new file mode 100644 index 00000000..a05e951b --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-urd_Arab.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: urd_Arab +doc_to_target: sentence_urd_Arab +doc_to_text: "As a Urdu and English linguist, translate the following English sentences\ + \ to Urdu \nEnglish: {{sentence_eng_Latn}} \nUrdu: " +include: ntrex +task: ntrex_eng_Latn-urd_Arab_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-ven_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-ven_Latn.yaml new file mode 100644 index 00000000..43452016 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-ven_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ven_Latn +doc_to_target: sentence_ven_Latn +doc_to_text: "As a Venda and English linguist, translate the following English sentences\ + \ to Venda \nEnglish: {{sentence_eng_Latn}} \nVenda: " +include: ntrex +task: ntrex_eng_Latn-ven_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-wol_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-wol_Latn.yaml new file mode 100644 index 00000000..48abbb33 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-wol_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: wol_Latn +doc_to_target: sentence_wol_Latn +doc_to_text: "As a Wolof and English linguist, translate the following English sentences\ + \ to Wolof \nEnglish: {{sentence_eng_Latn}} \nWolof: " +include: ntrex +task: ntrex_eng_Latn-wol_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-xho_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-xho_Latn.yaml new file mode 100644 index 00000000..b1071a5f --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-xho_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: xho_Latn +doc_to_target: sentence_xho_Latn +doc_to_text: "As a Xhosa and English linguist, translate the following English sentences\ + \ to Xhosa \nEnglish: {{sentence_eng_Latn}} \nXhosa: " +include: ntrex +task: ntrex_eng_Latn-xho_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-yor_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-yor_Latn.yaml new file mode 100644 index 00000000..43c1be35 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-yor_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: yor_Latn +doc_to_target: sentence_yor_Latn +doc_to_text: "As a Yoruba and English linguist, translate the following English sentences\ + \ to Yoruba \nEnglish: {{sentence_eng_Latn}} \nYoruba: " +include: ntrex +task: ntrex_eng_Latn-yor_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-zul_Latn.yaml b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-zul_Latn.yaml new file mode 100644 index 00000000..10e890a9 --- /dev/null +++ b/lm_eval/tasks/afrobench/ntrex/prompt_3/english-african/ntrex_eng_Latn-zul_Latn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: zul_Latn +doc_to_target: sentence_zul_Latn +doc_to_text: "As a Zulu and English linguist, translate the following English sentences\ + \ to Zulu \nEnglish: {{sentence_eng_Latn}} \nZulu: " +include: ntrex +task: ntrex_eng_Latn-zul_Latn_prompt_3 diff --git a/lm_eval/tasks/afrobench/openai_mmlu/README.md b/lm_eval/tasks/afrobench/openai_mmlu/README.md new file mode 100644 index 00000000..fe980e87 --- /dev/null +++ b/lm_eval/tasks/afrobench/openai_mmlu/README.md @@ -0,0 +1,25 @@ +# + +## Paper +Title: `Multilingual Massive Multitask Language Understanding (MMMLU)` + +Paper Link: https://arxiv.org/abs/2009.03300 + +## Abstract +>We propose a new test to measure a text model's multitask accuracy. The test covers 57 tasks including elementary mathematics, US history, computer science, law, and more. To attain high accuracy on this test, models must possess extensive world knowledge and problem solving ability. We find that while most recent models have near random-chance accuracy, the very largest GPT-3 model improves over random chance by almost 20 percentage points on average. However, on every one of the 57 tasks, the best models still need substantial improvements before they can reach expert-level accuracy. Models also have lopsided performance and frequently do not know when they are wrong. Worse, they still have near-random accuracy on some socially important subjects such as morality and law. By comprehensively evaluating the breadth and depth of a model's academic and professional understanding, our test can be used to analyze models across many tasks and to identify important shortcomings. + +HomePage: https://huggingface.co/datasets/openai/MMMLU + +### Citation + +``` +@misc{hendrycks2021measuringmassivemultitasklanguage, + title={Measuring Massive Multitask Language Understanding}, + author={Dan Hendrycks and Collin Burns and Steven Basart and Andy Zou and Mantas Mazeika and Dawn Song and Jacob Steinhardt}, + year={2021}, + eprint={2009.03300}, + archivePrefix={arXiv}, + primaryClass={cs.CY}, + url={https://arxiv.org/abs/2009.03300}, +} +``` diff --git a/lm_eval/tasks/afrobench/openai_mmlu/openai_mmlu.yaml b/lm_eval/tasks/afrobench/openai_mmlu/openai_mmlu.yaml new file mode 100644 index 00000000..541eb43c --- /dev/null +++ b/lm_eval/tasks/afrobench/openai_mmlu/openai_mmlu.yaml @@ -0,0 +1,13 @@ +group: openai_mmlu +task: + - openai_mmlu_prompt_1 + - openai_mmlu_prompt_2 + - openai_mmlu_prompt_3 + - openai_mmlu_prompt_4 + - openai_mmlu_prompt_5 +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 1 diff --git a/lm_eval/tasks/afrobench/openai_mmlu/prompt_1/openai_mmlu b/lm_eval/tasks/afrobench/openai_mmlu/prompt_1/openai_mmlu new file mode 100644 index 00000000..ce4f02ee --- /dev/null +++ b/lm_eval/tasks/afrobench/openai_mmlu/prompt_1/openai_mmlu @@ -0,0 +1,22 @@ +tag: + - openai_mmlu_tasks + - openai_mmlu_prompt_1 + - afrobench_mmlu_tasks +dataset_path: openai/MMMLU +output_type: multiple_choice +test_split: test +fewshot_config: + sampler: first_n +doc_to_target: "{{['A', 'B', 'C', 'D'].index(Answer.strip())}}" +should_decontaminate: true +doc_to_decontamination_query: "{{Question}}" +doc_to_choice: ["A", "B", "C", "D"] +metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/openai_mmlu/prompt_1/openai_mmlu_ara.yaml b/lm_eval/tasks/afrobench/openai_mmlu/prompt_1/openai_mmlu_ara.yaml new file mode 100644 index 00000000..5c9b86fc --- /dev/null +++ b/lm_eval/tasks/afrobench/openai_mmlu/prompt_1/openai_mmlu_ara.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: AR_XY +doc_to_text: 'Q: {{Question.strip()}} + + A: {{A}} + + B: {{B}} + + C: {{C}} + + D: {{D}} + + Please choose the correct answer from the options above:' +include: openai_mmlu +task: openai_mmlu_ara_prompt_1 diff --git a/lm_eval/tasks/afrobench/openai_mmlu/prompt_1/openai_mmlu_swa.yaml b/lm_eval/tasks/afrobench/openai_mmlu/prompt_1/openai_mmlu_swa.yaml new file mode 100644 index 00000000..1a3661d4 --- /dev/null +++ b/lm_eval/tasks/afrobench/openai_mmlu/prompt_1/openai_mmlu_swa.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: SW_KE +doc_to_text: 'Q: {{Question.strip()}} + + A: {{A}} + + B: {{B}} + + C: {{C}} + + D: {{D}} + + Please choose the correct answer from the options above:' +include: openai_mmlu +task: openai_mmlu_swa_prompt_1 diff --git a/lm_eval/tasks/afrobench/openai_mmlu/prompt_1/openai_mmlu_yor.yaml b/lm_eval/tasks/afrobench/openai_mmlu/prompt_1/openai_mmlu_yor.yaml new file mode 100644 index 00000000..4124252b --- /dev/null +++ b/lm_eval/tasks/afrobench/openai_mmlu/prompt_1/openai_mmlu_yor.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: YO_NG +doc_to_text: 'Q: {{Question.strip()}} + + A: {{A}} + + B: {{B}} + + C: {{C}} + + D: {{D}} + + Please choose the correct answer from the options above:' +include: openai_mmlu +task: openai_mmlu_yor_prompt_1 diff --git a/lm_eval/tasks/afrobench/openai_mmlu/prompt_2/openai_mmlu b/lm_eval/tasks/afrobench/openai_mmlu/prompt_2/openai_mmlu new file mode 100644 index 00000000..9f39b0a9 --- /dev/null +++ b/lm_eval/tasks/afrobench/openai_mmlu/prompt_2/openai_mmlu @@ -0,0 +1,22 @@ +tag: + - openai_mmlu_tasks + - openai_mmlu_prompt_2 + - afrobench_mmlu_tasks +dataset_path: openai/MMMLU +output_type: multiple_choice +test_split: test +fewshot_config: + sampler: first_n +doc_to_target: "{{['A', 'B', 'C', 'D'].index(Answer.strip())}}" +should_decontaminate: true +doc_to_decontamination_query: "{{Question}}" +doc_to_choice: ["A", "B", "C", "D"] +metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/openai_mmlu/prompt_2/openai_mmlu_ara.yaml b/lm_eval/tasks/afrobench/openai_mmlu/prompt_2/openai_mmlu_ara.yaml new file mode 100644 index 00000000..55083425 --- /dev/null +++ b/lm_eval/tasks/afrobench/openai_mmlu/prompt_2/openai_mmlu_ara.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: AR_XY +doc_to_text: 'Question: {{Question.strip()}} + + 1: {{A}} + + 2: {{B}} + + 3: {{C}} + + 4: {{D}} + + Please select the correct answer from the given choices:' +include: openai_mmlu +task: openai_mmlu_ara_prompt_2 diff --git a/lm_eval/tasks/afrobench/openai_mmlu/prompt_2/openai_mmlu_swa.yaml b/lm_eval/tasks/afrobench/openai_mmlu/prompt_2/openai_mmlu_swa.yaml new file mode 100644 index 00000000..9b3025fd --- /dev/null +++ b/lm_eval/tasks/afrobench/openai_mmlu/prompt_2/openai_mmlu_swa.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: SW_KE +doc_to_text: 'Question: {{Question.strip()}} + + 1: {{A}} + + 2: {{B}} + + 3: {{C}} + + 4: {{D}} + + Please select the correct answer from the given choices:' +include: openai_mmlu +task: openai_mmlu_swa_prompt_2 diff --git a/lm_eval/tasks/afrobench/openai_mmlu/prompt_2/openai_mmlu_yor.yaml b/lm_eval/tasks/afrobench/openai_mmlu/prompt_2/openai_mmlu_yor.yaml new file mode 100644 index 00000000..145b237e --- /dev/null +++ b/lm_eval/tasks/afrobench/openai_mmlu/prompt_2/openai_mmlu_yor.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: YO_NG +doc_to_text: 'Question: {{Question.strip()}} + + 1: {{A}} + + 2: {{B}} + + 3: {{C}} + + 4: {{D}} + + Please select the correct answer from the given choices:' +include: openai_mmlu +task: openai_mmlu_yor_prompt_2 diff --git a/lm_eval/tasks/afrobench/openai_mmlu/prompt_3/openai_mmlu b/lm_eval/tasks/afrobench/openai_mmlu/prompt_3/openai_mmlu new file mode 100644 index 00000000..95456656 --- /dev/null +++ b/lm_eval/tasks/afrobench/openai_mmlu/prompt_3/openai_mmlu @@ -0,0 +1,23 @@ +tag: + - openai_mmlu_tasks + - openai_mmlu_prompt_3 + - afrobench_mmlu_tasks +dataset_path: openai/MMMLU +dataset_name: null +output_type: multiple_choice +test_split: test +fewshot_config: + sampler: first_n +doc_to_target: "{{['A', 'B', 'C', 'D'].index(Answer.strip())}}" +should_decontaminate: true +doc_to_decontamination_query: "{{Question}}" +doc_to_choice: ["A", "B", "C", "D"] +metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/openai_mmlu/prompt_3/openai_mmlu_ara.yaml b/lm_eval/tasks/afrobench/openai_mmlu/prompt_3/openai_mmlu_ara.yaml new file mode 100644 index 00000000..012192ce --- /dev/null +++ b/lm_eval/tasks/afrobench/openai_mmlu/prompt_3/openai_mmlu_ara.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: AR_XY +doc_to_text: 'Input Question: {{Question.strip()}} + + Option A: {{A}} + + Option B: {{B}} + + Option C: {{C}} + + Option D: {{D}} + + Please indicate the correct option from the list above:' +include: openai_mmlu +task: openai_mmlu_ara_prompt_3 diff --git a/lm_eval/tasks/afrobench/openai_mmlu/prompt_3/openai_mmlu_swa.yaml b/lm_eval/tasks/afrobench/openai_mmlu/prompt_3/openai_mmlu_swa.yaml new file mode 100644 index 00000000..431bdb34 --- /dev/null +++ b/lm_eval/tasks/afrobench/openai_mmlu/prompt_3/openai_mmlu_swa.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: SW_KE +doc_to_text: 'Input Question: {{Question.strip()}} + + Option A: {{A}} + + Option B: {{B}} + + Option C: {{C}} + + Option D: {{D}} + + Please indicate the correct option from the list above:' +include: openai_mmlu +task: openai_mmlu_swa_prompt_3 diff --git a/lm_eval/tasks/afrobench/openai_mmlu/prompt_3/openai_mmlu_yor.yaml b/lm_eval/tasks/afrobench/openai_mmlu/prompt_3/openai_mmlu_yor.yaml new file mode 100644 index 00000000..814fe380 --- /dev/null +++ b/lm_eval/tasks/afrobench/openai_mmlu/prompt_3/openai_mmlu_yor.yaml @@ -0,0 +1,15 @@ +# Generated by utils.py +dataset_name: YO_NG +doc_to_text: 'Input Question: {{Question.strip()}} + + Option A: {{A}} + + Option B: {{B}} + + Option C: {{C}} + + Option D: {{D}} + + Please indicate the correct option from the list above:' +include: openai_mmlu +task: openai_mmlu_yor_prompt_3 diff --git a/lm_eval/tasks/afrobench/openai_mmlu/prompt_4/openai_mmlu b/lm_eval/tasks/afrobench/openai_mmlu/prompt_4/openai_mmlu new file mode 100644 index 00000000..37a5949f --- /dev/null +++ b/lm_eval/tasks/afrobench/openai_mmlu/prompt_4/openai_mmlu @@ -0,0 +1,23 @@ +tag: + - openai_mmlu_tasks + - openai_mmlu_prompt_4 + - afrobench_mmlu_tasks +dataset_path: openai/MMMLU +dataset_name: null +output_type: multiple_choice +test_split: test +fewshot_config: + sampler: first_n +doc_to_target: "{{['A', 'B', 'C', 'D'].index(Answer.strip())}}" +should_decontaminate: true +doc_to_decontamination_query: "{{Question}}" +doc_to_choice: ["A", "B", "C", "D"] +metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/openai_mmlu/prompt_4/openai_mmlu_ara.yaml b/lm_eval/tasks/afrobench/openai_mmlu/prompt_4/openai_mmlu_ara.yaml new file mode 100644 index 00000000..793eb744 --- /dev/null +++ b/lm_eval/tasks/afrobench/openai_mmlu/prompt_4/openai_mmlu_ara.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: AR_XY +doc_to_text: 'Critically analyze the question and select the most probable answer + from the list: + + {{Question.strip()}} + + Choices: + + A) {{A}} + + B) {{B}} + + C) {{C}} + + D) {{D}}' +include: openai_mmlu +task: openai_mmlu_ara_prompt_4 diff --git a/lm_eval/tasks/afrobench/openai_mmlu/prompt_4/openai_mmlu_swa.yaml b/lm_eval/tasks/afrobench/openai_mmlu/prompt_4/openai_mmlu_swa.yaml new file mode 100644 index 00000000..095dd7ff --- /dev/null +++ b/lm_eval/tasks/afrobench/openai_mmlu/prompt_4/openai_mmlu_swa.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: SW_KE +doc_to_text: 'Critically analyze the question and select the most probable answer + from the list: + + {{Question.strip()}} + + Choices: + + A) {{A}} + + B) {{B}} + + C) {{C}} + + D) {{D}}' +include: openai_mmlu +task: openai_mmlu_swa_prompt_4 diff --git a/lm_eval/tasks/afrobench/openai_mmlu/prompt_4/openai_mmlu_yor.yaml b/lm_eval/tasks/afrobench/openai_mmlu/prompt_4/openai_mmlu_yor.yaml new file mode 100644 index 00000000..dd0a9daa --- /dev/null +++ b/lm_eval/tasks/afrobench/openai_mmlu/prompt_4/openai_mmlu_yor.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: YO_NG +doc_to_text: 'Critically analyze the question and select the most probable answer + from the list: + + {{Question.strip()}} + + Choices: + + A) {{A}} + + B) {{B}} + + C) {{C}} + + D) {{D}}' +include: openai_mmlu +task: openai_mmlu_yor_prompt_4 diff --git a/lm_eval/tasks/afrobench/openai_mmlu/prompt_5/openai_mmlu b/lm_eval/tasks/afrobench/openai_mmlu/prompt_5/openai_mmlu new file mode 100644 index 00000000..77183eb0 --- /dev/null +++ b/lm_eval/tasks/afrobench/openai_mmlu/prompt_5/openai_mmlu @@ -0,0 +1,23 @@ +tag: + - openai_mmlu_tasks + - openai_mmlu_prompt_5 + - afrobench_mmlu_tasks +dataset_path: openai/MMMLU +dataset_name: null +output_type: multiple_choice +test_split: test +fewshot_config: + sampler: first_n +doc_to_target: "{{['A', 'B', 'C', 'D'].index(Answer.strip())}}" +should_decontaminate: true +doc_to_decontamination_query: "{{Question}}" +doc_to_choice: ["A", "B", "C", "D"] +metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/openai_mmlu/prompt_5/openai_mmlu_ara.yaml b/lm_eval/tasks/afrobench/openai_mmlu/prompt_5/openai_mmlu_ara.yaml new file mode 100644 index 00000000..50a6e74f --- /dev/null +++ b/lm_eval/tasks/afrobench/openai_mmlu/prompt_5/openai_mmlu_ara.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: AR_XY +doc_to_text: 'Answer the question and pick the correct answer from the options: {{Question.strip()}} + + Options: + + A. {{A}} + + B. {{B}} + + C. {{C}} + + D. {{D}} + + Please choose the correct option from the above list:' +include: openai_mmlu +task: openai_mmlu_ara_prompt_5 diff --git a/lm_eval/tasks/afrobench/openai_mmlu/prompt_5/openai_mmlu_swa.yaml b/lm_eval/tasks/afrobench/openai_mmlu/prompt_5/openai_mmlu_swa.yaml new file mode 100644 index 00000000..c0cc1986 --- /dev/null +++ b/lm_eval/tasks/afrobench/openai_mmlu/prompt_5/openai_mmlu_swa.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: SW_KE +doc_to_text: 'Answer the question and pick the correct answer from the options: {{Question.strip()}} + + Options: + + A. {{A}} + + B. {{B}} + + C. {{C}} + + D. {{D}} + + Please choose the correct option from the above list:' +include: openai_mmlu +task: openai_mmlu_swa_prompt_5 diff --git a/lm_eval/tasks/afrobench/openai_mmlu/prompt_5/openai_mmlu_yor.yaml b/lm_eval/tasks/afrobench/openai_mmlu/prompt_5/openai_mmlu_yor.yaml new file mode 100644 index 00000000..691657ef --- /dev/null +++ b/lm_eval/tasks/afrobench/openai_mmlu/prompt_5/openai_mmlu_yor.yaml @@ -0,0 +1,17 @@ +# Generated by utils.py +dataset_name: YO_NG +doc_to_text: 'Answer the question and pick the correct answer from the options: {{Question.strip()}} + + Options: + + A. {{A}} + + B. {{B}} + + C. {{C}} + + D. {{D}} + + Please choose the correct option from the above list:' +include: openai_mmlu +task: openai_mmlu_yor_prompt_5 diff --git a/lm_eval/tasks/afrobench/openai_mmlu/utils.py b/lm_eval/tasks/afrobench/openai_mmlu/utils.py new file mode 100644 index 00000000..0fc0fea9 --- /dev/null +++ b/lm_eval/tasks/afrobench/openai_mmlu/utils.py @@ -0,0 +1,99 @@ +import argparse +import os + +import yaml + + +def prompt_func(mode, lang): + prompt_map = { + "prompt_1": "Q: {{Question.strip()}}\nA: {{A}}\nB: {{B}}\nC: {{C}}\nD: {{D}}\nPlease choose the correct answer from the options above:", + "prompt_2": "Question: {{Question.strip()}}\n1: {{A}}\n2: {{B}}\n3: {{C}}\n4: {{D}}\nPlease select the correct answer from the given choices:", + "prompt_3": "Input Question: {{Question.strip()}}\nOption A: {{A}}\nOption B: {{B}}\nOption C: {{C}}\nOption D: {{D}}\nPlease indicate the correct option from the list above:", + "prompt_4": "Critically analyze the question and select the most probable answer from the list:\n{{Question.strip()}}\nChoices:\nA) {{A}}\nB) {{B}}\nC) {{C}}\nD) {{D}}", + "prompt_5": "Answer the question and pick the correct answer from the options: {{Question.strip()}}\nOptions:\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\nPlease choose the correct option from the above list:", + } + return prompt_map[mode] + + +def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None: + """ + Generate a yaml file for each language. + + :param output_dir: The directory to output the files to. + :param overwrite: Whether to overwrite files if they already exist. + """ + err = [] + languages = { + "ara": "Arabic", + "swa": "Swahili", + "yor": "Yoruba", + } + + lang2_code = { + "ara": "AR_XY", + "swa": "SW_KE", + "yor": "YO_NG", + } + + for lang in languages.keys(): + try: + file_name = f"openai_mmlu_{lang}.yaml" + task_name = f"openai_mmlu_{lang}_{mode}" + yaml_template = "openai_mmlu" + yaml_details = { + "include": yaml_template, + "task": task_name, + "dataset_name": lang2_code[lang], + "doc_to_text": prompt_func(mode, languages[lang]), + } + file_path = os.path.join(output_dir, mode) + os.makedirs(file_path, exist_ok=True) + + with open( + f"{output_dir}/{mode}/{file_name}", + "w" if overwrite else "x", + encoding="utf8", + ) as f: + f.write("# Generated by utils.py\n") + yaml.dump( + yaml_details, + f, + allow_unicode=True, + ) + except FileExistsError: + err.append(file_name) + + if len(err) > 0: + raise FileExistsError( + "Files were not created because they already exist (use --overwrite flag):" + f" {', '.join(err)}" + ) + + +def main() -> None: + """Parse CLI args and generate language-specific yaml files.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + default=True, + action="store_true", + help="Overwrite files if they already exist", + ) + parser.add_argument( + "--output-dir", + default="./", + help="Directory to write yaml files to", + ) + parser.add_argument( + "--mode", + default="prompt_1", + choices=["prompt_1", "prompt_2", "prompt_3", "prompt_4", "prompt_5"], + help="Prompt number", + ) + args = parser.parse_args() + + gen_lang_yamls(output_dir=args.output_dir, overwrite=args.overwrite, mode=args.mode) + + +if __name__ == "__main__": + main() diff --git a/lm_eval/tasks/afrobench/salt/README.md b/lm_eval/tasks/afrobench/salt/README.md new file mode 100644 index 00000000..3c5239a0 --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/README.md @@ -0,0 +1,17 @@ +# + +## Paper +Title: `Sunbird African Language Technology (SALT) dataset` + +Paper Link: https://aclanthology.org/2023.emnlp-main.862/ + +## Abstract +>SALT is a multi-way parallel text and speech corpus of Engish and six languages widely spoken in Uganda and East Africa: Luganda, Lugbara, Acholi, Runyankole, Ateso and Swahili. The core of the dataset is a set of 25,000 sentences covering a range of topics of local relevance, such as agriculture, health and society. Each sentence is translated into all languages, to support machine translation, and speech recordings are made for approximately 5,000 of the sentences both by a variety of speakers in natural settings (suitable for ASR) and by professionals in a studio setting (suitable for text-to-speech). + +HomePage: https://github.com/SunbirdAI/salt + +### Publications + +Multilingual Model and Data Resources for Text-To-Speech in Ugandan Languages. Isaac Owomugisha, Benjamin Akera, Ernest Tonny Mwebaze, John Quinn. 4th Workshop on African Natural Language Processing, 2023. [pdf](https://openreview.net/pdf?id=vaxG0WAPzL) + +Machine Translation For African Languages: Community Creation Of Datasets And Models In Uganda. Benjamin Akera, Jonathan Mukiibi, Lydia Sanyu Naggayi, Claire Babirye, Isaac Owomugisha, Solomon Nsumba, Joyce Nakatumba-Nabende, Engineer Bainomugisha, Ernest Mwebaze, John Quinn. 3rd Workshop on African Natural Language Processing, 2022. [pdf](https://openreview.net/pdf?id=BK-z5qzEU-9) diff --git a/lm_eval/tasks/afrobench/salt/gen_utils.py b/lm_eval/tasks/afrobench/salt/gen_utils.py new file mode 100644 index 00000000..6ac703a0 --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/gen_utils.py @@ -0,0 +1,149 @@ +import argparse +import os + +import yaml + + +class FunctionTag: + def __init__(self, value): + self.value = value + + +def prompt_func(mode, lang, lang_dict): + language_column_name = f"{lang}_text" + prompt_map = { + "prompt_1": f"{lang_dict[lang]} sentence: {{{{{language_column_name}}}}} \nEnglish sentence: ", + "prompt_1_reverse": "English sentence: {{eng_source_text}} " + f"\n{lang_dict[lang]} sentence: ", + "prompt_2": f"You are a translation expert. Translate the following {lang_dict[lang]} sentences to English \n" + f"{lang_dict[lang]} sentence: {{{{{language_column_name}}}}}\nEnglish sentence: ", + "prompt_2_reverse": f"You are a translation expert. Translate the following English sentences to " + f"{lang_dict[lang]} " + "\nEnglish sentence: {{eng_source_text}} " + f"\n{lang_dict[lang]} sentence: ", + "prompt_3": f"As a {lang_dict[lang]} and English linguist, translate the following {lang_dict[lang]} sentences " + f"to English. \n{lang_dict[lang]} sentence: {{{{{language_column_name}}}}}\nEnglish sentence: ", + "prompt_3_reverse": f"As a {lang_dict[lang]} and English linguist, translate the following English sentences to " + f"{lang_dict[lang]}. " + "\nEnglish sentence: {{eng_source_text}} " + f"\n{lang_dict[lang]} sentence: ", + } + return prompt_map[mode] + + +def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str, reverse: bool) -> None: + """ + Generate a yaml file for each language. + + :param output_dir: The directory to output the files to. + :param overwrite: Whether to overwrite files if they already exist. + """ + err = [] + languages = { + "eng": "English", + "lug": "Luganda", + "ach": "Acholi", + "lgg": "Lugbara", + "teo": "Ateso", + "nyn": "Runyankole", + "swa": "Swahili", + "ibo": "Igbo", + } + + for lang in languages.keys(): + try: + if lang != "eng": + if not reverse: + file_name = f"salt_{lang}-eng.yaml" + task_name = f"salt_{lang}-eng_{mode}" + yaml_template = "salt" + yaml_details = { + "include": yaml_template, + "task": task_name, + "dataset_name": "text-all", + "doc_to_target": "eng_target_text", + "doc_to_text": prompt_func(mode, lang, languages), + } + os.makedirs(f"{output_dir}/{mode}", exist_ok=True) + with open( + f"{output_dir}/{mode}/{file_name}", + "w" if overwrite else "x", + encoding="utf8", + ) as f: + f.write("# Generated by utils.py\n") + yaml.dump( + yaml_details, + f, + allow_unicode=True, + ) + else: + file_name = f"salt_eng-{lang}.yaml" + task_name = f"salt_eng-{lang}_{mode}" + yaml_template = "salt" + yaml_details = { + "include": yaml_template, + "task": task_name, + "dataset_name": "text-all", + "doc_to_target": f"{lang}_text", + "doc_to_text": prompt_func(f"{mode}_reverse", lang, languages), + } + os.makedirs(f"{output_dir}/{mode}", exist_ok=True) + with open( + f"{output_dir}/{mode}/{file_name}", + "w" if overwrite else "x", + encoding="utf8", + ) as f: + f.write("# Generated by utils.py\n") + yaml.dump( + yaml_details, + f, + allow_unicode=True, + ) + except FileExistsError: + err.append(file_name) + + if len(err) > 0: + raise FileExistsError( + "Files were not created because they already exist (use --overwrite flag):" + f" {', '.join(err)}" + ) + + +def main() -> None: + """Parse CLI args and generate language-specific yaml files.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + default=True, + action="store_true", + help="Overwrite files if they already exist", + ) + parser.add_argument( + "--output-dir", + default="./", + help="Directory to write yaml files to", + ) + parser.add_argument( + "--mode", + default="prompt_1", + choices=["prompt_1", "prompt_2", "prompt_3"], + help="Prompt number", + ) + parser.add_argument( + "--reverse", + default=True, + choices=[True, False], + help="Reverse the translation direction", + ) + args = parser.parse_args() + + gen_lang_yamls( + output_dir=args.output_dir, + overwrite=args.overwrite, + mode=args.mode, + reverse=args.reverse, + ) + + +if __name__ == "__main__": + main() diff --git a/lm_eval/tasks/afrobench/salt/prompt_1/salt b/lm_eval/tasks/afrobench/salt/prompt_1/salt new file mode 100644 index 00000000..a07d434a --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_1/salt @@ -0,0 +1,24 @@ +tag: +- salt_tasks +- salt_prompt_1 +- afrobench_MT_tasks +dataset_path: Sunbird/salt +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: dev +fewshot_split: dev +test_split: test +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/salt/prompt_1/salt_ach-eng.yaml b/lm_eval/tasks/afrobench/salt/prompt_1/salt_ach-eng.yaml new file mode 100644 index 00000000..41731279 --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_1/salt_ach-eng.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "Acholi sentence: {{ach_text}} \nEnglish sentence: " +include: salt +task: salt_ach-eng_prompt_1 diff --git a/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-ach.yaml b/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-ach.yaml new file mode 100644 index 00000000..219e5780 --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-ach.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: ach_text +doc_to_text: "English sentence: {{eng_source_text}} \nAcholi sentence: " +include: salt +task: salt_eng-ach_prompt_1 diff --git a/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-ibo.yaml b/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-ibo.yaml new file mode 100644 index 00000000..f9022059 --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-ibo.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: ibo_text +doc_to_text: "English sentence: {{eng_source_text}} \nIgbo sentence: " +include: salt +task: salt_eng-ibo_prompt_1 diff --git a/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-lgg.yaml b/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-lgg.yaml new file mode 100644 index 00000000..2a038ddb --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-lgg.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: lgg_text +doc_to_text: "English sentence: {{eng_source_text}} \nLugbara sentence: " +include: salt +task: salt_eng-lgg_prompt_1 diff --git a/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-lug.yaml b/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-lug.yaml new file mode 100644 index 00000000..45399137 --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-lug.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: lug_text +doc_to_text: "English sentence: {{eng_source_text}} \nLuganda sentence: " +include: salt +task: salt_eng-lug_prompt_1 diff --git a/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-nyn.yaml b/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-nyn.yaml new file mode 100644 index 00000000..448e1101 --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-nyn.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: nyn_text +doc_to_text: "English sentence: {{eng_source_text}} \nRunyankole sentence: " +include: salt +task: salt_eng-nyn_prompt_1 diff --git a/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-swa.yaml b/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-swa.yaml new file mode 100644 index 00000000..792b4840 --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-swa.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: swa_text +doc_to_text: "English sentence: {{eng_source_text}} \nSwahili sentence: " +include: salt +task: salt_eng-swa_prompt_1 diff --git a/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-teo.yaml b/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-teo.yaml new file mode 100644 index 00000000..810626c6 --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_1/salt_eng-teo.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: teo_text +doc_to_text: "English sentence: {{eng_source_text}} \nAteso sentence: " +include: salt +task: salt_eng-teo_prompt_1 diff --git a/lm_eval/tasks/afrobench/salt/prompt_1/salt_ibo-eng.yaml b/lm_eval/tasks/afrobench/salt/prompt_1/salt_ibo-eng.yaml new file mode 100644 index 00000000..0a98c864 --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_1/salt_ibo-eng.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "Igbo sentence: {{ibo_text}} \nEnglish sentence: " +include: salt +task: salt_ibo-eng_prompt_1 diff --git a/lm_eval/tasks/afrobench/salt/prompt_1/salt_lgg-eng.yaml b/lm_eval/tasks/afrobench/salt/prompt_1/salt_lgg-eng.yaml new file mode 100644 index 00000000..c8e281ac --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_1/salt_lgg-eng.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "Lugbara sentence: {{lgg_text}} \nEnglish sentence: " +include: salt +task: salt_lgg-eng_prompt_1 diff --git a/lm_eval/tasks/afrobench/salt/prompt_1/salt_lug-eng.yaml b/lm_eval/tasks/afrobench/salt/prompt_1/salt_lug-eng.yaml new file mode 100644 index 00000000..f924d5c5 --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_1/salt_lug-eng.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "Luganda sentence: {{lug_text}} \nEnglish sentence: " +include: salt +task: salt_lug-eng_prompt_1 diff --git a/lm_eval/tasks/afrobench/salt/prompt_1/salt_nyn-eng.yaml b/lm_eval/tasks/afrobench/salt/prompt_1/salt_nyn-eng.yaml new file mode 100644 index 00000000..bd936361 --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_1/salt_nyn-eng.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "Runyankole sentence: {{nyn_text}} \nEnglish sentence: " +include: salt +task: salt_nyn-eng_prompt_1 diff --git a/lm_eval/tasks/afrobench/salt/prompt_1/salt_swa-eng.yaml b/lm_eval/tasks/afrobench/salt/prompt_1/salt_swa-eng.yaml new file mode 100644 index 00000000..c2308593 --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_1/salt_swa-eng.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "Swahili sentence: {{swa_text}} \nEnglish sentence: " +include: salt +task: salt_swa-eng_prompt_1 diff --git a/lm_eval/tasks/afrobench/salt/prompt_1/salt_teo-eng.yaml b/lm_eval/tasks/afrobench/salt/prompt_1/salt_teo-eng.yaml new file mode 100644 index 00000000..6efb4ea0 --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_1/salt_teo-eng.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "Ateso sentence: {{teo_text}} \nEnglish sentence: " +include: salt +task: salt_teo-eng_prompt_1 diff --git a/lm_eval/tasks/afrobench/salt/prompt_2/salt b/lm_eval/tasks/afrobench/salt/prompt_2/salt new file mode 100644 index 00000000..66355878 --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_2/salt @@ -0,0 +1,24 @@ +tag: +- salt_tasks +- salt_prompt_2 +- afrobench_MT_tasks +dataset_path: Sunbird/salt +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: dev +fewshot_split: dev +test_split: test +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/salt/prompt_2/salt_ach-eng.yaml b/lm_eval/tasks/afrobench/salt/prompt_2/salt_ach-eng.yaml new file mode 100644 index 00000000..dda717b7 --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_2/salt_ach-eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "You are a translation expert. Translate the following Acholi sentences\ + \ to English \nAcholi sentence: {{ach_text}}\nEnglish sentence: " +include: salt +task: salt_ach-eng_prompt_2 diff --git a/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-ach.yaml b/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-ach.yaml new file mode 100644 index 00000000..1e4a72a5 --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-ach.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: ach_text +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Acholi \nEnglish sentence: {{eng_source_text}} \nAcholi sentence: " +include: salt +task: salt_eng-ach_prompt_2 diff --git a/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-ibo.yaml b/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-ibo.yaml new file mode 100644 index 00000000..04649c12 --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-ibo.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: ibo_text +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Igbo \nEnglish sentence: {{eng_source_text}} \nIgbo sentence: " +include: salt +task: salt_eng-ibo_prompt_2 diff --git a/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-lgg.yaml b/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-lgg.yaml new file mode 100644 index 00000000..0ac6becb --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-lgg.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: lgg_text +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Lugbara \nEnglish sentence: {{eng_source_text}} \nLugbara sentence: " +include: salt +task: salt_eng-lgg_prompt_2 diff --git a/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-lug.yaml b/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-lug.yaml new file mode 100644 index 00000000..1b5f6399 --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-lug.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: lug_text +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Luganda \nEnglish sentence: {{eng_source_text}} \nLuganda sentence: " +include: salt +task: salt_eng-lug_prompt_2 diff --git a/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-nyn.yaml b/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-nyn.yaml new file mode 100644 index 00000000..84452d5a --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-nyn.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: nyn_text +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Runyankole \nEnglish sentence: {{eng_source_text}} \nRunyankole sentence: " +include: salt +task: salt_eng-nyn_prompt_2 diff --git a/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-swa.yaml b/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-swa.yaml new file mode 100644 index 00000000..523db9fb --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-swa.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: swa_text +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Swahili \nEnglish sentence: {{eng_source_text}} \nSwahili sentence: " +include: salt +task: salt_eng-swa_prompt_2 diff --git a/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-teo.yaml b/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-teo.yaml new file mode 100644 index 00000000..000e8d04 --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_2/salt_eng-teo.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: teo_text +doc_to_text: "You are a translation expert. Translate the following English sentences\ + \ to Ateso \nEnglish sentence: {{eng_source_text}} \nAteso sentence: " +include: salt +task: salt_eng-teo_prompt_2 diff --git a/lm_eval/tasks/afrobench/salt/prompt_2/salt_ibo-eng.yaml b/lm_eval/tasks/afrobench/salt/prompt_2/salt_ibo-eng.yaml new file mode 100644 index 00000000..b4ec6601 --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_2/salt_ibo-eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "You are a translation expert. Translate the following Igbo sentences\ + \ to English \nIgbo sentence: {{ibo_text}}\nEnglish sentence: " +include: salt +task: salt_ibo-eng_prompt_2 diff --git a/lm_eval/tasks/afrobench/salt/prompt_2/salt_lgg-eng.yaml b/lm_eval/tasks/afrobench/salt/prompt_2/salt_lgg-eng.yaml new file mode 100644 index 00000000..2d802c0f --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_2/salt_lgg-eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "You are a translation expert. Translate the following Lugbara sentences\ + \ to English \nLugbara sentence: {{lgg_text}}\nEnglish sentence: " +include: salt +task: salt_lgg-eng_prompt_2 diff --git a/lm_eval/tasks/afrobench/salt/prompt_2/salt_lug-eng.yaml b/lm_eval/tasks/afrobench/salt/prompt_2/salt_lug-eng.yaml new file mode 100644 index 00000000..521bbf15 --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_2/salt_lug-eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "You are a translation expert. Translate the following Luganda sentences\ + \ to English \nLuganda sentence: {{lug_text}}\nEnglish sentence: " +include: salt +task: salt_lug-eng_prompt_2 diff --git a/lm_eval/tasks/afrobench/salt/prompt_2/salt_nyn-eng.yaml b/lm_eval/tasks/afrobench/salt/prompt_2/salt_nyn-eng.yaml new file mode 100644 index 00000000..4cc4abfc --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_2/salt_nyn-eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "You are a translation expert. Translate the following Runyankole sentences\ + \ to English \nRunyankole sentence: {{nyn_text}}\nEnglish sentence: " +include: salt +task: salt_nyn-eng_prompt_2 diff --git a/lm_eval/tasks/afrobench/salt/prompt_2/salt_swa-eng.yaml b/lm_eval/tasks/afrobench/salt/prompt_2/salt_swa-eng.yaml new file mode 100644 index 00000000..9e80b908 --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_2/salt_swa-eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "You are a translation expert. Translate the following Swahili sentences\ + \ to English \nSwahili sentence: {{swa_text}}\nEnglish sentence: " +include: salt +task: salt_swa-eng_prompt_2 diff --git a/lm_eval/tasks/afrobench/salt/prompt_2/salt_teo-eng.yaml b/lm_eval/tasks/afrobench/salt/prompt_2/salt_teo-eng.yaml new file mode 100644 index 00000000..d0b0d516 --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_2/salt_teo-eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "You are a translation expert. Translate the following Ateso sentences\ + \ to English \nAteso sentence: {{teo_text}}\nEnglish sentence: " +include: salt +task: salt_teo-eng_prompt_2 diff --git a/lm_eval/tasks/afrobench/salt/prompt_3/salt b/lm_eval/tasks/afrobench/salt/prompt_3/salt new file mode 100644 index 00000000..51dac9c5 --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_3/salt @@ -0,0 +1,24 @@ +tag: +- salt_tasks +- salt_prompt_3 +- afrobench_MT_tasks +dataset_path: Sunbird/salt +dataset_kwargs: {trust_remote_code: True} +output_type: generate_until +validation_split: dev +fewshot_split: dev +test_split: test +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: chrf + aggregation: chrf + higher_is_better: true +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/salt/prompt_3/salt_ach-eng.yaml b/lm_eval/tasks/afrobench/salt/prompt_3/salt_ach-eng.yaml new file mode 100644 index 00000000..c198a59f --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_3/salt_ach-eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "As a Acholi and English linguist, translate the following Acholi sentences\ + \ to English. \nAcholi sentence: {{ach_text}}\nEnglish sentence: " +include: salt +task: salt_ach-eng_prompt_3 diff --git a/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-ach.yaml b/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-ach.yaml new file mode 100644 index 00000000..636a77d8 --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-ach.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: ach_text +doc_to_text: "As a Acholi and English linguist, translate the following English sentences\ + \ to Acholi. \nEnglish sentence: {{eng_source_text}} \nAcholi sentence: " +include: salt +task: salt_eng-ach_prompt_3 diff --git a/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-ibo.yaml b/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-ibo.yaml new file mode 100644 index 00000000..44d015d6 --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-ibo.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: ibo_text +doc_to_text: "As a Igbo and English linguist, translate the following English sentences\ + \ to Igbo. \nEnglish sentence: {{eng_source_text}} \nIgbo sentence: " +include: salt +task: salt_eng-ibo_prompt_3 diff --git a/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-lgg.yaml b/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-lgg.yaml new file mode 100644 index 00000000..8f1e6f43 --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-lgg.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: lgg_text +doc_to_text: "As a Lugbara and English linguist, translate the following English sentences\ + \ to Lugbara. \nEnglish sentence: {{eng_source_text}} \nLugbara sentence: " +include: salt +task: salt_eng-lgg_prompt_3 diff --git a/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-lug.yaml b/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-lug.yaml new file mode 100644 index 00000000..e2065c30 --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-lug.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: lug_text +doc_to_text: "As a Luganda and English linguist, translate the following English sentences\ + \ to Luganda. \nEnglish sentence: {{eng_source_text}} \nLuganda sentence: " +include: salt +task: salt_eng-lug_prompt_3 diff --git a/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-nyn.yaml b/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-nyn.yaml new file mode 100644 index 00000000..9e48970a --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-nyn.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: nyn_text +doc_to_text: "As a Runyankole and English linguist, translate the following English\ + \ sentences to Runyankole. \nEnglish sentence: {{eng_source_text}} \nRunyankole\ + \ sentence: " +include: salt +task: salt_eng-nyn_prompt_3 diff --git a/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-swa.yaml b/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-swa.yaml new file mode 100644 index 00000000..cfd3f8ea --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-swa.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: swa_text +doc_to_text: "As a Swahili and English linguist, translate the following English sentences\ + \ to Swahili. \nEnglish sentence: {{eng_source_text}} \nSwahili sentence: " +include: salt +task: salt_eng-swa_prompt_3 diff --git a/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-teo.yaml b/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-teo.yaml new file mode 100644 index 00000000..f8d280bb --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_3/salt_eng-teo.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: teo_text +doc_to_text: "As a Ateso and English linguist, translate the following English sentences\ + \ to Ateso. \nEnglish sentence: {{eng_source_text}} \nAteso sentence: " +include: salt +task: salt_eng-teo_prompt_3 diff --git a/lm_eval/tasks/afrobench/salt/prompt_3/salt_ibo-eng.yaml b/lm_eval/tasks/afrobench/salt/prompt_3/salt_ibo-eng.yaml new file mode 100644 index 00000000..13be699c --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_3/salt_ibo-eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "As a Igbo and English linguist, translate the following Igbo sentences\ + \ to English. \nIgbo sentence: {{ibo_text}}\nEnglish sentence: " +include: salt +task: salt_ibo-eng_prompt_3 diff --git a/lm_eval/tasks/afrobench/salt/prompt_3/salt_lgg-eng.yaml b/lm_eval/tasks/afrobench/salt/prompt_3/salt_lgg-eng.yaml new file mode 100644 index 00000000..7aa4ffc4 --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_3/salt_lgg-eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "As a Lugbara and English linguist, translate the following Lugbara sentences\ + \ to English. \nLugbara sentence: {{lgg_text}}\nEnglish sentence: " +include: salt +task: salt_lgg-eng_prompt_3 diff --git a/lm_eval/tasks/afrobench/salt/prompt_3/salt_lug-eng.yaml b/lm_eval/tasks/afrobench/salt/prompt_3/salt_lug-eng.yaml new file mode 100644 index 00000000..da505f6d --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_3/salt_lug-eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "As a Luganda and English linguist, translate the following Luganda sentences\ + \ to English. \nLuganda sentence: {{lug_text}}\nEnglish sentence: " +include: salt +task: salt_lug-eng_prompt_3 diff --git a/lm_eval/tasks/afrobench/salt/prompt_3/salt_nyn-eng.yaml b/lm_eval/tasks/afrobench/salt/prompt_3/salt_nyn-eng.yaml new file mode 100644 index 00000000..9edba7c4 --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_3/salt_nyn-eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "As a Runyankole and English linguist, translate the following Runyankole\ + \ sentences to English. \nRunyankole sentence: {{nyn_text}}\nEnglish sentence: " +include: salt +task: salt_nyn-eng_prompt_3 diff --git a/lm_eval/tasks/afrobench/salt/prompt_3/salt_swa-eng.yaml b/lm_eval/tasks/afrobench/salt/prompt_3/salt_swa-eng.yaml new file mode 100644 index 00000000..3d01c917 --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_3/salt_swa-eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "As a Swahili and English linguist, translate the following Swahili sentences\ + \ to English. \nSwahili sentence: {{swa_text}}\nEnglish sentence: " +include: salt +task: salt_swa-eng_prompt_3 diff --git a/lm_eval/tasks/afrobench/salt/prompt_3/salt_teo-eng.yaml b/lm_eval/tasks/afrobench/salt/prompt_3/salt_teo-eng.yaml new file mode 100644 index 00000000..c81336ca --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/prompt_3/salt_teo-eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: text-all +doc_to_target: eng_target_text +doc_to_text: "As a Ateso and English linguist, translate the following Ateso sentences\ + \ to English. \nAteso sentence: {{teo_text}}\nEnglish sentence: " +include: salt +task: salt_teo-eng_prompt_3 diff --git a/lm_eval/tasks/afrobench/salt/salt.yaml b/lm_eval/tasks/afrobench/salt/salt.yaml new file mode 100644 index 00000000..edd3070d --- /dev/null +++ b/lm_eval/tasks/afrobench/salt/salt.yaml @@ -0,0 +1,11 @@ +group: salt +task: + - salt_prompt_1 + - salt_prompt_2 + - salt_prompt_3 +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 1 diff --git a/lm_eval/tasks/afrobench/sample_run_scripts/run_afrobench.sh b/lm_eval/tasks/afrobench/sample_run_scripts/run_afrobench.sh new file mode 100644 index 00000000..886c9495 --- /dev/null +++ b/lm_eval/tasks/afrobench/sample_run_scripts/run_afrobench.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +batch_size=5 +num_fewshot=0 + +export CUDA_VISIBLE_DEVICES=0,1 + +model_names=( + "google/gemma-1.1-7b-it", + "google/gemma-2-9b-it", + "google/gemma-2-27b-it", + "Jacaranda/AfroLlama_V1", + "LLaMAX/LLaMAX3-8B-Alpaca", + "meta-llama/Llama-2-7b-chat-hf", + "meta-llama/Llama-3.1-8B-Instruct", + "meta-llama/Llama-3.1-70B-Instruct", + "meta-llama/Meta-Llama-3-8B-Instruct", + "CohereForAI/aya-101" +) + +for model_name in "${model_names[@]}" +do + echo "Running model: $model_name" + lm_eval --model hf \ + --model_args pretrained=${model_names},parallelize=true \ + --tasks afrobench\ + --batch_size ${batch_size} \ + --num_fewshot ${num_fewshot} \ + --verbosity DEBUG \ + --output_path 'path_to_results/' \ + --log_samples +done diff --git a/lm_eval/tasks/afrobench/sample_run_scripts/run_afrobench_lite.sh b/lm_eval/tasks/afrobench/sample_run_scripts/run_afrobench_lite.sh new file mode 100644 index 00000000..89291faa --- /dev/null +++ b/lm_eval/tasks/afrobench/sample_run_scripts/run_afrobench_lite.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +batch_size=5 +num_fewshot=0 + +export CUDA_VISIBLE_DEVICES=0,1 + +model_names=( + "google/gemma-1.1-7b-it", + "google/gemma-2-9b-it", + "google/gemma-2-27b-it", + "Jacaranda/AfroLlama_V1", + "LLaMAX/LLaMAX3-8B-Alpaca", + "meta-llama/Llama-2-7b-chat-hf", + "meta-llama/Llama-3.1-8B-Instruct", + "meta-llama/Llama-3.1-70B-Instruct", + "meta-llama/Meta-Llama-3-8B-Instruct", + "CohereForAI/aya-101" +) + +for model_name in "${model_names[@]}" +do + echo "Running model: $model_name" + lm_eval --model hf \ + --model_args pretrained=${model_name},parallelize=true \ + --tasks afrobench_lite\ + --batch_size ${batch_size} \ + --num_fewshot ${num_fewshot} \ + --verbosity DEBUG \ + --output_path 'path_to_results/' \ + --log_samples +done diff --git a/lm_eval/tasks/afrobench/sib/README.md b/lm_eval/tasks/afrobench/sib/README.md new file mode 100644 index 00000000..732db84b --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/README.md @@ -0,0 +1,37 @@ +# + +## Paper +Title: `SIB-200: A Simple, Inclusive, and Big Evaluation Dataset for Topic Classification in 200+ Languages and Dialects` + +Paper Link: https://aclanthology.org/2024.eacl-long.14/ + +## Abstract +>Despite the progress in building multilingual language models, evaluation is often limited to a few languages with available datasets which excludes a large number of low-resource languages. In this paper, we create SIB-200—a large-scale open-sourced benchmark dataset for topic classification in 205 languages and dialects to address the lack of evaluation dataset for Natural Language Understanding (NLU). For many of the languages covered in SIB-200, this is the first publicly available evaluation dataset for NLU. The dataset is based on Flores-200 machine translation corpus. We annotated the English portion of the dataset and extended the sentence-level annotation to the remaining 204 languages covered in the corpus. Despite the simplicity of this task, our evaluation in full-supervised setting, cross-lingual transfer setting and prompting of large language model setting show that there is still a large gap between the performance of high-resource and low-resource languages when multilingual evaluation is scaled to numerous world languages. We found that languages unseen during the pre-training of multilingual language models, languages from under-represented families (like Nilotic and Altantic-Congo), and languages from the regions of Africa, Americas, Oceania and South East Asia, often have the lowest performance on our topic classification dataset. We hope our dataset %will encourages a more inclusive evaluation of multilingual language models on a more diverse set of languages. + +HomePage: https://github.com/dadelani/sib-200 + +### Citation + +``` +@inproceedings{adelani-etal-2024-sib, + title = "{SIB}-200: A Simple, Inclusive, and Big Evaluation Dataset for Topic Classification in 200+ Languages and Dialects", + author = "Adelani, David Ifeoluwa and + Liu, Hannah and + Shen, Xiaoyu and + Vassilyev, Nikita and + Alabi, Jesujoba O. and + Mao, Yanke and + Gao, Haonan and + Lee, En-Shiun Annie", + editor = "Graham, Yvette and + Purver, Matthew", + booktitle = "Proceedings of the 18th Conference of the European Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)", + month = mar, + year = "2024", + address = "St. Julian{'}s, Malta", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2024.eacl-long.14/", + pages = "226--245", + abstract = "Despite the progress in building multilingual language models, evaluation is often limited to a few languages with available datasets which excludes a large number of low-resource languages. In this paper, we create SIB-200{---}a large-scale open-sourced benchmark dataset for topic classification in 205 languages and dialects to address the lack of evaluation dataset for Natural Language Understanding (NLU). For many of the languages covered in SIB-200, this is the first publicly available evaluation dataset for NLU. The dataset is based on Flores-200 machine translation corpus. We annotated the English portion of the dataset and extended the sentence-level annotation to the remaining 204 languages covered in the corpus. Despite the simplicity of this task, our evaluation in full-supervised setting, cross-lingual transfer setting and prompting of large language model setting show that there is still a large gap between the performance of high-resource and low-resource languages when multilingual evaluation is scaled to numerous world languages. We found that languages unseen during the pre-training of multilingual language models, languages from under-represented families (like Nilotic and Altantic-Congo), and languages from the regions of Africa, Americas, Oceania and South East Asia, often have the lowest performance on our topic classification dataset. We hope our dataset {\%}will encourages a more inclusive evaluation of multilingual language models on a more diverse set of languages." +} +``` diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib b/lm_eval/tasks/afrobench/sib/prompt_1/sib new file mode 100644 index 00000000..37fda5d1 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib @@ -0,0 +1,43 @@ +tag: + - sib_tasks + - sib_prompt_1 + - afrobench_TC_tasks +dataset_path: Davlan/sib200 +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: category +doc_to_choice: + - "science/technology" + - "travel" + - "politics" + - "sports" + - "health" + - "entertainment" + - "geography" +should_decontaminate: true +doc_to_decontamination_query: text +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_aeb.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_aeb.yaml new file mode 100644 index 00000000..d4116035 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_aeb.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: aeb_Arab +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_aeb_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_afr.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_afr.yaml new file mode 100644 index 00000000..001eee84 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_afr.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: afr_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_afr_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_aka.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_aka.yaml new file mode 100644 index 00000000..907977dc --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_aka.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: aka_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_aka_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_amh.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_amh.yaml new file mode 100644 index 00000000..dde54207 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_amh.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: amh_Ethi +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_amh_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_ary.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_ary.yaml new file mode 100644 index 00000000..68347bd5 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_ary.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ary_Arab +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_ary_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_arz.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_arz.yaml new file mode 100644 index 00000000..2c032813 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_arz.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: arz_Arab +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_arz_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_bam.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_bam.yaml new file mode 100644 index 00000000..5469a8a1 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_bam.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: bam_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_bam_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_bem.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_bem.yaml new file mode 100644 index 00000000..01aaa1cb --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_bem.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: bem_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_bem_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_cjk.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_cjk.yaml new file mode 100644 index 00000000..6deaee75 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_cjk.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: cjk_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_cjk_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_dik.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_dik.yaml new file mode 100644 index 00000000..d80d0a08 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_dik.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: dik_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_dik_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_dyu.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_dyu.yaml new file mode 100644 index 00000000..1d72e632 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_dyu.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: dyu_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_dyu_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_eng.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_eng.yaml new file mode 100644 index 00000000..1e324696 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_eng_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_ewe.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_ewe.yaml new file mode 100644 index 00000000..60cf7db8 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_ewe.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ewe_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_ewe_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_fon.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_fon.yaml new file mode 100644 index 00000000..7ae76552 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_fon.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fon_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_fon_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_fra.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_fra.yaml new file mode 100644 index 00000000..4614e6d2 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_fra.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fra_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_fra_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_fuv.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_fuv.yaml new file mode 100644 index 00000000..24f1d28a --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_fuv.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fuv_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_fuv_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_gaz.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_gaz.yaml new file mode 100644 index 00000000..df904f95 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_gaz.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: gaz_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_gaz_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_hau.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_hau.yaml new file mode 100644 index 00000000..b160b8cf --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_hau.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: hau_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_hau_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_ibo.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_ibo.yaml new file mode 100644 index 00000000..e481aeac --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_ibo.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ibo_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_ibo_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_kab.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_kab.yaml new file mode 100644 index 00000000..a914b01c --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_kab.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kab_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_kab_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_kam.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_kam.yaml new file mode 100644 index 00000000..aaa05108 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_kam.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kam_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_kam_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_kbp.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_kbp.yaml new file mode 100644 index 00000000..d4284908 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_kbp.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kbp_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_kbp_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_kea.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_kea.yaml new file mode 100644 index 00000000..4e458fb2 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_kea.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kea_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_kea_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_kik.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_kik.yaml new file mode 100644 index 00000000..beb94a8e --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_kik.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kik_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_kik_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_kin.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_kin.yaml new file mode 100644 index 00000000..7c16432e --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_kin.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kin_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_kin_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_kmb.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_kmb.yaml new file mode 100644 index 00000000..c46477e3 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_kmb.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kmb_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_kmb_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_knc.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_knc.yaml new file mode 100644 index 00000000..9b43157e --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_knc.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: knc_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_knc_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_kon.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_kon.yaml new file mode 100644 index 00000000..def4a77d --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_kon.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kon_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_kon_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_lin.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_lin.yaml new file mode 100644 index 00000000..bbba95e0 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_lin.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lin_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_lin_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_lua.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_lua.yaml new file mode 100644 index 00000000..d4bc665b --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_lua.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lua_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_lua_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_lug.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_lug.yaml new file mode 100644 index 00000000..cbf42e18 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_lug.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lug_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_lug_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_luo.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_luo.yaml new file mode 100644 index 00000000..a62ea03c --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_luo.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: luo_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_luo_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_mos.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_mos.yaml new file mode 100644 index 00000000..54140a5d --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_mos.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: mos_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_mos_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_nso.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_nso.yaml new file mode 100644 index 00000000..7f7382d5 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_nso.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nso_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_nso_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_nus.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_nus.yaml new file mode 100644 index 00000000..28208912 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_nus.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nus_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_nus_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_nya.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_nya.yaml new file mode 100644 index 00000000..6ca90a92 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_nya.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nya_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_nya_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_plt.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_plt.yaml new file mode 100644 index 00000000..650b9a4b --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_plt.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: plt_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_plt_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_por.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_por.yaml new file mode 100644 index 00000000..7901e924 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_por.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: por_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_por_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_run.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_run.yaml new file mode 100644 index 00000000..510fc5c1 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_run.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: run_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_run_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_sag.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_sag.yaml new file mode 100644 index 00000000..e7c0bb31 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_sag.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sag_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_sag_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_sna.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_sna.yaml new file mode 100644 index 00000000..f4115112 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_sna.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sna_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_sna_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_som.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_som.yaml new file mode 100644 index 00000000..be9c19f1 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_som.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: som_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_som_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_sot.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_sot.yaml new file mode 100644 index 00000000..78d0e1f5 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_sot.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sot_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_sot_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_ssw.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_ssw.yaml new file mode 100644 index 00000000..988f6828 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_ssw.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ssw_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_ssw_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_swa.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_swa.yaml new file mode 100644 index 00000000..d4a92192 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_swa.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: swh_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_swa_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_taq.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_taq.yaml new file mode 100644 index 00000000..a860f019 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_taq.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: taq_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_taq_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_tir.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_tir.yaml new file mode 100644 index 00000000..606755c5 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_tir.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tir_Ethi +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_tir_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_tso.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_tso.yaml new file mode 100644 index 00000000..c6b2e463 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_tso.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tso_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_tso_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_tum.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_tum.yaml new file mode 100644 index 00000000..9e17521f --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_tum.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tum_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_tum_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_twi.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_twi.yaml new file mode 100644 index 00000000..bf818808 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_twi.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: twi_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_twi_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_tzm.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_tzm.yaml new file mode 100644 index 00000000..10cf4c5b --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_tzm.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tzm_Tfng +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_tzm_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_umb.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_umb.yaml new file mode 100644 index 00000000..d171c9c6 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_umb.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: umb_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_umb_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_wol.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_wol.yaml new file mode 100644 index 00000000..c3a6d7e6 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_wol.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: wol_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_wol_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_xho.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_xho.yaml new file mode 100644 index 00000000..57ce4d2d --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_xho.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: xho_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_xho_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_yor.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_yor.yaml new file mode 100644 index 00000000..cab81176 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_yor.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: yor_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_yor_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/sib_zul.yaml b/lm_eval/tasks/afrobench/sib/prompt_1/sib_zul.yaml new file mode 100644 index 00000000..694ddfc1 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/sib_zul.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: zul_Latn +doc_to_text: "Given the categories science/technology, travel, politics, sports, health,\ + \ entertainment, or geography; what category does the text: '{{text}}' belong to:\ + \ \n\n" +include: sib +task: sib_zul_prompt_1 diff --git a/lm_eval/tasks/afrobench/sib/prompt_1/utils.py b/lm_eval/tasks/afrobench/sib/prompt_1/utils.py new file mode 100644 index 00000000..3e735e2d --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_1/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib b/lm_eval/tasks/afrobench/sib/prompt_2/sib new file mode 100644 index 00000000..27dd7d1f --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib @@ -0,0 +1,43 @@ +tag: + - sib_tasks + - sib_prompt_2 + - afrobench_TC_tasks +dataset_path: Davlan/sib200 +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: category +doc_to_choice: + - "science/technology" + - "travel" + - "politics" + - "sports" + - "health" + - "entertainment" + - "geography" +should_decontaminate: true +doc_to_decontamination_query: text +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_aeb.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_aeb.yaml new file mode 100644 index 00000000..32b24439 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_aeb.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: aeb_Arab +doc_to_text: 'Does this Tunisian Arabic topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_aeb_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_afr.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_afr.yaml new file mode 100644 index 00000000..c212b13f --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_afr.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: afr_Latn +doc_to_text: 'Does this Afrikaans topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_afr_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_aka.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_aka.yaml new file mode 100644 index 00000000..dacfef07 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_aka.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: aka_Latn +doc_to_text: 'Does this Akan topic; ''{{text}}'' belong to one of the following categories: + science/technology, travel, politics, sports, health, entertainment, or geography? + category only + + + ' +include: sib +task: sib_aka_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_amh.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_amh.yaml new file mode 100644 index 00000000..259009f0 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_amh.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: amh_Ethi +doc_to_text: 'Does this Amharic topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_amh_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_ary.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_ary.yaml new file mode 100644 index 00000000..141a6691 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_ary.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: ary_Arab +doc_to_text: 'Does this Moroccan Arabic topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_ary_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_arz.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_arz.yaml new file mode 100644 index 00000000..b2fee5ee --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_arz.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: arz_Arab +doc_to_text: 'Does this Egyptian Arabic topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_arz_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_bam.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_bam.yaml new file mode 100644 index 00000000..0ae5ddd0 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_bam.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: bam_Latn +doc_to_text: 'Does this Bambara topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_bam_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_bem.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_bem.yaml new file mode 100644 index 00000000..1631a349 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_bem.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: bem_Latn +doc_to_text: 'Does this Bemba topic; ''{{text}}'' belong to one of the following categories: + science/technology, travel, politics, sports, health, entertainment, or geography? + category only + + + ' +include: sib +task: sib_bem_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_cjk.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_cjk.yaml new file mode 100644 index 00000000..85521f13 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_cjk.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: cjk_Latn +doc_to_text: 'Does this Chokwe topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_cjk_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_dik.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_dik.yaml new file mode 100644 index 00000000..c660516f --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_dik.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: dik_Latn +doc_to_text: 'Does this Southwestern Dinka topic; ''{{text}}'' belong to one of the + following categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_dik_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_dyu.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_dyu.yaml new file mode 100644 index 00000000..698782fd --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_dyu.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: dyu_Latn +doc_to_text: 'Does this Dyula topic; ''{{text}}'' belong to one of the following categories: + science/technology, travel, politics, sports, health, entertainment, or geography? + category only + + + ' +include: sib +task: sib_dyu_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_eng.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_eng.yaml new file mode 100644 index 00000000..564d8656 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_eng.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: eng_Latn +doc_to_text: 'Does this English topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_eng_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_ewe.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_ewe.yaml new file mode 100644 index 00000000..ba064082 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_ewe.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: ewe_Latn +doc_to_text: 'Does this Ewe topic; ''{{text}}'' belong to one of the following categories: + science/technology, travel, politics, sports, health, entertainment, or geography? + category only + + + ' +include: sib +task: sib_ewe_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_fon.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_fon.yaml new file mode 100644 index 00000000..9bb542dd --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_fon.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: fon_Latn +doc_to_text: 'Does this Fon topic; ''{{text}}'' belong to one of the following categories: + science/technology, travel, politics, sports, health, entertainment, or geography? + category only + + + ' +include: sib +task: sib_fon_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_fra.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_fra.yaml new file mode 100644 index 00000000..cf279d61 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_fra.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: fra_Latn +doc_to_text: 'Does this French topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_fra_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_fuv.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_fuv.yaml new file mode 100644 index 00000000..50bb4b82 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_fuv.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: fuv_Latn +doc_to_text: 'Does this Nigerian Fulfulde topic; ''{{text}}'' belong to one of the + following categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_fuv_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_gaz.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_gaz.yaml new file mode 100644 index 00000000..601d5f79 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_gaz.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: gaz_Latn +doc_to_text: 'Does this West Central Oromo topic; ''{{text}}'' belong to one of the + following categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_gaz_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_hau.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_hau.yaml new file mode 100644 index 00000000..2c7255d4 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_hau.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: hau_Latn +doc_to_text: 'Does this Hausa topic; ''{{text}}'' belong to one of the following categories: + science/technology, travel, politics, sports, health, entertainment, or geography? + category only + + + ' +include: sib +task: sib_hau_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_ibo.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_ibo.yaml new file mode 100644 index 00000000..714c132f --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_ibo.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: ibo_Latn +doc_to_text: 'Does this Igbo topic; ''{{text}}'' belong to one of the following categories: + science/technology, travel, politics, sports, health, entertainment, or geography? + category only + + + ' +include: sib +task: sib_ibo_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_kab.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_kab.yaml new file mode 100644 index 00000000..22303a3f --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_kab.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: kab_Latn +doc_to_text: 'Does this Kabyle topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_kab_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_kam.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_kam.yaml new file mode 100644 index 00000000..498781d6 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_kam.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: kam_Latn +doc_to_text: 'Does this Kamba topic; ''{{text}}'' belong to one of the following categories: + science/technology, travel, politics, sports, health, entertainment, or geography? + category only + + + ' +include: sib +task: sib_kam_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_kbp.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_kbp.yaml new file mode 100644 index 00000000..679d7ccd --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_kbp.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: kbp_Latn +doc_to_text: 'Does this Kabiye topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_kbp_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_kea.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_kea.yaml new file mode 100644 index 00000000..aee33cf2 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_kea.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: kea_Latn +doc_to_text: 'Does this Kabuverdianu topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_kea_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_kik.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_kik.yaml new file mode 100644 index 00000000..77c87bc1 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_kik.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: kik_Latn +doc_to_text: 'Does this Kikuyu topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_kik_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_kin.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_kin.yaml new file mode 100644 index 00000000..5be0643e --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_kin.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: kin_Latn +doc_to_text: 'Does this Kinyarwanda topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_kin_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_kmb.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_kmb.yaml new file mode 100644 index 00000000..02f4e9d2 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_kmb.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: kmb_Latn +doc_to_text: 'Does this Kimbundu topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_kmb_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_knc.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_knc.yaml new file mode 100644 index 00000000..2623c480 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_knc.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: knc_Latn +doc_to_text: 'Does this Central Kanuri topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_knc_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_kon.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_kon.yaml new file mode 100644 index 00000000..7ec3bcf9 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_kon.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: kon_Latn +doc_to_text: 'Does this Kikongo topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_kon_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_lin.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_lin.yaml new file mode 100644 index 00000000..ec2fa57a --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_lin.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: lin_Latn +doc_to_text: 'Does this Lingala topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_lin_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_lua.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_lua.yaml new file mode 100644 index 00000000..4f3acc3d --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_lua.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: lua_Latn +doc_to_text: 'Does this Luba-Kasai topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_lua_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_lug.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_lug.yaml new file mode 100644 index 00000000..1d6e7b9f --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_lug.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: lug_Latn +doc_to_text: 'Does this Luganda topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_lug_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_luo.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_luo.yaml new file mode 100644 index 00000000..9d1a4385 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_luo.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: luo_Latn +doc_to_text: 'Does this Luo topic; ''{{text}}'' belong to one of the following categories: + science/technology, travel, politics, sports, health, entertainment, or geography? + category only + + + ' +include: sib +task: sib_luo_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_mos.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_mos.yaml new file mode 100644 index 00000000..cc025905 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_mos.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: mos_Latn +doc_to_text: 'Does this Mossi topic; ''{{text}}'' belong to one of the following categories: + science/technology, travel, politics, sports, health, entertainment, or geography? + category only + + + ' +include: sib +task: sib_mos_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_nso.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_nso.yaml new file mode 100644 index 00000000..75021cc5 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_nso.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: nso_Latn +doc_to_text: 'Does this Northern Sotho topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_nso_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_nus.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_nus.yaml new file mode 100644 index 00000000..abca40e8 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_nus.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: nus_Latn +doc_to_text: 'Does this Nuer topic; ''{{text}}'' belong to one of the following categories: + science/technology, travel, politics, sports, health, entertainment, or geography? + category only + + + ' +include: sib +task: sib_nus_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_nya.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_nya.yaml new file mode 100644 index 00000000..e09e2733 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_nya.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: nya_Latn +doc_to_text: 'Does this Nyanga topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_nya_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_plt.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_plt.yaml new file mode 100644 index 00000000..b5b385ca --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_plt.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: plt_Latn +doc_to_text: 'Does this Plateau Malagasy topic; ''{{text}}'' belong to one of the + following categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_plt_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_por.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_por.yaml new file mode 100644 index 00000000..a297c05a --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_por.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: por_Latn +doc_to_text: 'Does this Portuguese topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_por_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_run.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_run.yaml new file mode 100644 index 00000000..a4bb3224 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_run.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: run_Latn +doc_to_text: 'Does this Rundi topic; ''{{text}}'' belong to one of the following categories: + science/technology, travel, politics, sports, health, entertainment, or geography? + category only + + + ' +include: sib +task: sib_run_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_sag.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_sag.yaml new file mode 100644 index 00000000..979b4d84 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_sag.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: sag_Latn +doc_to_text: 'Does this Sango topic; ''{{text}}'' belong to one of the following categories: + science/technology, travel, politics, sports, health, entertainment, or geography? + category only + + + ' +include: sib +task: sib_sag_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_sna.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_sna.yaml new file mode 100644 index 00000000..b41184b3 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_sna.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: sna_Latn +doc_to_text: 'Does this Shona topic; ''{{text}}'' belong to one of the following categories: + science/technology, travel, politics, sports, health, entertainment, or geography? + category only + + + ' +include: sib +task: sib_sna_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_som.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_som.yaml new file mode 100644 index 00000000..cda1fb41 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_som.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: som_Latn +doc_to_text: 'Does this Somali topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_som_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_sot.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_sot.yaml new file mode 100644 index 00000000..08d0dbec --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_sot.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: sot_Latn +doc_to_text: 'Does this Southern Sotho topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_sot_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_ssw.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_ssw.yaml new file mode 100644 index 00000000..0d3b99e7 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_ssw.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: ssw_Latn +doc_to_text: 'Does this Swazi topic; ''{{text}}'' belong to one of the following categories: + science/technology, travel, politics, sports, health, entertainment, or geography? + category only + + + ' +include: sib +task: sib_ssw_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_swa.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_swa.yaml new file mode 100644 index 00000000..8e9faa83 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_swa.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: swh_Latn +doc_to_text: 'Does this Swahili topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_swa_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_taq.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_taq.yaml new file mode 100644 index 00000000..f1862c46 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_taq.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: taq_Latn +doc_to_text: 'Does this Tamasheq topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_taq_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_tir.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_tir.yaml new file mode 100644 index 00000000..80dcc1bb --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_tir.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: tir_Ethi +doc_to_text: 'Does this Tigrinya topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_tir_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_tso.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_tso.yaml new file mode 100644 index 00000000..fad909b4 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_tso.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: tso_Latn +doc_to_text: 'Does this Tsonga topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_tso_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_tum.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_tum.yaml new file mode 100644 index 00000000..613535bc --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_tum.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: tum_Latn +doc_to_text: 'Does this Tumbuka topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_tum_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_twi.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_twi.yaml new file mode 100644 index 00000000..064edb4c --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_twi.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: twi_Latn +doc_to_text: 'Does this Twi topic; ''{{text}}'' belong to one of the following categories: + science/technology, travel, politics, sports, health, entertainment, or geography? + category only + + + ' +include: sib +task: sib_twi_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_tzm.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_tzm.yaml new file mode 100644 index 00000000..7ec8adc2 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_tzm.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: tzm_Tfng +doc_to_text: 'Does this Tamazight topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_tzm_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_umb.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_umb.yaml new file mode 100644 index 00000000..5a910abc --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_umb.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: umb_Latn +doc_to_text: 'Does this Umbundu topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_umb_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_wol.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_wol.yaml new file mode 100644 index 00000000..4453b345 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_wol.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: wol_Latn +doc_to_text: 'Does this Wolof topic; ''{{text}}'' belong to one of the following categories: + science/technology, travel, politics, sports, health, entertainment, or geography? + category only + + + ' +include: sib +task: sib_wol_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_xho.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_xho.yaml new file mode 100644 index 00000000..e038cc94 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_xho.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: xho_Latn +doc_to_text: 'Does this Xhosa topic; ''{{text}}'' belong to one of the following categories: + science/technology, travel, politics, sports, health, entertainment, or geography? + category only + + + ' +include: sib +task: sib_xho_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_yor.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_yor.yaml new file mode 100644 index 00000000..e831b311 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_yor.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: yor_Latn +doc_to_text: 'Does this Yoruba topic; ''{{text}}'' belong to one of the following + categories: science/technology, travel, politics, sports, health, entertainment, + or geography? category only + + + ' +include: sib +task: sib_yor_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/sib_zul.yaml b/lm_eval/tasks/afrobench/sib/prompt_2/sib_zul.yaml new file mode 100644 index 00000000..f61a4061 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/sib_zul.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: zul_Latn +doc_to_text: 'Does this Zulu topic; ''{{text}}'' belong to one of the following categories: + science/technology, travel, politics, sports, health, entertainment, or geography? + category only + + + ' +include: sib +task: sib_zul_prompt_2 diff --git a/lm_eval/tasks/afrobench/sib/prompt_2/utils.py b/lm_eval/tasks/afrobench/sib/prompt_2/utils.py new file mode 100644 index 00000000..3e735e2d --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_2/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib b/lm_eval/tasks/afrobench/sib/prompt_3/sib new file mode 100644 index 00000000..fed4e5c5 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib @@ -0,0 +1,43 @@ +tag: + - sib_tasks + - sib_prompt_3 + - afrobench_TC_tasks +dataset_path: Davlan/sib200 +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: category +doc_to_choice: + - "science/technology" + - "travel" + - "politics" + - "sports" + - "health" + - "entertainment" + - "geography" +should_decontaminate: true +doc_to_decontamination_query: text +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_aeb.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_aeb.yaml new file mode 100644 index 00000000..7b82cc4e --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_aeb.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: aeb_Arab +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Tunisian Arabic statement below? Return\ + \ only the category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_aeb_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_afr.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_afr.yaml new file mode 100644 index 00000000..f8187596 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_afr.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: afr_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Afrikaans statement below? Return only\ + \ the category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_afr_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_aka.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_aka.yaml new file mode 100644 index 00000000..6d4ff4e4 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_aka.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: aka_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Akan statement below? Return only the category.\ + \ \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_aka_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_amh.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_amh.yaml new file mode 100644 index 00000000..58207e9e --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_amh.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: amh_Ethi +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Amharic statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_amh_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_ary.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_ary.yaml new file mode 100644 index 00000000..ccb9a068 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_ary.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: ary_Arab +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Moroccan Arabic statement below? Return\ + \ only the category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_ary_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_arz.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_arz.yaml new file mode 100644 index 00000000..19ebbed7 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_arz.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: arz_Arab +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Egyptian Arabic statement below? Return\ + \ only the category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_arz_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_bam.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_bam.yaml new file mode 100644 index 00000000..d2c1a18d --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_bam.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: bam_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Bambara statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_bam_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_bem.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_bem.yaml new file mode 100644 index 00000000..99750497 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_bem.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: bem_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Bemba statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_bem_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_cjk.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_cjk.yaml new file mode 100644 index 00000000..470612b5 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_cjk.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: cjk_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Chokwe statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_cjk_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_dik.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_dik.yaml new file mode 100644 index 00000000..c5269b02 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_dik.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: dik_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Southwestern Dinka statement below? Return\ + \ only the category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_dik_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_dyu.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_dyu.yaml new file mode 100644 index 00000000..f04a1c17 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_dyu.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: dyu_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Dyula statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_dyu_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_eng.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_eng.yaml new file mode 100644 index 00000000..bf22d08f --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_eng.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: eng_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the English statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_eng_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_ewe.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_ewe.yaml new file mode 100644 index 00000000..4cc99104 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_ewe.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: ewe_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Ewe statement below? Return only the category.\ + \ \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_ewe_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_fon.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_fon.yaml new file mode 100644 index 00000000..3127fde2 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_fon.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: fon_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Fon statement below? Return only the category.\ + \ \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_fon_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_fra.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_fra.yaml new file mode 100644 index 00000000..9a24ff30 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_fra.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: fra_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the French statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_fra_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_fuv.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_fuv.yaml new file mode 100644 index 00000000..405838c7 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_fuv.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: fuv_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Nigerian Fulfulde statement below? Return\ + \ only the category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_fuv_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_gaz.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_gaz.yaml new file mode 100644 index 00000000..282b439a --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_gaz.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: gaz_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the West Central Oromo statement below? Return\ + \ only the category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_gaz_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_hau.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_hau.yaml new file mode 100644 index 00000000..239181bf --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_hau.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: hau_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Hausa statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_hau_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_ibo.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_ibo.yaml new file mode 100644 index 00000000..0581291d --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_ibo.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: ibo_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Igbo statement below? Return only the category.\ + \ \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_ibo_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_kab.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_kab.yaml new file mode 100644 index 00000000..32fbbf44 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_kab.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: kab_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Kabyle statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_kab_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_kam.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_kam.yaml new file mode 100644 index 00000000..3f745ba5 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_kam.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: kam_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Kamba statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_kam_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_kbp.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_kbp.yaml new file mode 100644 index 00000000..c5be1035 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_kbp.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: kbp_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Kabiye statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_kbp_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_kea.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_kea.yaml new file mode 100644 index 00000000..a1d3e2a6 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_kea.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: kea_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Kabuverdianu statement below? Return only\ + \ the category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_kea_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_kik.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_kik.yaml new file mode 100644 index 00000000..521a0f89 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_kik.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: kik_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Kikuyu statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_kik_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_kin.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_kin.yaml new file mode 100644 index 00000000..066bfb63 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_kin.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: kin_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Kinyarwanda statement below? Return only\ + \ the category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_kin_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_kmb.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_kmb.yaml new file mode 100644 index 00000000..c548af89 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_kmb.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: kmb_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Kimbundu statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_kmb_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_knc.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_knc.yaml new file mode 100644 index 00000000..91368237 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_knc.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: knc_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Central Kanuri statement below? Return\ + \ only the category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_knc_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_kon.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_kon.yaml new file mode 100644 index 00000000..d8777511 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_kon.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: kon_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Kikongo statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_kon_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_lin.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_lin.yaml new file mode 100644 index 00000000..8688cb87 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_lin.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: lin_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Lingala statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_lin_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_lua.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_lua.yaml new file mode 100644 index 00000000..e71ac2aa --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_lua.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: lua_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Luba-Kasai statement below? Return only\ + \ the category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_lua_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_lug.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_lug.yaml new file mode 100644 index 00000000..c3554267 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_lug.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: lug_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Luganda statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_lug_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_luo.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_luo.yaml new file mode 100644 index 00000000..161814d3 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_luo.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: luo_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Luo statement below? Return only the category.\ + \ \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_luo_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_mos.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_mos.yaml new file mode 100644 index 00000000..7b80d500 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_mos.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: mos_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Mossi statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_mos_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_nso.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_nso.yaml new file mode 100644 index 00000000..5c9dd8bd --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_nso.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: nso_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Northern Sotho statement below? Return\ + \ only the category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_nso_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_nus.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_nus.yaml new file mode 100644 index 00000000..361698af --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_nus.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: nus_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Nuer statement below? Return only the category.\ + \ \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_nus_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_nya.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_nya.yaml new file mode 100644 index 00000000..0c455c78 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_nya.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: nya_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Nyanga statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_nya_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_plt.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_plt.yaml new file mode 100644 index 00000000..bb90a034 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_plt.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: plt_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Plateau Malagasy statement below? Return\ + \ only the category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_plt_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_por.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_por.yaml new file mode 100644 index 00000000..65b8c2da --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_por.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: por_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Portuguese statement below? Return only\ + \ the category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_por_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_run.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_run.yaml new file mode 100644 index 00000000..19f3681c --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_run.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: run_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Rundi statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_run_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_sag.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_sag.yaml new file mode 100644 index 00000000..8dfdcbd4 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_sag.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: sag_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Sango statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_sag_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_sna.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_sna.yaml new file mode 100644 index 00000000..f30ff0d2 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_sna.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: sna_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Shona statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_sna_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_som.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_som.yaml new file mode 100644 index 00000000..0ea27fd2 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_som.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: som_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Somali statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_som_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_sot.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_sot.yaml new file mode 100644 index 00000000..d4ad477d --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_sot.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: sot_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Southern Sotho statement below? Return\ + \ only the category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_sot_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_ssw.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_ssw.yaml new file mode 100644 index 00000000..25b7f85e --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_ssw.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: ssw_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Swazi statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_ssw_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_swa.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_swa.yaml new file mode 100644 index 00000000..7be0be9d --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_swa.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: swh_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Swahili statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_swa_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_taq.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_taq.yaml new file mode 100644 index 00000000..a7e7b3ab --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_taq.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: taq_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Tamasheq statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_taq_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_tir.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_tir.yaml new file mode 100644 index 00000000..aceb3525 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_tir.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: tir_Ethi +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Tigrinya statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_tir_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_tso.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_tso.yaml new file mode 100644 index 00000000..025b7163 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_tso.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: tso_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Tsonga statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_tso_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_tum.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_tum.yaml new file mode 100644 index 00000000..35092ea7 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_tum.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: tum_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Tumbuka statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_tum_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_twi.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_twi.yaml new file mode 100644 index 00000000..fc75f657 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_twi.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: twi_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Twi statement below? Return only the category.\ + \ \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_twi_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_tzm.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_tzm.yaml new file mode 100644 index 00000000..b9b3044c --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_tzm.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: tzm_Tfng +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Tamazight statement below? Return only\ + \ the category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_tzm_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_umb.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_umb.yaml new file mode 100644 index 00000000..d8bb8540 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_umb.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: umb_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Umbundu statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_umb_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_wol.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_wol.yaml new file mode 100644 index 00000000..115796d5 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_wol.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: wol_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Wolof statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_wol_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_xho.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_xho.yaml new file mode 100644 index 00000000..b61c84b7 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_xho.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: xho_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Xhosa statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_xho_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_yor.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_yor.yaml new file mode 100644 index 00000000..c5ccd0c7 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_yor.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: yor_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Yoruba statement below? Return only the\ + \ category. \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_yor_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/sib_zul.yaml b/lm_eval/tasks/afrobench/sib/prompt_3/sib_zul.yaml new file mode 100644 index 00000000..4915989d --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/sib_zul.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: zul_Latn +doc_to_text: "You are an assistant able to classify topics in texts. \n\nGiven the\ + \ categories science/technology, travel, politics, sports, health, entertainment,\ + \ or geography; what is the topic of the Zulu statement below? Return only the category.\ + \ \n\ntext: {{text}} \\category:\n\n" +include: sib +task: sib_zul_prompt_3 diff --git a/lm_eval/tasks/afrobench/sib/prompt_3/utils.py b/lm_eval/tasks/afrobench/sib/prompt_3/utils.py new file mode 100644 index 00000000..3e735e2d --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_3/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib b/lm_eval/tasks/afrobench/sib/prompt_4/sib new file mode 100644 index 00000000..28ed8f4a --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib @@ -0,0 +1,43 @@ +tag: + - sib_tasks + - sib_prompt_4 + - afrobench_TC_tasks +dataset_path: Davlan/sib200 +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: category +doc_to_choice: + - "science/technology" + - "travel" + - "politics" + - "sports" + - "health" + - "entertainment" + - "geography" +should_decontaminate: true +doc_to_decontamination_query: text +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_aeb.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_aeb.yaml new file mode 100644 index 00000000..e8c737f2 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_aeb.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: aeb_Arab +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_aeb_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_afr.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_afr.yaml new file mode 100644 index 00000000..7139d04e --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_afr.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: afr_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_afr_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_aka.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_aka.yaml new file mode 100644 index 00000000..59c8c56a --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_aka.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: aka_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_aka_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_amh.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_amh.yaml new file mode 100644 index 00000000..cec6b6c4 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_amh.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: amh_Ethi +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_amh_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_ary.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_ary.yaml new file mode 100644 index 00000000..2c107434 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_ary.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ary_Arab +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_ary_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_arz.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_arz.yaml new file mode 100644 index 00000000..1740975a --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_arz.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: arz_Arab +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_arz_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_bam.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_bam.yaml new file mode 100644 index 00000000..33ee240e --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_bam.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: bam_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_bam_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_bem.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_bem.yaml new file mode 100644 index 00000000..aa5608e8 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_bem.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: bem_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_bem_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_cjk.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_cjk.yaml new file mode 100644 index 00000000..52e08d7b --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_cjk.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: cjk_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_cjk_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_dik.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_dik.yaml new file mode 100644 index 00000000..c8db6013 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_dik.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: dik_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_dik_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_dyu.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_dyu.yaml new file mode 100644 index 00000000..e9bbc0b5 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_dyu.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: dyu_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_dyu_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_eng.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_eng.yaml new file mode 100644 index 00000000..1c847491 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_eng.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: eng_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_eng_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_ewe.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_ewe.yaml new file mode 100644 index 00000000..02e7ea82 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_ewe.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ewe_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_ewe_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_fon.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_fon.yaml new file mode 100644 index 00000000..67053ed8 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_fon.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fon_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_fon_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_fra.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_fra.yaml new file mode 100644 index 00000000..c2b858ce --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_fra.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fra_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_fra_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_fuv.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_fuv.yaml new file mode 100644 index 00000000..0c73f826 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_fuv.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: fuv_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_fuv_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_gaz.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_gaz.yaml new file mode 100644 index 00000000..ba95ef5d --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_gaz.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: gaz_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_gaz_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_hau.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_hau.yaml new file mode 100644 index 00000000..1d537948 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_hau.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: hau_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_hau_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_ibo.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_ibo.yaml new file mode 100644 index 00000000..2683d98d --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_ibo.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ibo_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_ibo_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_kab.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_kab.yaml new file mode 100644 index 00000000..9f645a45 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_kab.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kab_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_kab_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_kam.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_kam.yaml new file mode 100644 index 00000000..7f035b89 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_kam.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kam_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_kam_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_kbp.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_kbp.yaml new file mode 100644 index 00000000..c65b6352 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_kbp.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kbp_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_kbp_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_kea.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_kea.yaml new file mode 100644 index 00000000..0e7bba4a --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_kea.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kea_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_kea_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_kik.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_kik.yaml new file mode 100644 index 00000000..06480d18 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_kik.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kik_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_kik_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_kin.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_kin.yaml new file mode 100644 index 00000000..7b447219 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_kin.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kin_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_kin_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_kmb.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_kmb.yaml new file mode 100644 index 00000000..5fc51890 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_kmb.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kmb_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_kmb_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_knc.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_knc.yaml new file mode 100644 index 00000000..32644331 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_knc.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: knc_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_knc_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_kon.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_kon.yaml new file mode 100644 index 00000000..6da4ab39 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_kon.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: kon_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_kon_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_lin.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_lin.yaml new file mode 100644 index 00000000..51076dbd --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_lin.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lin_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_lin_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_lua.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_lua.yaml new file mode 100644 index 00000000..95973f7d --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_lua.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lua_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_lua_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_lug.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_lug.yaml new file mode 100644 index 00000000..a570b584 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_lug.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: lug_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_lug_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_luo.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_luo.yaml new file mode 100644 index 00000000..76d79985 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_luo.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: luo_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_luo_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_mos.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_mos.yaml new file mode 100644 index 00000000..aeb058ac --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_mos.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: mos_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_mos_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_nso.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_nso.yaml new file mode 100644 index 00000000..f76e016a --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_nso.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nso_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_nso_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_nus.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_nus.yaml new file mode 100644 index 00000000..255c1861 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_nus.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nus_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_nus_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_nya.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_nya.yaml new file mode 100644 index 00000000..bc7a48ab --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_nya.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nya_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_nya_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_plt.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_plt.yaml new file mode 100644 index 00000000..880c3d03 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_plt.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: plt_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_plt_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_por.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_por.yaml new file mode 100644 index 00000000..16a25836 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_por.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: por_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_por_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_run.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_run.yaml new file mode 100644 index 00000000..a97737ed --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_run.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: run_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_run_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_sag.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_sag.yaml new file mode 100644 index 00000000..8c689779 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_sag.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sag_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_sag_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_sna.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_sna.yaml new file mode 100644 index 00000000..da13a6ec --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_sna.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sna_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_sna_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_som.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_som.yaml new file mode 100644 index 00000000..b6c35f3c --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_som.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: som_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_som_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_sot.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_sot.yaml new file mode 100644 index 00000000..1385e058 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_sot.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sot_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_sot_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_ssw.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_ssw.yaml new file mode 100644 index 00000000..d678c124 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_ssw.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ssw_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_ssw_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_swa.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_swa.yaml new file mode 100644 index 00000000..7492cfa3 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_swa.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: swh_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_swa_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_taq.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_taq.yaml new file mode 100644 index 00000000..783be833 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_taq.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: taq_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_taq_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_tir.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_tir.yaml new file mode 100644 index 00000000..931ede56 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_tir.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tir_Ethi +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_tir_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_tso.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_tso.yaml new file mode 100644 index 00000000..fc4c0f1a --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_tso.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tso_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_tso_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_tum.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_tum.yaml new file mode 100644 index 00000000..c099dc6f --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_tum.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tum_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_tum_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_twi.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_twi.yaml new file mode 100644 index 00000000..00501281 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_twi.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: twi_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_twi_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_tzm.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_tzm.yaml new file mode 100644 index 00000000..3915fa18 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_tzm.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: tzm_Tfng +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_tzm_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_umb.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_umb.yaml new file mode 100644 index 00000000..a7f1cc79 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_umb.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: umb_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_umb_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_wol.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_wol.yaml new file mode 100644 index 00000000..fc244024 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_wol.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: wol_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_wol_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_xho.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_xho.yaml new file mode 100644 index 00000000..e075b84c --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_xho.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: xho_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_xho_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_yor.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_yor.yaml new file mode 100644 index 00000000..41ef0620 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_yor.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: yor_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_yor_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/sib_zul.yaml b/lm_eval/tasks/afrobench/sib/prompt_4/sib_zul.yaml new file mode 100644 index 00000000..7fc2f85e --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/sib_zul.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: zul_Latn +doc_to_text: "Label the following text as science/technology, travel, politics, sports,\ + \ health, entertainment, or geography. Provide only the category as your response.\ + \ \n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_zul_prompt_4 diff --git a/lm_eval/tasks/afrobench/sib/prompt_4/utils.py b/lm_eval/tasks/afrobench/sib/prompt_4/utils.py new file mode 100644 index 00000000..3e735e2d --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_4/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib b/lm_eval/tasks/afrobench/sib/prompt_5/sib new file mode 100644 index 00000000..812df7f6 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib @@ -0,0 +1,43 @@ +tag: + - sib_tasks + - sib_prompt_5 + - afrobench_TC_tasks +dataset_path: Davlan/sib200 +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: category +doc_to_choice: + - "science/technology" + - "travel" + - "politics" + - "sports" + - "health" + - "entertainment" + - "geography" +should_decontaminate: true +doc_to_decontamination_query: text +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_aeb.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_aeb.yaml new file mode 100644 index 00000000..3c88c0a2 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_aeb.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: aeb_Arab +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Tunisian Arabic text. For each input, classify the topic as science/technology,\ + \ travel, politics, sports, health, entertainment, or geography. Use the following\ + \ guidelines: \n\n science/technology: The text discusses scientific discoveries,\ + \ technological advancements, or related topics. \ntravel: The text describes travel\ + \ experiences, destinations, or related topics. \npolitics: The text covers political\ + \ events, policies, or related topics. \nsports: The text talks about sports events,\ + \ athletes, or related topics. \nhealth: The text addresses health issues, medical\ + \ advancements, or related topics. \nentertainment: The text pertains to movies,\ + \ music, celebrities, or related topics. \ngeography: The text involves geographical\ + \ information, locations, or related topics. \n\nIf the text contains multiple topics,\ + \ choose the dominant topic. For ambiguous or unclear topics, select the category\ + \ that best reflects the overall content. Please provide a single classification\ + \ for each input.\n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_aeb_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_afr.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_afr.yaml new file mode 100644 index 00000000..0d585478 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_afr.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: afr_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Afrikaans text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_afr_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_aka.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_aka.yaml new file mode 100644 index 00000000..4644bfa3 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_aka.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: aka_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Akan text. For each input, classify the topic as science/technology, travel, politics,\ + \ sports, health, entertainment, or geography. Use the following guidelines: \n\n\ + \ science/technology: The text discusses scientific discoveries, technological advancements,\ + \ or related topics. \ntravel: The text describes travel experiences, destinations,\ + \ or related topics. \npolitics: The text covers political events, policies, or\ + \ related topics. \nsports: The text talks about sports events, athletes, or related\ + \ topics. \nhealth: The text addresses health issues, medical advancements, or related\ + \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\ + \ topics. \ngeography: The text involves geographical information, locations, or\ + \ related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_aka_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_amh.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_amh.yaml new file mode 100644 index 00000000..f2b5e6f9 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_amh.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: amh_Ethi +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Amharic text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_amh_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_ary.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_ary.yaml new file mode 100644 index 00000000..348c849d --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_ary.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: ary_Arab +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Moroccan Arabic text. For each input, classify the topic as science/technology,\ + \ travel, politics, sports, health, entertainment, or geography. Use the following\ + \ guidelines: \n\n science/technology: The text discusses scientific discoveries,\ + \ technological advancements, or related topics. \ntravel: The text describes travel\ + \ experiences, destinations, or related topics. \npolitics: The text covers political\ + \ events, policies, or related topics. \nsports: The text talks about sports events,\ + \ athletes, or related topics. \nhealth: The text addresses health issues, medical\ + \ advancements, or related topics. \nentertainment: The text pertains to movies,\ + \ music, celebrities, or related topics. \ngeography: The text involves geographical\ + \ information, locations, or related topics. \n\nIf the text contains multiple topics,\ + \ choose the dominant topic. For ambiguous or unclear topics, select the category\ + \ that best reflects the overall content. Please provide a single classification\ + \ for each input.\n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_ary_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_arz.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_arz.yaml new file mode 100644 index 00000000..10057042 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_arz.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: arz_Arab +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Egyptian Arabic text. For each input, classify the topic as science/technology,\ + \ travel, politics, sports, health, entertainment, or geography. Use the following\ + \ guidelines: \n\n science/technology: The text discusses scientific discoveries,\ + \ technological advancements, or related topics. \ntravel: The text describes travel\ + \ experiences, destinations, or related topics. \npolitics: The text covers political\ + \ events, policies, or related topics. \nsports: The text talks about sports events,\ + \ athletes, or related topics. \nhealth: The text addresses health issues, medical\ + \ advancements, or related topics. \nentertainment: The text pertains to movies,\ + \ music, celebrities, or related topics. \ngeography: The text involves geographical\ + \ information, locations, or related topics. \n\nIf the text contains multiple topics,\ + \ choose the dominant topic. For ambiguous or unclear topics, select the category\ + \ that best reflects the overall content. Please provide a single classification\ + \ for each input.\n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_arz_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_bam.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_bam.yaml new file mode 100644 index 00000000..bdc65500 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_bam.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: bam_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Bambara text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_bam_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_bem.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_bem.yaml new file mode 100644 index 00000000..d42ea873 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_bem.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: bem_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Bemba text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_bem_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_cjk.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_cjk.yaml new file mode 100644 index 00000000..9623b8c5 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_cjk.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: cjk_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Chokwe text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_cjk_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_dik.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_dik.yaml new file mode 100644 index 00000000..83e76e96 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_dik.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: dik_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Southwestern Dinka text. For each input, classify the topic as science/technology,\ + \ travel, politics, sports, health, entertainment, or geography. Use the following\ + \ guidelines: \n\n science/technology: The text discusses scientific discoveries,\ + \ technological advancements, or related topics. \ntravel: The text describes travel\ + \ experiences, destinations, or related topics. \npolitics: The text covers political\ + \ events, policies, or related topics. \nsports: The text talks about sports events,\ + \ athletes, or related topics. \nhealth: The text addresses health issues, medical\ + \ advancements, or related topics. \nentertainment: The text pertains to movies,\ + \ music, celebrities, or related topics. \ngeography: The text involves geographical\ + \ information, locations, or related topics. \n\nIf the text contains multiple topics,\ + \ choose the dominant topic. For ambiguous or unclear topics, select the category\ + \ that best reflects the overall content. Please provide a single classification\ + \ for each input.\n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_dik_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_dyu.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_dyu.yaml new file mode 100644 index 00000000..8ab215e8 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_dyu.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: dyu_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Dyula text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_dyu_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_eng.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_eng.yaml new file mode 100644 index 00000000..a17a006d --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_eng.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: eng_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ English text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_eng_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_ewe.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_ewe.yaml new file mode 100644 index 00000000..19587699 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_ewe.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: ewe_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Ewe text. For each input, classify the topic as science/technology, travel, politics,\ + \ sports, health, entertainment, or geography. Use the following guidelines: \n\n\ + \ science/technology: The text discusses scientific discoveries, technological advancements,\ + \ or related topics. \ntravel: The text describes travel experiences, destinations,\ + \ or related topics. \npolitics: The text covers political events, policies, or\ + \ related topics. \nsports: The text talks about sports events, athletes, or related\ + \ topics. \nhealth: The text addresses health issues, medical advancements, or related\ + \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\ + \ topics. \ngeography: The text involves geographical information, locations, or\ + \ related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_ewe_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_fon.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_fon.yaml new file mode 100644 index 00000000..61980b51 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_fon.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: fon_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Fon text. For each input, classify the topic as science/technology, travel, politics,\ + \ sports, health, entertainment, or geography. Use the following guidelines: \n\n\ + \ science/technology: The text discusses scientific discoveries, technological advancements,\ + \ or related topics. \ntravel: The text describes travel experiences, destinations,\ + \ or related topics. \npolitics: The text covers political events, policies, or\ + \ related topics. \nsports: The text talks about sports events, athletes, or related\ + \ topics. \nhealth: The text addresses health issues, medical advancements, or related\ + \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\ + \ topics. \ngeography: The text involves geographical information, locations, or\ + \ related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_fon_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_fra.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_fra.yaml new file mode 100644 index 00000000..29573054 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_fra.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: fra_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ French text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_fra_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_fuv.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_fuv.yaml new file mode 100644 index 00000000..3b48f9f4 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_fuv.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: fuv_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Nigerian Fulfulde text. For each input, classify the topic as science/technology,\ + \ travel, politics, sports, health, entertainment, or geography. Use the following\ + \ guidelines: \n\n science/technology: The text discusses scientific discoveries,\ + \ technological advancements, or related topics. \ntravel: The text describes travel\ + \ experiences, destinations, or related topics. \npolitics: The text covers political\ + \ events, policies, or related topics. \nsports: The text talks about sports events,\ + \ athletes, or related topics. \nhealth: The text addresses health issues, medical\ + \ advancements, or related topics. \nentertainment: The text pertains to movies,\ + \ music, celebrities, or related topics. \ngeography: The text involves geographical\ + \ information, locations, or related topics. \n\nIf the text contains multiple topics,\ + \ choose the dominant topic. For ambiguous or unclear topics, select the category\ + \ that best reflects the overall content. Please provide a single classification\ + \ for each input.\n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_fuv_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_gaz.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_gaz.yaml new file mode 100644 index 00000000..37e2a4f9 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_gaz.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: gaz_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ West Central Oromo text. For each input, classify the topic as science/technology,\ + \ travel, politics, sports, health, entertainment, or geography. Use the following\ + \ guidelines: \n\n science/technology: The text discusses scientific discoveries,\ + \ technological advancements, or related topics. \ntravel: The text describes travel\ + \ experiences, destinations, or related topics. \npolitics: The text covers political\ + \ events, policies, or related topics. \nsports: The text talks about sports events,\ + \ athletes, or related topics. \nhealth: The text addresses health issues, medical\ + \ advancements, or related topics. \nentertainment: The text pertains to movies,\ + \ music, celebrities, or related topics. \ngeography: The text involves geographical\ + \ information, locations, or related topics. \n\nIf the text contains multiple topics,\ + \ choose the dominant topic. For ambiguous or unclear topics, select the category\ + \ that best reflects the overall content. Please provide a single classification\ + \ for each input.\n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_gaz_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_hau.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_hau.yaml new file mode 100644 index 00000000..24ce0970 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_hau.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: hau_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Hausa text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_hau_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_ibo.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_ibo.yaml new file mode 100644 index 00000000..a39ee75c --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_ibo.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: ibo_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Igbo text. For each input, classify the topic as science/technology, travel, politics,\ + \ sports, health, entertainment, or geography. Use the following guidelines: \n\n\ + \ science/technology: The text discusses scientific discoveries, technological advancements,\ + \ or related topics. \ntravel: The text describes travel experiences, destinations,\ + \ or related topics. \npolitics: The text covers political events, policies, or\ + \ related topics. \nsports: The text talks about sports events, athletes, or related\ + \ topics. \nhealth: The text addresses health issues, medical advancements, or related\ + \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\ + \ topics. \ngeography: The text involves geographical information, locations, or\ + \ related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_ibo_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_kab.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_kab.yaml new file mode 100644 index 00000000..d29da033 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_kab.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: kab_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Kabyle text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_kab_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_kam.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_kam.yaml new file mode 100644 index 00000000..1e55d121 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_kam.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: kam_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Kamba text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_kam_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_kbp.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_kbp.yaml new file mode 100644 index 00000000..210baea8 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_kbp.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: kbp_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Kabiye text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_kbp_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_kea.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_kea.yaml new file mode 100644 index 00000000..34a6813c --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_kea.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: kea_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Kabuverdianu text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_kea_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_kik.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_kik.yaml new file mode 100644 index 00000000..55fdcb00 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_kik.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: kik_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Kikuyu text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_kik_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_kin.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_kin.yaml new file mode 100644 index 00000000..6567d52b --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_kin.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: kin_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Kinyarwanda text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_kin_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_kmb.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_kmb.yaml new file mode 100644 index 00000000..4ae05cd0 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_kmb.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: kmb_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Kimbundu text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_kmb_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_knc.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_knc.yaml new file mode 100644 index 00000000..9870bd64 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_knc.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: knc_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Central Kanuri text. For each input, classify the topic as science/technology,\ + \ travel, politics, sports, health, entertainment, or geography. Use the following\ + \ guidelines: \n\n science/technology: The text discusses scientific discoveries,\ + \ technological advancements, or related topics. \ntravel: The text describes travel\ + \ experiences, destinations, or related topics. \npolitics: The text covers political\ + \ events, policies, or related topics. \nsports: The text talks about sports events,\ + \ athletes, or related topics. \nhealth: The text addresses health issues, medical\ + \ advancements, or related topics. \nentertainment: The text pertains to movies,\ + \ music, celebrities, or related topics. \ngeography: The text involves geographical\ + \ information, locations, or related topics. \n\nIf the text contains multiple topics,\ + \ choose the dominant topic. For ambiguous or unclear topics, select the category\ + \ that best reflects the overall content. Please provide a single classification\ + \ for each input.\n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_knc_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_kon.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_kon.yaml new file mode 100644 index 00000000..afcab8b8 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_kon.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: kon_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Kikongo text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_kon_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_lin.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_lin.yaml new file mode 100644 index 00000000..4c1611fd --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_lin.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: lin_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Lingala text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_lin_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_lua.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_lua.yaml new file mode 100644 index 00000000..b3b2b9ed --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_lua.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: lua_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Luba-Kasai text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_lua_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_lug.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_lug.yaml new file mode 100644 index 00000000..f8ca880a --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_lug.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: lug_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Luganda text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_lug_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_luo.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_luo.yaml new file mode 100644 index 00000000..b942d69d --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_luo.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: luo_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Luo text. For each input, classify the topic as science/technology, travel, politics,\ + \ sports, health, entertainment, or geography. Use the following guidelines: \n\n\ + \ science/technology: The text discusses scientific discoveries, technological advancements,\ + \ or related topics. \ntravel: The text describes travel experiences, destinations,\ + \ or related topics. \npolitics: The text covers political events, policies, or\ + \ related topics. \nsports: The text talks about sports events, athletes, or related\ + \ topics. \nhealth: The text addresses health issues, medical advancements, or related\ + \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\ + \ topics. \ngeography: The text involves geographical information, locations, or\ + \ related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_luo_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_mos.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_mos.yaml new file mode 100644 index 00000000..daccd62e --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_mos.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: mos_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Mossi text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_mos_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_nso.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_nso.yaml new file mode 100644 index 00000000..09936e3c --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_nso.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: nso_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Northern Sotho text. For each input, classify the topic as science/technology,\ + \ travel, politics, sports, health, entertainment, or geography. Use the following\ + \ guidelines: \n\n science/technology: The text discusses scientific discoveries,\ + \ technological advancements, or related topics. \ntravel: The text describes travel\ + \ experiences, destinations, or related topics. \npolitics: The text covers political\ + \ events, policies, or related topics. \nsports: The text talks about sports events,\ + \ athletes, or related topics. \nhealth: The text addresses health issues, medical\ + \ advancements, or related topics. \nentertainment: The text pertains to movies,\ + \ music, celebrities, or related topics. \ngeography: The text involves geographical\ + \ information, locations, or related topics. \n\nIf the text contains multiple topics,\ + \ choose the dominant topic. For ambiguous or unclear topics, select the category\ + \ that best reflects the overall content. Please provide a single classification\ + \ for each input.\n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_nso_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_nus.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_nus.yaml new file mode 100644 index 00000000..a5f8e101 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_nus.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: nus_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Nuer text. For each input, classify the topic as science/technology, travel, politics,\ + \ sports, health, entertainment, or geography. Use the following guidelines: \n\n\ + \ science/technology: The text discusses scientific discoveries, technological advancements,\ + \ or related topics. \ntravel: The text describes travel experiences, destinations,\ + \ or related topics. \npolitics: The text covers political events, policies, or\ + \ related topics. \nsports: The text talks about sports events, athletes, or related\ + \ topics. \nhealth: The text addresses health issues, medical advancements, or related\ + \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\ + \ topics. \ngeography: The text involves geographical information, locations, or\ + \ related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_nus_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_nya.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_nya.yaml new file mode 100644 index 00000000..65737777 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_nya.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: nya_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Nyanga text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_nya_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_plt.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_plt.yaml new file mode 100644 index 00000000..24f6ea33 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_plt.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: plt_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Plateau Malagasy text. For each input, classify the topic as science/technology,\ + \ travel, politics, sports, health, entertainment, or geography. Use the following\ + \ guidelines: \n\n science/technology: The text discusses scientific discoveries,\ + \ technological advancements, or related topics. \ntravel: The text describes travel\ + \ experiences, destinations, or related topics. \npolitics: The text covers political\ + \ events, policies, or related topics. \nsports: The text talks about sports events,\ + \ athletes, or related topics. \nhealth: The text addresses health issues, medical\ + \ advancements, or related topics. \nentertainment: The text pertains to movies,\ + \ music, celebrities, or related topics. \ngeography: The text involves geographical\ + \ information, locations, or related topics. \n\nIf the text contains multiple topics,\ + \ choose the dominant topic. For ambiguous or unclear topics, select the category\ + \ that best reflects the overall content. Please provide a single classification\ + \ for each input.\n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_plt_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_por.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_por.yaml new file mode 100644 index 00000000..d98ee118 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_por.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: por_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Portuguese text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_por_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_run.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_run.yaml new file mode 100644 index 00000000..01820da5 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_run.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: run_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Rundi text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_run_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_sag.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_sag.yaml new file mode 100644 index 00000000..fdabdcb6 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_sag.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: sag_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Sango text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_sag_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_sna.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_sna.yaml new file mode 100644 index 00000000..3d66f53a --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_sna.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: sna_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Shona text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_sna_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_som.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_som.yaml new file mode 100644 index 00000000..c0c34f97 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_som.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: som_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Somali text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_som_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_sot.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_sot.yaml new file mode 100644 index 00000000..81ab5c3f --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_sot.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: sot_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Southern Sotho text. For each input, classify the topic as science/technology,\ + \ travel, politics, sports, health, entertainment, or geography. Use the following\ + \ guidelines: \n\n science/technology: The text discusses scientific discoveries,\ + \ technological advancements, or related topics. \ntravel: The text describes travel\ + \ experiences, destinations, or related topics. \npolitics: The text covers political\ + \ events, policies, or related topics. \nsports: The text talks about sports events,\ + \ athletes, or related topics. \nhealth: The text addresses health issues, medical\ + \ advancements, or related topics. \nentertainment: The text pertains to movies,\ + \ music, celebrities, or related topics. \ngeography: The text involves geographical\ + \ information, locations, or related topics. \n\nIf the text contains multiple topics,\ + \ choose the dominant topic. For ambiguous or unclear topics, select the category\ + \ that best reflects the overall content. Please provide a single classification\ + \ for each input.\n\ntext: {{text}} \\category: \n\n" +include: sib +task: sib_sot_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_ssw.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_ssw.yaml new file mode 100644 index 00000000..f662d2ab --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_ssw.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: ssw_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Swazi text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_ssw_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_swa.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_swa.yaml new file mode 100644 index 00000000..ee47ca51 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_swa.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: swh_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Swahili text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_swa_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_taq.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_taq.yaml new file mode 100644 index 00000000..3fa1380d --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_taq.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: taq_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Tamasheq text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_taq_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_tir.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_tir.yaml new file mode 100644 index 00000000..20ec0638 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_tir.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: tir_Ethi +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Tigrinya text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_tir_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_tso.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_tso.yaml new file mode 100644 index 00000000..44b3b867 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_tso.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: tso_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Tsonga text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_tso_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_tum.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_tum.yaml new file mode 100644 index 00000000..bb15fb71 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_tum.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: tum_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Tumbuka text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_tum_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_twi.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_twi.yaml new file mode 100644 index 00000000..44bca619 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_twi.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: twi_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Twi text. For each input, classify the topic as science/technology, travel, politics,\ + \ sports, health, entertainment, or geography. Use the following guidelines: \n\n\ + \ science/technology: The text discusses scientific discoveries, technological advancements,\ + \ or related topics. \ntravel: The text describes travel experiences, destinations,\ + \ or related topics. \npolitics: The text covers political events, policies, or\ + \ related topics. \nsports: The text talks about sports events, athletes, or related\ + \ topics. \nhealth: The text addresses health issues, medical advancements, or related\ + \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\ + \ topics. \ngeography: The text involves geographical information, locations, or\ + \ related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_twi_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_tzm.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_tzm.yaml new file mode 100644 index 00000000..5d1af77d --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_tzm.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: tzm_Tfng +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Tamazight text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_tzm_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_umb.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_umb.yaml new file mode 100644 index 00000000..a756680c --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_umb.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: umb_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Umbundu text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_umb_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_wol.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_wol.yaml new file mode 100644 index 00000000..8062b55d --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_wol.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: wol_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Wolof text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_wol_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_xho.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_xho.yaml new file mode 100644 index 00000000..22c27b71 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_xho.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: xho_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Xhosa text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_xho_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_yor.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_yor.yaml new file mode 100644 index 00000000..df519782 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_yor.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: yor_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Yoruba text. For each input, classify the topic as science/technology, travel,\ + \ politics, sports, health, entertainment, or geography. Use the following guidelines:\ + \ \n\n science/technology: The text discusses scientific discoveries, technological\ + \ advancements, or related topics. \ntravel: The text describes travel experiences,\ + \ destinations, or related topics. \npolitics: The text covers political events,\ + \ policies, or related topics. \nsports: The text talks about sports events, athletes,\ + \ or related topics. \nhealth: The text addresses health issues, medical advancements,\ + \ or related topics. \nentertainment: The text pertains to movies, music, celebrities,\ + \ or related topics. \ngeography: The text involves geographical information, locations,\ + \ or related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_yor_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/sib_zul.yaml b/lm_eval/tasks/afrobench/sib/prompt_5/sib_zul.yaml new file mode 100644 index 00000000..03fb9af9 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/sib_zul.yaml @@ -0,0 +1,18 @@ +# Generated by utils.py +dataset_name: zul_Latn +doc_to_text: "You are tasked with performing topic classification on the following\ + \ Zulu text. For each input, classify the topic as science/technology, travel, politics,\ + \ sports, health, entertainment, or geography. Use the following guidelines: \n\n\ + \ science/technology: The text discusses scientific discoveries, technological advancements,\ + \ or related topics. \ntravel: The text describes travel experiences, destinations,\ + \ or related topics. \npolitics: The text covers political events, policies, or\ + \ related topics. \nsports: The text talks about sports events, athletes, or related\ + \ topics. \nhealth: The text addresses health issues, medical advancements, or related\ + \ topics. \nentertainment: The text pertains to movies, music, celebrities, or related\ + \ topics. \ngeography: The text involves geographical information, locations, or\ + \ related topics. \n\nIf the text contains multiple topics, choose the dominant\ + \ topic. For ambiguous or unclear topics, select the category that best reflects\ + \ the overall content. Please provide a single classification for each input.\n\n\ + text: {{text}} \\category: \n\n" +include: sib +task: sib_zul_prompt_5 diff --git a/lm_eval/tasks/afrobench/sib/prompt_5/utils.py b/lm_eval/tasks/afrobench/sib/prompt_5/utils.py new file mode 100644 index 00000000..3e735e2d --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/prompt_5/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm_eval/tasks/afrobench/sib/sib.yaml b/lm_eval/tasks/afrobench/sib/sib.yaml new file mode 100644 index 00000000..d6935fee --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/sib.yaml @@ -0,0 +1,13 @@ +group: sib +task: + - sib_prompt_1 + - sib_prompt_2 + - sib_prompt_3 + - sib_prompt_4 + - sib_prompt_5 +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 1 diff --git a/lm_eval/tasks/afrobench/sib/utils.py b/lm_eval/tasks/afrobench/sib/utils.py new file mode 100644 index 00000000..d99649e3 --- /dev/null +++ b/lm_eval/tasks/afrobench/sib/utils.py @@ -0,0 +1,227 @@ +import argparse +import os + +import yaml + + +class FunctionTag: + def __init__(self, value): + self.value = value + + +def prompt_func(mode, lang): + prompt_map = { + "prompt_1": "Given the categories science/technology, travel, politics, sports, health, entertainment, or geography; what category does the text: '{{text}}' belong to: \n\n", + "prompt_2": f"Does this {lang} topic; " + "'{{text}}' belong to one of the following categories: science/technology, travel, politics, sports, health, entertainment, or geography? category only\n\n", + "prompt_3": f"You are an assistant able to classify topics in texts. \n\n" + f"Given the categories science/technology, travel, politics, sports, health, entertainment, or geography; what is " + f"the topic of the {lang} statement below? Return only the category. " + "\n\ntext: {{text}} \category:\n\n", + "prompt_4": "Label the following text as science/technology, travel, politics, sports, health, entertainment, or geography. Provide only the category as your " + "response. \n\ntext: {{text}} \category: \n\n", + "prompt_5": f"You are tasked with performing topic classification on the following {lang} text. " + f"For each input, classify the topic as science/technology, travel, politics, sports, health, entertainment, or geography. " + f"Use the following guidelines: \n\n " + f"science/technology: The text discusses scientific discoveries, technological advancements, or related topics. \n" + f"travel: The text describes travel experiences, destinations, or related topics. \n" + f"politics: The text covers political events, policies, or related topics. \n" + f"sports: The text talks about sports events, athletes, or related topics. \n" + f"health: The text addresses health issues, medical advancements, or related topics. \n" + f"entertainment: The text pertains to movies, music, celebrities, or related topics. \n" + f"geography: The text involves geographical information, locations, or related topics. \n\n" + f"If the text contains multiple topics, choose the dominant topic. " + f"For ambiguous or unclear topics, select the category that best reflects the overall content. " + "Please provide a single classification for each input.\n\ntext: {{text}} \category: \n\n", + } + return prompt_map[mode] + + +def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None: + """ + Generate a yaml file for each language. + + :param output_dir: The directory to output the files to. + :param overwrite: Whether to overwrite files if they already exist. + """ + err = [] + languages = { + "aeb": "Tunisian Arabic", + "afr": "Afrikaans", + "aka": "Akan", + "amh": "Amharic", + "ary": "Moroccan Arabic", + "arz": "Egyptian Arabic", + "bam": "Bambara", + "bem": "Bemba", + "cjk": "Chokwe", + "dik": "Southwestern Dinka", + "dyu": "Dyula", + "eng": "English", + "ewe": "Ewe", + "fon": "Fon", + "fra": "French", + "fuv": "Nigerian Fulfulde", + "gaz": "West Central Oromo", + "hau": "Hausa", + "ibo": "Igbo", + "kab": "Kabyle", + "kam": "Kamba", + "kmb": "Kimbundu", + "kbp": "Kabiye", + "kea": "Kabuverdianu", + "kik": "Kikuyu", + "kin": "Kinyarwanda", + "kon": "Kikongo", + "knc": "Central Kanuri", + "lua": "Luba-Kasai", + "lug": "Luganda", + "luo": "Luo", + "lin": "Lingala", + "mos": "Mossi", + "nus": "Nuer", + "nso": "Northern Sotho", + "nya": "Nyanga", + "plt": "Plateau Malagasy", + "por": "Portuguese", + "run": "Rundi", + "sag": "Sango", + "sna": "Shona", + "som": "Somali", + "sot": "Southern Sotho", + "ssw": "Swazi", + "swa": "Swahili", + "taq": "Tamasheq", + "tir": "Tigrinya", + "tum": "Tumbuka", + "tso": "Tsonga", + "twi": "Twi", + "tzm": "Tamazight", + "umb": "Umbundu", + "wol": "Wolof", + "xho": "Xhosa", + "yor": "Yoruba", + "zul": "Zulu", + } + + lang_2_dataset_lang_code = { + "aeb": "aeb_Arab", + "afr": "afr_Latn", + "aka": "aka_Latn", + "amh": "amh_Ethi", + "ary": "ary_Arab", + "arz": "arz_Arab", + "bam": "bam_Latn", + "bem": "bem_Latn", + "cjk": "cjk_Latn", + "dik": "dik_Latn", + "dyu": "dyu_Latn", + "eng": "eng_Latn", + "ewe": "ewe_Latn", + "fon": "fon_Latn", + "fra": "fra_Latn", + "fuv": "fuv_Latn", + "gaz": "gaz_Latn", + "hau": "hau_Latn", + "ibo": "ibo_Latn", + "kab": "kab_Latn", + "kam": "kam_Latn", + "kmb": "kmb_Latn", + "kbp": "kbp_Latn", + "kea": "kea_Latn", + "kik": "kik_Latn", + "kin": "kin_Latn", + "kon": "kon_Latn", + "knc": "knc_Latn", + "lua": "lua_Latn", + "lug": "lug_Latn", + "luo": "luo_Latn", + "lin": "lin_Latn", + "mos": "mos_Latn", + "nus": "nus_Latn", + "nso": "nso_Latn", + "nya": "nya_Latn", + "plt": "plt_Latn", + "por": "por_Latn", + "run": "run_Latn", + "sag": "sag_Latn", + "sna": "sna_Latn", + "som": "som_Latn", + "sot": "sot_Latn", + "ssw": "ssw_Latn", + "swa": "swh_Latn", + "taq": "taq_Latn", + "tir": "tir_Ethi", + "tum": "tum_Latn", + "tso": "tso_Latn", + "twi": "twi_Latn", + "tzm": "tzm_Tfng", + "umb": "umb_Latn", + "wol": "wol_Latn", + "xho": "xho_Latn", + "yor": "yor_Latn", + "zul": "zul_Latn", + } + + for lang in languages.keys(): + try: + file_name = f"sib_{lang}.yaml" + task_name = f"sib_{lang}_{mode}" + yaml_template = "sib" + yaml_details = { + "include": yaml_template, + "task": task_name, + "dataset_name": lang_2_dataset_lang_code[lang], + "doc_to_text": prompt_func(mode, languages[lang]), + } + file_path = os.path.join(output_dir, mode) + os.makedirs(file_path, exist_ok=True) + + with open( + f"{output_dir}/{mode}/{file_name}", + "w" if overwrite else "x", + encoding="utf8", + ) as f: + f.write("# Generated by utils.py\n") + yaml.dump( + yaml_details, + f, + allow_unicode=True, + ) + except FileExistsError: + err.append(file_name) + + if len(err) > 0: + raise FileExistsError( + "Files were not created because they already exist (use --overwrite flag):" + f" {', '.join(err)}" + ) + + +def main() -> None: + """Parse CLI args and generate language-specific yaml files.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + default=True, + action="store_true", + help="Overwrite files if they already exist", + ) + parser.add_argument( + "--output-dir", + default="./", + help="Directory to write yaml files to", + ) + parser.add_argument( + "--mode", + default="prompt_3", + choices=["prompt_1", "prompt_2", "prompt_3", "prompt_4", "prompt_5"], + help="Prompt number", + ) + args = parser.parse_args() + + gen_lang_yamls(output_dir=args.output_dir, overwrite=args.overwrite, mode=args.mode) + + +if __name__ == "__main__": + main() diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/README.md b/lm_eval/tasks/afrobench/uhura-arc-easy/README.md new file mode 100644 index 00000000..a0253f98 --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/README.md @@ -0,0 +1,25 @@ +# + +## Paper +Title: `Uhura: A Benchmark for Evaluating Scientific Question Answering and Truthfulness in Low-Resource African Languages` + +Paper Link: https://arxiv.org/abs/2412.00948 + +## Abstract +>Evaluations of Large Language Models (LLMs) on knowledge-intensive tasks and factual accuracy often focus on high-resource languages primarily because datasets for low-resource languages (LRLs) are scarce. In this paper, we present Uhura -- a new benchmark that focuses on two tasks in six typologically-diverse African languages, created via human translation of existing English benchmarks. The first dataset, Uhura-ARC-Easy, is composed of multiple-choice science questions. The second, Uhura-TruthfulQA, is a safety benchmark testing the truthfulness of models on topics including health, law, finance, and politics. We highlight the challenges creating benchmarks with highly technical content for LRLs and outline mitigation strategies. Our evaluation reveals a significant performance gap between proprietary models such as GPT-4o and o1-preview, and Claude models, and open-source models like Meta's LLaMA and Google's Gemma. Additionally, all models perform better in English than in African languages. These results indicate that LMs struggle with answering scientific questions and are more prone to generating false claims in low-resource African languages. Our findings underscore the necessity for continuous improvement of multilingual LM capabilities in LRL settings to ensure safe and reliable use in real-world contexts. We open-source the Uhura Benchmark and Uhura Platform to foster further research and development in NLP for LRLs. + +HomePage: https://huggingface.co/datasets/masakhane/uhura-arc-easy + +### Citation + +``` +@misc{bayes2024uhurabenchmarkevaluatingscientific, + title={Uhura: A Benchmark for Evaluating Scientific Question Answering and Truthfulness in Low-Resource African Languages}, + author={Edward Bayes and Israel Abebe Azime and Jesujoba O. Alabi and Jonas Kgomo and Tyna Eloundou and Elizabeth Proehl and Kai Chen and Imaan Khadir and Naome A. Etori and Shamsuddeen Hassan Muhammad and Choice Mpanza and Igneciah Pocia Thete and Dietrich Klakow and David Ifeoluwa Adelani}, + year={2024}, + eprint={2412.00948}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2412.00948}, +} +``` diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy new file mode 100644 index 00000000..a7e37181 --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy @@ -0,0 +1,39 @@ +tag: + - uhura_arc_easy_tasks + - uhura_arc_easy_prompt_1 +task: null +dataset_path: masakhane/uhura-arc-easy +dataset_name: null +output_type: multiple_choice +doc_to_target: "{{['A', 'B', 'C', 'D'].index(answerKey)}}" +doc_to_choice: + - A + - B + - C + - D +test_split: test +fewshot_split: validation +should_decontaminate: false +doc_to_decontamination_query: "Question: {{question}}\nAnswer:" +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_am.yaml b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_am.yaml new file mode 100644 index 00000000..f61efe4e --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_am.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: am_multiple_choice +doc_to_text: "You are a virtual assistant that answers multiple-choice questions with\ + \ the correct option only.\n\nQuestion: {{question}}\n\nChoices:\n\n{% for i in\ + \ range(choices['text']|length) %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n\ + {% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_am_prompt_1 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_en.yaml b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_en.yaml new file mode 100644 index 00000000..f1b879e0 --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_en.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: en_multiple_choice +doc_to_text: "You are a virtual assistant that answers multiple-choice questions with\ + \ the correct option only.\n\nQuestion: {{question}}\n\nChoices:\n\n{% for i in\ + \ range(choices['text']|length) %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n\ + {% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_en_prompt_1 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_ha.yaml b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_ha.yaml new file mode 100644 index 00000000..986ac507 --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_ha.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: ha_multiple_choice +doc_to_text: "You are a virtual assistant that answers multiple-choice questions with\ + \ the correct option only.\n\nQuestion: {{question}}\n\nChoices:\n\n{% for i in\ + \ range(choices['text']|length) %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n\ + {% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_ha_prompt_1 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_nso.yaml b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_nso.yaml new file mode 100644 index 00000000..ead6d97d --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_nso.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: nso_multiple_choice +doc_to_text: "You are a virtual assistant that answers multiple-choice questions with\ + \ the correct option only.\n\nQuestion: {{question}}\n\nChoices:\n\n{% for i in\ + \ range(choices['text']|length) %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n\ + {% endfor %}\nAnswer: " +fewshot_split: train +include: uhura-arc-easy +task: uhura-arc-easy_nso_prompt_1 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_sw.yaml b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_sw.yaml new file mode 100644 index 00000000..2e07bb23 --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_sw.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: sw_multiple_choice +doc_to_text: "You are a virtual assistant that answers multiple-choice questions with\ + \ the correct option only.\n\nQuestion: {{question}}\n\nChoices:\n\n{% for i in\ + \ range(choices['text']|length) %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n\ + {% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_sw_prompt_1 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_yo.yaml b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_yo.yaml new file mode 100644 index 00000000..f96113e4 --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_yo.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: yo_multiple_choice +doc_to_text: "You are a virtual assistant that answers multiple-choice questions with\ + \ the correct option only.\n\nQuestion: {{question}}\n\nChoices:\n\n{% for i in\ + \ range(choices['text']|length) %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n\ + {% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_yo_prompt_1 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_zu.yaml b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_zu.yaml new file mode 100644 index 00000000..41c965a0 --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/uhura-arc-easy_zu.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: zu_multiple_choice +doc_to_text: "You are a virtual assistant that answers multiple-choice questions with\ + \ the correct option only.\n\nQuestion: {{question}}\n\nChoices:\n\n{% for i in\ + \ range(choices['text']|length) %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n\ + {% endfor %}\nAnswer: " +fewshot_split: train +include: uhura-arc-easy +task: uhura-arc-easy_zu_prompt_1 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/utils.py b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/utils.py new file mode 100644 index 00000000..3e735e2d --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_1/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy new file mode 100644 index 00000000..295d9c8e --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy @@ -0,0 +1,38 @@ +tag: + - uhura_arc_easy_tasks + - uhura_arc_easy_prompt_2 +dataset_path: masakhane/uhura-arc-easy +dataset_name: null +output_type: multiple_choice +doc_to_target: "{{['A', 'B', 'C', 'D'].index(answerKey)}}" +doc_to_choice: + - A + - B + - C + - D +test_split: test +fewshot_split: validation +should_decontaminate: false +doc_to_decontamination_query: "Question: {{question}}\nAnswer:" +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_am.yaml b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_am.yaml new file mode 100644 index 00000000..2596bd48 --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_am.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: am_multiple_choice +doc_to_text: "Choose the correct option that answers the question below:\n\nQuestion:\ + \ {{question}}\n\nChoices:\n\n{% for i in range(choices['text']|length) %}\t{{ 'ABCD'[i]\ + \ }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_am_prompt_2 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_en.yaml b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_en.yaml new file mode 100644 index 00000000..f3edfc10 --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_en.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: en_multiple_choice +doc_to_text: "Choose the correct option that answers the question below:\n\nQuestion:\ + \ {{question}}\n\nChoices:\n\n{% for i in range(choices['text']|length) %}\t{{ 'ABCD'[i]\ + \ }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_en_prompt_2 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_ha.yaml b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_ha.yaml new file mode 100644 index 00000000..d857b2e4 --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_ha.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ha_multiple_choice +doc_to_text: "Choose the correct option that answers the question below:\n\nQuestion:\ + \ {{question}}\n\nChoices:\n\n{% for i in range(choices['text']|length) %}\t{{ 'ABCD'[i]\ + \ }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_ha_prompt_2 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_nso.yaml b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_nso.yaml new file mode 100644 index 00000000..93fbfe58 --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_nso.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: nso_multiple_choice +doc_to_text: "Choose the correct option that answers the question below:\n\nQuestion:\ + \ {{question}}\n\nChoices:\n\n{% for i in range(choices['text']|length) %}\t{{ 'ABCD'[i]\ + \ }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +fewshot_split: train +include: uhura-arc-easy +task: uhura-arc-easy_nso_prompt_2 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_sw.yaml b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_sw.yaml new file mode 100644 index 00000000..b5fc929f --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_sw.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sw_multiple_choice +doc_to_text: "Choose the correct option that answers the question below:\n\nQuestion:\ + \ {{question}}\n\nChoices:\n\n{% for i in range(choices['text']|length) %}\t{{ 'ABCD'[i]\ + \ }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_sw_prompt_2 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_yo.yaml b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_yo.yaml new file mode 100644 index 00000000..67b09752 --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_yo.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: yo_multiple_choice +doc_to_text: "Choose the correct option that answers the question below:\n\nQuestion:\ + \ {{question}}\n\nChoices:\n\n{% for i in range(choices['text']|length) %}\t{{ 'ABCD'[i]\ + \ }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_yo_prompt_2 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_zu.yaml b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_zu.yaml new file mode 100644 index 00000000..4b261b51 --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/uhura-arc-easy_zu.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: zu_multiple_choice +doc_to_text: "Choose the correct option that answers the question below:\n\nQuestion:\ + \ {{question}}\n\nChoices:\n\n{% for i in range(choices['text']|length) %}\t{{ 'ABCD'[i]\ + \ }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +fewshot_split: train +include: uhura-arc-easy +task: uhura-arc-easy_zu_prompt_2 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/utils.py b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/utils.py new file mode 100644 index 00000000..3e735e2d --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_2/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy new file mode 100644 index 00000000..23e2c373 --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy @@ -0,0 +1,38 @@ +tag: + - uhura_arc_easy_tasks + - uhura_arc_easy_prompt_3 +dataset_path: masakhane/uhura-arc-easy +dataset_name: null +output_type: multiple_choice +doc_to_target: "{{['A', 'B', 'C', 'D'].index(answerKey)}}" +doc_to_choice: + - A + - B + - C + - D +test_split: test +fewshot_split: validation +should_decontaminate: false +doc_to_decontamination_query: "Question: {{question}}\nAnswer:" +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_am.yaml b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_am.yaml new file mode 100644 index 00000000..42716a7c --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_am.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: am_multiple_choice +doc_to_text: "Answer the following multiple-choice question by picking 'A', 'B', 'C',\ + \ or 'D'.\n\nQuestion: {{question}}\n\nOptions:\n\n{% for i in range(choices['text']|length)\ + \ %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_am_prompt_3 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_en.yaml b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_en.yaml new file mode 100644 index 00000000..a89312e0 --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_en.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: en_multiple_choice +doc_to_text: "Answer the following multiple-choice question by picking 'A', 'B', 'C',\ + \ or 'D'.\n\nQuestion: {{question}}\n\nOptions:\n\n{% for i in range(choices['text']|length)\ + \ %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_en_prompt_3 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_ha.yaml b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_ha.yaml new file mode 100644 index 00000000..de511a8a --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_ha.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ha_multiple_choice +doc_to_text: "Answer the following multiple-choice question by picking 'A', 'B', 'C',\ + \ or 'D'.\n\nQuestion: {{question}}\n\nOptions:\n\n{% for i in range(choices['text']|length)\ + \ %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_ha_prompt_3 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_nso.yaml b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_nso.yaml new file mode 100644 index 00000000..358d084c --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_nso.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: nso_multiple_choice +doc_to_text: "Answer the following multiple-choice question by picking 'A', 'B', 'C',\ + \ or 'D'.\n\nQuestion: {{question}}\n\nOptions:\n\n{% for i in range(choices['text']|length)\ + \ %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +fewshot_split: train +include: uhura-arc-easy +task: uhura-arc-easy_nso_prompt_3 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_sw.yaml b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_sw.yaml new file mode 100644 index 00000000..d4a8785d --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_sw.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sw_multiple_choice +doc_to_text: "Answer the following multiple-choice question by picking 'A', 'B', 'C',\ + \ or 'D'.\n\nQuestion: {{question}}\n\nOptions:\n\n{% for i in range(choices['text']|length)\ + \ %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_sw_prompt_3 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_yo.yaml b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_yo.yaml new file mode 100644 index 00000000..e9416362 --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_yo.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: yo_multiple_choice +doc_to_text: "Answer the following multiple-choice question by picking 'A', 'B', 'C',\ + \ or 'D'.\n\nQuestion: {{question}}\n\nOptions:\n\n{% for i in range(choices['text']|length)\ + \ %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_yo_prompt_3 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_zu.yaml b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_zu.yaml new file mode 100644 index 00000000..6a44b8c0 --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/uhura-arc-easy_zu.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: zu_multiple_choice +doc_to_text: "Answer the following multiple-choice question by picking 'A', 'B', 'C',\ + \ or 'D'.\n\nQuestion: {{question}}\n\nOptions:\n\n{% for i in range(choices['text']|length)\ + \ %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +fewshot_split: train +include: uhura-arc-easy +task: uhura-arc-easy_zu_prompt_3 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/utils.py b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/utils.py new file mode 100644 index 00000000..3e735e2d --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_3/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy new file mode 100644 index 00000000..e697f4c7 --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy @@ -0,0 +1,38 @@ +tag: + - uhura_arc_easy_tasks + - uhura_arc_easy_prompt_4 +dataset_path: masakhane/uhura-arc-easy +dataset_name: null +output_type: multiple_choice +doc_to_target: "{{['A', 'B', 'C', 'D'].index(answerKey)}}" +doc_to_choice: + - A + - B + - C + - D +test_split: test +fewshot_split: validation +should_decontaminate: false +doc_to_decontamination_query: "Question: {{question}}\nAnswer:" +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_am.yaml b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_am.yaml new file mode 100644 index 00000000..4eaa02f5 --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_am.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: am_multiple_choice +doc_to_text: "Question: {{question}}\n\nOptions:\n\n{% for i in range(choices['text']|length)\ + \ %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_am_prompt_4 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_en.yaml b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_en.yaml new file mode 100644 index 00000000..461e6f9e --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_en.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: en_multiple_choice +doc_to_text: "Question: {{question}}\n\nOptions:\n\n{% for i in range(choices['text']|length)\ + \ %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_en_prompt_4 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_ha.yaml b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_ha.yaml new file mode 100644 index 00000000..435ea73b --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_ha.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ha_multiple_choice +doc_to_text: "Question: {{question}}\n\nOptions:\n\n{% for i in range(choices['text']|length)\ + \ %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_ha_prompt_4 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_nso.yaml b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_nso.yaml new file mode 100644 index 00000000..09112d5a --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_nso.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: nso_multiple_choice +doc_to_text: "Question: {{question}}\n\nOptions:\n\n{% for i in range(choices['text']|length)\ + \ %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +fewshot_split: train +include: uhura-arc-easy +task: uhura-arc-easy_nso_prompt_4 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_sw.yaml b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_sw.yaml new file mode 100644 index 00000000..264770ee --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_sw.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: sw_multiple_choice +doc_to_text: "Question: {{question}}\n\nOptions:\n\n{% for i in range(choices['text']|length)\ + \ %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_sw_prompt_4 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_yo.yaml b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_yo.yaml new file mode 100644 index 00000000..10af53de --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_yo.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: yo_multiple_choice +doc_to_text: "Question: {{question}}\n\nOptions:\n\n{% for i in range(choices['text']|length)\ + \ %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_yo_prompt_4 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_zu.yaml b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_zu.yaml new file mode 100644 index 00000000..239b1648 --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/uhura-arc-easy_zu.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: zu_multiple_choice +doc_to_text: "Question: {{question}}\n\nOptions:\n\n{% for i in range(choices['text']|length)\ + \ %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n{% endfor %}\nAnswer: " +fewshot_split: train +include: uhura-arc-easy +task: uhura-arc-easy_zu_prompt_4 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/utils.py b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/utils.py new file mode 100644 index 00000000..3e735e2d --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_4/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy new file mode 100644 index 00000000..3f5ac554 --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy @@ -0,0 +1,38 @@ +tag: + - uhura_arc_easy_tasks + - uhura_arc_easy_prompt_5 +dataset_path: masakhane/uhura-arc-easy +dataset_name: null +output_type: multiple_choice +doc_to_target: "{{['A', 'B', 'C', 'D'].index(answerKey)}}" +doc_to_choice: + - A + - B + - C + - D +test_split: test +fewshot_split: validation +should_decontaminate: false +doc_to_decontamination_query: "Question: {{question}}\nAnswer:" +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_am.yaml b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_am.yaml new file mode 100644 index 00000000..f7f02310 --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_am.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: am_multiple_choice +doc_to_text: "Which of the following options answers this question: {{question}}\n\ + \n{% for i in range(choices['text']|length) %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i]\ + \ }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_am_prompt_5 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_en.yaml b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_en.yaml new file mode 100644 index 00000000..5aea6aba --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_en.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: en_multiple_choice +doc_to_text: "Which of the following options answers this question: {{question}}\n\ + \n{% for i in range(choices['text']|length) %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i]\ + \ }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_en_prompt_5 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_ha.yaml b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_ha.yaml new file mode 100644 index 00000000..6293bda2 --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_ha.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: ha_multiple_choice +doc_to_text: "Which of the following options answers this question: {{question}}\n\ + \n{% for i in range(choices['text']|length) %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i]\ + \ }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_ha_prompt_5 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_nso.yaml b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_nso.yaml new file mode 100644 index 00000000..80aff706 --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_nso.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: nso_multiple_choice +doc_to_text: "Which of the following options answers this question: {{question}}\n\ + \n{% for i in range(choices['text']|length) %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i]\ + \ }}\n{% endfor %}\nAnswer: " +fewshot_split: train +include: uhura-arc-easy +task: uhura-arc-easy_nso_prompt_5 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_sw.yaml b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_sw.yaml new file mode 100644 index 00000000..a5bc7d2e --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_sw.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: sw_multiple_choice +doc_to_text: "Which of the following options answers this question: {{question}}\n\ + \n{% for i in range(choices['text']|length) %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i]\ + \ }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_sw_prompt_5 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_yo.yaml b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_yo.yaml new file mode 100644 index 00000000..a267e987 --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_yo.yaml @@ -0,0 +1,7 @@ +# Generated by utils.py +dataset_name: yo_multiple_choice +doc_to_text: "Which of the following options answers this question: {{question}}\n\ + \n{% for i in range(choices['text']|length) %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i]\ + \ }}\n{% endfor %}\nAnswer: " +include: uhura-arc-easy +task: uhura-arc-easy_yo_prompt_5 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_zu.yaml b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_zu.yaml new file mode 100644 index 00000000..69ce4a39 --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/uhura-arc-easy_zu.yaml @@ -0,0 +1,8 @@ +# Generated by utils.py +dataset_name: zu_multiple_choice +doc_to_text: "Which of the following options answers this question: {{question}}\n\ + \n{% for i in range(choices['text']|length) %}\t{{ 'ABCD'[i] }}: {{ choices['text'][i]\ + \ }}\n{% endfor %}\nAnswer: " +fewshot_split: train +include: uhura-arc-easy +task: uhura-arc-easy_zu_prompt_5 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/utils.py b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/utils.py new file mode 100644 index 00000000..3e735e2d --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/prompt_5/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/uhura.yaml b/lm_eval/tasks/afrobench/uhura-arc-easy/uhura.yaml new file mode 100644 index 00000000..e2e2fea5 --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/uhura.yaml @@ -0,0 +1,13 @@ +group: uhura_arc_easy +task: + - uhura_arc_easy_prompt_1 + - uhura_arc_easy_prompt_2 + - uhura_arc_easy_prompt_3 + - uhura_arc_easy_prompt_4 + - uhura_arc_easy_prompt_5 +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 1 diff --git a/lm_eval/tasks/afrobench/uhura-arc-easy/utils.py b/lm_eval/tasks/afrobench/uhura-arc-easy/utils.py new file mode 100644 index 00000000..1216618c --- /dev/null +++ b/lm_eval/tasks/afrobench/uhura-arc-easy/utils.py @@ -0,0 +1,129 @@ +import argparse +import os + +import pycountry +import yaml + + +def get_language_from_code(code: str) -> str: + language_tuple = pycountry.languages.get(**{f"alpha_{len(code)}": code}) + return language_tuple.name + + +def prompt_func(mode): + prompt_map = { + "prompt_1": "You are a virtual assistant that answers multiple-choice questions with the correct option only.\n\n" + "Question: {{question}}\n\n" + "Choices:\n\n" + "{% for i in range(choices['text']|length) %}" + "\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n" + "{% endfor %}\n" + "Answer: ", + "prompt_2": "Choose the correct option that answers the question below:\n\n" + "Question: {{question}}\n\n" + "Choices:\n\n" + "{% for i in range(choices['text']|length) %}" + "\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n" + "{% endfor %}\n" + "Answer: ", + "prompt_3": "Answer the following multiple-choice question by picking 'A', 'B', 'C', or 'D'.\n\n" + "Question: {{question}}\n\n" + "Options:\n\n" + "{% for i in range(choices['text']|length) %}" + "\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n" + "{% endfor %}\n" + "Answer: ", + "prompt_4": "Question: {{question}}\n\n" + "Options:\n\n" + "{% for i in range(choices['text']|length) %}" + "\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n" + "{% endfor %}\n" + "Answer: ", + "prompt_5": "Which of the following options answers this question: {{question}}\n\n" + "{% for i in range(choices['text']|length) %}" + "\t{{ 'ABCD'[i] }}: {{ choices['text'][i] }}\n" + "{% endfor %}\n" + "Answer: ", + } + return prompt_map[mode] + + +def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None: + """ + Generate a yaml file for each language. + + :param output_dir: The directory to output the files to. + :param overwrite: Whether to overwrite files if they already exist. + """ + err = [] + languages = {"am", "en", "ha", "nso", "sw", "yo", "zu"} + + for lang in languages: + try: + file_name = f"uhura-arc-easy_{lang}.yaml" + task_name = f"uhura-arc-easy_{lang}_{mode}" + yaml_template = "uhura-arc-easy" + yaml_details = { + "include": yaml_template, + "task": task_name, + "dataset_name": f"{lang}_multiple_choice{'_unmatched' if lang == 'nso' else ''}", + "doc_to_text": prompt_func(mode), + } + if lang in ("nso", "zu"): + yaml_details["fewshot_split"] = "train" + + file_path = os.path.join(output_dir, mode) + os.makedirs(file_path, exist_ok=True) + + with open( + f"{output_dir}/{mode}/{file_name}", + "w" if overwrite else "x", + encoding="utf8", + ) as f: + f.write("# Generated by utils.py\n") + yaml.dump( + yaml_details, + f, + allow_unicode=True, + ) + except FileExistsError: + err.append(file_name) + + if len(err) > 0: + raise FileExistsError( + "Files were not created because they already exist (use --overwrite flag):" + f" {', '.join(err)}" + ) + + +def main() -> None: + """Parse CLI args and generate language-specific yaml files.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + default=True, + action="store_true", + help="Overwrite files if they already exist", + ) + parser.add_argument( + "--output-dir", + default="./", + help="Directory to write yaml files to", + ) + + PROMPT_CHOICES = ["prompt_1", "prompt_2", "prompt_3", "prompt_4", "prompt_5"] + parser.add_argument( + "--mode", + nargs="*", + default=PROMPT_CHOICES, + choices=PROMPT_CHOICES, + help="Prompt number(s)", + ) + args = parser.parse_args() + + for mode in args.mode: + gen_lang_yamls(output_dir=args.output_dir, overwrite=args.overwrite, mode=mode) + + +if __name__ == "__main__": + main() diff --git a/lm_eval/tasks/afrobench/xlsum/README.md b/lm_eval/tasks/afrobench/xlsum/README.md new file mode 100644 index 00000000..d9a47076 --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/README.md @@ -0,0 +1,34 @@ +# + +## Paper +Title: `XL-Sum: Large-Scale Multilingual Abstractive Summarization for 44 Languages` + +Paper Link: https://aclanthology.org/2021.findings-acl.413/ + +## Abstract +>Contemporary works on abstractive text summarization have focused primarily on high-resource languages like English, mostly due to the limited availability of datasets for low/mid-resource ones. In this work, we present XL-Sum, a comprehensive and diverse dataset comprising 1 million professionally annotated article-summary pairs from BBC, extracted using a set of carefully designed heuristics. The dataset covers 44 languages ranging from low to high-resource, for many of which no public dataset is currently available. XL-Sum is highly abstractive, concise, and of high quality, as indicated by human and intrinsic evaluation. We fine-tune mT5, a state-of-the-art pretrained multilingual model, with XL-Sum and experiment on multilingual and low-resource summarization tasks. XL-Sum induces competitive results compared to the ones obtained using similar monolingual datasets: we show higher than 11 ROUGE-2 scores on 10 languages we benchmark on, with some of them exceeding 15, as obtained by multilingual training. Additionally, training on low-resource languages individually also provides competitive performance. To the best of our knowledge, XL-Sum is the largest abstractive summarization dataset in terms of the number of samples collected from a single source and the number of languages covered. We are releasing our dataset and models to encourage future research on multilingual abstractive summarization. + +HomePage: https://github.com/csebuetnlp/xl-sum + +### Citation + +``` +@inproceedings{hasan-etal-2021-xl, + title = "{XL}-Sum: Large-Scale Multilingual Abstractive Summarization for 44 Languages", + author = "Hasan, Tahmid and + Bhattacharjee, Abhik and + Islam, Md. Saiful and + Mubasshir, Kazi and + Li, Yuan-Fang and + Kang, Yong-Bin and + Rahman, M. Sohel and + Shahriyar, Rifat", + booktitle = "Findings of the Association for Computational Linguistics: ACL-IJCNLP 2021", + month = aug, + year = "2021", + address = "Online", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2021.findings-acl.413", + pages = "4693--4703", +} +``` diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_1/utils.py b/lm_eval/tasks/afrobench/xlsum/prompt_1/utils.py new file mode 100644 index 00000000..85db4d4f --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_1/utils.py @@ -0,0 +1,18 @@ +import evaluate + + +def rougeL(items): + """ + # passthrough for efficiency + """ + return items + + +def rougeL_agg(items): + """ + Higher is better + """ + refs = list(zip(*items))[0] + preds = list(zip(*items))[1] + rouge_scorer = evaluate.load("rouge") + return rouge_scorer.compute(predictions=preds, references=refs)["rougeL"] diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum b/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum new file mode 100644 index 00000000..f6b0421e --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum @@ -0,0 +1,22 @@ +tag: + - xlsum_tasks + - xlsum_prompt_1 +task: null +dataset_path: csebuetnlp/xlsum +dataset_name: null +dataset_kwargs: + trust_remote_code: true +output_type: generate_until +generation_kwargs: + until: + - "" +validation_split: validation +fewshot_split: validation +test_split: test +should_decontaminate: false +metric_list: + - metric: !function utils.rougeL + higher_is_better: true + aggregation: !function utils.rougeL_agg +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_amharic.yaml b/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_amharic.yaml new file mode 100644 index 00000000..8ab68805 --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_amharic.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: amharic +doc_to_target: '{{summary}}' +doc_to_text: 'Provide a summary of the document written in Amharic. Ensure that you + provide the summary in Amharic and nothing else. + + Document in Amharic: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_amharic_prompt_1 diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_arabic.yaml b/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_arabic.yaml new file mode 100644 index 00000000..af7df7d9 --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_arabic.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: arabic +doc_to_target: '{{summary}}' +doc_to_text: 'Provide a summary of the document written in Arabic. Ensure that you + provide the summary in Arabic and nothing else. + + Document in Arabic: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_arabic_prompt_1 diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_hausa.yaml b/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_hausa.yaml new file mode 100644 index 00000000..37f6b3e5 --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_hausa.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: hausa +doc_to_target: '{{summary}}' +doc_to_text: 'Provide a summary of the document written in Hausa. Ensure that you + provide the summary in Hausa and nothing else. + + Document in Hausa: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_hausa_prompt_1 diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_igbo.yaml b/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_igbo.yaml new file mode 100644 index 00000000..04644b5d --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_igbo.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: igbo +doc_to_target: '{{summary}}' +doc_to_text: 'Provide a summary of the document written in Igbo. Ensure that you provide + the summary in Igbo and nothing else. + + Document in Igbo: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_igbo_prompt_1 diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_kirundi.yaml b/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_kirundi.yaml new file mode 100644 index 00000000..7c434296 --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_kirundi.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: kirundi +doc_to_target: '{{summary}}' +doc_to_text: 'Provide a summary of the document written in Kirundi. Ensure that you + provide the summary in Kirundi and nothing else. + + Document in Kirundi: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_kirundi_prompt_1 diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_oromo.yaml b/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_oromo.yaml new file mode 100644 index 00000000..78fb14ec --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_oromo.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: oromo +doc_to_target: '{{summary}}' +doc_to_text: 'Provide a summary of the document written in Oromo. Ensure that you + provide the summary in Oromo and nothing else. + + Document in Oromo: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_oromo_prompt_1 diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_pidgin.yaml b/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_pidgin.yaml new file mode 100644 index 00000000..68f2c17f --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_pidgin.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: pidgin +doc_to_target: '{{summary}}' +doc_to_text: 'Provide a summary of the document written in Nigerian pidgin. Ensure + that you provide the summary in Nigerian pidgin and nothing else. + + Document in Nigerian pidgin: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_pidgin_prompt_1 diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_somali.yaml b/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_somali.yaml new file mode 100644 index 00000000..d699dc19 --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_somali.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: somali +doc_to_target: '{{summary}}' +doc_to_text: 'Provide a summary of the document written in Somali. Ensure that you + provide the summary in Somali and nothing else. + + Document in Somali: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_somali_prompt_1 diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_swahili.yaml b/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_swahili.yaml new file mode 100644 index 00000000..6a951c11 --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_swahili.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: swahili +doc_to_target: '{{summary}}' +doc_to_text: 'Provide a summary of the document written in Swahili. Ensure that you + provide the summary in Swahili and nothing else. + + Document in Swahili: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_swahili_prompt_1 diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_telugu.yaml b/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_telugu.yaml new file mode 100644 index 00000000..82a60171 --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_telugu.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: telugu +doc_to_target: '{{summary}}' +doc_to_text: 'Provide a summary of the document written in Telugu. Ensure that you + provide the summary in Telugu and nothing else. + + Document in Telugu: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_telugu_prompt_1 diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_tigrinya.yaml b/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_tigrinya.yaml new file mode 100644 index 00000000..31630982 --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_tigrinya.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: tigrinya +doc_to_target: '{{summary}}' +doc_to_text: 'Provide a summary of the document written in Tigrinya. Ensure that you + provide the summary in Tigrinya and nothing else. + + Document in Tigrinya: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_tigrinya_prompt_1 diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_yoruba.yaml b/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_yoruba.yaml new file mode 100644 index 00000000..9c14a911 --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_1/xlsum_yoruba.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: yoruba +doc_to_target: '{{summary}}' +doc_to_text: 'Provide a summary of the document written in Yoruba. Ensure that you + provide the summary in Yoruba and nothing else. + + Document in Yoruba: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_yoruba_prompt_1 diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_2/utils.py b/lm_eval/tasks/afrobench/xlsum/prompt_2/utils.py new file mode 100644 index 00000000..85db4d4f --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_2/utils.py @@ -0,0 +1,18 @@ +import evaluate + + +def rougeL(items): + """ + # passthrough for efficiency + """ + return items + + +def rougeL_agg(items): + """ + Higher is better + """ + refs = list(zip(*items))[0] + preds = list(zip(*items))[1] + rouge_scorer = evaluate.load("rouge") + return rouge_scorer.compute(predictions=preds, references=refs)["rougeL"] diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum b/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum new file mode 100644 index 00000000..e572c00c --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum @@ -0,0 +1,22 @@ +tag: + - xlsum_tasks + - xlsum_prompt_2 +task: null +dataset_path: csebuetnlp/xlsum +dataset_name: null +dataset_kwargs: + trust_remote_code: true +output_type: generate_until +generation_kwargs: + until: + - "" +validation_split: validation +fewshot_split: validation +test_split: test +should_decontaminate: false +metric_list: + - metric: !function utils.rougeL + higher_is_better: true + aggregation: !function utils.rougeL_agg +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_amharic.yaml b/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_amharic.yaml new file mode 100644 index 00000000..0f2275c6 --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_amharic.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: amharic +doc_to_target: '{{summary}}' +doc_to_text: 'Summarize the document below in triple backticks and return only the + summary and nothing else. + + ```{{''text''}}```\n' +include: xlsum +task: xlsum_amharic_prompt_2 diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_arabic.yaml b/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_arabic.yaml new file mode 100644 index 00000000..a4f772c3 --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_arabic.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: arabic +doc_to_target: '{{summary}}' +doc_to_text: 'Summarize the document below in triple backticks and return only the + summary and nothing else. + + ```{{''text''}}```\n' +include: xlsum +task: xlsum_arabic_prompt_2 diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_hausa.yaml b/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_hausa.yaml new file mode 100644 index 00000000..7485672c --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_hausa.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: hausa +doc_to_target: '{{summary}}' +doc_to_text: 'Summarize the document below in triple backticks and return only the + summary and nothing else. + + ```{{''text''}}```\n' +include: xlsum +task: xlsum_hausa_prompt_2 diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_igbo.yaml b/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_igbo.yaml new file mode 100644 index 00000000..2cf7fafe --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_igbo.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: igbo +doc_to_target: '{{summary}}' +doc_to_text: 'Summarize the document below in triple backticks and return only the + summary and nothing else. + + ```{{''text''}}```\n' +include: xlsum +task: xlsum_igbo_prompt_2 diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_kirundi.yaml b/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_kirundi.yaml new file mode 100644 index 00000000..63021d7d --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_kirundi.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: kirundi +doc_to_target: '{{summary}}' +doc_to_text: 'Summarize the document below in triple backticks and return only the + summary and nothing else. + + ```{{''text''}}```\n' +include: xlsum +task: xlsum_kirundi_prompt_2 diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_oromo.yaml b/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_oromo.yaml new file mode 100644 index 00000000..b637b10d --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_oromo.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: oromo +doc_to_target: '{{summary}}' +doc_to_text: 'Summarize the document below in triple backticks and return only the + summary and nothing else. + + ```{{''text''}}```\n' +include: xlsum +task: xlsum_oromo_prompt_2 diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_pidgin.yaml b/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_pidgin.yaml new file mode 100644 index 00000000..6c13d93d --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_pidgin.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: pidgin +doc_to_target: '{{summary}}' +doc_to_text: 'Summarize the document below in triple backticks and return only the + summary and nothing else. + + ```{{''text''}}```\n' +include: xlsum +task: xlsum_pidgin_prompt_2 diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_somali.yaml b/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_somali.yaml new file mode 100644 index 00000000..b7245ddc --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_somali.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: somali +doc_to_target: '{{summary}}' +doc_to_text: 'Summarize the document below in triple backticks and return only the + summary and nothing else. + + ```{{''text''}}```\n' +include: xlsum +task: xlsum_somali_prompt_2 diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_swahili.yaml b/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_swahili.yaml new file mode 100644 index 00000000..65f176fb --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_swahili.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: swahili +doc_to_target: '{{summary}}' +doc_to_text: 'Summarize the document below in triple backticks and return only the + summary and nothing else. + + ```{{''text''}}```\n' +include: xlsum +task: xlsum_swahili_prompt_2 diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_telugu.yaml b/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_telugu.yaml new file mode 100644 index 00000000..0ecbdde5 --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_telugu.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: telugu +doc_to_target: '{{summary}}' +doc_to_text: 'Summarize the document below in triple backticks and return only the + summary and nothing else. + + ```{{''text''}}```\n' +include: xlsum +task: xlsum_telugu_prompt_2 diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_tigrinya.yaml b/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_tigrinya.yaml new file mode 100644 index 00000000..d46e2fb5 --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_tigrinya.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: tigrinya +doc_to_target: '{{summary}}' +doc_to_text: 'Summarize the document below in triple backticks and return only the + summary and nothing else. + + ```{{''text''}}```\n' +include: xlsum +task: xlsum_tigrinya_prompt_2 diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_yoruba.yaml b/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_yoruba.yaml new file mode 100644 index 00000000..7ea0ef50 --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_2/xlsum_yoruba.yaml @@ -0,0 +1,9 @@ +# Generated by utils.py +dataset_name: yoruba +doc_to_target: '{{summary}}' +doc_to_text: 'Summarize the document below in triple backticks and return only the + summary and nothing else. + + ```{{''text''}}```\n' +include: xlsum +task: xlsum_yoruba_prompt_2 diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_3/utils.py b/lm_eval/tasks/afrobench/xlsum/prompt_3/utils.py new file mode 100644 index 00000000..85db4d4f --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_3/utils.py @@ -0,0 +1,18 @@ +import evaluate + + +def rougeL(items): + """ + # passthrough for efficiency + """ + return items + + +def rougeL_agg(items): + """ + Higher is better + """ + refs = list(zip(*items))[0] + preds = list(zip(*items))[1] + rouge_scorer = evaluate.load("rouge") + return rouge_scorer.compute(predictions=preds, references=refs)["rougeL"] diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum b/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum new file mode 100644 index 00000000..08842ef8 --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum @@ -0,0 +1,22 @@ +tag: + - xlsum_tasks + - xlsum_prompt_3 +task: null +dataset_path: csebuetnlp/xlsum +dataset_name: null +dataset_kwargs: + trust_remote_code: true +output_type: generate_until +generation_kwargs: + until: + - "" +validation_split: validation +fewshot_split: validation +test_split: test +should_decontaminate: false +metric_list: + - metric: !function utils.rougeL + higher_is_better: true + aggregation: !function utils.rougeL_agg +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_amharic.yaml b/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_amharic.yaml new file mode 100644 index 00000000..6fc85e7c --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_amharic.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: amharic +doc_to_target: '{{summary}}' +doc_to_text: 'You are an advanced Summarizer, a specialized assistant designed to + summarize documents in Amharic. Your main goal is to ensure summaries are concise + and informative. Ensure you return the summary only and nothing else. + + Document: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_amharic_prompt_3 diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_arabic.yaml b/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_arabic.yaml new file mode 100644 index 00000000..d4f2b1f5 --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_arabic.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: arabic +doc_to_target: '{{summary}}' +doc_to_text: 'You are an advanced Summarizer, a specialized assistant designed to + summarize documents in Arabic. Your main goal is to ensure summaries are concise + and informative. Ensure you return the summary only and nothing else. + + Document: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_arabic_prompt_3 diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_hausa.yaml b/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_hausa.yaml new file mode 100644 index 00000000..e1a06037 --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_hausa.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: hausa +doc_to_target: '{{summary}}' +doc_to_text: 'You are an advanced Summarizer, a specialized assistant designed to + summarize documents in Hausa. Your main goal is to ensure summaries are concise + and informative. Ensure you return the summary only and nothing else. + + Document: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_hausa_prompt_3 diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_igbo.yaml b/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_igbo.yaml new file mode 100644 index 00000000..6b23f8f3 --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_igbo.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: igbo +doc_to_target: '{{summary}}' +doc_to_text: 'You are an advanced Summarizer, a specialized assistant designed to + summarize documents in Igbo. Your main goal is to ensure summaries are concise and + informative. Ensure you return the summary only and nothing else. + + Document: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_igbo_prompt_3 diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_kirundi.yaml b/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_kirundi.yaml new file mode 100644 index 00000000..8f40b2a7 --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_kirundi.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: kirundi +doc_to_target: '{{summary}}' +doc_to_text: 'You are an advanced Summarizer, a specialized assistant designed to + summarize documents in Kirundi. Your main goal is to ensure summaries are concise + and informative. Ensure you return the summary only and nothing else. + + Document: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_kirundi_prompt_3 diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_oromo.yaml b/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_oromo.yaml new file mode 100644 index 00000000..bbc91285 --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_oromo.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: oromo +doc_to_target: '{{summary}}' +doc_to_text: 'You are an advanced Summarizer, a specialized assistant designed to + summarize documents in Oromo. Your main goal is to ensure summaries are concise + and informative. Ensure you return the summary only and nothing else. + + Document: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_oromo_prompt_3 diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_pidgin.yaml b/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_pidgin.yaml new file mode 100644 index 00000000..8149e441 --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_pidgin.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: pidgin +doc_to_target: '{{summary}}' +doc_to_text: 'You are an advanced Summarizer, a specialized assistant designed to + summarize documents in Nigerian pidgin. Your main goal is to ensure summaries are + concise and informative. Ensure you return the summary only and nothing else. + + Document: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_pidgin_prompt_3 diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_somali.yaml b/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_somali.yaml new file mode 100644 index 00000000..a2936da1 --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_somali.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: somali +doc_to_target: '{{summary}}' +doc_to_text: 'You are an advanced Summarizer, a specialized assistant designed to + summarize documents in Somali. Your main goal is to ensure summaries are concise + and informative. Ensure you return the summary only and nothing else. + + Document: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_somali_prompt_3 diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_swahili.yaml b/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_swahili.yaml new file mode 100644 index 00000000..6f90f4cf --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_swahili.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: swahili +doc_to_target: '{{summary}}' +doc_to_text: 'You are an advanced Summarizer, a specialized assistant designed to + summarize documents in Swahili. Your main goal is to ensure summaries are concise + and informative. Ensure you return the summary only and nothing else. + + Document: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_swahili_prompt_3 diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_telugu.yaml b/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_telugu.yaml new file mode 100644 index 00000000..67d116dc --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_telugu.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: telugu +doc_to_target: '{{summary}}' +doc_to_text: 'You are an advanced Summarizer, a specialized assistant designed to + summarize documents in Telugu. Your main goal is to ensure summaries are concise + and informative. Ensure you return the summary only and nothing else. + + Document: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_telugu_prompt_3 diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_tigrinya.yaml b/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_tigrinya.yaml new file mode 100644 index 00000000..5b20d6e3 --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_tigrinya.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: tigrinya +doc_to_target: '{{summary}}' +doc_to_text: 'You are an advanced Summarizer, a specialized assistant designed to + summarize documents in Tigrinya. Your main goal is to ensure summaries are concise + and informative. Ensure you return the summary only and nothing else. + + Document: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_tigrinya_prompt_3 diff --git a/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_yoruba.yaml b/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_yoruba.yaml new file mode 100644 index 00000000..353be14c --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/prompt_3/xlsum_yoruba.yaml @@ -0,0 +1,10 @@ +# Generated by utils.py +dataset_name: yoruba +doc_to_target: '{{summary}}' +doc_to_text: 'You are an advanced Summarizer, a specialized assistant designed to + summarize documents in Yoruba. Your main goal is to ensure summaries are concise + and informative. Ensure you return the summary only and nothing else. + + Document: {{''text''}}\nSummary: ' +include: xlsum +task: xlsum_yoruba_prompt_3 diff --git a/lm_eval/tasks/afrobench/xlsum/utils.py b/lm_eval/tasks/afrobench/xlsum/utils.py new file mode 100644 index 00000000..8df1e12e --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/utils.py @@ -0,0 +1,118 @@ +import argparse +import os + +import yaml + + +def prompt_func(mode, lang): + if lang == "pidgin": + lang = "Nigerian Pidgin" + + prompt_map = { + "prompt_1": f"Provide a summary of the document written in {lang.capitalize()}. Ensure that you provide the summary in {lang.capitalize()} and nothing else.\n" + f"Document in {lang.capitalize()}: " + r"{{'text'}}\n" + "Summary: ", + "prompt_2": "Summarize the document below in triple backticks and return only the summary and nothing else.\n" + + r"```{{'text'}}```\n", + "prompt_3": f"You are an advanced Summarizer, a specialized assistant designed to summarize documents in {lang.capitalize()}. " + f"Your main goal is to ensure summaries are concise and informative. Ensure you return the summary only and nothing else.\n" + f"Document: " + r"{{'text'}}\n" + "Summary: ", + "prompt_4": f"Summarize this {lang.capitalize()} document:\n" + r"{{'text'}}\n" + "Summary: ", + "prompt_5": f"{lang.capitalize()} document: " + r"{{'text'}}\n" + "Summary: ", + } + return prompt_map[mode] + + +def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None: + """ + Generate a yaml file for each language. + + :param output_dir: The directory to output the files to. + :param overwrite: Whether to overwrite files if they already exist. + """ + err = [] + XLSUM_LANGUAGES = ( + "amharic", + "arabic", + "hausa", + "igbo", + "kirundi", + "oromo", + "pidgin", + "somali", + "swahili", + "telugu", + "tigrinya", + "yoruba", + ) + + for lang in XLSUM_LANGUAGES: + try: + file_name = f"xlsum_{lang}.yaml" + task_name = f"xlsum_{lang}_{mode}" + yaml_template = "xlsum" + yaml_details = { + "include": yaml_template, + "task": task_name, + "dataset_name": lang, + "doc_to_text": prompt_func(mode, lang), + "doc_to_target": "{{summary}}", + } + file_path = os.path.join(output_dir, mode) + os.makedirs(file_path, exist_ok=True) + + with open( + f"{output_dir}/{mode}/{file_name}", + "w" if overwrite else "x", + encoding="utf8", + ) as f: + f.write("# Generated by utils.py\n") + yaml.dump( + yaml_details, + f, + allow_unicode=True, + ) + except FileExistsError: + err.append(file_name) + + if len(err) > 0: + raise FileExistsError( + "Files were not created because they already exist (use --overwrite flag):" + f" {', '.join(err)}" + ) + + +def main() -> None: + """Parse CLI args and generate language-specific yaml files.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + default=True, + action="store_true", + help="Overwrite files if they already exist", + ) + parser.add_argument( + "--output-dir", + default="./", + help="Directory to write yaml files to", + ) + + PROMPT_CHOICES = ["prompt_1", "prompt_2", "prompt_3", "prompt_4", "prompt_5"] + parser.add_argument( + "--mode", + nargs="*", + default=PROMPT_CHOICES, + choices=PROMPT_CHOICES, + help="Prompt number(s)", + ) + args = parser.parse_args() + + for mode in args.mode: + gen_lang_yamls(output_dir=args.output_dir, overwrite=args.overwrite, mode=mode) + + +if __name__ == "__main__": + main() diff --git a/lm_eval/tasks/afrobench/xlsum/xlsum.yaml b/lm_eval/tasks/afrobench/xlsum/xlsum.yaml new file mode 100644 index 00000000..8d877175 --- /dev/null +++ b/lm_eval/tasks/afrobench/xlsum/xlsum.yaml @@ -0,0 +1,11 @@ +group: xlum +task: + - xlsum_prompt_1 + - xlsum_prompt_2 + - xlsum_prompt_3 +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 2 -- GitLab From 86a3b2709114d5b3df7c166798f9ef7199d6fce3 Mon Sep 17 00:00:00 2001 From: Yufeng Xu Date: Thu, 15 May 2025 07:53:48 -0400 Subject: [PATCH 13/46] Added C4 Support (#2889) * added c4 dataset (working) * fixed bugs in c4 * fixed loading bugs in c4 dataset; using partial loading * cleaned the code * added version number for c4 * removed irrelevant files --- lm_eval/tasks/README.md | 1 + lm_eval/tasks/c4/README.md | 51 +++++++++++++++++++++++++++++++ lm_eval/tasks/c4/c4.yaml | 24 +++++++++++++++ lm_eval/tasks/c4/preprocess_c4.py | 48 +++++++++++++++++++++++++++++ 4 files changed, 124 insertions(+) create mode 100644 lm_eval/tasks/c4/README.md create mode 100644 lm_eval/tasks/c4/c4.yaml create mode 100644 lm_eval/tasks/c4/preprocess_c4.py diff --git a/lm_eval/tasks/README.md b/lm_eval/tasks/README.md index c4fc5a55..5ebccef4 100644 --- a/lm_eval/tasks/README.md +++ b/lm_eval/tasks/README.md @@ -29,6 +29,7 @@ | [bertaqa](bertaqa/README.md) | Local Basque cultural trivia QA tests in English and Basque languages. | English, Basque, Basque (MT) | | [bigbench](bigbench/README.md) | Broad tasks from the BIG-bench benchmark designed to push the boundaries of large models. | Multiple | | [blimp](blimp/README.md) | Tasks testing grammatical phenomena to evaluate language model's linguistic capabilities. | English | +| [c4](c4/README.md) | Tasks based on a colossal, cleaned version of Common Crawl's web crawl corpus to assess models' language modeling capabilities. | English | | [careqa](careqa/README.md) | Multiple choice and open-ended medical question answering based on the Spanish Specialised Healthcare Training (MIR) exams. | English, Spanish | | [catalan_bench](catalan_bench/README.md) | Collection of tasks in Catalan encompassing various evaluation areas. | Catalan | | [ceval](ceval/README.md) | Tasks that evaluate language understanding and reasoning in an educational context. | Chinese | diff --git a/lm_eval/tasks/c4/README.md b/lm_eval/tasks/c4/README.md new file mode 100644 index 00000000..5953415b --- /dev/null +++ b/lm_eval/tasks/c4/README.md @@ -0,0 +1,51 @@ +# Colossal Clean Crawled Corpus(C4) + +### Paper + +[Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/abs/1910.10683) + +A colossal, cleaned version of Common Crawl's web crawl corpus. Based on [Common Crawl dataset](https://commoncrawl.org). + +This is the processed version of Google's C4 dataset. + +[Homepage](https://huggingface.co/datasets/allenai/c4) + +### Citation + +```text +@misc{raffel2023exploringlimitstransferlearning, + title={Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer}, + author={Colin Raffel and Noam Shazeer and Adam Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu}, + year={2023}, + eprint={1910.10683}, + archivePrefix={arXiv}, + primaryClass={cs.LG}, + url={https://arxiv.org/abs/1910.10683}, +} +``` + +### Groups, Tags, and Tasks + +#### Groups + +* Not part of a group yet. + +#### Tasks + +* `c4`: measure perplexity on the C4 dataset, via rolling loglikelihoods. + +### Checklist + +For adding novel benchmarks/datasets to the library: + +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + +If other tasks on this dataset are already supported: + +* [ ] Is the "Main" variant of this task clearly denoted? +* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? + +### Changelog diff --git a/lm_eval/tasks/c4/c4.yaml b/lm_eval/tasks/c4/c4.yaml new file mode 100644 index 00000000..daea7dd1 --- /dev/null +++ b/lm_eval/tasks/c4/c4.yaml @@ -0,0 +1,24 @@ +task: c4 +dataset_path: allenai/c4 +dataset_name: en +output_type: loglikelihood_rolling +training_split: train +validation_split: validation +doc_to_text: "" +doc_to_target: !function preprocess_c4.c4_detokenizer +process_results: !function preprocess_c4.process_results +should_decontaminate: true +doc_to_decontamination_query: "{{page}}" +metric_list: + - metric: word_perplexity + - metric: byte_perplexity + - metric: bits_per_byte +metadata: + version: 0.0 +dataset_kwargs: + data_files: + train: en/c4-train.00000-of-01024.json.gz + validation: en/c4-validation.00000-of-00008.json.gz + # following the choice of https://arxiv.org/abs/2410.07461 + trust_remote_code: true + verification_mode: "no_checks" \ No newline at end of file diff --git a/lm_eval/tasks/c4/preprocess_c4.py b/lm_eval/tasks/c4/preprocess_c4.py new file mode 100644 index 00000000..5ab0d32f --- /dev/null +++ b/lm_eval/tasks/c4/preprocess_c4.py @@ -0,0 +1,48 @@ +import re + + +def c4_detokenizer(doc): + string = doc["text"] + # contractions + string = string.replace("s '", "s'") + string = re.sub(r"/' [0-9]/", r"/'[0-9]/", string) + # number separators + string = string.replace(" @-@ ", "-") + string = string.replace(" @,@ ", ",") + string = string.replace(" @.@ ", ".") + # punctuation + string = string.replace(" : ", ": ") + string = string.replace(" ; ", "; ") + string = string.replace(" . ", ". ") + string = string.replace(" ! ", "! ") + string = string.replace(" ? ", "? ") + string = string.replace(" , ", ", ") + # double brackets + string = re.sub(r"\(\s*([^\)]*?)\s*\)", r"(\1)", string) + string = re.sub(r"\[\s*([^\]]*?)\s*\]", r"[\1]", string) + string = re.sub(r"{\s*([^}]*?)\s*}", r"{\1}", string) + string = re.sub(r"\"\s*([^\"]*?)\s*\"", r'"\1"', string) + string = re.sub(r"'\s*([^']*?)\s*'", r"'\1'", string) + # miscellaneous + string = string.replace("= = = =", "====") + string = string.replace("= = =", "===") + string = string.replace("= =", "==") + string = string.replace(" " + chr(176) + " ", chr(176)) + string = string.replace(" \n", "\n") + string = string.replace("\n ", "\n") + string = string.replace(" N ", " 1 ") + string = string.replace(" 's", "'s") + + return string + + +def process_results(doc, results): + (loglikelihood,) = results + # IMPORTANT: wikitext counts number of words in *original doc before detokenization* + _words = len(re.split(r"\s+", doc["text"])) + _bytes = len(doc["text"].encode("utf-8")) + return { + "word_perplexity": (loglikelihood, _words), + "byte_perplexity": (loglikelihood, _bytes), + "bits_per_byte": (loglikelihood, _bytes), + } -- GitLab From 2bde99e48c401338beba76e2c33cde732f548ace Mon Sep 17 00:00:00 2001 From: tawsif Date: Thu, 15 May 2025 17:55:58 +0600 Subject: [PATCH 14/46] Update utils.py (#2870) --- lm_eval/tasks/mmlu_pro/utils.py | 29 ++++++----------------------- 1 file changed, 6 insertions(+), 23 deletions(-) diff --git a/lm_eval/tasks/mmlu_pro/utils.py b/lm_eval/tasks/mmlu_pro/utils.py index 03117be5..94bd6f73 100644 --- a/lm_eval/tasks/mmlu_pro/utils.py +++ b/lm_eval/tasks/mmlu_pro/utils.py @@ -1,25 +1,6 @@ from functools import partial - -choices = [ - "A", - "B", - "C", - "D", - "E", - "F", - "G", - "H", - "I", - "J", - "K", - "L", - "M", - "N", - "O", - "P", -] - +choices = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J"] def format_cot_example(example, including_answer=True): prompt = "Question:\n" @@ -27,8 +8,12 @@ def format_cot_example(example, including_answer=True): options = example["options"] prompt += question + "\n" prompt += "Options:\n" + for i, opt in enumerate(options): + if i >= len(choices): + break prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: cot_content = example["cot_content"].replace( "A: Let's think step by step.", "Answer: Let's think step by step." @@ -36,17 +21,15 @@ def format_cot_example(example, including_answer=True): prompt += cot_content + "\n\n" else: prompt += "Answer: Let's think step by step." + return prompt - doc_to_text = partial(format_cot_example, including_answer=False) fewshot_to_text = partial(format_cot_example, including_answer=True) - def process_docs(dataset, subject): return dataset.filter(lambda x: x["category"] == subject) - process_biology = partial(process_docs, subject="biology") process_business = partial(process_docs, subject="business") process_chemistry = partial(process_docs, subject="chemistry") -- GitLab From 4dbd5ec994d8477dc4a755cbc13f67b0186ca560 Mon Sep 17 00:00:00 2001 From: Tingchen Fu <48080217+TingchenFu@users.noreply.github.com> Date: Thu, 15 May 2025 19:58:00 +0800 Subject: [PATCH 15/46] feat: add question suffix (#2876) --- lm_eval/__main__.py | 6 ++++++ lm_eval/api/task.py | 14 +++++++++++--- lm_eval/evaluator.py | 2 ++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/lm_eval/__main__.py b/lm_eval/__main__.py index b97e0b17..406faddf 100644 --- a/lm_eval/__main__.py +++ b/lm_eval/__main__.py @@ -261,6 +261,12 @@ def setup_parser() -> argparse.ArgumentParser: default="", help="Comma separated string arguments passed to Hugging Face Hub's log function, e.g. `hub_results_org=EleutherAI,hub_repo_name=lm-eval-results`", ) + parser.add_argument( + "--question_suffix", + type=str, + default=None, + help="Suffix to append to the target question before the <|assistant|>, e.g., Think for maximum 128 tokens", + ) parser.add_argument( "--predict_only", "-x", diff --git a/lm_eval/api/task.py b/lm_eval/api/task.py index c1bc967a..a0dc389b 100644 --- a/lm_eval/api/task.py +++ b/lm_eval/api/task.py @@ -401,6 +401,7 @@ class Task(abc.ABC): fewshot_as_multiturn: bool = False, chat_template: Optional[Callable] = None, tokenizer_name: str = "", + question_suffix: str = "" ) -> None: """Build a set of Instances for a task, and store them in task.instances""" @@ -464,6 +465,7 @@ class Task(abc.ABC): fewshot_as_multiturn, chat_template, gen_prefix=self.doc_to_prefix(doc), + question_suffix=question_suffix, ) # TODO: we should override self.config.repeats if doing greedy gen so users don't waste time+compute @@ -1066,6 +1068,7 @@ class ConfigurableTask(Task): question: str, fewshot_as_multiturn: bool = False, gen_prefix: Optional[str] = None, + question_suffix: Optional[str] = None, ) -> None: """Adds a target question to the labeled examples list. If fewshot_as_multiturn is True, or labeled_examples is empty, or the last entry is a system turn, appends the question as a new user entry. @@ -1074,13 +1077,13 @@ class ConfigurableTask(Task): if not fewshot_as_multiturn: # if no messages or last message is system, append as new user entry if len(labeled_examples) == 0 or labeled_examples[-1]["role"] == "system": - labeled_examples.append({"role": "user", "content": question}) + labeled_examples.append({"role": "user", "content": question + question_suffix} if question_suffix else {"role": "user", "content": question} ) # if last message is user, append to it to avoid two user messages in a row else: - labeled_examples[-1]["content"] += question + labeled_examples[-1]["content"] += question + question_suffix if question_suffix else question else: # if fewshot_as_multiturn is True, append as next user entry (last is always assistant) - labeled_examples.append({"role": "user", "content": question}) + labeled_examples.append({"role": "user", "content": question + question_suffix} if question_suffix else {"role": "user", "content": question} ) if gen_prefix: labeled_examples.append({"role": "assistant", "content": gen_prefix}) @@ -1094,6 +1097,7 @@ class ConfigurableTask(Task): fewshot_as_multiturn: bool = False, chat_template: Optional[Callable] = None, gen_prefix: Optional[str] = None, + question_suffix: Optional[str] = None, ) -> Union[str, List[str]]: """Returns a fewshot context string that is made up of a prepended description (if provided), the `num_fewshot` number of examples, and an appended prompt example. @@ -1171,6 +1175,7 @@ class ConfigurableTask(Task): example, fewshot_as_multiturn, gen_prefix=gen_prefix, + question_suffix=question_suffix, ) # for loglikelihood create a list of questions with appended choices elif isinstance(example, list): @@ -1183,6 +1188,7 @@ class ConfigurableTask(Task): ex, fewshot_as_multiturn, gen_prefix=gen_prefix, + question_suffix=question_suffix, ) # TODO: append prefill? labeled_examples_list.append( @@ -1201,6 +1207,7 @@ class ConfigurableTask(Task): choices[example], fewshot_as_multiturn, gen_prefix=gen_prefix, + question_suffix=question_suffix, ) else: self.append_target_question( @@ -1208,6 +1215,7 @@ class ConfigurableTask(Task): str(example), fewshot_as_multiturn, gen_prefix=gen_prefix, + question_suffix=question_suffix, ) # return lm.apply_chat_template(labeled_examples) return chat_template( diff --git a/lm_eval/evaluator.py b/lm_eval/evaluator.py index d1312b28..f9875f3d 100644 --- a/lm_eval/evaluator.py +++ b/lm_eval/evaluator.py @@ -413,6 +413,7 @@ def evaluate( fewshot_as_multiturn: bool = False, verbosity: str = "INFO", confirm_run_unsafe_code: bool = False, + question_suffix: Optional[str] = None, ): """Instantiate and evaluate a model on a list of tasks. @@ -526,6 +527,7 @@ def evaluate( tokenizer_name=getattr(lm, "tokenizer_name", "") if apply_chat_template else "", + question_suffix=question_suffix, ) eval_logger.debug( f"Task: {task_output.task_name}; number of requests on this rank: {len(task.instances)}" -- GitLab From 96966f5323ea7ade57a83a169eaed44b274f48b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filippo=20Moment=C3=A8?= <68816087+momentino@users.noreply.github.com> Date: Thu, 15 May 2025 13:59:25 +0200 Subject: [PATCH 16/46] Add device arg to model_args passed to LLM object in VLLM model class (#2879) * fix: pass device arg in model_ar in vllm_causallms * casting device arg to str in vLLM model args --- lm_eval/models/vllm_causallms.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lm_eval/models/vllm_causallms.py b/lm_eval/models/vllm_causallms.py index b02c541a..8d71a7b8 100644 --- a/lm_eval/models/vllm_causallms.py +++ b/lm_eval/models/vllm_causallms.py @@ -100,6 +100,7 @@ class VLLM(TemplateLM): "swap_space": int(swap_space), "quantization": quantization, "seed": int(seed), + "device": str(device), } self.model_args.update(kwargs) self.batch_size = ( -- GitLab From 0126f6d15e6d5f9b93f244b8ece5a44bdbac9c2c Mon Sep 17 00:00:00 2001 From: Baber Abbasi <92168766+baberabb@users.noreply.github.com> Date: Thu, 15 May 2025 17:01:12 +0500 Subject: [PATCH 17/46] fix formatting (#2759) --- lm_eval/tasks/paws-x/paws_de.yaml | 1 + lm_eval/tasks/paws-x/paws_en.yaml | 1 + lm_eval/tasks/paws-x/paws_es.yaml | 1 + lm_eval/tasks/paws-x/paws_fr.yaml | 1 + lm_eval/tasks/paws-x/pawsx_template_yaml | 1 + lm_eval/tasks/paws-x/utils.py | 43 ++++++++++++++++++++++++ 6 files changed, 48 insertions(+) create mode 100644 lm_eval/tasks/paws-x/utils.py diff --git a/lm_eval/tasks/paws-x/paws_de.yaml b/lm_eval/tasks/paws-x/paws_de.yaml index 055b3735..52def7a7 100644 --- a/lm_eval/tasks/paws-x/paws_de.yaml +++ b/lm_eval/tasks/paws-x/paws_de.yaml @@ -4,4 +4,5 @@ doc_to_choice: '{{[sentence1+", richtig? Nein, "+sentence2, sentence1+", richtig "+sentence2]}}' doc_to_text: '' include: pawsx_template_yaml +process_docs: !function utils.process_docs_paraphrases task: paws_de diff --git a/lm_eval/tasks/paws-x/paws_en.yaml b/lm_eval/tasks/paws-x/paws_en.yaml index b5955b03..fdc0e2ec 100644 --- a/lm_eval/tasks/paws-x/paws_en.yaml +++ b/lm_eval/tasks/paws-x/paws_en.yaml @@ -3,4 +3,5 @@ dataset_name: en doc_to_choice: '{{[sentence1+", right? No, "+sentence2, sentence1+", right? Yes, "+sentence2]}}' doc_to_text: '' include: pawsx_template_yaml +process_docs: !function utils.process_docs_paraphrases task: paws_en diff --git a/lm_eval/tasks/paws-x/paws_es.yaml b/lm_eval/tasks/paws-x/paws_es.yaml index 65189a37..4df52f7c 100644 --- a/lm_eval/tasks/paws-x/paws_es.yaml +++ b/lm_eval/tasks/paws-x/paws_es.yaml @@ -4,4 +4,5 @@ doc_to_choice: '{{[sentence1+", verdad? No, "+sentence2, sentence1+", verdad? S "+sentence2]}}' doc_to_text: '' include: pawsx_template_yaml +process_docs: !function utils.process_docs_paraphrases task: paws_es diff --git a/lm_eval/tasks/paws-x/paws_fr.yaml b/lm_eval/tasks/paws-x/paws_fr.yaml index a8f599a1..7cc55c1e 100644 --- a/lm_eval/tasks/paws-x/paws_fr.yaml +++ b/lm_eval/tasks/paws-x/paws_fr.yaml @@ -4,4 +4,5 @@ doc_to_choice: '{{[sentence1+", n''est-ce pas? Non, "+sentence2, sentence1+", n' pas? Oui, "+sentence2]}}' doc_to_text: '' include: pawsx_template_yaml +process_docs: !function utils.process_docs_paraphrases task: paws_fr diff --git a/lm_eval/tasks/paws-x/pawsx_template_yaml b/lm_eval/tasks/paws-x/pawsx_template_yaml index 6f82e4a5..ccf4c543 100644 --- a/lm_eval/tasks/paws-x/pawsx_template_yaml +++ b/lm_eval/tasks/paws-x/pawsx_template_yaml @@ -11,6 +11,7 @@ test_split: test doc_to_text: null doc_to_target: label doc_to_choice: null +target_delimiter: "" metric_list: - metric: acc aggregation: mean diff --git a/lm_eval/tasks/paws-x/utils.py b/lm_eval/tasks/paws-x/utils.py new file mode 100644 index 00000000..6f5a077a --- /dev/null +++ b/lm_eval/tasks/paws-x/utils.py @@ -0,0 +1,43 @@ +import re + + +def general_detokenize(string): + string = string.replace(" n't", "n't") + string = string.replace(" )", ")") + string = string.replace("( ", "(") + string = string.replace('" ', '"') + string = string.replace(' "', '"') + string = re.sub(r" (['.,])", r"\1", string) + return string + + +def lowercase_first_letter(text): + return text[0].lower() + text[1:] + + +def process_docs_paraphrases(dataset): + empty_docs = [] + + def _process_doc(doc): + if doc["sentence1"] not in [None, ""] and doc["sentence2"] not in [None, ""]: + doc["sentence1"] = general_detokenize(doc["sentence1"]).strip() + doc["sentence2"] = general_detokenize(doc["sentence2"]).strip() + # Remove final punctuation mark in the first sentence + if doc["sentence1"].endswith((".", ",", ";")): + doc["sentence1"] = doc["sentence1"][:-1] + # Start the second sentence in lowercase (to be used after "Yes, ...") + doc["sentence2"] = lowercase_first_letter(doc["sentence2"]) + return doc + else: + empty_docs.append(doc) + return doc + + if empty_docs != []: + len_empty_docs = len(empty_docs) + print( + f"Found {len_empty_docs} empty documents out of the {len(dataset)} total docs in the dataset: {empty_docs}" + ) + return dataset.filter( + lambda doc: doc["sentence1"] not in [None, ""] + and doc["sentence2"] not in [None, ""] + ).map(_process_doc) -- GitLab From 86c266a1b3d9f51a43d277032da9f16a632d0bc3 Mon Sep 17 00:00:00 2001 From: Stella Biderman Date: Sat, 17 May 2025 17:54:43 -0400 Subject: [PATCH 18/46] Delete scripts/cost_estimate.py (#2985) This function was written years ago when the cost of running an OpenAI model was easy to compute. It is no longer viable to support this. --- scripts/cost_estimate.py | 99 ---------------------------------------- 1 file changed, 99 deletions(-) delete mode 100644 scripts/cost_estimate.py diff --git a/scripts/cost_estimate.py b/scripts/cost_estimate.py deleted file mode 100644 index baf81147..00000000 --- a/scripts/cost_estimate.py +++ /dev/null @@ -1,99 +0,0 @@ -import random - -import transformers - -from lm_eval import evaluator, tasks -from lm_eval.api.model import LM - - -class DryrunLM(LM): - def __init__(self): - self.tokencost = 0 - self.tokenizer = transformers.GPT2TokenizerFast.from_pretrained("gpt2") - self.tokenizer.pad_token = "<|endoftext|>" - - @classmethod - def create_from_arg_string(cls, arg_string): - return cls() - - def loglikelihood(self, requests): - res = [] - - for ctx, cont in requests: - res.append((-random.random(), False)) - self.tokencost += len(self.tokenizer.tokenize(ctx + cont)) - - return res - - def generate_until(self, requests): - res = [] - - for ctx, _ in requests: - res.append("lol") - - # assume worst case - generates until 256 - self.tokencost += len(self.tokenizer.tokenize(ctx)) + 256 - - return res - - def loglikelihood_rolling(self, requests): - res = [] - - for (s,) in requests: - # assume worst case: extra full context - self.tokencost += len(self.tokenizer.tokenize(s)) + 2048 - - return res - - -def main(): - lm = DryrunLM() - - task_list = "arc_challenge,arc_easy,boolq,cola,copa,headqa,hellaswag,lambada,logiqa,mathqa,mc_taco,mrpc,multirc,openbookqa,piqa,prost,pubmedqa,qnli,qqp,race,record,rte,sciq,sst,triviaqa,webqs,wic,wikitext,winogrande,wnli,wsc" - values = [] - for taskname in task_list.split(","): - lm.tokencost = 0 - evaluator.simple_evaluate( - lm=lm, - task_dict={taskname: tasks.get_task(taskname)()}, - num_fewshot=0, - limit=None, - bootstrap_iters=10, - ) - - print(taskname, lm.tokencost) - values.append( - [ - taskname, - lm.tokencost, - lm.tokencost / 1000 * 0.0008, - lm.tokencost / 1000 * 0.0012, - lm.tokencost / 1000 * 0.006, - lm.tokencost / 1000 * 0.06, - ] - ) - from pytablewriter import MarkdownTableWriter - - writer = MarkdownTableWriter() - writer.headers = ["Task", "Tokens", "Ada", "Babbage", "Curie", "Davinci"] - - values.sort(key=lambda x: -x[1]) - totcost = sum([x[1] for x in values]) - values.append( - [ - "**Total**", - totcost, - totcost / 1000 * 0.0008, - totcost / 1000 * 0.0012, - totcost / 1000 * 0.006, - totcost / 1000 * 0.06, - ] - ) - - writer.value_matrix = values - - print(writer.dumps()) - - -if __name__ == "__main__": - main() -- GitLab From 0daf28fda71dc17ca4cf93407b9601fa42cce498 Mon Sep 17 00:00:00 2001 From: Harsha <858059+harshakokel@users.noreply.github.com> Date: Mon, 19 May 2025 03:39:34 -0700 Subject: [PATCH 19/46] Adding ACPBench Hard tasks (#2980) * adding ACPBench_hard * adding Clingo * changing tarski to tarski[clingo] * denoting the main variants in each paper --- lm_eval/tasks/README.md | 1 + lm_eval/tasks/acpbench/README.md | 61 +- .../tasks/acpbench/gen_2shot/_gen_yaml_2shot | 25 + .../tasks/acpbench/gen_2shot/acp_grammar.lark | 23 + lm_eval/tasks/acpbench/gen_2shot/acp_utils.py | 1128 +++++++++++++++++ .../tasks/acpbench/gen_2shot/act_reach.yaml | 19 + lm_eval/tasks/acpbench/gen_2shot/app.yaml | 19 + lm_eval/tasks/acpbench/gen_2shot/just.yaml | 20 + lm_eval/tasks/acpbench/gen_2shot/land.yaml | 19 + .../tasks/acpbench/gen_2shot/next_act.yaml | 19 + lm_eval/tasks/acpbench/gen_2shot/prog.yaml | 20 + lm_eval/tasks/acpbench/gen_2shot/reach.yaml | 19 + lm_eval/tasks/acpbench/gen_2shot/val.yaml | 19 + .../gen_2shot_with_pddl/_gen_yaml_2shot | 26 + .../gen_2shot_with_pddl/acp_grammar.lark | 23 + .../acpbench/gen_2shot_with_pddl/acp_utils.py | 1128 +++++++++++++++++ .../gen_2shot_with_pddl/act_reach.yaml | 23 + .../acpbench/gen_2shot_with_pddl/app.yaml | 23 + .../acpbench/gen_2shot_with_pddl/just.yaml | 24 + .../acpbench/gen_2shot_with_pddl/land.yaml | 23 + .../gen_2shot_with_pddl/next_act.yaml | 23 + .../acpbench/gen_2shot_with_pddl/prog.yaml | 24 + .../acpbench/gen_2shot_with_pddl/reach.yaml | 23 + .../acpbench/gen_2shot_with_pddl/val.yaml | 23 + pyproject.toml | 2 + 25 files changed, 2752 insertions(+), 5 deletions(-) create mode 100644 lm_eval/tasks/acpbench/gen_2shot/_gen_yaml_2shot create mode 100644 lm_eval/tasks/acpbench/gen_2shot/acp_grammar.lark create mode 100644 lm_eval/tasks/acpbench/gen_2shot/acp_utils.py create mode 100644 lm_eval/tasks/acpbench/gen_2shot/act_reach.yaml create mode 100644 lm_eval/tasks/acpbench/gen_2shot/app.yaml create mode 100644 lm_eval/tasks/acpbench/gen_2shot/just.yaml create mode 100644 lm_eval/tasks/acpbench/gen_2shot/land.yaml create mode 100644 lm_eval/tasks/acpbench/gen_2shot/next_act.yaml create mode 100644 lm_eval/tasks/acpbench/gen_2shot/prog.yaml create mode 100644 lm_eval/tasks/acpbench/gen_2shot/reach.yaml create mode 100644 lm_eval/tasks/acpbench/gen_2shot/val.yaml create mode 100644 lm_eval/tasks/acpbench/gen_2shot_with_pddl/_gen_yaml_2shot create mode 100644 lm_eval/tasks/acpbench/gen_2shot_with_pddl/acp_grammar.lark create mode 100644 lm_eval/tasks/acpbench/gen_2shot_with_pddl/acp_utils.py create mode 100644 lm_eval/tasks/acpbench/gen_2shot_with_pddl/act_reach.yaml create mode 100644 lm_eval/tasks/acpbench/gen_2shot_with_pddl/app.yaml create mode 100644 lm_eval/tasks/acpbench/gen_2shot_with_pddl/just.yaml create mode 100644 lm_eval/tasks/acpbench/gen_2shot_with_pddl/land.yaml create mode 100644 lm_eval/tasks/acpbench/gen_2shot_with_pddl/next_act.yaml create mode 100644 lm_eval/tasks/acpbench/gen_2shot_with_pddl/prog.yaml create mode 100644 lm_eval/tasks/acpbench/gen_2shot_with_pddl/reach.yaml create mode 100644 lm_eval/tasks/acpbench/gen_2shot_with_pddl/val.yaml diff --git a/lm_eval/tasks/README.md b/lm_eval/tasks/README.md index 5ebccef4..8578a8af 100644 --- a/lm_eval/tasks/README.md +++ b/lm_eval/tasks/README.md @@ -9,6 +9,7 @@ |--------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------| | [aclue](aclue/README.md) | Tasks focusing on ancient Chinese language understanding and cultural aspects. | Ancient Chinese | | [acp_bench](acpbench/README.md) | Tasks evaluating the reasoning ability about Action, Change, and Planning | English | +| [acp_bench_hard](acpbench/README.md) | Tasks evaluating the reasoning ability about Action, Change, and Planning | English | | [aexams](aexams/README.md) | Tasks in Arabic related to various academic exams covering a range of subjects. | Arabic | | [agieval](agieval/README.md) | Tasks involving historical data or questions related to history and historical texts. | English, Chinese | | [anli](anli/README.md) | Adversarial natural language inference tasks designed to test model robustness. | English | diff --git a/lm_eval/tasks/acpbench/README.md b/lm_eval/tasks/acpbench/README.md index 5ff19061..264b54ce 100644 --- a/lm_eval/tasks/acpbench/README.md +++ b/lm_eval/tasks/acpbench/README.md @@ -1,13 +1,26 @@ # ACPBench -### Paper +**Homepage:** https://ibm.github.io/ACPBench/ -Title: ACPBench: Reasoning About Action, Change, and Planning -Abstract: https://arxiv.org/pdf/2410.05669 +### Papers + +**Title:** ACPBench: Reasoning About Action, Change, and Planning +**Pdf:** https://arxiv.org/pdf/2410.05669 +**Task:** `acp_bench` +**Abstract:** There is an increasing body of work using Large Language Models (LLMs) as agents for orchestrating workflows and making decisions in domains that require planning and multi-step reasoning. As a result, it is imperative to evaluate LMs on core skills required for planning. ACPBench is a benchmark for evaluating the reasoning tasks in the field of planning. The benchmark consists of 7 reasoning tasks over 13 planning domains. The collection is constructed from planning domains described in a formal language. This allows the synthesized problems to have provably correct solutions across many tasks and domains. Further, it allows the luxury to scale without additional human effort, i.e., many additional problems can be created automatically. -Homepage: https://ibm.github.io/ACPBench/ + + +**Title:** ACPBench Hard: Unrestrained Reasoning about Action, Change, and Planning +**Pdf:** https://arxiv.org/abs/2503.24378 +**Task:** `acp_bench_hard` +**Abstract:** + +We introduce ACPBench Hard, a dataset of generative, open-ended questions which LLM models needs to answer in order to plan. Models that perform well on these tasks could in principle be integrated into a planner or be used directly as a policy. We discuss the complexity of these tasks as well as the complexity of validating the correctness of their answers and present validation algorithms for each task. Equipped with these validators, we test the performance of a variety of models on our tasks and find that for most of these tasks, the performance of even the largest models is still subpar. Our experiments show that no model outperforms any other in these tasks, and with a few exceptions, all tested language models score below 65\%, indicating that even the current frontier language models as well as so-called reasoning models have a long way to go before they can reliably reason about planning. + +The dataset is available on [HuggingFace](https://huggingface.co/datasets/ibm-research/acp_bench). ### Citation @@ -23,6 +36,19 @@ Homepage: https://ibm.github.io/ACPBench/ publisher = {{AAAI} Press}, year = {2025} } + +@misc{KokelKSS25ACPHard, + title = {ACPBench Hard: Unrestrained Reasoning about Action, Change, and Planning}, + author = {Harsha Kokel and + Michael Katz and + Kavitha Srinivas and + Shirin Sohrabi}, + year = {2025}, + eprint = {2503.24378}, + archivePrefix = {arXiv}, + primaryClass = {cs.AI}, + url = {https://arxiv.org/abs/2503.24378}, +} ``` ### Groups, Tags, and Tasks @@ -33,9 +59,13 @@ Homepage: https://ibm.github.io/ACPBench/ #### Tags -* `acp_bench` : Evaluates `acp_bool_cot_2shot` and `acp_mcq_cot_2shot` +* `acp_bench` : Evaluates `acp_bool_cot_2shot` and `acp_mcq_cot_2shot` (Main variant for ACPBench paper) * `acp_bool_cot_2shot` : Evaluates `acp_areach_bool`, `acp_app_bool`, `acp_just_bool`, `acp_land_bool`, `acp_prog_bool`, `acp_reach_bool`, `acp_val_bool` with chain-of-thought and 2 shots * `acp_mcq_cot_2shot` : Evaluates `acp_areach_mcq`, `acp_app_mcq`, `acp_just_mcq`, `acp_land_mcq`, `acp_prog_mcq`, `acp_reach_mcq`, `acp_val_mcq` with chain-of-thought and 2 shots +* `acp_bench_hard` : Evaluates `acp_gen_2shot` (Main variant for ACPBench Hard paper) +* `acp_gen_2shot` : Evaluates `acp_areach_gen`, `acp_app_gen`, `acp_just_gen`, `acp_land_gen`, `acp_nexta_gen`, `acp_prog_gen`, `acp_reach_gen`, `acp_val_gen` with 2 shots +* `acp_bench_hard_with_pddl` : Evaluates `acp_gen_2shot_with_pddl` +* `acp_gen_2shot_with_pddl` : Evaluates `acp_areach_gen_with_pddl`, `acp_app_gen_with_pddl`, `acp_just_gen_with_pddl`, `acp_land_gen_with_pddl`, `acp_nexta_gen_with_pddl`, `acp_prog_gen_with_pddl`, `acp_reach_gen_with_pddl`, `acp_val_gen_with_pddl` with 2 shots #### Tasks @@ -57,6 +87,26 @@ Homepage: https://ibm.github.io/ACPBench/ * `acp_reach_mcq` * `acp_val_mcq` +8 Generative tasks (with just natural language description in context) +* `acp_areach_gen` +* `acp_app_gen` +* `acp_just_gen` +* `acp_land_gen` +* `acp_nexta_gen` +* `acp_prog_gen` +* `acp_reach_gen` +* `acp_val_gen` + +and the same 8 generative tasks with natural language as well as the PDDL description of the domain and problem in context. +* `acp_areach_gen_with_pddl` +* `acp_app_gen_with_pddl` +* `acp_just_gen_with_pddl` +* `acp_land_gen_with_pddl` +* `acp_nexta_gen_with_pddl` +* `acp_prog_gen_with_pddl` +* `acp_reach_gen_with_pddl` +* `acp_val_gen_with_pddl` + > ! The evaluation scripts are taken from original github https://github.com/IBM/ACPBench @@ -77,3 +127,4 @@ If other tasks on this dataset are already supported: ### Change Log * 03/17/2025 Initial Commit +* 05/13/2025 Adding ACPBench Hard tasks (with and without PDDL) diff --git a/lm_eval/tasks/acpbench/gen_2shot/_gen_yaml_2shot b/lm_eval/tasks/acpbench/gen_2shot/_gen_yaml_2shot new file mode 100644 index 00000000..d7316051 --- /dev/null +++ b/lm_eval/tasks/acpbench/gen_2shot/_gen_yaml_2shot @@ -0,0 +1,25 @@ +tag: + - acp_gen_2shot + - acp_bench_hard +dataset_path: ibm-research/acp_bench +test_split: test +doc_to_target: "{{answer}}" +output_type: generate_until +num_fewshot: 2 +generation_kwargs: + until: + - "\n\n\n\n" + - "\n\n" + - "**Question**:" + - "**Question:**" + - "Q:" + do_sample: false + max_gen_toks: 1000 + temperature: 0.0 +metadata: + version: 1.0 +process_results: !function acp_utils.process_acp_results +metric_list: + - metric: "score" + aggregation: mean + higher_is_better: True diff --git a/lm_eval/tasks/acpbench/gen_2shot/acp_grammar.lark b/lm_eval/tasks/acpbench/gen_2shot/acp_grammar.lark new file mode 100644 index 00000000..036bd675 --- /dev/null +++ b/lm_eval/tasks/acpbench/gen_2shot/acp_grammar.lark @@ -0,0 +1,23 @@ +NAME: /[a-zA-Z][a-zA-Z0-9-_]*/ +LPAR : "(" +RPAR : ")" +LSPAR: "[" +RSPAR: "]" +COMMA: "," +WS: /[ \n]/ + +action_none : "None" + +action_name : LPAR NAME (WS NAME)* RPAR + +action_list : (action_name WS?)* + +prog_list : action_name* (COMMA action_name)* + +progression_list : LSPAR prog_list RSPAR LSPAR prog_list RSPAR + +act : action_name | action_none + +index: /[0-9]+[0-9]*/ + +start: action_list diff --git a/lm_eval/tasks/acpbench/gen_2shot/acp_utils.py b/lm_eval/tasks/acpbench/gen_2shot/acp_utils.py new file mode 100644 index 00000000..5051b68c --- /dev/null +++ b/lm_eval/tasks/acpbench/gen_2shot/acp_utils.py @@ -0,0 +1,1128 @@ +import json +import os +from abc import ABC, abstractmethod +from collections import defaultdict +from pathlib import Path + +from lm_eval.api.registry import register_filter +from lm_eval.filters.extraction import RegexFilter + + +try: + import tempfile + + import tarski + from kstar_planner import planners as kp + from lark import Lark + from lark.lexer import Token + from lark.visitors import Visitor + from pddl.core import Problem + from pddl.parser.domain import DomainParser + from pddl.parser.problem import ProblemParser + from tarski.grounding.common import StateVariableLite + from tarski.grounding.lp_grounding import LPGroundingStrategy + from tarski.io import PDDLReader + from tarski.io import fstrips as iofs + from tarski.syntax.formulas import is_atom + from tarski.syntax.transform.action_grounding import ( + ground_schema_into_plain_operator_from_grounding, + ) + from tarski.util import SymbolIndex +except ModuleNotFoundError: + raise ModuleNotFoundError( + "`lark>=1.1.9`, `tarski[clingo]==0.8.2`, `pddl==0.4.2` and `kstar-planner==1.4.2` are required for evaluating the generative tasks. \ +Please install via pip install lm-eval[acpbench] or pip install -e .[acpbench]", + ) + + +######################################################################### +# Grammar + + +GRAMMAR_FILE = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "acp_grammar.lark" +) + + +class ACPBench_Visitor(Visitor): + def __init__(self) -> None: + super().__init__() + self.action_lists = None + self.action_names = None + self.progression_lists = None + self.prog_lists = None + self.indexes = None + + def action_list(self, tree): + self.action_lists = [] + + def prog_list(self, tree): + if self.prog_lists is not None: + self.progression_lists.append(self.prog_lists) + self.prog_lists = [] + + def progression_list(self, tree): + self.progression_lists = [] + + def action_none(self, tree): + self.action_names = "None" + + def action_name(self, tree): + act_name = "(" + "".join(tree.children[1:-1]) + ")" + self.action_names = act_name + if self.action_lists is not None: + self.action_lists.append(act_name) + if self.prog_lists is not None: + self.prog_lists.append(act_name) + + def index(self, tree): + self.indexes = "".join(tree.children) + if not self.indexes.isnumeric(): + self.indexes = None + + +class ACPGrammarParser(object): + def __init__(self, task) -> None: + self.task = task + with open(GRAMMAR_FILE) as f: + grammar = f.read() + self.acp_parser = Lark(grammar, start=task, parser="lalr") + + def parse(self, input, debug=False): + def ignore_errors(e): + if hasattr(e, "token") and e.token.type == "$END": + for x in e.expected: + if x != "WS": + e.interactive_parser.feed_token( + Token(x, self.acp_parser.get_terminal(x).pattern.value) + ) + + return True + + input = input.replace("\n", "") + input = input.strip() + try: + tree = self.acp_parser.parse(input, on_error=ignore_errors) + + if debug: + print(tree) + visitor = ACPBench_Visitor() + visitor.visit_topdown(tree) + if self.task == "action_list": + return visitor.action_lists + elif self.task == "act": + return visitor.action_names + elif self.task == "action_name": + return visitor.action_names + elif self.task == "index": + return visitor.indexes + elif self.task == "progression_list": + if visitor.prog_lists not in visitor.progression_lists: + visitor.progression_lists.append(visitor.prog_lists) + return visitor.progression_lists + except Exception as e: + if debug: + print("exception") + print(e) + return None + + +############################################################################## +# Utils + + +# Used in next action +def is_on_optimal_plan(domain, problem, action, opt): + with ( + tempfile.NamedTemporaryFile() as domain_temp, + tempfile.NamedTemporaryFile() as problem_temp, + ): + with open(str(domain_temp.name), "w", encoding="utf8") as file: + file.write(domain.lower()) + with open(str(problem_temp.name), "w", encoding="utf8") as file: + file.write(problem.lower()) + + # Here, we need to keep the temp files live until the end of the function + try: + P = STRIPS(str(domain_temp.name), str(problem_temp.name)) + except Exception: + # Unsolvable + return False + + a = P.get_action_or_none(action[1:-1]) + if a is None: + return False + state = P.init + next_state = progress(state, a) + if opt is None: + # Get an optimal plan cost + plans = generate_optimal_plans_for_problem_state( + P, state, num_plans=1, timeout=5 + ) + opt = len(plans[0]["actions"]) + else: + opt = int(opt) + + # Getting an optimal plan for the next state + next_plans = generate_optimal_plans_for_problem_state( + P, next_state, num_plans=1, timeout=5 + ) + if next_plans is None: + return False + next_opt = len(next_plans[0]["actions"]) + return next_opt + 1 == opt + + +# Used in justification +def is_plan(domain, problem, new_plan): + P = get_STRIPS(domain, problem) + if P is None: + # Unsolvable + return False + + # Check if new_plan is a plan + current_state = P.init + for action in new_plan: + applicable_actions = P.get_applicable_actions(current_state) + app_actions_list = [f"({a.name.lower()})" for a in applicable_actions] + if action.lower() not in app_actions_list: + return False + a = applicable_actions[app_actions_list.index(action.lower())] + current_state = progress(current_state, a) + return entails(current_state, P.goal) + + +# Used in action reachability +def get_action_preconditions(domain, problem, action): + P = get_STRIPS(domain, problem) + + assert P is not None, f"Domain\n{domain}\nProblem\n{problem}\nAction: {action}" + a = P.get_action_or_none(action[1:-1]) + if a is None: + return a + + return [f"({f})" for f in a.pres] + + +def generate_optimal_plans_for_problem_state(P, state, num_plans, timeout): + import tempfile + + with ( + tempfile.NamedTemporaryFile() as domain_temp, + tempfile.NamedTemporaryFile() as problem_temp, + ): + create_tmp_dom_prob_replace_init(P, state, domain_temp, problem_temp) + plans = generate_top_q_plans( + domain=str(domain_temp.name), + problem=str(problem_temp.name), + num_plans=num_plans, + quality_bound=1.0, + timeout=timeout, + ) + # print(plans) + if plans is None or len(plans["plans"]) == 0: + return None + return plans["plans"] + + +def generate_top_q_plans(domain, problem, num_plans=10, quality_bound=1.0, timeout=30): + # print("Running K* planner") + plans = kp.plan_unordered_topq( + domain_file=Path(domain), + problem_file=Path(problem), + number_of_plans_bound=num_plans, + quality_bound=quality_bound, + timeout=timeout, + ) + return plans + + +# Used in (action) reachability +def is_unsolvable_new_goal(domain, problem, new_goal): + goal = extract_goal(problem) + new_problem = problem.replace(goal, f"(:goal {new_goal} )") + return is_unsolvable(domain, new_problem) + + +def is_unsolvable(domain, problem): + with ( + tempfile.NamedTemporaryFile() as domain_temp, + tempfile.NamedTemporaryFile() as problem_temp, + ): + with open(str(domain_temp.name), "w", encoding="utf8") as file: + file.write(str(domain)) + with open(str(problem_temp.name), "w", encoding="utf8") as file: + file.write(str(problem)) + + plans = kp.plan_unordered_topq( + domain_file=Path(str(domain_temp.name)), + problem_file=Path(str(problem_temp.name)), + quality_bound=1.0, + number_of_plans_bound=1, + timeout=3, + ) + + if len(plans["planner_error"]) > 0: + fl = plans["planner_error"].split("\n")[0] + print(f"Planner error: {fl}") + return False + if plans is None or len(plans["plans"]) == 0: + return plans["unsolvable"] + return False + + +def extract_goal(prob): + a = prob.split("(:goal")[1] + cp = 1 + for i, c in enumerate(a): + if c == ")": + cp -= 1 + if c == "(": + cp += 1 + if cp == 0: + return "(:goal" + a[: i + 1] + + assert False + + +def entails(state, partialstate): + return partialstate <= state + + +def progress(state, act): + assert entails(state, act.pres), ( + "Cannot progress with inconsistent state / action precondition:\n\t Action: " + + act.name + + "\n\t State: \n\t\t" + + "\n\t\t".join(state) + ) + return (state - act.dels) | act.adds + + +def regress(state, act): + assert len(state & act.dels) == 0, ( + "Cannot regress with inconsistent state / action delete effect:\n\t Action: " + + act.name + + "\n\t State: \n\t\t" + + "\n\t\t".join(state) + ) + return (state - act.adds) | act.pres + + +def get_STRIPS(domain, problem): + with ( + tempfile.NamedTemporaryFile() as domain_temp, + tempfile.NamedTemporaryFile() as problem_temp, + ): + with open(str(domain_temp.name), "w", encoding="utf8") as file: + file.write(domain.lower()) + with open(str(problem_temp.name), "w", encoding="utf8") as file: + file.write(problem.lower()) + + try: + P = STRIPS(str(domain_temp.name), str(problem_temp.name)) + return P + except Exception as e: + print(f"||{e}||") + return None + + +def create_tmp_dom_prob_replace_init(P, state, result_domain_file, result_problem_file): + d, p = P.PDDL_replace_init_pddl_parser(state) + with open(str(result_domain_file.name), "w", encoding="utf8") as file: + file.write(str(d)) + with open(str(result_problem_file.name), "w", encoding="utf8") as file: + file.write(str(p)) + + return d, p + + +def fix_name(s): + # (act param) + if "(" == s[0] and ")" == s[-1]: + return s[1:-1] + # make it space separated + s = s.replace(", ", " ").replace(",", " ") + # act(param) + if "(" in s: + assert ")" == s[-1], f"Broken name? {s}" + s = s.replace("(", " ").replace(")", "") + # act param + return s + + +def get_atoms_pddl(d, p, atoms): + objs = set() + preds = defaultdict(list) + for atom in atoms: + a = atom.lower().strip().split(" ") + args = a[1:] + preds[a[0]].append(args) + objs |= set(args) + + constants = [o for o in p.objects | d.constants if o.name.lower() in objs] + constants_dict = {} + for c in constants: + constants_dict[c.name.lower()] = c + assert len(objs) == len(constants), ( + f"Could not identify all objects: {objs - set(constants_dict.keys())} not found, {set(constants_dict.keys()) - objs} should not be there" + ) + + state = [] + covered_preds = set() + for f in d.predicates: + name = f.name.lower() + if name in preds: + covered_preds.add(name) + assert len(preds[name][0]) == f.arity, ( + f"The arity does not match: {preds[name]} vs {f.terms}" + ) + # Going over the lists of objects, adding ground predicate for each + for ob in preds[name]: + c = [constants_dict[o] for o in ob] + state.append(f(*c)) + assert len(covered_preds) == len(preds.keys()), ( + f"Covered predicates: \n{sorted(list(covered_preds))} vs \n{sorted(list(preds.keys()))}" + ) + return set(state) + + +class Action: + def __init__(self, name, pre, add, delete): + self.name = name + self.pres = pre + self.adds = add + self.dels = delete + + def __str__(self): + pres = "{" + ", ".join([f"({a})" for a in self.pres]) + "}" + adds = "{" + ", ".join([f"({a})" for a in self.adds]) + "}" + dels = "{" + ", ".join([f"({a})" for a in self.dels]) + "}" + + return f"< {self.name}, {pres}, {adds}, {dels} >" + + def toJSON(self): + return json.dumps( + { + "name": self.name, + "preconditions": [f"({a})" for a in self.pres], + "add_effects": [f"({a})" for a in self.adds], + "delete_effects": [f"({a})" for a in self.dels], + }, + sort_keys=True, + indent=4, + ) + + def __repr__(self): + return self.name + + def __eq__(self, action): + return self.name == action.name + + def __hash__(self): + return hash(self.name) + + +class STRIPS: + def __init__(self, domain, problem): + self.domain_file = domain + self.problem_file = problem + self.reader = PDDLReader(raise_on_error=True) + self.reader.parse_domain(domain) + self.problem = self.reader.parse_instance(problem) + (self.grounded_fluents, init, goal, self.operators, self.grounder) = ( + self.ground_problem(self.problem) + ) + + self.fluents = set([fix_name(str(f)) for f in self.grounded_fluents]) + self.fluents_map = dict() + for f in self.grounded_fluents: + self.fluents_map[fix_name(str(f))] = f + self.init = set([fix_name(str(f)) for f in init]) + self.goal = set([fix_name(str(f)) for f in goal]) + self.actions = set() + self.action_map = {} + self.init_fluents = [self.fluents_map[f] for f in self.init] + + self.static_predicates = [i.name for i in self.grounder.static_symbols] + for op in self.operators: + act = self.operator_to_action(op) + self.actions.add(act) + self.action_map[act.name.lower()] = act + + def __str__(self): + fluents = "P = {" + ", ".join([f"({a})" for a in self.fluents]) + "}" + init = "I = {" + ", ".join([f"({a})" for a in self.init]) + "}" + goal = "G = {" + ", ".join([f"({a})" for a in self.goal]) + "}" + actions = "A = {" + "\n ".join([a.__str__() for a in self.actions]) + "}" + return fluents + ",\n" + init + "\n" + goal + "\n" + actions + + def toJSON(self): + actions = [a.toJSON() for a in self.actions] + return json.dumps( + { + "fluents": list(self.fluents), + "initial_state": list(self.init), + "goal": list(self.goal), + "actions": actions, + }, + sort_keys=True, + indent=4, + ) + + def operator_to_action(self, op, check_fluents=True, check_static=False): + adds = { + fix_name(str(f.atom)) for f in op.effects if isinstance(f, iofs.AddEffect) + } & self.fluents + dels = { + fix_name(str(f.atom)) for f in op.effects if isinstance(f, iofs.DelEffect) + } & self.fluents + pre = self.fix_pre_name(op.precondition) + if check_fluents: + pre = pre & self.fluents + if check_static: + pre = {p for p in pre if p.split()[0] not in self.static_predicates} + act = Action(fix_name(str(op)), pre, adds, dels) + return act + + def fix_pre_name(self, precondition): + if not is_atom(precondition): + return {fix_name(str(f)) for f in precondition.subformulas} + return {fix_name(str(precondition))} + + def action(self, name): + return self.action_map[fix_name(name).lower()] + + def get_action_or_none(self, name): + if "(" in name and ")" != name[-1]: + return None + return self.action_map.get(fix_name(name).lower(), None) + + def fluent(self, name): + return fix_name(name) + + def static_symbols(self): + return list(self.grounder.static_symbols) + + def fluent_symbols(self): + return list(self.grounder.fluent_symbols) + + def get_grounded_atoms(self, symbol): + variables = SymbolIndex() + lang = symbol.language + key = "atom_" + symbol.name + model = self.grounder._solve_lp() + if ( + key in model + ): # in case there is no reachable ground state variable from that fluent symbol + for binding in model[key]: + binding_with_constants = tuple(lang.get(c) for c in binding) + variables.add(StateVariableLite(symbol, binding_with_constants)) + return variables + + def get_applicable_actions(self, s): + return [a for a in self.actions if entails(s, a.pres)] + + def ground_problem(self, problem): + grounder = LPGroundingStrategy(problem, include_variable_inequalities=True) + action_groundings = grounder.ground_actions() + operators = [] + for action_name, groundings in action_groundings.items(): + action = problem.get_action(action_name) + for grounding in groundings: + operators.append( + ground_schema_into_plain_operator_from_grounding(action, grounding) + ) + + grounded_fluents = set( + [ + grounded_fluent.to_atom() + for grounded_fluent in grounder.ground_state_variables().objects + ] + ) + init = [f for f in problem.init.as_atoms() if f in grounded_fluents] + if isinstance(problem.goal, tarski.syntax.Atom): + goal = [problem.goal] + else: + goal = [f for f in problem.goal.subformulas if f in grounded_fluents] + + return (grounded_fluents, init, goal, operators, grounder) + + def get_static(self): + static_symbols = self.static_symbols() + ret = [] + for symbol in static_symbols: + ret.extend(self.get_grounded_atoms(symbol)) + return set([fix_name(str(x)) for x in ret]) + + def PDDL_replace_init_pddl_parser(self, s): + d = DomainParser()(open(self.domain_file, "r").read().lower()) + p = ProblemParser()(open(self.problem_file, "r").read().lower()) + + new_state = get_atoms_pddl(d, p, s | self.get_static()) + + new_p = Problem( + p.name, domain=d, objects=p.objects, init=new_state, goal=p.goal + ) + + return d, new_p + + +def parse_ans(response: str, parser: ACPGrammarParser, task: str): + return [parser.parse(clean_answer(resp, task)) for resp in response] + + +# def parse_ans(response : str, parser : ACPGrammarParser, task : str): +# ans = [parser.parse(clean_answer(resp, task), debug=True) for resp in response] +# if any(elem is None for elem in ans) or any(elem is None for elem in ans[0]): +# return None +# return ans + + +def remove_garbage(s): + while True: + if s.endswith("."): + s = s[:-1] + elif s.endswith("\n"): + s = s[:-2] + else: + break + return s.rstrip() + + +def compare_str(s1, s2): + return remove_garbage(s1).lower() == remove_garbage(s2).lower() + + +def compare(l1, l2): + if not isinstance(l1, list): + return compare_str(l1, l2) + if not isinstance(l2, list): + return False + for i, v in enumerate(l1): + if not compare(v, l2[i]): + return False + return True + + +def check_prog_response(resp): + if ( + "Positive Effects".lower() in resp.lower() + and "Negative Effects".lower() in resp.lower() + ): + if "[" not in resp: + return True + return False + + +def clean_answer(resp, task): + # Minor cleanup + if "progression_gen" in task: + # Check for Positive Effects and Negative Effects instead of separation + if check_prog_response(resp): + # replace **Positive Effects** with "[" + # replace **Negative Effects** with "] [" + # append "]" to the end + resp2 = resp.lower() + resp2 = resp2.replace("*", "") + resp2 = resp2.replace("positive effects", "[") + resp2 = resp2.replace("negative effects", "] [") + resp2 = resp2 + "]" + return resp2 + if "action_justification_gen" in task: + # Check for "simplified plan:" + if "simplified plan:" in resp.lower(): + resp2 = resp.lower() + resp2 = resp2.replace("*", "") + resp2 = resp2.split("simplified plan:")[1] + return resp2 + return resp + + +def get_grammar_task(task): + # print(task) + if task == "reachable_atom_gen": + return "act" + elif task == "progression_gen": + return "progression_list" + elif task == "validation_gen": + return "index" + elif task == "reachable_action_gen": + return "act" + elif task == "action_justification_gen": + return "action_list" + elif task == "landmarks_gen": + return "act" + elif task == "goal_closer_gen": + return "action_name" + elif task == "applicable_actions_gen": + return "action_list" + + +############################################################################## +# Evaluators + + +def fix_action_name(a): + assert a.startswith("(") and a.endswith(")") + return "(" + " ".join([x.strip() for x in a[1:-1].split(" ") if len(x) > 0]) + ")" + + +def str_remove_before_first_parentheses(s): + if s.startswith("("): + return s + try: + return s[s.index("(") :] + except Exception: + return "" + + +def str_remove_after_last_parentheses(s): + if s.endswith(")"): + return s + + i = s.rfind(")") + + if i == -1: + return "" + return s[: i + 1] + + +def cleanup_answer(ans): + if isinstance(ans, str): + ans = str_remove_before_first_parentheses(ans) + ans = str_remove_after_last_parentheses(ans) + ans = ans.lower() + ans = ( + ans.replace(")\n(", ")######(") + .replace("),(", ")######(") + .replace(") (", ")######(") + .split("######") + ) + return ans + if isinstance(ans, list): + res = [] + for x in ans: + res.extend(cleanup_answer(x)) + return res + + +def set_equal(ans1, ans2): + return set(ans1) == set(ans2) + + +class BaseEvaluator(ABC): + def __init__(self) -> None: + self.scores = [] + + @abstractmethod + def get_score(self, ans, doc): + pass + + def add_scores(self, scores): + self.scores.extend(scores) + + def get_avg_score(self): + avg_score = sum(self.scores) / len(self.scores) + return avg_score + + +def get_evaluator(group): + if group == "applicable_actions_gen": + return ApplicabilityEvaluator() + elif group == "progression_gen": + return ProgressionEvaluator() + elif group == "validation_gen": + return ValidationEvaluator() + elif group == "reachable_atom_gen": + return ReachabilityEvaluator() + elif group == "goal_closer_gen": + return NextActionEvaluator() + elif group == "action_justification_gen": + return JustificationEvaluator() + elif group == "landmarks_gen": + return LandmarksEvaluator() + elif group == "reachable_action_gen": + return ActionReachabilityEvaluator() + assert True, f"Group {group} not found" + + +""" +Action Reachability task: generate a valid action that is not applicable to any reachable state. +answer: A subset of actions that are known to be unreachable (not an exhaustive set). + It is empty only when we *know* that there are no such actions. +""" + + +class ActionReachabilityEvaluator(BaseEvaluator): + def get_score(self, ans, doc): + real_answer = doc["answer"] + if not real_answer or len(real_answer) == 0: + # The correct answer is None + self.add_scores( + ["none" == x.strip().lower() if x is not None else False for x in ans] + ) + else: + for x in ans: + if x is None: + self.scores.append(False) + continue + action = x.strip().lower() + if action in real_answer: + # The answer is in the subset of stored correct answers + self.scores.append(True) + continue + prec = get_action_preconditions( + doc["PDDL_domain"].lower(), doc["PDDL_problem"].lower(), action + ) + if prec is None: + # The answer does not correspond to a valid action + self.scores.append(False) + else: + # Need to run a planner on a task with the answer action preconditions as the new goal + prec = f"(and {' '.join(prec)})" + self.scores.append( + is_unsolvable_new_goal( + doc["PDDL_domain"].lower(), + doc["PDDL_problem"].lower(), + prec, + ) + ) + + return self.get_avg_score() + + +""" +Action Applicability task: generate all actions that are applicable in the current state. +answer: A set of all applicable actions. +""" + + +class ApplicabilityEvaluator(BaseEvaluator): + def get_score(self, ans, doc): + real_answer = doc["answer"] + real_answer = [a.lower() for a in real_answer] + ans = [[fix_action_name(a) for a in x] if x is not None else None for x in ans] + + # Check if the answer is equal (as a set) to the real stored answer + self.add_scores( + [ + set_equal(real_answer, cleanup_answer(x)) if x is not None else False + for x in ans + ] + ) + return self.get_avg_score() + + +def is_subsequence(plan, new_plan): + i = 0 + for a in plan: + if a == new_plan[i]: + i += 1 + if len(new_plan) == i: + # Done + return True + return False + + +def is_subsequence_and_plan(domain, problem, plan, new_plan): + if len(plan) <= len(new_plan): + return False + if not is_subsequence(plan, new_plan): + return False + return is_plan(domain, problem, new_plan) + + +""" +Justification task: generate a proper subsequence of the given plan that is also a plan. +answer: A list of examples of actions that can be removed (ignored in evaluation). +""" + + +class JustificationEvaluator(BaseEvaluator): + def get_score(self, ans, doc): + # Sequence of actions (plan) from the question + if "inputs" in doc: # old field name + seq = doc["inputs"][19:-147] + else: + seq = doc["question"][19:-147] + seq = seq.replace(") (", ")######(").split("######") + for x in ans: + if x is None: + self.scores.append(False) + continue + # An answer plan candidate + x = [fix_action_name(a) for a in x] + if len(x) == 0: + # Wrong answer - never an empty sequence + self.scores.append(0) + continue + # Check if the plan candidate from the answer (a) is a proper subsequence of the plan in the question and (b) is a plan. + self.scores.append( + is_subsequence_and_plan( + doc["PDDL_domain"].lower(), doc["PDDL_problem"].lower(), seq, x + ) + ) + return self.get_avg_score() + + +""" +Landmarks task: generate a fact that is a non-trivial landmark for the current state. +answer: A list of facts that are found to be landmarks and a list of facts that are found to be non-landmarks. + +The questions are generated only for cases where all facts either + (a) hold in the current state, + (b) true in goal, + (c) are found to be landmarks, or + (d) are found to be non-landmarks. +In such cases, the evaluation is simple, it does not require checking whether a fact is a landmark, it was +already done during question generation. +""" + + +class LandmarksEvaluator(BaseEvaluator): + def get_score(self, ans, doc): + # The set of facts that are found to be landmarks + real_answer = doc["answer"] + real_answer_yes = [a.lower() for a in real_answer["yes"]] + + for x in ans: + if x is None: + self.scores.append(False) + continue + if x.strip().lower() in real_answer_yes: + # The answer fact is known to be landmark + self.scores.append(True) + elif x.strip().lower() == "none": + # The answer is none, correct only if there are no known landmarks, + # since we only generate questions when that means that there are no non-trivial landmarks + self.scores.append(len(real_answer_yes) == 0) + else: + # All other cases the answer is incorrect + self.scores.append(False) + + return self.get_avg_score() + + +""" +Next Action task: generate an action that takes us closer to the goal. +answer: + (a) A list of applicable actions that are known to be correct answers + (b) A list of applicable actions that are known to be incorrect answers + (c) The rest of the applicable actions (maybe). +""" + + +class NextActionEvaluator(BaseEvaluator): + def get_score(self, ans, doc): + real_answer = doc["answer"] + real_answer_yes = [a.lower() for a in real_answer["yes"]] + real_answer_no = [a.lower() for a in real_answer["no"]] + real_answer_maybe = [a.lower() for a in real_answer["maybe"]] + # The cost of the optimal plan from the current state + opt = real_answer.get("opt", None) + for x in ans: + if x is None: + self.scores.append(False) + continue + action = x.strip().lower() + if action in real_answer_yes: + # Known to be correct + self.scores.append(True) + elif action in real_answer_no: + # Known to be incorrect + self.scores.append(False) + elif action not in real_answer_maybe: + # Not applicable, must be incorrect + self.scores.append(False) + else: + # Unknown, need to run a planner to check whether the state that results from applying the action is closer to the goal + # meaning has smaller optimal plan cost. + self.scores.append( + is_on_optimal_plan( + doc["PDDL_domain"].lower(), + doc["PDDL_problem"].lower(), + action, + opt, + ) + ) + + return self.get_avg_score() + + +""" +Progression task: generate the positive and negative effects of an action in the current state. +answer: + (a) A list of facts that were false and become true, when the action is applied + (b) A list of facts that were true and become false, when the action is applied +""" + + +class ProgressionEvaluator(BaseEvaluator): + def get_score(self, ans, doc): + real_answer = doc["answer"] + real_answer_pos = [a.lower() for a in real_answer["pos"]] + real_answer_neg = [a.lower() for a in real_answer["neg"]] + + for x in ans: + # The answer should be two lists. We allow for a single list and assume that the second one is empty (relaxed evaluation). + if x is None or len(x) > 2 or len(x) < 1: + self.scores.append(False) + else: + p = cleanup_answer(x[0]) + if len(x) == 2: + n = cleanup_answer(x[1]) + else: + # Assuming the last element is dropped because it is empty + n = [] + # Check if the answer is equal as sets to the correct answers. + ans = [set_equal(real_answer_pos, p), set_equal(real_answer_neg, n)] + self.scores.append(all(ans)) + + return self.get_avg_score() + + +""" +Reachability task: generate a valid fact that will never become true in any reachable state. +answer: A subset of facts that are known to be unreachable (not an exhaustive set). + It is empty only when we *know* that there are no such facts. +""" + + +class ReachabilityEvaluator(BaseEvaluator): + def get_score(self, ans, doc): + real_answer = doc["answer"] + real_answer = [f"({x.strip().lower()})" for x in real_answer] + + if len(real_answer) == 0: + # The correct answer is None + self.add_scores( + ["none" == x.strip().lower() if x is not None else False for x in ans] + ) + else: + for x in ans: + if x is None: + self.scores.append(False) + elif x.strip().lower() in real_answer: + # The answer is in the subset of stored correct answers + self.scores.append(True) + else: + # Need to run a planner on a task with the answer fact as the new goal + atom = x.strip().lower() + self.scores.append( + is_unsolvable_new_goal( + doc["PDDL_domain"].lower(), + doc["PDDL_problem"].lower(), + atom, + ) + ) + + return self.get_avg_score() + + +""" +Validation task: generate an index of the first inapplicable action in the given sequence. +answer: the correct index. +""" + + +class ValidationEvaluator(BaseEvaluator): + def get_score(self, ans, doc): + real_answer = str(doc["answer"]) + assert int(real_answer) >= 0, ( + f"The index must be non-negative, received {real_answer}" + ) + # Exact match + self.add_scores( + [ + real_answer.lower() == x.strip().lower() if x is not None else False + for x in ans + ] + ) + + return self.get_avg_score() + + +############################################################################## + + +def dump_item(item, **kwargs): + return json.dumps(item) + + +def parse_prediction(prediction): + try: + ans = json.loads(prediction.strip()) + response = ans.get("answer", None) + return response + except Exception as e: + print(f"Exception occurred {e}") + return prediction + + +@register_filter("ACP_grammar_filter") +class ACPGrammarFilter(RegexFilter): + """Filtering Index using""" + + def __init__(self, *args, **kwargs): + self.parser = ACPGrammarParser(kwargs["grammar_task"]) + self.clean = kwargs["clean"] if "clean" in kwargs else None + + def clean_pos_neg(self, resp): + # Check for Positive Effects and Negative Effects instead of separation + if check_prog_response(resp): + resp2 = resp.lower() + resp2 = resp2.replace("*", "") + resp2 = resp2.replace("positive effects", "[") + resp2 = resp2.replace("negative effects", "] [") + resp2 = resp2 + "]" + return resp2 + return resp + + def clean_simplified_plan(self, resp): + # Check for "simplified plan:" + if "simplified plan:" in resp.lower(): + resp2 = resp.lower() + resp2 = resp2.replace("*", "") + resp2 = resp2.split("simplified plan:")[1] + return resp2 + return resp + + def apply(self, resps, docs): + if self.clean == "pos_neg": + filtered_resps = [ + [self.parser.parse(self.clean_pos_neg(r)) for r in resp] + for resp in resps + ] + elif self.clean == "simplified plan": + filtered_resps = [ + [self.parser.parse(self.clean_simplified_plan(r)) for r in resp] + for resp in resps + ] + else: + filtered_resps = [[self.parser.parse(r) for r in resp] for resp in resps] + return filtered_resps + + +def process_acp_results(doc, results): + return {"score": get_evaluator(doc["group"]).get_score(results, doc)} + + +def get_score(references, predictions, **kwargs): + # print(f"References: {references}") + # print(f"Predictions: {predictions}") + data = json.loads(references[0].strip()) + real_ans = data["answer"] + task = data["group"] + + responses = [parse_prediction(prediction) for prediction in predictions] + + print(f"Real answer: {real_ans}") + print(f"Model answers: {responses}") + parser = ACPGrammarParser(get_grammar_task(task)) + ans = parse_ans(responses, parser, task) + + print(f"Parsed model answers: {ans}") + score = get_evaluator(task).get_score(ans, data) + + return {"get_score": score} diff --git a/lm_eval/tasks/acpbench/gen_2shot/act_reach.yaml b/lm_eval/tasks/acpbench/gen_2shot/act_reach.yaml new file mode 100644 index 00000000..36850e9c --- /dev/null +++ b/lm_eval/tasks/acpbench/gen_2shot/act_reach.yaml @@ -0,0 +1,19 @@ +task: acp_areach_gen +dataset_name: acp_areach_gen +include: _gen_yaml_2shot +fewshot_config: + sampler: first_n + samples: + - context: "A robot is in a grid and can only move to places that are connected to its current position. The grid size is 5x5, and the locations are of the form fi-jf (e.g., f3-2f or f0-1f). The grid cells are connected to their neighbors (e.g., f1-2f is connected to the four neighbors f0-2f, f2-2f, f1-1f, and f1-3f). Some positions on the grid are locked and can be opened with a key of a matching shape. The robot has an arm that can pick up a key when the key is in same location as the robot and the arm is empty. There are 2 keys in 0 different shapes: Key key0-0 is of shape shape0, Key key0-1 is of shape shape0. Currently, the robot is at position f2-2f and its arm is empty. All the positions are open except the following: f2-0f has shape0 shaped lock, f4-2f has shape0 shaped lock. Key key0-0 is at position f1-2f. Key key0-1 is at position f1-3f. The available actions are: (unlock ?curpos ?lockpos ?key ?shape) - unlock the place ?lockpos with key ?key of shape ?shape from the current position place ?curpos, (move ?curpos ?nextpos) - move from place ?curpos to place ?nextpos, (pickup ?curpos ?key) - retrieve the key ?key from its current position ?curpos, (pickup-and-loose ?curpos ?newkey ?oldkey) - pick up the key ?newkey from the current position ?curpos and loose the key ?oldkey which is being held, and (putdown ?curpos ?key) - put the key ?key at the current position place ?curpos." + question: "What action can never become applicable, in any state reachable from the current state?" + answer: "(unlock f0-3f f0-4f key0-0 shape0)" + - context: "There are several cities, each containing several locations, some of which are airports. There are also trucks, which can drive within a single city, and airplanes, which can fly between airports. The goal is to get some packages from various locations to various new locations. There are 2 trucks and 1 airplane, as well as 4 packages. There are 4 locations across 2 cities. The locations are in cities as follows: l1-1 and l1-0 are in c1; l0-0 and l0-1 are in c0. Currently, t1 is at l1-1, a0 is at l1-0, p0 is at l0-0, t0 is at l0-1, p2 is in a0, p1 is in t1, p3 is in t0. The available actions are: (load-truck ?obj ?truck ?loc) - load the object ?obj from location ?loc into the truck ?truck, (load-airplane ?obj ?airplane ?loc) - load object ?obj into airplane ?airplane at location ?loc, (unload-truck ?obj ?truck ?loc) - offload the object ?obj from the truck ?truck at location ?loc, (unload-airplane ?obj ?airplane ?loc) - offload the object ?obj from the airplane ?airplane at location ?loc, (drive-truck ?truck ?loc-from ?loc-to ?city) - navigate the truck ?truck from location ?loc-from in city ?city to location ?loc-to in the same city, and (fly-airplane ?airplane ?loc-from ?loc-to) - operate the airplane ?airplane from airport ?loc-from to airport ?loc-to." + question: "What action can never become applicable, in any state reachable from the current state?" + answer: "(drive-truck t0 l1-1 l0-0 c0)" +doc_to_text: "**Question**: {{context}} {{question}} Each action starts with an opening parenthesis and ends with closing parenthesis. Provide one action or None. **Final Answer**:" +filter_list: + - name: "acp_grammar_parse" + filter: + - function: "ACP_grammar_filter" + grammar_task: "act" + - function: "take_first" diff --git a/lm_eval/tasks/acpbench/gen_2shot/app.yaml b/lm_eval/tasks/acpbench/gen_2shot/app.yaml new file mode 100644 index 00000000..452254ab --- /dev/null +++ b/lm_eval/tasks/acpbench/gen_2shot/app.yaml @@ -0,0 +1,19 @@ +task: acp_app_gen +dataset_name: acp_app_gen +include: _gen_yaml_2shot +fewshot_config: + sampler: first_n + samples: + - context: "A robot is in a grid and can only move to places that are connected to its current position. The grid size is 5x5, and the locations are of the form fi-jf (e.g., f3-2f or f0-1f). The grid cells are connected to their neighbors (e.g., f1-2f is connected to the four neighbors f0-2f, f2-2f, f1-1f, and f1-3f). Some positions on the grid are locked and can be opened with a key of a matching shape. The robot has an arm that can pick up a key when the key is in same location as the robot and the arm is empty. There are 2 keys in 1 different shapes: Key key0-1 is of shape shape0, Key key0-0 is of shape shape0. Currently, the robot is at position f3-2f and its arm is empty. All the positions are open except the following: f2-0f has shape0 shaped lock, f4-2f has shape0 shaped lock. Key key0-0 is at position f2-2f. Key key0-1 is at position f1-3f. The available actions are: (unlock ?curpos ?lockpos ?key ?shape) - unlock the place ?lockpos with key ?key of shape ?shape from the current position place ?curpos, (move ?curpos ?nextpos) - travel from the current position ?curpos to the next position ?nextpos, (pickup ?curpos ?key) - pick up key ?key from place ?curpos, (pickup-and-loose ?curpos ?newkey ?oldkey) - pick up the key ?newkey at the current position place ?curpos and loose the key ?oldkey being held, and (putdown ?curpos ?key) - put down key ?key at current position place ?curpos." + question: "Generate the list of all ground actions that are applicable in this state." + answer: "[(move f3-2f f3-1f), (move f3-2f f2-2f), (move f3-2f f3-3f)]" + - context: "There are several cities, each containing several locations, some of which are airports. There are also trucks, which can drive within a single city, and airplanes, which can fly between airports. The goal is to get some packages from various locations to various new locations. There are 2 trucks and 1 airplane, as well as 4 packages. There are 4 locations across 2 cities. The locations are in cities as follows: l0-0 and l0-1 are in c0; l1-0 and l1-1 are in c1. Currently, t1, p2, and p3 are at l1-0, a0 is at l0-0, t0 is at l0-1, p1 and p0 are in t1. The available actions are: (load-truck ?obj ?truck ?loc) - load object ?obj into truck ?truck at location ?loc, (load-airplane ?obj ?airplane ?loc) - load object ?obj into airplane ?airplane at location ?loc, (unload-truck ?obj ?truck ?loc) - unload object ?obj from truck ?truck at location ?loc, (unload-airplane ?obj ?airplane ?loc) - remove the object ?obj from the airplane ?airplane and place it on the location ?loc, (drive-truck ?truck ?loc-from ?loc-to ?city) - navigate the truck ?truck from its current location ?loc-from in city ?city to the new location ?loc-to within the same city, and (fly-airplane ?airplane ?loc-from ?loc-to) - fly airplane ?airplane from airport ?loc-from to airport ?loc-to." + question: "Generate the list of all ground actions that are applicable in this state." + answer: "[(drive-truck t1 l1-0 l1-0 c1), (drive-truck t0 l0-1 l0-0 c0), (load-truck p2 t1 l1-0), (unload-truck p0 t1 l1-0), (drive-truck t0 l0-1 l0-1 c0), (fly-airplane a0 l0-0 l1-0), (fly-airplane a0 l0-0 l0-0), (unload-truck p1 t1 l1-0), (drive-truck t1 l1-0 l1-1 c1), (load-truck p3 t1 l1-0)]" +doc_to_text: "**Question**: {{context}} {{question}} Each action starts with an opening parenthesis and ends with closing parenthesis. Provide only the actions. **Final Answer**:" +filter_list: + - name: "acp_grammar_parse" + filter: + - function: "ACP_grammar_filter" + grammar_task: "action_list" + - function: "take_first" diff --git a/lm_eval/tasks/acpbench/gen_2shot/just.yaml b/lm_eval/tasks/acpbench/gen_2shot/just.yaml new file mode 100644 index 00000000..78daa393 --- /dev/null +++ b/lm_eval/tasks/acpbench/gen_2shot/just.yaml @@ -0,0 +1,20 @@ +task: acp_just_gen +dataset_name: acp_just_gen +include: _gen_yaml_2shot +fewshot_config: + sampler: first_n + samples: + - context: "A robot is in a grid and can only move to places that are connected to its current position. The grid size is 5x5, and the locations are of the form fi-jf (e.g., f3-2f or f0-1f). The grid cells are connected to their neighbors (e.g., f1-2f is connected to the four neighbors f0-2f, f2-2f, f1-1f, and f1-3f). Some positions on the grid are locked and can be opened with a key of a matching shape. The robot has an arm that can pick up a key when the key is in same location as the robot and the arm is empty. There are 2 keys in 1 different shapes: Key key0-0 is of shape shape0, Key key0-1 is of shape shape0. Currently, the robot is at position f3-3f and its arm is empty. All the positions are open except the following: f4-2f has shape0 shaped lock, f2-0f has shape0 shaped lock. Key key0-0 is at position f2-2f. Key key0-1 is at position f1-3f. The available actions are: (unlock ?curpos ?lockpos ?key ?shape) - unlock place ?lockpos with key ?key of shape ?shape from current position place ?curpos, (move ?curpos ?nextpos) - move from ?curpos to ?nextpos, (pickup ?curpos ?key) - retrieve the key ?key from its current position ?curpos, (pickup-and-loose ?curpos ?newkey ?oldkey) - pick up key ?newkey at current position place ?curpos and loose key ?oldkey being held, and (putdown ?curpos ?key) - put down the key ?key at the current position ?curpos. The goal is to reach a state where the following facts hold: Key key0-0 is at f2-0f location and Key key0-1 is at f1-3f location." + question: "Simplify the plan [(move f3-3f f3-2f), (move f3-2f f2-2f), (pickup f2-2f key0-0), (move f2-2f f2-1f), (putdown f2-1f key0-0), (pickup f2-1f key0-0), (unlock f2-1f f2-0f key0-0 shape0), (move f2-1f f2-0f), (putdown f2-0f key0-0)] by removing either a single action or a pair of consecutive actions, while still maintaining a valid plan. Provide the resulting simplified plan." + answer: "[(move f3-3f f3-2f), (move f3-2f f2-2f), (pickup f2-2f key0-0), (move f2-2f f2-1f), (unlock f2-1f f2-0f key0-0 shape0), (move f2-1f f2-0f), (putdown f2-0f key0-0)]" + - context: "There are several cities, each containing several locations, some of which are airports. There are also trucks, which can drive within a single city, and airplanes, which can fly between airports. The goal is to get some packages from various locations to various new locations. There are 2 trucks and 1 airplane, as well as 4 packages. There are 4 locations across 2 cities. The locations are in cities as follows: l1-1 and l1-0 are in c1; l0-0 and l0-1 are in c0. Currently, p2, p1, and p3 are at l1-0, p0 and t1 are at l1-1, t0 is at l0-1, a0 is at l0-0. The available actions are: (load-truck ?obj ?truck ?loc) - load the object ?obj from location ?loc into the truck ?truck, (load-airplane ?obj ?airplane ?loc) - load the object ?obj from location ?loc onto the airplane ?airplane, (unload-truck ?obj ?truck ?loc) - unload the object ?obj from the truck ?truck at location ?loc, (unload-airplane ?obj ?airplane ?loc) - remove the object ?obj from the airplane ?airplane and place it on the location ?loc, (drive-truck ?truck ?loc-from ?loc-to ?city) - drive truck ?truck from location ?loc-from in city ?city to location ?loc-to in the same city, and (fly-airplane ?airplane ?loc-from ?loc-to) - fly the airplane ?airplane from location ?loc-from to location ?loc-to. The goal is to reach a state where the following facts hold: p3 is at l0-1, p2 is at l1-0, p0 is at l0-0, and p1 is at l1-0." + question: "Simplify the plan [(fly-airplane a0 l0-0 l1-0), (fly-airplane a0 l1-0 l0-0), (load-truck p0 t1 l1-1), (drive-truck t1 l1-1 l1-0 c1), (unload-truck p0 t1 l1-0), (fly-airplane a0 l0-0 l1-0), (load-airplane p0 a0 l1-0), (load-airplane p3 a0 l1-0), (fly-airplane a0 l1-0 l0-0), (unload-airplane p0 a0 l0-0), (unload-airplane p3 a0 l0-0), (drive-truck t0 l0-1 l0-0 c0), (load-truck p3 t0 l0-0), (drive-truck t0 l0-0 l0-1 c0), (unload-truck p3 t0 l0-1)] by removing either a single action or a pair of consecutive actions, while still maintaining a valid plan. Provide the resulting simplified plan." + answer: "[(load-truck p0 t1 l1-1), (drive-truck t1 l1-1 l1-0 c1), (unload-truck p0 t1 l1-0), (fly-airplane a0 l0-0 l1-0), (load-airplane p0 a0 l1-0), (load-airplane p3 a0 l1-0), (fly-airplane a0 l1-0 l0-0), (unload-airplane p0 a0 l0-0), (unload-airplane p3 a0 l0-0), (drive-truck t0 l0-1 l0-0 c0), (load-truck p3 t0 l0-0), (drive-truck t0 l0-0 l0-1 c0), (unload-truck p3 t0 l0-1)]" +doc_to_text: "**Question**: {{context}} {{question}} **Final Answer**:" +filter_list: + - name: "acp_grammar_parse" + filter: + - function: "ACP_grammar_filter" + grammar_task: "action_list" + clean: "simplified plan" + - function: "take_first" diff --git a/lm_eval/tasks/acpbench/gen_2shot/land.yaml b/lm_eval/tasks/acpbench/gen_2shot/land.yaml new file mode 100644 index 00000000..a3c65fd3 --- /dev/null +++ b/lm_eval/tasks/acpbench/gen_2shot/land.yaml @@ -0,0 +1,19 @@ +task: acp_land_gen +dataset_name: acp_land_gen +include: _gen_yaml_2shot +fewshot_config: + sampler: first_n + samples: + - context: "A robot is in a grid and can only move to places that are connected to its current position. The grid size is 5x5, and the locations are of the form fi-jf (e.g., f3-2f or f0-1f). The grid cells are connected to their neighbors (e.g., f1-2f is connected to the four neighbors f0-2f, f2-2f, f1-1f, and f1-3f). Some positions on the grid are locked and can be opened with a key of a matching shape. The robot has an arm that can pick up a key when the key is in same location as the robot and the arm is empty. There are 2 keys in 1 different shapes: Key key0-1 is of shape shape0, Key key0-0 is of shape shape0. Currently, the robot is at position f3-0f and its arm is empty. All the positions are open except the following: f4-2f has shape0 shaped lock. Key key0-0 is at position f3-0f. Key key0-1 is at position f1-3f. The goal is to reach a state where the following facts hold: Key key0-0 is at f2-0f location and Key key0-1 is at f1-3f location. The available propositions are: (at ?r ?x) - Key ?r is at ?x location, (at-robot ?x) - Robot is at ?x location, (locked ?x) - Location ?x is locked, (holding ?k) - Robot is holding ?k, (open ?x) - Location ?x is open, and (arm-empty) - Robot's arm is empty." + question: "Generate a non-trivial fact landmark, one that does not hold in the initial state or goal." + answer: "(holding key0-0)" + - context: "There are several cities, each containing several locations, some of which are airports. There are also trucks, which can drive within a single city, and airplanes, which can fly between airports. The goal is to get some packages from various locations to various new locations. There are 2 trucks and 1 airplane, as well as 4 packages. There are 4 locations across 2 cities. The locations are in cities as follows: l1-0 and l1-1 are in c1; l0-1 and l0-0 are in c0. Currently, a0 and p2 are at l1-0, t0 is at l0-0, t1 is at l1-1, p3 and p1 are in a0, p0 is in t1. The goal is to reach a state where the following facts hold: p0 is at l0-0, p2 is at l1-0, p1 is at l1-0, and p3 is at l0-1. The available propositions are: (at ?obj ?loc) - ?obj is at ?loc and (in ?obj1 ?obj2) - ?obj1 is in ?obj2." + question: "Generate a non-trivial fact landmark, one that does not hold in the initial state or goal." + answer: "(in p3 t0)" +doc_to_text: "**Question**: {{context}} {{question}} Provide only the ground proposition or None. **Final Answer**:" +filter_list: + - name: "acp_grammar_parse" + filter: + - function: "ACP_grammar_filter" + grammar_task: "act" + - function: "take_first" diff --git a/lm_eval/tasks/acpbench/gen_2shot/next_act.yaml b/lm_eval/tasks/acpbench/gen_2shot/next_act.yaml new file mode 100644 index 00000000..a264d644 --- /dev/null +++ b/lm_eval/tasks/acpbench/gen_2shot/next_act.yaml @@ -0,0 +1,19 @@ +task: acp_nexta_gen +dataset_name: acp_nexta_gen +include: _gen_yaml_2shot +fewshot_config: + sampler: first_n + samples: + - context: "A robot is in a grid and can only move to places that are connected to its current position. The grid size is 5x5, and the locations are of the form fi-jf (e.g., f3-2f or f0-1f). The grid cells are connected to their neighbors (e.g., f1-2f is connected to the four neighbors f0-2f, f2-2f, f1-1f, and f1-3f). Some positions on the grid are locked and can be opened with a key of a matching shape. The robot has an arm that can pick up a key when the key is in same location as the robot and the arm is empty. There are 2 keys in 1 different shapes: Key key0-1 is of shape shape0, Key key0-0 is of shape shape0. Currently, the robot is at position f4-0f and its arm is empty. All the positions are open except the following: f4-2f has shape0 shaped lock. Key key0-0 is at position f3-0f. Key key0-1 is at position f1-3f. The goal is to reach a state where the following facts hold: Key key0-0 is at f2-0f location and Key key0-1 is at f1-3f location. The available actions are: (unlock ?curpos ?lockpos ?key ?shape) - unlock place ?lockpos with key ?key of shape ?shape from current position place ?curpos, (move ?curpos ?nextpos) - travel from the current position ?curpos to the next position ?nextpos, (pickup ?curpos ?key) - pick up key ?key from place ?curpos, (pickup-and-loose ?curpos ?newkey ?oldkey) - pick up the key ?newkey at the current position place ?curpos and loose the key ?oldkey being held, and (putdown ?curpos ?key) - put down the key ?key at the current position ?curpos." + question: "What is the next action that takes us towards the goal?" + answer: "(move f4-0f f3-0f)" + - context: "There are several cities, each containing several locations, some of which are airports. There are also trucks, which can drive within a single city, and airplanes, which can fly between airports. The goal is to get some packages from various locations to various new locations. There are 2 trucks and 1 airplane, as well as 4 packages. There are 4 locations across 2 cities. The locations are in cities as follows: l0-1 and l0-0 are in c0; l1-1 and l1-0 are in c1. Currently, t0 is at l0-1, a0 is at l0-0, t1 and p1 are at l1-0, p2, p0, and p3 are in t1. The goal is to reach a state where the following facts hold: p3 is at l0-1, p2 is at l1-0, p1 is at l1-0, and p0 is at l0-0. The available actions are: (load-truck ?obj ?truck ?loc) - load object ?obj into truck ?truck at location ?loc, (load-airplane ?obj ?airplane ?loc) - load the object ?obj from location ?loc onto the airplane ?airplane, (unload-truck ?obj ?truck ?loc) - unload the object ?obj from the truck ?truck at location ?loc, (unload-airplane ?obj ?airplane ?loc) - unload object ?obj from airplane ?airplane at location ?loc, (drive-truck ?truck ?loc-from ?loc-to ?city) - drive the truck ?truck in city ?city from location ?loc-from to location ?loc-to, and (fly-airplane ?airplane ?loc-from ?loc-to) - operate the airplane ?airplane from airport ?loc-from to airport ?loc-to." + question: "What is the next action that takes us towards the goal?" + answer: "(drive-truck t0 l0-1 l0-0 c0)" +doc_to_text: "**Question**: {{context}} {{question}} Each action starts with an opening parenthesis and ends with closing parenthesis. Provide only the action. **Final Answer**:" +filter_list: + - name: "acp_grammar_parse" + filter: + - function: "ACP_grammar_filter" + grammar_task: "action_name" + - function: "take_first" diff --git a/lm_eval/tasks/acpbench/gen_2shot/prog.yaml b/lm_eval/tasks/acpbench/gen_2shot/prog.yaml new file mode 100644 index 00000000..6267f29a --- /dev/null +++ b/lm_eval/tasks/acpbench/gen_2shot/prog.yaml @@ -0,0 +1,20 @@ +task: acp_prog_gen +dataset_name: acp_prog_gen +include: _gen_yaml_2shot +fewshot_config: + sampler: first_n + samples: + - context: "A robot is in a grid and can only move to places that are connected to its current position. \nThe grid size is 5x5, and the locations are of the form fi-jf (e.g., f3-2f or f0-1f). The grid cells are connected to their neighbors (e.g., f1-2f is connected to the four neighbors f0-2f, f2-2f, f1-1f, and f1-3f). Some positions on the grid are locked and can be opened with a key of a matching shape. The robot has an arm that can pick up a key when the key is in same location as the robot and the arm is empty. \nThere are 2 keys in 0 different shapes: Key key0-0 is of shape shape0, Key key0-1 is of shape shape0. \nCurrently, the robot is at position f0-1f and its arm is empty. All the positions are open except the following: f4-2f has shape0 shaped lock. Key key0-1 is at position f1-3f. Key key0-0 is at position f0-1f. The available propositions are: (at ?r ?x) - Key ?r is at ?x location, (at-robot ?x) - Robot is at ?x location, (locked ?x) - Location ?x is locked, (holding ?k) - Robot is holding ?k, (open ?x) - Location ?x is open, and (arm-empty) - Robot's arm is empty." + question: "Break down the outcomes of performing the action \"retrieve the key key0-0 from its current position f0-1f\" into two lists, positive effects and negative effects. Positive effects are the propositions that are false in the current state but will become true after performing the action. Negative effects are the propositions that are true in the current state and will become false after performing the action." + answer: "[(holding key0-0)] [(arm-empty), (at key0-0 f0-1f)]" + - context: "There are several cities, each containing several locations, some of which are airports. There are also trucks, which can drive within a single city, and airplanes, which can fly between airports. The goal is to get some packages from various locations to various new locations. There are 2 trucks and 1 airplane, as well as 4 packages. There are 4 locations across 2 cities. The locations are in cities as follows: l1-1 and l1-0 are in c1; l0-1 and l0-0 are in c0. Currently, p2, t1, p1, p3, a0, and p0 are at l1-0, t0 is at l0-1. The available propositions are: (at ?obj ?loc) - ?obj is at ?loc and (in ?obj1 ?obj2) - ?obj1 is in ?obj2." + question: "Break down the outcomes of performing the action \"load object p3 into truck t1 at location l1-0\" into two lists, positive effects and negative effects. Positive effects are the propositions that are false in the current state but will become true after performing the action. Negative effects are the propositions that are true in the current state and will become false after performing the action." + answer: "[(in p3 t1)] [(at p3 l1-0)]" +doc_to_text: "**Question**: {{context}} {{question}} Provide only the two lists with the ground propositions. **Final Answer**:" +filter_list: + - name: "acp_grammar_parse" + filter: + - function: "ACP_grammar_filter" + grammar_task: "progression_list" + clean: "pos_neg" + - function: "take_first" diff --git a/lm_eval/tasks/acpbench/gen_2shot/reach.yaml b/lm_eval/tasks/acpbench/gen_2shot/reach.yaml new file mode 100644 index 00000000..c3a192fc --- /dev/null +++ b/lm_eval/tasks/acpbench/gen_2shot/reach.yaml @@ -0,0 +1,19 @@ +task: acp_reach_gen +dataset_name: acp_reach_gen +include: _gen_yaml_2shot +fewshot_config: + sampler: first_n + samples: + - context: "A robot is in a grid and can only move to places that are connected to its current position. The grid size is 5x5, and the locations are of the form fi-jf (e.g., f3-2f or f0-1f). The grid cells are connected to their neighbors (e.g., f1-2f is connected to the four neighbors f0-2f, f2-2f, f1-1f, and f1-3f). Some positions on the grid are locked and can be opened with a key of a matching shape. The robot has an arm that can pick up a key when the key is in same location as the robot and the arm is empty. There are 2 keys in 0 different shapes: Key key0-1 is of shape shape0, Key key0-0 is of shape shape0. Currently, the robot is at position f1-2f and its arm is empty. All the positions are open except the following: f4-2f has shape0 shaped lock. Key key0-0 is at position f1-0f. Key key0-1 is at position f1-3f. The available propositions are: (at ?r ?x) - Key ?r is at ?x location, (at-robot ?x) - Robot is at ?x location, (locked ?x) - Location ?x is locked, (holding ?k) - Robot is holding ?k, (open ?x) - Location ?x is open, and (arm-empty) - Robot's arm is empty." + question: "What proposition can never hold in any potentially reachable state?" + answer: "(locked f3-1f)" + - context: "There are several cities, each containing several locations, some of which are airports. There are also trucks, which can drive within a single city, and airplanes, which can fly between airports. The goal is to get some packages from various locations to various new locations. There are 2 trucks and 1 airplane, as well as 4 packages. There are 4 locations across 2 cities. The locations are in cities as follows: l0-0 and l0-1 are in c0; l1-0 and l1-1 are in c1. Currently, a0, p2, and t1 are at l1-0, p3 and p0 are at l0-0, t0 is at l0-1, p1 is in t1. The available propositions are: (at ?obj ?loc) - ?obj is at ?loc and (in ?obj1 ?obj2) - ?obj1 is in ?obj2." + question: "What proposition can never hold in any potentially reachable state?" + answer: "(at t0 l1-1)" +doc_to_text: "**Question**: {{context}} {{question}} Provide one proposition or None. **Final Answer**:" +filter_list: + - name: "acp_grammar_parse" + filter: + - function: "ACP_grammar_filter" + grammar_task: "act" + - function: "take_first" diff --git a/lm_eval/tasks/acpbench/gen_2shot/val.yaml b/lm_eval/tasks/acpbench/gen_2shot/val.yaml new file mode 100644 index 00000000..5dc02acf --- /dev/null +++ b/lm_eval/tasks/acpbench/gen_2shot/val.yaml @@ -0,0 +1,19 @@ +task: acp_val_gen +dataset_name: acp_val_gen +include: _gen_yaml_2shot +fewshot_config: + sampler: first_n + samples: + - context: "A robot is in a grid and can only move to places that are connected to its current position. The grid size is 5x5, and the locations are of the form fi-jf (e.g., f3-2f or f0-1f). The grid cells are connected to their neighbors (e.g., f1-2f is connected to the four neighbors f0-2f, f2-2f, f1-1f, and f1-3f). Some positions on the grid are locked and can be opened with a key of a matching shape. The robot has an arm that can pick up a key when the key is in same location as the robot and the arm is empty. There are 2 keys in 1 different shapes: Key key0-0 is of shape shape0, Key key0-1 is of shape shape0. Currently, the robot is at position f3-3f and its arm is empty. All the positions are open except the following: f2-0f has shape0 shaped lock, f4-2f has shape0 shaped lock. Key key0-1 is at position f1-3f. Key key0-0 is at position f2-2f. The goal is to reach a state where the following facts hold: Key key0-0 is at f2-0f location and Key key0-1 is at f1-3f location. The available actions are: (unlock ?curpos ?lockpos ?key ?shape) - unlock the place ?lockpos with the key ?key of the shape ?shape from the current position place ?curpos, (move ?curpos ?nextpos) - travel from the current position ?curpos to the next position ?nextpos, (pickup ?curpos ?key) - pick up key ?key from place ?curpos, (pickup-and-loose ?curpos ?newkey ?oldkey) - pick up the key ?newkey from the current position ?curpos and loose the key ?oldkey which is being held, and (putdown ?curpos ?key) - put down key ?key at current position place ?curpos." + question: "What is the first inapplicable action in the next sequence of actions: [(move f3-3f f3-2f), (move f3-2f f2-2f), (pickup f2-2f key0-0), (pickup-and-loose f4-0f key0-0 key0-1), (unlock f2-1f f2-0f key0-0 shape0), (move f2-1f f2-0f), (putdown f2-0f key0-0), (move f2-0f f2-1f)]?" + answer: "3" + - context: "There are several cities, each containing several locations, some of which are airports. There are also trucks, which can drive within a single city, and airplanes, which can fly between airports. The goal is to get some packages from various locations to various new locations. There are 2 trucks and 1 airplane, as well as 4 packages. There are 4 locations across 2 cities. The locations are in cities as follows: l0-1 and l0-0 are in c0; l1-1 and l1-0 are in c1. Currently, t1 and p0 are at l1-1, t0 is at l0-1, p3, p2, and p1 are at l1-0, a0 is at l0-0. The goal is to reach a state where the following facts hold: p2 is at l1-0, p3 is at l0-1, p0 is at l0-0, and p1 is at l1-0. The available actions are: (load-truck ?obj ?truck ?loc) - load object ?obj into truck ?truck at location ?loc, (load-airplane ?obj ?airplane ?loc) - load the object ?obj from location ?loc onto the airplane ?airplane, (unload-truck ?obj ?truck ?loc) - unload the object ?obj from the truck ?truck at location ?loc, (unload-airplane ?obj ?airplane ?loc) - unload object ?obj from airplane ?airplane at location ?loc, (drive-truck ?truck ?loc-from ?loc-to ?city) - navigate the truck ?truck from its current location ?loc-from in city ?city to the new location ?loc-to within the same city, and (fly-airplane ?airplane ?loc-from ?loc-to) - fly the airplane ?airplane from location ?loc-from to location ?loc-to." + question: "What is the first inapplicable action in the next sequence of actions: [(load-truck p0 t1 l1-1), (drive-truck t1 l1-1 l1-0 c1), (unload-truck p0 t1 l1-0), (fly-airplane a0 l0-0 l1-0), (unload-truck p3 t0 l0-1), (load-airplane p3 a0 l1-0), (fly-airplane a0 l1-0 l0-0), (unload-airplane p0 a0 l0-0), (unload-airplane p3 a0 l0-0), (drive-truck t0 l0-1 l0-0 c0), (load-truck p3 t0 l0-0), (drive-truck t0 l0-0 l0-1 c0), (unload-truck p3 t0 l0-1)]?" + answer: "4" +doc_to_text: "**Question**: {{context}} {{question}} Provide only the index of the action. **Final Answer**:" +filter_list: + - name: "acp_grammar_parse" + filter: + - function: "ACP_grammar_filter" + grammar_task: "index" + - function: "take_first" diff --git a/lm_eval/tasks/acpbench/gen_2shot_with_pddl/_gen_yaml_2shot b/lm_eval/tasks/acpbench/gen_2shot_with_pddl/_gen_yaml_2shot new file mode 100644 index 00000000..710b3604 --- /dev/null +++ b/lm_eval/tasks/acpbench/gen_2shot_with_pddl/_gen_yaml_2shot @@ -0,0 +1,26 @@ +tag: + - acp_gen_2shot_with_pddl + - acp_bench_hard_with_pddl +dataset_path: ibm-research/acp_bench +test_split: test +description: "Answer the question based on the provided PDDL domain and PDDL problem. The current state is the initial state described in the PDDL problem below.\n\n" +doc_to_target: "{{answer}}" +output_type: generate_until +num_fewshot: 2 +generation_kwargs: + until: + - "\n\n\n\n" + - "\n\n" + - "**Question**:" + - "**Question:**" + - "Q:" + do_sample: false + max_gen_toks: 1000 + temperature: 0.0 +metadata: + version: 1.0 +process_results: !function acp_utils.process_acp_results +metric_list: + - metric: "score" + aggregation: mean + higher_is_better: True diff --git a/lm_eval/tasks/acpbench/gen_2shot_with_pddl/acp_grammar.lark b/lm_eval/tasks/acpbench/gen_2shot_with_pddl/acp_grammar.lark new file mode 100644 index 00000000..036bd675 --- /dev/null +++ b/lm_eval/tasks/acpbench/gen_2shot_with_pddl/acp_grammar.lark @@ -0,0 +1,23 @@ +NAME: /[a-zA-Z][a-zA-Z0-9-_]*/ +LPAR : "(" +RPAR : ")" +LSPAR: "[" +RSPAR: "]" +COMMA: "," +WS: /[ \n]/ + +action_none : "None" + +action_name : LPAR NAME (WS NAME)* RPAR + +action_list : (action_name WS?)* + +prog_list : action_name* (COMMA action_name)* + +progression_list : LSPAR prog_list RSPAR LSPAR prog_list RSPAR + +act : action_name | action_none + +index: /[0-9]+[0-9]*/ + +start: action_list diff --git a/lm_eval/tasks/acpbench/gen_2shot_with_pddl/acp_utils.py b/lm_eval/tasks/acpbench/gen_2shot_with_pddl/acp_utils.py new file mode 100644 index 00000000..5051b68c --- /dev/null +++ b/lm_eval/tasks/acpbench/gen_2shot_with_pddl/acp_utils.py @@ -0,0 +1,1128 @@ +import json +import os +from abc import ABC, abstractmethod +from collections import defaultdict +from pathlib import Path + +from lm_eval.api.registry import register_filter +from lm_eval.filters.extraction import RegexFilter + + +try: + import tempfile + + import tarski + from kstar_planner import planners as kp + from lark import Lark + from lark.lexer import Token + from lark.visitors import Visitor + from pddl.core import Problem + from pddl.parser.domain import DomainParser + from pddl.parser.problem import ProblemParser + from tarski.grounding.common import StateVariableLite + from tarski.grounding.lp_grounding import LPGroundingStrategy + from tarski.io import PDDLReader + from tarski.io import fstrips as iofs + from tarski.syntax.formulas import is_atom + from tarski.syntax.transform.action_grounding import ( + ground_schema_into_plain_operator_from_grounding, + ) + from tarski.util import SymbolIndex +except ModuleNotFoundError: + raise ModuleNotFoundError( + "`lark>=1.1.9`, `tarski[clingo]==0.8.2`, `pddl==0.4.2` and `kstar-planner==1.4.2` are required for evaluating the generative tasks. \ +Please install via pip install lm-eval[acpbench] or pip install -e .[acpbench]", + ) + + +######################################################################### +# Grammar + + +GRAMMAR_FILE = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "acp_grammar.lark" +) + + +class ACPBench_Visitor(Visitor): + def __init__(self) -> None: + super().__init__() + self.action_lists = None + self.action_names = None + self.progression_lists = None + self.prog_lists = None + self.indexes = None + + def action_list(self, tree): + self.action_lists = [] + + def prog_list(self, tree): + if self.prog_lists is not None: + self.progression_lists.append(self.prog_lists) + self.prog_lists = [] + + def progression_list(self, tree): + self.progression_lists = [] + + def action_none(self, tree): + self.action_names = "None" + + def action_name(self, tree): + act_name = "(" + "".join(tree.children[1:-1]) + ")" + self.action_names = act_name + if self.action_lists is not None: + self.action_lists.append(act_name) + if self.prog_lists is not None: + self.prog_lists.append(act_name) + + def index(self, tree): + self.indexes = "".join(tree.children) + if not self.indexes.isnumeric(): + self.indexes = None + + +class ACPGrammarParser(object): + def __init__(self, task) -> None: + self.task = task + with open(GRAMMAR_FILE) as f: + grammar = f.read() + self.acp_parser = Lark(grammar, start=task, parser="lalr") + + def parse(self, input, debug=False): + def ignore_errors(e): + if hasattr(e, "token") and e.token.type == "$END": + for x in e.expected: + if x != "WS": + e.interactive_parser.feed_token( + Token(x, self.acp_parser.get_terminal(x).pattern.value) + ) + + return True + + input = input.replace("\n", "") + input = input.strip() + try: + tree = self.acp_parser.parse(input, on_error=ignore_errors) + + if debug: + print(tree) + visitor = ACPBench_Visitor() + visitor.visit_topdown(tree) + if self.task == "action_list": + return visitor.action_lists + elif self.task == "act": + return visitor.action_names + elif self.task == "action_name": + return visitor.action_names + elif self.task == "index": + return visitor.indexes + elif self.task == "progression_list": + if visitor.prog_lists not in visitor.progression_lists: + visitor.progression_lists.append(visitor.prog_lists) + return visitor.progression_lists + except Exception as e: + if debug: + print("exception") + print(e) + return None + + +############################################################################## +# Utils + + +# Used in next action +def is_on_optimal_plan(domain, problem, action, opt): + with ( + tempfile.NamedTemporaryFile() as domain_temp, + tempfile.NamedTemporaryFile() as problem_temp, + ): + with open(str(domain_temp.name), "w", encoding="utf8") as file: + file.write(domain.lower()) + with open(str(problem_temp.name), "w", encoding="utf8") as file: + file.write(problem.lower()) + + # Here, we need to keep the temp files live until the end of the function + try: + P = STRIPS(str(domain_temp.name), str(problem_temp.name)) + except Exception: + # Unsolvable + return False + + a = P.get_action_or_none(action[1:-1]) + if a is None: + return False + state = P.init + next_state = progress(state, a) + if opt is None: + # Get an optimal plan cost + plans = generate_optimal_plans_for_problem_state( + P, state, num_plans=1, timeout=5 + ) + opt = len(plans[0]["actions"]) + else: + opt = int(opt) + + # Getting an optimal plan for the next state + next_plans = generate_optimal_plans_for_problem_state( + P, next_state, num_plans=1, timeout=5 + ) + if next_plans is None: + return False + next_opt = len(next_plans[0]["actions"]) + return next_opt + 1 == opt + + +# Used in justification +def is_plan(domain, problem, new_plan): + P = get_STRIPS(domain, problem) + if P is None: + # Unsolvable + return False + + # Check if new_plan is a plan + current_state = P.init + for action in new_plan: + applicable_actions = P.get_applicable_actions(current_state) + app_actions_list = [f"({a.name.lower()})" for a in applicable_actions] + if action.lower() not in app_actions_list: + return False + a = applicable_actions[app_actions_list.index(action.lower())] + current_state = progress(current_state, a) + return entails(current_state, P.goal) + + +# Used in action reachability +def get_action_preconditions(domain, problem, action): + P = get_STRIPS(domain, problem) + + assert P is not None, f"Domain\n{domain}\nProblem\n{problem}\nAction: {action}" + a = P.get_action_or_none(action[1:-1]) + if a is None: + return a + + return [f"({f})" for f in a.pres] + + +def generate_optimal_plans_for_problem_state(P, state, num_plans, timeout): + import tempfile + + with ( + tempfile.NamedTemporaryFile() as domain_temp, + tempfile.NamedTemporaryFile() as problem_temp, + ): + create_tmp_dom_prob_replace_init(P, state, domain_temp, problem_temp) + plans = generate_top_q_plans( + domain=str(domain_temp.name), + problem=str(problem_temp.name), + num_plans=num_plans, + quality_bound=1.0, + timeout=timeout, + ) + # print(plans) + if plans is None or len(plans["plans"]) == 0: + return None + return plans["plans"] + + +def generate_top_q_plans(domain, problem, num_plans=10, quality_bound=1.0, timeout=30): + # print("Running K* planner") + plans = kp.plan_unordered_topq( + domain_file=Path(domain), + problem_file=Path(problem), + number_of_plans_bound=num_plans, + quality_bound=quality_bound, + timeout=timeout, + ) + return plans + + +# Used in (action) reachability +def is_unsolvable_new_goal(domain, problem, new_goal): + goal = extract_goal(problem) + new_problem = problem.replace(goal, f"(:goal {new_goal} )") + return is_unsolvable(domain, new_problem) + + +def is_unsolvable(domain, problem): + with ( + tempfile.NamedTemporaryFile() as domain_temp, + tempfile.NamedTemporaryFile() as problem_temp, + ): + with open(str(domain_temp.name), "w", encoding="utf8") as file: + file.write(str(domain)) + with open(str(problem_temp.name), "w", encoding="utf8") as file: + file.write(str(problem)) + + plans = kp.plan_unordered_topq( + domain_file=Path(str(domain_temp.name)), + problem_file=Path(str(problem_temp.name)), + quality_bound=1.0, + number_of_plans_bound=1, + timeout=3, + ) + + if len(plans["planner_error"]) > 0: + fl = plans["planner_error"].split("\n")[0] + print(f"Planner error: {fl}") + return False + if plans is None or len(plans["plans"]) == 0: + return plans["unsolvable"] + return False + + +def extract_goal(prob): + a = prob.split("(:goal")[1] + cp = 1 + for i, c in enumerate(a): + if c == ")": + cp -= 1 + if c == "(": + cp += 1 + if cp == 0: + return "(:goal" + a[: i + 1] + + assert False + + +def entails(state, partialstate): + return partialstate <= state + + +def progress(state, act): + assert entails(state, act.pres), ( + "Cannot progress with inconsistent state / action precondition:\n\t Action: " + + act.name + + "\n\t State: \n\t\t" + + "\n\t\t".join(state) + ) + return (state - act.dels) | act.adds + + +def regress(state, act): + assert len(state & act.dels) == 0, ( + "Cannot regress with inconsistent state / action delete effect:\n\t Action: " + + act.name + + "\n\t State: \n\t\t" + + "\n\t\t".join(state) + ) + return (state - act.adds) | act.pres + + +def get_STRIPS(domain, problem): + with ( + tempfile.NamedTemporaryFile() as domain_temp, + tempfile.NamedTemporaryFile() as problem_temp, + ): + with open(str(domain_temp.name), "w", encoding="utf8") as file: + file.write(domain.lower()) + with open(str(problem_temp.name), "w", encoding="utf8") as file: + file.write(problem.lower()) + + try: + P = STRIPS(str(domain_temp.name), str(problem_temp.name)) + return P + except Exception as e: + print(f"||{e}||") + return None + + +def create_tmp_dom_prob_replace_init(P, state, result_domain_file, result_problem_file): + d, p = P.PDDL_replace_init_pddl_parser(state) + with open(str(result_domain_file.name), "w", encoding="utf8") as file: + file.write(str(d)) + with open(str(result_problem_file.name), "w", encoding="utf8") as file: + file.write(str(p)) + + return d, p + + +def fix_name(s): + # (act param) + if "(" == s[0] and ")" == s[-1]: + return s[1:-1] + # make it space separated + s = s.replace(", ", " ").replace(",", " ") + # act(param) + if "(" in s: + assert ")" == s[-1], f"Broken name? {s}" + s = s.replace("(", " ").replace(")", "") + # act param + return s + + +def get_atoms_pddl(d, p, atoms): + objs = set() + preds = defaultdict(list) + for atom in atoms: + a = atom.lower().strip().split(" ") + args = a[1:] + preds[a[0]].append(args) + objs |= set(args) + + constants = [o for o in p.objects | d.constants if o.name.lower() in objs] + constants_dict = {} + for c in constants: + constants_dict[c.name.lower()] = c + assert len(objs) == len(constants), ( + f"Could not identify all objects: {objs - set(constants_dict.keys())} not found, {set(constants_dict.keys()) - objs} should not be there" + ) + + state = [] + covered_preds = set() + for f in d.predicates: + name = f.name.lower() + if name in preds: + covered_preds.add(name) + assert len(preds[name][0]) == f.arity, ( + f"The arity does not match: {preds[name]} vs {f.terms}" + ) + # Going over the lists of objects, adding ground predicate for each + for ob in preds[name]: + c = [constants_dict[o] for o in ob] + state.append(f(*c)) + assert len(covered_preds) == len(preds.keys()), ( + f"Covered predicates: \n{sorted(list(covered_preds))} vs \n{sorted(list(preds.keys()))}" + ) + return set(state) + + +class Action: + def __init__(self, name, pre, add, delete): + self.name = name + self.pres = pre + self.adds = add + self.dels = delete + + def __str__(self): + pres = "{" + ", ".join([f"({a})" for a in self.pres]) + "}" + adds = "{" + ", ".join([f"({a})" for a in self.adds]) + "}" + dels = "{" + ", ".join([f"({a})" for a in self.dels]) + "}" + + return f"< {self.name}, {pres}, {adds}, {dels} >" + + def toJSON(self): + return json.dumps( + { + "name": self.name, + "preconditions": [f"({a})" for a in self.pres], + "add_effects": [f"({a})" for a in self.adds], + "delete_effects": [f"({a})" for a in self.dels], + }, + sort_keys=True, + indent=4, + ) + + def __repr__(self): + return self.name + + def __eq__(self, action): + return self.name == action.name + + def __hash__(self): + return hash(self.name) + + +class STRIPS: + def __init__(self, domain, problem): + self.domain_file = domain + self.problem_file = problem + self.reader = PDDLReader(raise_on_error=True) + self.reader.parse_domain(domain) + self.problem = self.reader.parse_instance(problem) + (self.grounded_fluents, init, goal, self.operators, self.grounder) = ( + self.ground_problem(self.problem) + ) + + self.fluents = set([fix_name(str(f)) for f in self.grounded_fluents]) + self.fluents_map = dict() + for f in self.grounded_fluents: + self.fluents_map[fix_name(str(f))] = f + self.init = set([fix_name(str(f)) for f in init]) + self.goal = set([fix_name(str(f)) for f in goal]) + self.actions = set() + self.action_map = {} + self.init_fluents = [self.fluents_map[f] for f in self.init] + + self.static_predicates = [i.name for i in self.grounder.static_symbols] + for op in self.operators: + act = self.operator_to_action(op) + self.actions.add(act) + self.action_map[act.name.lower()] = act + + def __str__(self): + fluents = "P = {" + ", ".join([f"({a})" for a in self.fluents]) + "}" + init = "I = {" + ", ".join([f"({a})" for a in self.init]) + "}" + goal = "G = {" + ", ".join([f"({a})" for a in self.goal]) + "}" + actions = "A = {" + "\n ".join([a.__str__() for a in self.actions]) + "}" + return fluents + ",\n" + init + "\n" + goal + "\n" + actions + + def toJSON(self): + actions = [a.toJSON() for a in self.actions] + return json.dumps( + { + "fluents": list(self.fluents), + "initial_state": list(self.init), + "goal": list(self.goal), + "actions": actions, + }, + sort_keys=True, + indent=4, + ) + + def operator_to_action(self, op, check_fluents=True, check_static=False): + adds = { + fix_name(str(f.atom)) for f in op.effects if isinstance(f, iofs.AddEffect) + } & self.fluents + dels = { + fix_name(str(f.atom)) for f in op.effects if isinstance(f, iofs.DelEffect) + } & self.fluents + pre = self.fix_pre_name(op.precondition) + if check_fluents: + pre = pre & self.fluents + if check_static: + pre = {p for p in pre if p.split()[0] not in self.static_predicates} + act = Action(fix_name(str(op)), pre, adds, dels) + return act + + def fix_pre_name(self, precondition): + if not is_atom(precondition): + return {fix_name(str(f)) for f in precondition.subformulas} + return {fix_name(str(precondition))} + + def action(self, name): + return self.action_map[fix_name(name).lower()] + + def get_action_or_none(self, name): + if "(" in name and ")" != name[-1]: + return None + return self.action_map.get(fix_name(name).lower(), None) + + def fluent(self, name): + return fix_name(name) + + def static_symbols(self): + return list(self.grounder.static_symbols) + + def fluent_symbols(self): + return list(self.grounder.fluent_symbols) + + def get_grounded_atoms(self, symbol): + variables = SymbolIndex() + lang = symbol.language + key = "atom_" + symbol.name + model = self.grounder._solve_lp() + if ( + key in model + ): # in case there is no reachable ground state variable from that fluent symbol + for binding in model[key]: + binding_with_constants = tuple(lang.get(c) for c in binding) + variables.add(StateVariableLite(symbol, binding_with_constants)) + return variables + + def get_applicable_actions(self, s): + return [a for a in self.actions if entails(s, a.pres)] + + def ground_problem(self, problem): + grounder = LPGroundingStrategy(problem, include_variable_inequalities=True) + action_groundings = grounder.ground_actions() + operators = [] + for action_name, groundings in action_groundings.items(): + action = problem.get_action(action_name) + for grounding in groundings: + operators.append( + ground_schema_into_plain_operator_from_grounding(action, grounding) + ) + + grounded_fluents = set( + [ + grounded_fluent.to_atom() + for grounded_fluent in grounder.ground_state_variables().objects + ] + ) + init = [f for f in problem.init.as_atoms() if f in grounded_fluents] + if isinstance(problem.goal, tarski.syntax.Atom): + goal = [problem.goal] + else: + goal = [f for f in problem.goal.subformulas if f in grounded_fluents] + + return (grounded_fluents, init, goal, operators, grounder) + + def get_static(self): + static_symbols = self.static_symbols() + ret = [] + for symbol in static_symbols: + ret.extend(self.get_grounded_atoms(symbol)) + return set([fix_name(str(x)) for x in ret]) + + def PDDL_replace_init_pddl_parser(self, s): + d = DomainParser()(open(self.domain_file, "r").read().lower()) + p = ProblemParser()(open(self.problem_file, "r").read().lower()) + + new_state = get_atoms_pddl(d, p, s | self.get_static()) + + new_p = Problem( + p.name, domain=d, objects=p.objects, init=new_state, goal=p.goal + ) + + return d, new_p + + +def parse_ans(response: str, parser: ACPGrammarParser, task: str): + return [parser.parse(clean_answer(resp, task)) for resp in response] + + +# def parse_ans(response : str, parser : ACPGrammarParser, task : str): +# ans = [parser.parse(clean_answer(resp, task), debug=True) for resp in response] +# if any(elem is None for elem in ans) or any(elem is None for elem in ans[0]): +# return None +# return ans + + +def remove_garbage(s): + while True: + if s.endswith("."): + s = s[:-1] + elif s.endswith("\n"): + s = s[:-2] + else: + break + return s.rstrip() + + +def compare_str(s1, s2): + return remove_garbage(s1).lower() == remove_garbage(s2).lower() + + +def compare(l1, l2): + if not isinstance(l1, list): + return compare_str(l1, l2) + if not isinstance(l2, list): + return False + for i, v in enumerate(l1): + if not compare(v, l2[i]): + return False + return True + + +def check_prog_response(resp): + if ( + "Positive Effects".lower() in resp.lower() + and "Negative Effects".lower() in resp.lower() + ): + if "[" not in resp: + return True + return False + + +def clean_answer(resp, task): + # Minor cleanup + if "progression_gen" in task: + # Check for Positive Effects and Negative Effects instead of separation + if check_prog_response(resp): + # replace **Positive Effects** with "[" + # replace **Negative Effects** with "] [" + # append "]" to the end + resp2 = resp.lower() + resp2 = resp2.replace("*", "") + resp2 = resp2.replace("positive effects", "[") + resp2 = resp2.replace("negative effects", "] [") + resp2 = resp2 + "]" + return resp2 + if "action_justification_gen" in task: + # Check for "simplified plan:" + if "simplified plan:" in resp.lower(): + resp2 = resp.lower() + resp2 = resp2.replace("*", "") + resp2 = resp2.split("simplified plan:")[1] + return resp2 + return resp + + +def get_grammar_task(task): + # print(task) + if task == "reachable_atom_gen": + return "act" + elif task == "progression_gen": + return "progression_list" + elif task == "validation_gen": + return "index" + elif task == "reachable_action_gen": + return "act" + elif task == "action_justification_gen": + return "action_list" + elif task == "landmarks_gen": + return "act" + elif task == "goal_closer_gen": + return "action_name" + elif task == "applicable_actions_gen": + return "action_list" + + +############################################################################## +# Evaluators + + +def fix_action_name(a): + assert a.startswith("(") and a.endswith(")") + return "(" + " ".join([x.strip() for x in a[1:-1].split(" ") if len(x) > 0]) + ")" + + +def str_remove_before_first_parentheses(s): + if s.startswith("("): + return s + try: + return s[s.index("(") :] + except Exception: + return "" + + +def str_remove_after_last_parentheses(s): + if s.endswith(")"): + return s + + i = s.rfind(")") + + if i == -1: + return "" + return s[: i + 1] + + +def cleanup_answer(ans): + if isinstance(ans, str): + ans = str_remove_before_first_parentheses(ans) + ans = str_remove_after_last_parentheses(ans) + ans = ans.lower() + ans = ( + ans.replace(")\n(", ")######(") + .replace("),(", ")######(") + .replace(") (", ")######(") + .split("######") + ) + return ans + if isinstance(ans, list): + res = [] + for x in ans: + res.extend(cleanup_answer(x)) + return res + + +def set_equal(ans1, ans2): + return set(ans1) == set(ans2) + + +class BaseEvaluator(ABC): + def __init__(self) -> None: + self.scores = [] + + @abstractmethod + def get_score(self, ans, doc): + pass + + def add_scores(self, scores): + self.scores.extend(scores) + + def get_avg_score(self): + avg_score = sum(self.scores) / len(self.scores) + return avg_score + + +def get_evaluator(group): + if group == "applicable_actions_gen": + return ApplicabilityEvaluator() + elif group == "progression_gen": + return ProgressionEvaluator() + elif group == "validation_gen": + return ValidationEvaluator() + elif group == "reachable_atom_gen": + return ReachabilityEvaluator() + elif group == "goal_closer_gen": + return NextActionEvaluator() + elif group == "action_justification_gen": + return JustificationEvaluator() + elif group == "landmarks_gen": + return LandmarksEvaluator() + elif group == "reachable_action_gen": + return ActionReachabilityEvaluator() + assert True, f"Group {group} not found" + + +""" +Action Reachability task: generate a valid action that is not applicable to any reachable state. +answer: A subset of actions that are known to be unreachable (not an exhaustive set). + It is empty only when we *know* that there are no such actions. +""" + + +class ActionReachabilityEvaluator(BaseEvaluator): + def get_score(self, ans, doc): + real_answer = doc["answer"] + if not real_answer or len(real_answer) == 0: + # The correct answer is None + self.add_scores( + ["none" == x.strip().lower() if x is not None else False for x in ans] + ) + else: + for x in ans: + if x is None: + self.scores.append(False) + continue + action = x.strip().lower() + if action in real_answer: + # The answer is in the subset of stored correct answers + self.scores.append(True) + continue + prec = get_action_preconditions( + doc["PDDL_domain"].lower(), doc["PDDL_problem"].lower(), action + ) + if prec is None: + # The answer does not correspond to a valid action + self.scores.append(False) + else: + # Need to run a planner on a task with the answer action preconditions as the new goal + prec = f"(and {' '.join(prec)})" + self.scores.append( + is_unsolvable_new_goal( + doc["PDDL_domain"].lower(), + doc["PDDL_problem"].lower(), + prec, + ) + ) + + return self.get_avg_score() + + +""" +Action Applicability task: generate all actions that are applicable in the current state. +answer: A set of all applicable actions. +""" + + +class ApplicabilityEvaluator(BaseEvaluator): + def get_score(self, ans, doc): + real_answer = doc["answer"] + real_answer = [a.lower() for a in real_answer] + ans = [[fix_action_name(a) for a in x] if x is not None else None for x in ans] + + # Check if the answer is equal (as a set) to the real stored answer + self.add_scores( + [ + set_equal(real_answer, cleanup_answer(x)) if x is not None else False + for x in ans + ] + ) + return self.get_avg_score() + + +def is_subsequence(plan, new_plan): + i = 0 + for a in plan: + if a == new_plan[i]: + i += 1 + if len(new_plan) == i: + # Done + return True + return False + + +def is_subsequence_and_plan(domain, problem, plan, new_plan): + if len(plan) <= len(new_plan): + return False + if not is_subsequence(plan, new_plan): + return False + return is_plan(domain, problem, new_plan) + + +""" +Justification task: generate a proper subsequence of the given plan that is also a plan. +answer: A list of examples of actions that can be removed (ignored in evaluation). +""" + + +class JustificationEvaluator(BaseEvaluator): + def get_score(self, ans, doc): + # Sequence of actions (plan) from the question + if "inputs" in doc: # old field name + seq = doc["inputs"][19:-147] + else: + seq = doc["question"][19:-147] + seq = seq.replace(") (", ")######(").split("######") + for x in ans: + if x is None: + self.scores.append(False) + continue + # An answer plan candidate + x = [fix_action_name(a) for a in x] + if len(x) == 0: + # Wrong answer - never an empty sequence + self.scores.append(0) + continue + # Check if the plan candidate from the answer (a) is a proper subsequence of the plan in the question and (b) is a plan. + self.scores.append( + is_subsequence_and_plan( + doc["PDDL_domain"].lower(), doc["PDDL_problem"].lower(), seq, x + ) + ) + return self.get_avg_score() + + +""" +Landmarks task: generate a fact that is a non-trivial landmark for the current state. +answer: A list of facts that are found to be landmarks and a list of facts that are found to be non-landmarks. + +The questions are generated only for cases where all facts either + (a) hold in the current state, + (b) true in goal, + (c) are found to be landmarks, or + (d) are found to be non-landmarks. +In such cases, the evaluation is simple, it does not require checking whether a fact is a landmark, it was +already done during question generation. +""" + + +class LandmarksEvaluator(BaseEvaluator): + def get_score(self, ans, doc): + # The set of facts that are found to be landmarks + real_answer = doc["answer"] + real_answer_yes = [a.lower() for a in real_answer["yes"]] + + for x in ans: + if x is None: + self.scores.append(False) + continue + if x.strip().lower() in real_answer_yes: + # The answer fact is known to be landmark + self.scores.append(True) + elif x.strip().lower() == "none": + # The answer is none, correct only if there are no known landmarks, + # since we only generate questions when that means that there are no non-trivial landmarks + self.scores.append(len(real_answer_yes) == 0) + else: + # All other cases the answer is incorrect + self.scores.append(False) + + return self.get_avg_score() + + +""" +Next Action task: generate an action that takes us closer to the goal. +answer: + (a) A list of applicable actions that are known to be correct answers + (b) A list of applicable actions that are known to be incorrect answers + (c) The rest of the applicable actions (maybe). +""" + + +class NextActionEvaluator(BaseEvaluator): + def get_score(self, ans, doc): + real_answer = doc["answer"] + real_answer_yes = [a.lower() for a in real_answer["yes"]] + real_answer_no = [a.lower() for a in real_answer["no"]] + real_answer_maybe = [a.lower() for a in real_answer["maybe"]] + # The cost of the optimal plan from the current state + opt = real_answer.get("opt", None) + for x in ans: + if x is None: + self.scores.append(False) + continue + action = x.strip().lower() + if action in real_answer_yes: + # Known to be correct + self.scores.append(True) + elif action in real_answer_no: + # Known to be incorrect + self.scores.append(False) + elif action not in real_answer_maybe: + # Not applicable, must be incorrect + self.scores.append(False) + else: + # Unknown, need to run a planner to check whether the state that results from applying the action is closer to the goal + # meaning has smaller optimal plan cost. + self.scores.append( + is_on_optimal_plan( + doc["PDDL_domain"].lower(), + doc["PDDL_problem"].lower(), + action, + opt, + ) + ) + + return self.get_avg_score() + + +""" +Progression task: generate the positive and negative effects of an action in the current state. +answer: + (a) A list of facts that were false and become true, when the action is applied + (b) A list of facts that were true and become false, when the action is applied +""" + + +class ProgressionEvaluator(BaseEvaluator): + def get_score(self, ans, doc): + real_answer = doc["answer"] + real_answer_pos = [a.lower() for a in real_answer["pos"]] + real_answer_neg = [a.lower() for a in real_answer["neg"]] + + for x in ans: + # The answer should be two lists. We allow for a single list and assume that the second one is empty (relaxed evaluation). + if x is None or len(x) > 2 or len(x) < 1: + self.scores.append(False) + else: + p = cleanup_answer(x[0]) + if len(x) == 2: + n = cleanup_answer(x[1]) + else: + # Assuming the last element is dropped because it is empty + n = [] + # Check if the answer is equal as sets to the correct answers. + ans = [set_equal(real_answer_pos, p), set_equal(real_answer_neg, n)] + self.scores.append(all(ans)) + + return self.get_avg_score() + + +""" +Reachability task: generate a valid fact that will never become true in any reachable state. +answer: A subset of facts that are known to be unreachable (not an exhaustive set). + It is empty only when we *know* that there are no such facts. +""" + + +class ReachabilityEvaluator(BaseEvaluator): + def get_score(self, ans, doc): + real_answer = doc["answer"] + real_answer = [f"({x.strip().lower()})" for x in real_answer] + + if len(real_answer) == 0: + # The correct answer is None + self.add_scores( + ["none" == x.strip().lower() if x is not None else False for x in ans] + ) + else: + for x in ans: + if x is None: + self.scores.append(False) + elif x.strip().lower() in real_answer: + # The answer is in the subset of stored correct answers + self.scores.append(True) + else: + # Need to run a planner on a task with the answer fact as the new goal + atom = x.strip().lower() + self.scores.append( + is_unsolvable_new_goal( + doc["PDDL_domain"].lower(), + doc["PDDL_problem"].lower(), + atom, + ) + ) + + return self.get_avg_score() + + +""" +Validation task: generate an index of the first inapplicable action in the given sequence. +answer: the correct index. +""" + + +class ValidationEvaluator(BaseEvaluator): + def get_score(self, ans, doc): + real_answer = str(doc["answer"]) + assert int(real_answer) >= 0, ( + f"The index must be non-negative, received {real_answer}" + ) + # Exact match + self.add_scores( + [ + real_answer.lower() == x.strip().lower() if x is not None else False + for x in ans + ] + ) + + return self.get_avg_score() + + +############################################################################## + + +def dump_item(item, **kwargs): + return json.dumps(item) + + +def parse_prediction(prediction): + try: + ans = json.loads(prediction.strip()) + response = ans.get("answer", None) + return response + except Exception as e: + print(f"Exception occurred {e}") + return prediction + + +@register_filter("ACP_grammar_filter") +class ACPGrammarFilter(RegexFilter): + """Filtering Index using""" + + def __init__(self, *args, **kwargs): + self.parser = ACPGrammarParser(kwargs["grammar_task"]) + self.clean = kwargs["clean"] if "clean" in kwargs else None + + def clean_pos_neg(self, resp): + # Check for Positive Effects and Negative Effects instead of separation + if check_prog_response(resp): + resp2 = resp.lower() + resp2 = resp2.replace("*", "") + resp2 = resp2.replace("positive effects", "[") + resp2 = resp2.replace("negative effects", "] [") + resp2 = resp2 + "]" + return resp2 + return resp + + def clean_simplified_plan(self, resp): + # Check for "simplified plan:" + if "simplified plan:" in resp.lower(): + resp2 = resp.lower() + resp2 = resp2.replace("*", "") + resp2 = resp2.split("simplified plan:")[1] + return resp2 + return resp + + def apply(self, resps, docs): + if self.clean == "pos_neg": + filtered_resps = [ + [self.parser.parse(self.clean_pos_neg(r)) for r in resp] + for resp in resps + ] + elif self.clean == "simplified plan": + filtered_resps = [ + [self.parser.parse(self.clean_simplified_plan(r)) for r in resp] + for resp in resps + ] + else: + filtered_resps = [[self.parser.parse(r) for r in resp] for resp in resps] + return filtered_resps + + +def process_acp_results(doc, results): + return {"score": get_evaluator(doc["group"]).get_score(results, doc)} + + +def get_score(references, predictions, **kwargs): + # print(f"References: {references}") + # print(f"Predictions: {predictions}") + data = json.loads(references[0].strip()) + real_ans = data["answer"] + task = data["group"] + + responses = [parse_prediction(prediction) for prediction in predictions] + + print(f"Real answer: {real_ans}") + print(f"Model answers: {responses}") + parser = ACPGrammarParser(get_grammar_task(task)) + ans = parse_ans(responses, parser, task) + + print(f"Parsed model answers: {ans}") + score = get_evaluator(task).get_score(ans, data) + + return {"get_score": score} diff --git a/lm_eval/tasks/acpbench/gen_2shot_with_pddl/act_reach.yaml b/lm_eval/tasks/acpbench/gen_2shot_with_pddl/act_reach.yaml new file mode 100644 index 00000000..b05d7223 --- /dev/null +++ b/lm_eval/tasks/acpbench/gen_2shot_with_pddl/act_reach.yaml @@ -0,0 +1,23 @@ +task: acp_areach_gen_with_pddl +dataset_name: acp_areach_gen +include: _gen_yaml_2shot +fewshot_config: + sampler: first_n + samples: + - context: "A robot is in a grid and can only move to places that are connected to its current position. \nThe grid size is 5x5, and the locations are of the form fi-jf (e.g., f3-2f or f0-1f). The grid cells are connected to their neighbors (e.g., f1-2f is connected to the four neighbors f0-2f, f2-2f, f1-1f, and f1-3f). Some positions on the grid are locked and can be opened with a key of a matching shape. The robot has an arm that can pick up a key when the key is in same location as the robot and the arm is empty. \nThere are 2 keys in 0 different shapes: Key key0-1 is of shape shape0, Key key0-0 is of shape shape0. \nCurrently, the robot is at position f3-2f and its arm is empty. All the positions are open except the following: f2-0f has shape0 shaped lock, f4-2f has shape0 shaped lock. Key key0-1 is at position f1-3f. Key key0-0 is at position f2-2f. The available actions are: (unlock ?curpos ?lockpos ?key ?shape) - unlock the place ?lockpos with key ?key of shape ?shape from the current position place ?curpos, (move ?curpos ?nextpos) - move to place ?nextpos from place ?curpos, (pickup ?curpos ?key) - acquire the key ?key from the place ?curpos, (pickup-and-loose ?curpos ?newkey ?oldkey) - pick up key ?newkey at current position place ?curpos and loose key ?oldkey being held, and (putdown ?curpos ?key) - put down key ?key at current position place ?curpos." + question: "What action can never become applicable, in any state reachable from the current state?" + answer: "(pickup-and-loose f0-1f key0-0 key0-0)" + PDDL_domain: "(define (domain grid)\n (:requirements :strips :typing)\n (:types key place shape - object)\n (:predicates (arm-empty) (at ?r - key ?x - place) (at-robot ?x - place) (conn ?x - place ?y - place) (holding ?k - key) (key-shape ?k - key ?s - shape) (lock-shape ?x - place ?s - shape) (locked ?x - place) (open ?x - place))\n (:action move\n :parameters (?curpos - place ?nextpos - place)\n :precondition (and (at-robot ?curpos) (conn ?curpos ?nextpos) (open ?nextpos))\n :effect (and (at-robot ?nextpos) (not (at-robot ?curpos)))\n )\n (:action pickup\n :parameters (?curpos - place ?key - key)\n :precondition (and (at-robot ?curpos) (at ?key ?curpos) (arm-empty))\n :effect (and (holding ?key) (not (at ?key ?curpos)) (not (arm-empty)))\n )\n (:action pickup-and-loose\n :parameters (?curpos - place ?newkey - key ?oldkey - key)\n :precondition (and (at-robot ?curpos) (holding ?oldkey) (at ?newkey ?curpos))\n :effect (and (holding ?newkey) (at ?oldkey ?curpos) (not (holding ?oldkey)) (not (at ?newkey ?curpos)))\n )\n (:action putdown\n :parameters (?curpos - place ?key - key)\n :precondition (and (at-robot ?curpos) (holding ?key))\n :effect (and (arm-empty) (at ?key ?curpos) (not (holding ?key)))\n )\n (:action unlock\n :parameters (?curpos - place ?lockpos - place ?key - key ?shape - shape)\n :precondition (and (conn ?curpos ?lockpos) (key-shape ?key ?shape) (lock-shape ?lockpos ?shape) (at-robot ?curpos) (locked ?lockpos) (holding ?key))\n :effect (and (open ?lockpos) (not (locked ?lockpos)))\n )\n)" + PDDL_problem: "(define (problem grid-x5-y5-t1-k2-l2-p100)\n (:domain grid)\n (:requirements :strips :typing)\n (:objects key0-0 key0-1 - key f0-0f f0-1f f0-2f f0-3f f0-4f f1-0f f1-1f f1-2f f1-3f f1-4f f2-0f f2-1f f2-2f f2-3f f2-4f f3-0f f3-1f f3-2f f3-3f f3-4f f4-0f f4-1f f4-2f f4-3f f4-4f - place shape0 - shape)\n (:init (arm-empty) (at key0-0 f2-2f) (at key0-1 f1-3f) (at-robot f3-2f) (conn f0-0f f0-1f) (conn f0-0f f1-0f) (conn f0-1f f0-0f) (conn f0-1f f0-2f) (conn f0-1f f1-1f) (conn f0-2f f0-1f) (conn f0-2f f0-3f) (conn f0-2f f1-2f) (conn f0-3f f0-2f) (conn f0-3f f0-4f) (conn f0-3f f1-3f) (conn f0-4f f0-3f) (conn f0-4f f1-4f) (conn f1-0f f0-0f) (conn f1-0f f1-1f) (conn f1-0f f2-0f) (conn f1-1f f0-1f) (conn f1-1f f1-0f) (conn f1-1f f1-2f) (conn f1-1f f2-1f) (conn f1-2f f0-2f) (conn f1-2f f1-1f) (conn f1-2f f1-3f) (conn f1-2f f2-2f) (conn f1-3f f0-3f) (conn f1-3f f1-2f) (conn f1-3f f1-4f) (conn f1-3f f2-3f) (conn f1-4f f0-4f) (conn f1-4f f1-3f) (conn f1-4f f2-4f) (conn f2-0f f1-0f) (conn f2-0f f2-1f) (conn f2-0f f3-0f) (conn f2-1f f1-1f) (conn f2-1f f2-0f) (conn f2-1f f2-2f) (conn f2-1f f3-1f) (conn f2-2f f1-2f) (conn f2-2f f2-1f) (conn f2-2f f2-3f) (conn f2-2f f3-2f) (conn f2-3f f1-3f) (conn f2-3f f2-2f) (conn f2-3f f2-4f) (conn f2-3f f3-3f) (conn f2-4f f1-4f) (conn f2-4f f2-3f) (conn f2-4f f3-4f) (conn f3-0f f2-0f) (conn f3-0f f3-1f) (conn f3-0f f4-0f) (conn f3-1f f2-1f) (conn f3-1f f3-0f) (conn f3-1f f3-2f) (conn f3-1f f4-1f) (conn f3-2f f2-2f) (conn f3-2f f3-1f) (conn f3-2f f3-3f) (conn f3-2f f4-2f) (conn f3-3f f2-3f) (conn f3-3f f3-2f) (conn f3-3f f3-4f) (conn f3-3f f4-3f) (conn f3-4f f2-4f) (conn f3-4f f3-3f) (conn f3-4f f4-4f) (conn f4-0f f3-0f) (conn f4-0f f4-1f) (conn f4-1f f3-1f) (conn f4-1f f4-0f) (conn f4-1f f4-2f) (conn f4-2f f3-2f) (conn f4-2f f4-1f) (conn f4-2f f4-3f) (conn f4-3f f3-3f) (conn f4-3f f4-2f) (conn f4-3f f4-4f) (conn f4-4f f3-4f) (conn f4-4f f4-3f) (key-shape key0-0 shape0) (key-shape key0-1 shape0) (lock-shape f2-0f shape0) (lock-shape f4-2f shape0) (locked f2-0f) (locked f4-2f) (open f0-0f) (open f0-1f) (open f0-2f) (open f0-3f) (open f0-4f) (open f1-0f) (open f1-1f) (open f1-2f) (open f1-3f) (open f1-4f) (open f2-1f) (open f2-2f) (open f2-3f) (open f2-4f) (open f3-0f) (open f3-1f) (open f3-2f) (open f3-3f) (open f3-4f) (open f4-0f) (open f4-1f) (open f4-3f) (open f4-4f))\n (:goal (and (at key0-0 f2-0f) (at key0-1 f1-3f)))\n)" + - context: "There are several cities, each containing several locations, some of which are airports. There are also trucks, which can drive within a single city, and airplanes, which can fly between airports. The goal is to get some packages from various locations to various new locations. \nThere are 2 trucks and 1 airplane, as well as 4 packages. There are 4 locations across 2 cities. \nThe locations are in cities as follows: l1-0 and l1-1 are in c1; l0-0 and l0-1 are in c0. \nCurrently, a0, p1, and p2 are at l1-0, t0 is at l0-1, p3 and p0 are at l0-0, t1 is at l1-1. The available actions are: (load-truck ?obj ?truck ?loc) - place the object ?obj into the truck ?truck at location ?loc, (load-airplane ?obj ?airplane ?loc) - load the object ?obj from location ?loc into the airplane ?airplane, (unload-truck ?obj ?truck ?loc) - unload object ?obj from truck ?truck at location ?loc, (unload-airplane ?obj ?airplane ?loc) - unload object ?obj from airplane ?airplane at location ?loc, (drive-truck ?truck ?loc-from ?loc-to ?city) - navigate the truck ?truck from its current location ?loc-from in city ?city to the new location ?loc-to within the same city, and (fly-airplane ?airplane ?loc-from ?loc-to) - fly airplane ?airplane from airport ?loc-from to airport ?loc-to." + question: "What action can never become applicable, in any state reachable from the current state??" + answer: "(load-truck p2 t0 l1-1)" + PDDL_domain: "(define (domain logistics-strips)\n (:requirements :strips :typing) \n\n (:types \n location locatable city - object \n package movable - locatable\n airport - location\n airplane truck - movable \n )\t\t\n \n (:predicates \t\n\t\t(at ?obj - locatable ?loc - location)\n\t\t(in ?obj1 - package ?obj2 - movable)\n\t\t(in-city ?obj - location ?city - city))\n\n\n(:action LOAD-TRUCK\n :parameters\n (?obj - package\n ?truck - truck\n ?loc - location)\n :precondition\n (and \n (at ?truck ?loc) (at ?obj ?loc))\n :effect\n (and (not (at ?obj ?loc)) (in ?obj ?truck)))\n\n(:action LOAD-AIRPLANE\n :parameters\n (?obj - package\n ?airplane - airplane\n ?loc - location)\n :precondition\n (and \n (at ?obj ?loc) (at ?airplane ?loc))\n :effect\n (and (not (at ?obj ?loc)) (in ?obj ?airplane)))\n\n\n\n(:action UNLOAD-TRUCK\n :parameters\n (?obj - package\n ?truck - truck\n ?loc - location)\n :precondition\n (and \n (at ?truck ?loc) (in ?obj ?truck))\n :effect\n (and (not (in ?obj ?truck)) (at ?obj ?loc)))\n\n(:action UNLOAD-AIRPLANE\n :parameters\n (?obj - package\n ?airplane - airplane\n ?loc - location)\n :precondition\n (and \n (in ?obj ?airplane) (at ?airplane ?loc))\n :effect\n (and (not (in ?obj ?airplane)) (at ?obj ?loc)))\n\n(:action DRIVE-TRUCK\n :parameters\n (?truck - truck\n ?loc-from - location\n ?loc-to - location\n ?city - city)\n :precondition\n (and \n (at ?truck ?loc-from)\n (in-city ?loc-from ?city)\n (in-city ?loc-to ?city))\n :effect\n (and (not (at ?truck ?loc-from)) (at ?truck ?loc-to)))\n\n(:action FLY-AIRPLANE\n :parameters\n (?airplane - airplane\n ?loc-from - airport\n ?loc-to - airport)\n :precondition\n (and \n\t(at ?airplane ?loc-from))\n :effect\n (and (not (at ?airplane ?loc-from)) (at ?airplane ?loc-to)))\n)" + PDDL_problem: "(define (problem logistics-c2-s2-p4-a1)\n (:domain logistics-strips)\n (:requirements :strips :typing)\n (:objects a0 - airplane l0-0 l1-0 - airport c0 c1 - city l0-1 l1-1 - location p0 p1 p2 p3 - package t0 t1 - truck)\n (:init (at a0 l1-0) (at p0 l0-0) (at p1 l1-0) (at p2 l1-0) (at p3 l0-0) (at t0 l0-1) (at t1 l1-1) (in-city l0-0 c0) (in-city l0-1 c0) (in-city l1-0 c1) (in-city l1-1 c1))\n (:goal (and (at p0 l0-0) (at p1 l1-0) (at p2 l1-0) (at p3 l0-1)))\n)" +doc_to_text: "# PDDL DOMAIN \n\n```\n{{PDDL_domain}}\n```\n\n# PDDL PROBLEM \n\n```\n{{PDDL_problem}}\n```\n\n**Question**: {{context}} {{question}} Each action starts with an opening parenthesis and ends with closing parenthesis. Provide one action or None. **Final Answer**:" +filter_list: + - name: "acp_grammar_parse" + filter: + - function: "ACP_grammar_filter" + grammar_task: "act" + - function: "take_first" diff --git a/lm_eval/tasks/acpbench/gen_2shot_with_pddl/app.yaml b/lm_eval/tasks/acpbench/gen_2shot_with_pddl/app.yaml new file mode 100644 index 00000000..c212924c --- /dev/null +++ b/lm_eval/tasks/acpbench/gen_2shot_with_pddl/app.yaml @@ -0,0 +1,23 @@ +task: acp_app_gen_with_pddl +dataset_name: acp_app_gen +include: _gen_yaml_2shot +fewshot_config: + sampler: first_n + samples: + - context: "A robot is in a grid and can only move to places that are connected to its current position. \nThe grid size is 5x5, and the locations are of the form fi-jf (e.g., f3-2f or f0-1f). The grid cells are connected to their neighbors (e.g., f1-2f is connected to the four neighbors f0-2f, f2-2f, f1-1f, and f1-3f). Some positions on the grid are locked and can be opened with a key of a matching shape. The robot has an arm that can pick up a key when the key is in same location as the robot and the arm is empty. \nThere are 2 keys in 0 different shapes: Key key0-0 is of shape shape0, Key key0-1 is of shape shape0. \nCurrently, the robot is at position f4-3f and its arm is empty. All the positions are open except the following: f2-0f has shape0 shaped lock, f4-2f has shape0 shaped lock. Key key0-0 is at position f3-1f. Key key0-1 is at position f1-3f. The available actions are: (unlock ?curpos ?lockpos ?key ?shape) - use the key ?key of shape ?shape to unlock the place ?lockpos from the current position ?curpos, (move ?curpos ?nextpos) - transition from the current position ?curpos to the next position ?nextpos, (pickup ?curpos ?key) - pick up key ?key from place ?curpos, (pickup-and-loose ?curpos ?newkey ?oldkey) - pick up the key ?newkey from the current position ?curpos and loose the key ?oldkey which is being held, and (putdown ?curpos ?key) - place the key ?key at the current position ?curpos." + question: "Generate the list of all ground actions that are applicable in this state." + answer: "[(move f4-3f f3-3f), (move f4-3f f4-4f)]" + PDDL_domain: "(define (domain grid)\n (:requirements :strips :typing)\n (:types key place shape - object)\n (:predicates (arm-empty) (at ?r - key ?x - place) (at-robot ?x - place) (conn ?x - place ?y - place) (holding ?k - key) (key-shape ?k - key ?s - shape) (lock-shape ?x - place ?s - shape) (locked ?x - place) (open ?x - place))\n (:action move\n :parameters (?curpos - place ?nextpos - place)\n :precondition (and (at-robot ?curpos) (conn ?curpos ?nextpos) (open ?nextpos))\n :effect (and (at-robot ?nextpos) (not (at-robot ?curpos)))\n )\n (:action pickup\n :parameters (?curpos - place ?key - key)\n :precondition (and (at-robot ?curpos) (at ?key ?curpos) (arm-empty))\n :effect (and (holding ?key) (not (at ?key ?curpos)) (not (arm-empty)))\n )\n (:action pickup-and-loose\n :parameters (?curpos - place ?newkey - key ?oldkey - key)\n :precondition (and (at-robot ?curpos) (holding ?oldkey) (at ?newkey ?curpos))\n :effect (and (holding ?newkey) (at ?oldkey ?curpos) (not (holding ?oldkey)) (not (at ?newkey ?curpos)))\n )\n (:action putdown\n :parameters (?curpos - place ?key - key)\n :precondition (and (at-robot ?curpos) (holding ?key))\n :effect (and (arm-empty) (at ?key ?curpos) (not (holding ?key)))\n )\n (:action unlock\n :parameters (?curpos - place ?lockpos - place ?key - key ?shape - shape)\n :precondition (and (conn ?curpos ?lockpos) (key-shape ?key ?shape) (lock-shape ?lockpos ?shape) (at-robot ?curpos) (locked ?lockpos) (holding ?key))\n :effect (and (open ?lockpos) (not (locked ?lockpos)))\n )\n)" + PDDL_problem: "(define (problem grid-x5-y5-t1-k2-l2-p100)\n (:domain grid)\n (:requirements :strips :typing)\n (:objects key0-0 key0-1 - key f0-0f f0-1f f0-2f f0-3f f0-4f f1-0f f1-1f f1-2f f1-3f f1-4f f2-0f f2-1f f2-2f f2-3f f2-4f f3-0f f3-1f f3-2f f3-3f f3-4f f4-0f f4-1f f4-2f f4-3f f4-4f - place shape0 - shape)\n (:init (arm-empty) (at key0-0 f3-1f) (at key0-1 f1-3f) (at-robot f4-3f) (conn f0-0f f0-1f) (conn f0-0f f1-0f) (conn f0-1f f0-0f) (conn f0-1f f0-2f) (conn f0-1f f1-1f) (conn f0-2f f0-1f) (conn f0-2f f0-3f) (conn f0-2f f1-2f) (conn f0-3f f0-2f) (conn f0-3f f0-4f) (conn f0-3f f1-3f) (conn f0-4f f0-3f) (conn f0-4f f1-4f) (conn f1-0f f0-0f) (conn f1-0f f1-1f) (conn f1-0f f2-0f) (conn f1-1f f0-1f) (conn f1-1f f1-0f) (conn f1-1f f1-2f) (conn f1-1f f2-1f) (conn f1-2f f0-2f) (conn f1-2f f1-1f) (conn f1-2f f1-3f) (conn f1-2f f2-2f) (conn f1-3f f0-3f) (conn f1-3f f1-2f) (conn f1-3f f1-4f) (conn f1-3f f2-3f) (conn f1-4f f0-4f) (conn f1-4f f1-3f) (conn f1-4f f2-4f) (conn f2-0f f1-0f) (conn f2-0f f2-1f) (conn f2-0f f3-0f) (conn f2-1f f1-1f) (conn f2-1f f2-0f) (conn f2-1f f2-2f) (conn f2-1f f3-1f) (conn f2-2f f1-2f) (conn f2-2f f2-1f) (conn f2-2f f2-3f) (conn f2-2f f3-2f) (conn f2-3f f1-3f) (conn f2-3f f2-2f) (conn f2-3f f2-4f) (conn f2-3f f3-3f) (conn f2-4f f1-4f) (conn f2-4f f2-3f) (conn f2-4f f3-4f) (conn f3-0f f2-0f) (conn f3-0f f3-1f) (conn f3-0f f4-0f) (conn f3-1f f2-1f) (conn f3-1f f3-0f) (conn f3-1f f3-2f) (conn f3-1f f4-1f) (conn f3-2f f2-2f) (conn f3-2f f3-1f) (conn f3-2f f3-3f) (conn f3-2f f4-2f) (conn f3-3f f2-3f) (conn f3-3f f3-2f) (conn f3-3f f3-4f) (conn f3-3f f4-3f) (conn f3-4f f2-4f) (conn f3-4f f3-3f) (conn f3-4f f4-4f) (conn f4-0f f3-0f) (conn f4-0f f4-1f) (conn f4-1f f3-1f) (conn f4-1f f4-0f) (conn f4-1f f4-2f) (conn f4-2f f3-2f) (conn f4-2f f4-1f) (conn f4-2f f4-3f) (conn f4-3f f3-3f) (conn f4-3f f4-2f) (conn f4-3f f4-4f) (conn f4-4f f3-4f) (conn f4-4f f4-3f) (key-shape key0-0 shape0) (key-shape key0-1 shape0) (lock-shape f2-0f shape0) (lock-shape f4-2f shape0) (locked f2-0f) (locked f4-2f) (open f0-0f) (open f0-1f) (open f0-2f) (open f0-3f) (open f0-4f) (open f1-0f) (open f1-1f) (open f1-2f) (open f1-3f) (open f1-4f) (open f2-1f) (open f2-2f) (open f2-3f) (open f2-4f) (open f3-0f) (open f3-1f) (open f3-2f) (open f3-3f) (open f3-4f) (open f4-0f) (open f4-1f) (open f4-3f) (open f4-4f))\n (:goal (and (at key0-0 f2-0f) (at key0-1 f1-3f)))\n)" + - context: "There are several cities, each containing several locations, some of which are airports. There are also trucks, which can drive within a single city, and airplanes, which can fly between airports. The goal is to get some packages from various locations to various new locations. \nThere are 2 trucks and 1 airplane, as well as 4 packages. There are 4 locations across 2 cities. \nThe locations are in cities as follows: l0-1 and l0-0 are in c0; l1-1 and l1-0 are in c1. \nCurrently, t1 is at l1-0, p0, a0, t0, and p3 are at l0-0, p1 and p2 are in t1. The available actions are: (load-truck ?obj ?truck ?loc) - load the object ?obj from location ?loc into the truck ?truck, (load-airplane ?obj ?airplane ?loc) - place the object ?obj onto the airplane ?airplane at location ?loc, (unload-truck ?obj ?truck ?loc) - remove the object ?obj from the truck ?truck and place it on the location ?loc, (unload-airplane ?obj ?airplane ?loc) - unload object ?obj from airplane ?airplane at location ?loc, (drive-truck ?truck ?loc-from ?loc-to ?city) - navigate the truck ?truck from location ?loc-from in city ?city to location ?loc-to in the same city, and (fly-airplane ?airplane ?loc-from ?loc-to) - fly airplane ?airplane from airport ?loc-from to airport ?loc-to." + question: "Generate the list of all ground actions that are applicable in this state." + answer: "[(unload-truck p2 t1 l1-0), (drive-truck t0 l0-0 l0-0 c0), (load-airplane p0 a0 l0-0), (load-truck p0 t0 l0-0), (unload-truck p1 t1 l1-0), (drive-truck t1 l1-0 l1-0 c1), (drive-truck t0 l0-0 l0-1 c0), (drive-truck t1 l1-0 l1-1 c1), (fly-airplane a0 l0-0 l0-0), (load-truck p3 t0 l0-0), (fly-airplane a0 l0-0 l1-0), (load-airplane p3 a0 l0-0)]" + PDDL_domain: "(define (domain logistics-strips)\n (:requirements :strips :typing) \n\n (:types \n location locatable city - object \n package movable - locatable\n airport - location\n airplane truck - movable \n )\t\t\n \n (:predicates \t\n\t\t(at ?obj - locatable ?loc - location)\n\t\t(in ?obj1 - package ?obj2 - movable)\n\t\t(in-city ?obj - location ?city - city))\n\n\n(:action LOAD-TRUCK\n :parameters\n (?obj - package\n ?truck - truck\n ?loc - location)\n :precondition\n (and \n (at ?truck ?loc) (at ?obj ?loc))\n :effect\n (and (not (at ?obj ?loc)) (in ?obj ?truck)))\n\n(:action LOAD-AIRPLANE\n :parameters\n (?obj - package\n ?airplane - airplane\n ?loc - location)\n :precondition\n (and \n (at ?obj ?loc) (at ?airplane ?loc))\n :effect\n (and (not (at ?obj ?loc)) (in ?obj ?airplane)))\n\n\n\n(:action UNLOAD-TRUCK\n :parameters\n (?obj - package\n ?truck - truck\n ?loc - location)\n :precondition\n (and \n (at ?truck ?loc) (in ?obj ?truck))\n :effect\n (and (not (in ?obj ?truck)) (at ?obj ?loc)))\n\n(:action UNLOAD-AIRPLANE\n :parameters\n (?obj - package\n ?airplane - airplane\n ?loc - location)\n :precondition\n (and \n (in ?obj ?airplane) (at ?airplane ?loc))\n :effect\n (and (not (in ?obj ?airplane)) (at ?obj ?loc)))\n\n(:action DRIVE-TRUCK\n :parameters\n (?truck - truck\n ?loc-from - location\n ?loc-to - location\n ?city - city)\n :precondition\n (and \n (at ?truck ?loc-from)\n (in-city ?loc-from ?city)\n (in-city ?loc-to ?city))\n :effect\n (and (not (at ?truck ?loc-from)) (at ?truck ?loc-to)))\n\n(:action FLY-AIRPLANE\n :parameters\n (?airplane - airplane\n ?loc-from - airport\n ?loc-to - airport)\n :precondition\n (and \n\t(at ?airplane ?loc-from))\n :effect\n (and (not (at ?airplane ?loc-from)) (at ?airplane ?loc-to)))\n)" + PDDL_problem: "(define (problem logistics-c2-s2-p4-a1)\n (:domain logistics-strips)\n (:requirements :strips :typing)\n (:objects a0 - airplane l0-0 l1-0 - airport c0 c1 - city l0-1 l1-1 - location p0 p1 p2 p3 - package t0 t1 - truck)\n (:init (at a0 l0-0) (at p0 l0-0) (at p3 l0-0) (at t0 l0-0) (at t1 l1-0) (in p1 t1) (in p2 t1) (in-city l0-0 c0) (in-city l0-1 c0) (in-city l1-0 c1) (in-city l1-1 c1))\n (:goal (and (at p0 l0-0) (at p1 l1-0) (at p2 l1-0) (at p3 l0-1)))\n)" +doc_to_text: "# PDDL DOMAIN \n\n```\n{{PDDL_domain}}\n```\n\n# PDDL PROBLEM \n\n```\n{{PDDL_problem}}\n```\n\n**Question**: {{context}} {{question}} Each action starts with an opening parenthesis and ends with closing parenthesis. Provide only the actions. \n**Final Answer**:" +filter_list: + - name: "acp_grammar_parse" + filter: + - function: "ACP_grammar_filter" + grammar_task: "action_list" + - function: "take_first" diff --git a/lm_eval/tasks/acpbench/gen_2shot_with_pddl/just.yaml b/lm_eval/tasks/acpbench/gen_2shot_with_pddl/just.yaml new file mode 100644 index 00000000..9685b8b8 --- /dev/null +++ b/lm_eval/tasks/acpbench/gen_2shot_with_pddl/just.yaml @@ -0,0 +1,24 @@ +task: acp_just_gen_with_pddl +dataset_name: acp_just_gen +include: _gen_yaml_2shot +fewshot_config: + sampler: first_n + samples: + - context: "A robot is in a grid and can only move to places that are connected to its current position. \nThe grid size is 5x5, and the locations are of the form fi-jf (e.g., f3-2f or f0-1f). The grid cells are connected to their neighbors (e.g., f1-2f is connected to the four neighbors f0-2f, f2-2f, f1-1f, and f1-3f). Some positions on the grid are locked and can be opened with a key of a matching shape. The robot has an arm that can pick up a key when the key is in same location as the robot and the arm is empty. \nThere are 2 keys in 0 different shapes: Key key0-0 is of shape shape0, Key key0-1 is of shape shape0. \nCurrently, the robot is at position f3-3f and its arm is empty. All the positions are open except the following: f4-2f has shape0 shaped lock, f2-0f has shape0 shaped lock. Key key0-1 is at position f1-3f. Key key0-0 is at position f2-2f. The available actions are: (unlock ?curpos ?lockpos ?key ?shape) - unlock place ?lockpos with key ?key of shape ?shape from current position place ?curpos, (move ?curpos ?nextpos) - transition from the current position ?curpos to the next position ?nextpos, (pickup ?curpos ?key) - acquire the key ?key from the place ?curpos, (pickup-and-loose ?curpos ?newkey ?oldkey) - pick up the key ?newkey from the current position ?curpos and loose the key ?oldkey which is being held, and (putdown ?curpos ?key) - place the key ?key at the current position place ?curpos. The goal is to reach a state where the following facts hold: Key key0-1 is at f1-3f location and Key key0-0 is at f2-0f location." + question: "Simplify the plan \"(move f3-3f f3-2f) (move f3-2f f2-2f) (pickup f2-2f key0-0) (move f2-2f f2-1f) (putdown f2-1f key0-0) (pickup f2-1f key0-0) (unlock f2-1f f2-0f key0-0 shape0) (move f2-1f f2-0f) (putdown f2-0f key0-0)\" by removing either a single action or a pair of consecutive actions, while still maintaining a valid plan. Provide the resulting simplified plan." + answer: "[(move f3-3f f3-2f), (move f3-2f f2-2f), (pickup f2-2f key0-0), (move f2-2f f2-1f), (unlock f2-1f f2-0f key0-0 shape0), (move f2-1f f2-0f), (putdown f2-0f key0-0)]" + PDDL_domain: "(define (domain grid)\n (:requirements :strips :typing)\n (:types key place shape - object)\n (:predicates (arm-empty) (at ?r - key ?x - place) (at-robot ?x - place) (conn ?x - place ?y - place) (holding ?k - key) (key-shape ?k - key ?s - shape) (lock-shape ?x - place ?s - shape) (locked ?x - place) (open ?x - place))\n (:action move\n :parameters (?curpos - place ?nextpos - place)\n :precondition (and (at-robot ?curpos) (conn ?curpos ?nextpos) (open ?nextpos))\n :effect (and (at-robot ?nextpos) (not (at-robot ?curpos)))\n )\n (:action pickup\n :parameters (?curpos - place ?key - key)\n :precondition (and (at-robot ?curpos) (at ?key ?curpos) (arm-empty))\n :effect (and (holding ?key) (not (at ?key ?curpos)) (not (arm-empty)))\n )\n (:action pickup-and-loose\n :parameters (?curpos - place ?newkey - key ?oldkey - key)\n :precondition (and (at-robot ?curpos) (holding ?oldkey) (at ?newkey ?curpos))\n :effect (and (holding ?newkey) (at ?oldkey ?curpos) (not (holding ?oldkey)) (not (at ?newkey ?curpos)))\n )\n (:action putdown\n :parameters (?curpos - place ?key - key)\n :precondition (and (at-robot ?curpos) (holding ?key))\n :effect (and (arm-empty) (at ?key ?curpos) (not (holding ?key)))\n )\n (:action unlock\n :parameters (?curpos - place ?lockpos - place ?key - key ?shape - shape)\n :precondition (and (conn ?curpos ?lockpos) (key-shape ?key ?shape) (lock-shape ?lockpos ?shape) (at-robot ?curpos) (locked ?lockpos) (holding ?key))\n :effect (and (open ?lockpos) (not (locked ?lockpos)))\n )\n)" + PDDL_problem: "(define (problem grid-x5-y5-t1-k2-l2-p100)\n (:domain grid)\n (:requirements :strips :typing)\n (:objects key0-0 key0-1 - key f0-0f f0-1f f0-2f f0-3f f0-4f f1-0f f1-1f f1-2f f1-3f f1-4f f2-0f f2-1f f2-2f f2-3f f2-4f f3-0f f3-1f f3-2f f3-3f f3-4f f4-0f f4-1f f4-2f f4-3f f4-4f - place shape0 - shape)\n (:init (arm-empty) (at key0-0 f2-2f) (at key0-1 f1-3f) (at-robot f3-3f) (conn f0-0f f0-1f) (conn f0-0f f1-0f) (conn f0-1f f0-0f) (conn f0-1f f0-2f) (conn f0-1f f1-1f) (conn f0-2f f0-1f) (conn f0-2f f0-3f) (conn f0-2f f1-2f) (conn f0-3f f0-2f) (conn f0-3f f0-4f) (conn f0-3f f1-3f) (conn f0-4f f0-3f) (conn f0-4f f1-4f) (conn f1-0f f0-0f) (conn f1-0f f1-1f) (conn f1-0f f2-0f) (conn f1-1f f0-1f) (conn f1-1f f1-0f) (conn f1-1f f1-2f) (conn f1-1f f2-1f) (conn f1-2f f0-2f) (conn f1-2f f1-1f) (conn f1-2f f1-3f) (conn f1-2f f2-2f) (conn f1-3f f0-3f) (conn f1-3f f1-2f) (conn f1-3f f1-4f) (conn f1-3f f2-3f) (conn f1-4f f0-4f) (conn f1-4f f1-3f) (conn f1-4f f2-4f) (conn f2-0f f1-0f) (conn f2-0f f2-1f) (conn f2-0f f3-0f) (conn f2-1f f1-1f) (conn f2-1f f2-0f) (conn f2-1f f2-2f) (conn f2-1f f3-1f) (conn f2-2f f1-2f) (conn f2-2f f2-1f) (conn f2-2f f2-3f) (conn f2-2f f3-2f) (conn f2-3f f1-3f) (conn f2-3f f2-2f) (conn f2-3f f2-4f) (conn f2-3f f3-3f) (conn f2-4f f1-4f) (conn f2-4f f2-3f) (conn f2-4f f3-4f) (conn f3-0f f2-0f) (conn f3-0f f3-1f) (conn f3-0f f4-0f) (conn f3-1f f2-1f) (conn f3-1f f3-0f) (conn f3-1f f3-2f) (conn f3-1f f4-1f) (conn f3-2f f2-2f) (conn f3-2f f3-1f) (conn f3-2f f3-3f) (conn f3-2f f4-2f) (conn f3-3f f2-3f) (conn f3-3f f3-2f) (conn f3-3f f3-4f) (conn f3-3f f4-3f) (conn f3-4f f2-4f) (conn f3-4f f3-3f) (conn f3-4f f4-4f) (conn f4-0f f3-0f) (conn f4-0f f4-1f) (conn f4-1f f3-1f) (conn f4-1f f4-0f) (conn f4-1f f4-2f) (conn f4-2f f3-2f) (conn f4-2f f4-1f) (conn f4-2f f4-3f) (conn f4-3f f3-3f) (conn f4-3f f4-2f) (conn f4-3f f4-4f) (conn f4-4f f3-4f) (conn f4-4f f4-3f) (key-shape key0-0 shape0) (key-shape key0-1 shape0) (lock-shape f2-0f shape0) (lock-shape f4-2f shape0) (locked f2-0f) (locked f4-2f) (open f0-0f) (open f0-1f) (open f0-2f) (open f0-3f) (open f0-4f) (open f1-0f) (open f1-1f) (open f1-2f) (open f1-3f) (open f1-4f) (open f2-1f) (open f2-2f) (open f2-3f) (open f2-4f) (open f3-0f) (open f3-1f) (open f3-2f) (open f3-3f) (open f3-4f) (open f4-0f) (open f4-1f) (open f4-3f) (open f4-4f))\n (:goal (and (at key0-0 f2-0f) (at key0-1 f1-3f)))\n)" + - context: "There are several cities, each containing several locations, some of which are airports. There are also trucks, which can drive within a single city, and airplanes, which can fly between airports. The goal is to get some packages from various locations to various new locations. \nThere are 2 trucks and 1 airplane, as well as 4 packages. There are 4 locations across 2 cities. \nThe locations are in cities as follows: l1-0 and l1-1 are in c1; l0-1 and l0-0 are in c0. \nCurrently, p3, p2, and p1 are at l1-0, t0 is at l0-1, p0 and t1 are at l1-1, a0 is at l0-0. The available actions are: (load-truck ?obj ?truck ?loc) - place the object ?obj into the truck ?truck at location ?loc, (load-airplane ?obj ?airplane ?loc) - load the object ?obj from location ?loc into the airplane ?airplane, (unload-truck ?obj ?truck ?loc) - unload the object ?obj from the truck ?truck at location ?loc, (unload-airplane ?obj ?airplane ?loc) - remove the object ?obj from the airplane ?airplane and place it on the location ?loc, (drive-truck ?truck ?loc-from ?loc-to ?city) - navigate the truck ?truck which is in location ?loc-from in city ?city to another location ?loc-to in the same city, and (fly-airplane ?airplane ?loc-from ?loc-to) - fly the airplane ?airplane from airport ?loc-from to airport ?loc-to. The goal is to reach a state where the following facts hold: p2 is at l1-0, p0 is at l0-0, p3 is at l0-1, and p1 is at l1-0." + question: "Simplify the plan \"(load-truck p0 t1 l1-1) (unload-truck p0 t1 l1-1) (load-truck p0 t1 l1-1) (drive-truck t1 l1-1 l1-0 c1) (unload-truck p0 t1 l1-0) (fly-airplane a0 l0-0 l1-0) (load-airplane p0 a0 l1-0) (load-airplane p3 a0 l1-0) (fly-airplane a0 l1-0 l0-0) (unload-airplane p0 a0 l0-0) (unload-airplane p3 a0 l0-0) (drive-truck t0 l0-1 l0-0 c0) (load-truck p3 t0 l0-0) (drive-truck t0 l0-0 l0-1 c0) (unload-truck p3 t0 l0-1)\" by removing either a single action or a pair of consecutive actions, while still maintaining a valid plan. Provide the resulting simplified plan." + answer: "[(load-truck p0 t1 l1-1), (drive-truck t1 l1-1 l1-0 c1), (unload-truck p0 t1 l1-0), (fly-airplane a0 l0-0 l1-0), (load-airplane p0 a0 l1-0), (load-airplane p3 a0 l1-0), (fly-airplane a0 l1-0 l0-0), (unload-airplane p0 a0 l0-0), (unload-airplane p3 a0 l0-0), (drive-truck t0 l0-1 l0-0 c0), (load-truck p3 t0 l0-0), (drive-truck t0 l0-0 l0-1 c0), (unload-truck p3 t0 l0-1)]" + PDDL_domain: "(define (domain logistics-strips)\n (:requirements :strips :typing) \n\n (:types \n location locatable city - object \n package movable - locatable\n airport - location\n airplane truck - movable \n )\t\t\n \n (:predicates \t\n\t\t(at ?obj - locatable ?loc - location)\n\t\t(in ?obj1 - package ?obj2 - movable)\n\t\t(in-city ?obj - location ?city - city))\n\n\n(:action LOAD-TRUCK\n :parameters\n (?obj - package\n ?truck - truck\n ?loc - location)\n :precondition\n (and \n (at ?truck ?loc) (at ?obj ?loc))\n :effect\n (and (not (at ?obj ?loc)) (in ?obj ?truck)))\n\n(:action LOAD-AIRPLANE\n :parameters\n (?obj - package\n ?airplane - airplane\n ?loc - location)\n :precondition\n (and \n (at ?obj ?loc) (at ?airplane ?loc))\n :effect\n (and (not (at ?obj ?loc)) (in ?obj ?airplane)))\n\n\n\n(:action UNLOAD-TRUCK\n :parameters\n (?obj - package\n ?truck - truck\n ?loc - location)\n :precondition\n (and \n (at ?truck ?loc) (in ?obj ?truck))\n :effect\n (and (not (in ?obj ?truck)) (at ?obj ?loc)))\n\n(:action UNLOAD-AIRPLANE\n :parameters\n (?obj - package\n ?airplane - airplane\n ?loc - location)\n :precondition\n (and \n (in ?obj ?airplane) (at ?airplane ?loc))\n :effect\n (and (not (in ?obj ?airplane)) (at ?obj ?loc)))\n\n(:action DRIVE-TRUCK\n :parameters\n (?truck - truck\n ?loc-from - location\n ?loc-to - location\n ?city - city)\n :precondition\n (and \n (at ?truck ?loc-from)\n (in-city ?loc-from ?city)\n (in-city ?loc-to ?city))\n :effect\n (and (not (at ?truck ?loc-from)) (at ?truck ?loc-to)))\n\n(:action FLY-AIRPLANE\n :parameters\n (?airplane - airplane\n ?loc-from - airport\n ?loc-to - airport)\n :precondition\n (and \n\t(at ?airplane ?loc-from))\n :effect\n (and (not (at ?airplane ?loc-from)) (at ?airplane ?loc-to)))\n)" + PDDL_problem: "(define (problem logistics-c2-s2-p4-a1)\n (:domain logistics-strips)\n (:requirements :strips :typing)\n (:objects a0 - airplane l0-0 l1-0 - airport c0 c1 - city l0-1 l1-1 - location p0 p1 p2 p3 - package t0 t1 - truck)\n (:init (at a0 l0-0) (at p0 l1-1) (at p1 l1-0) (at p2 l1-0) (at p3 l1-0) (at t0 l0-1) (at t1 l1-1) (in-city l0-0 c0) (in-city l0-1 c0) (in-city l1-0 c1) (in-city l1-1 c1))\n (:goal (and (at p0 l0-0) (at p1 l1-0) (at p2 l1-0) (at p3 l0-1)))\n)" +doc_to_text: "# PDDL DOMAIN \n\n```\n{{PDDL_domain}}\n```\n\n# PDDL PROBLEM \n\n```\n{{PDDL_problem}}\n```\n\n**Question**: {{context}} {{question}} **Final Answer**:" +filter_list: + - name: "acp_grammar_parse" + filter: + - function: "ACP_grammar_filter" + grammar_task: "action_list" + clean: "simplified plan" + - function: "take_first" diff --git a/lm_eval/tasks/acpbench/gen_2shot_with_pddl/land.yaml b/lm_eval/tasks/acpbench/gen_2shot_with_pddl/land.yaml new file mode 100644 index 00000000..da7bf1c6 --- /dev/null +++ b/lm_eval/tasks/acpbench/gen_2shot_with_pddl/land.yaml @@ -0,0 +1,23 @@ +task: acp_land_gen_with_pddl +dataset_name: acp_land_gen +include: _gen_yaml_2shot +fewshot_config: + sampler: first_n + samples: + - context: "A robot is in a grid and can only move to places that are connected to its current position. \nThe grid size is 5x5, and the locations are of the form fi-jf (e.g., f3-2f or f0-1f). The grid cells are connected to their neighbors (e.g., f1-2f is connected to the four neighbors f0-2f, f2-2f, f1-1f, and f1-3f). Some positions on the grid are locked and can be opened with a key of a matching shape. The robot has an arm that can pick up a key when the key is in same location as the robot and the arm is empty. \nThere are 2 keys in 0 different shapes: Key key0-0 is of shape shape0, Key key0-1 is of shape shape0. \nCurrently, the robot is at position f1-0f and its arm is empty. All the positions are open except the following: f4-2f has shape0 shaped lock. Key key0-1 is at position f1-3f. Key key0-0 is at position f1-0f. The goal is to reach a state where the following facts hold: Key key0-1 is at f1-3f location and Key key0-0 is at f2-0f location. The available propositions are: (at ?r ?x) - Key ?r is at ?x location, (at-robot ?x) - Robot is at ?x location, (locked ?x) - Location ?x is locked, (holding ?k) - Robot is holding ?k, (open ?x) - Location ?x is open, and (arm-empty) - Robot's arm is empty." + question: "Generate a non-trivial fact landmark, one that does not hold in the initial state or goal." + answer: "(at-robot f2-0f)" + PDDL_domain: "(define (domain grid)\n (:requirements :strips :typing)\n (:types key place shape - object)\n (:predicates (arm-empty) (at ?r - key ?x - place) (at-robot ?x - place) (conn ?x - place ?y - place) (holding ?k - key) (key-shape ?k - key ?s - shape) (lock-shape ?x - place ?s - shape) (locked ?x - place) (open ?x - place))\n (:action move\n :parameters (?curpos - place ?nextpos - place)\n :precondition (and (at-robot ?curpos) (conn ?curpos ?nextpos) (open ?nextpos))\n :effect (and (at-robot ?nextpos) (not (at-robot ?curpos)))\n )\n (:action pickup\n :parameters (?curpos - place ?key - key)\n :precondition (and (at-robot ?curpos) (at ?key ?curpos) (arm-empty))\n :effect (and (holding ?key) (not (at ?key ?curpos)) (not (arm-empty)))\n )\n (:action pickup-and-loose\n :parameters (?curpos - place ?newkey - key ?oldkey - key)\n :precondition (and (at-robot ?curpos) (holding ?oldkey) (at ?newkey ?curpos))\n :effect (and (holding ?newkey) (at ?oldkey ?curpos) (not (holding ?oldkey)) (not (at ?newkey ?curpos)))\n )\n (:action putdown\n :parameters (?curpos - place ?key - key)\n :precondition (and (at-robot ?curpos) (holding ?key))\n :effect (and (arm-empty) (at ?key ?curpos) (not (holding ?key)))\n )\n (:action unlock\n :parameters (?curpos - place ?lockpos - place ?key - key ?shape - shape)\n :precondition (and (conn ?curpos ?lockpos) (key-shape ?key ?shape) (lock-shape ?lockpos ?shape) (at-robot ?curpos) (locked ?lockpos) (holding ?key))\n :effect (and (open ?lockpos) (not (locked ?lockpos)))\n )\n)" + PDDL_problem: "(define (problem grid-x5-y5-t1-k2-l2-p100)\n (:domain grid)\n (:requirements :strips :typing)\n (:objects key0-0 key0-1 - key f0-0f f0-1f f0-2f f0-3f f0-4f f1-0f f1-1f f1-2f f1-3f f1-4f f2-0f f2-1f f2-2f f2-3f f2-4f f3-0f f3-1f f3-2f f3-3f f3-4f f4-0f f4-1f f4-2f f4-3f f4-4f - place shape0 - shape)\n (:init (arm-empty) (at key0-0 f1-0f) (at key0-1 f1-3f) (at-robot f1-0f) (conn f0-0f f0-1f) (conn f0-0f f1-0f) (conn f0-1f f0-0f) (conn f0-1f f0-2f) (conn f0-1f f1-1f) (conn f0-2f f0-1f) (conn f0-2f f0-3f) (conn f0-2f f1-2f) (conn f0-3f f0-2f) (conn f0-3f f0-4f) (conn f0-3f f1-3f) (conn f0-4f f0-3f) (conn f0-4f f1-4f) (conn f1-0f f0-0f) (conn f1-0f f1-1f) (conn f1-0f f2-0f) (conn f1-1f f0-1f) (conn f1-1f f1-0f) (conn f1-1f f1-2f) (conn f1-1f f2-1f) (conn f1-2f f0-2f) (conn f1-2f f1-1f) (conn f1-2f f1-3f) (conn f1-2f f2-2f) (conn f1-3f f0-3f) (conn f1-3f f1-2f) (conn f1-3f f1-4f) (conn f1-3f f2-3f) (conn f1-4f f0-4f) (conn f1-4f f1-3f) (conn f1-4f f2-4f) (conn f2-0f f1-0f) (conn f2-0f f2-1f) (conn f2-0f f3-0f) (conn f2-1f f1-1f) (conn f2-1f f2-0f) (conn f2-1f f2-2f) (conn f2-1f f3-1f) (conn f2-2f f1-2f) (conn f2-2f f2-1f) (conn f2-2f f2-3f) (conn f2-2f f3-2f) (conn f2-3f f1-3f) (conn f2-3f f2-2f) (conn f2-3f f2-4f) (conn f2-3f f3-3f) (conn f2-4f f1-4f) (conn f2-4f f2-3f) (conn f2-4f f3-4f) (conn f3-0f f2-0f) (conn f3-0f f3-1f) (conn f3-0f f4-0f) (conn f3-1f f2-1f) (conn f3-1f f3-0f) (conn f3-1f f3-2f) (conn f3-1f f4-1f) (conn f3-2f f2-2f) (conn f3-2f f3-1f) (conn f3-2f f3-3f) (conn f3-2f f4-2f) (conn f3-3f f2-3f) (conn f3-3f f3-2f) (conn f3-3f f3-4f) (conn f3-3f f4-3f) (conn f3-4f f2-4f) (conn f3-4f f3-3f) (conn f3-4f f4-4f) (conn f4-0f f3-0f) (conn f4-0f f4-1f) (conn f4-1f f3-1f) (conn f4-1f f4-0f) (conn f4-1f f4-2f) (conn f4-2f f3-2f) (conn f4-2f f4-1f) (conn f4-2f f4-3f) (conn f4-3f f3-3f) (conn f4-3f f4-2f) (conn f4-3f f4-4f) (conn f4-4f f3-4f) (conn f4-4f f4-3f) (key-shape key0-0 shape0) (key-shape key0-1 shape0) (lock-shape f2-0f shape0) (lock-shape f4-2f shape0) (locked f4-2f) (open f0-0f) (open f0-1f) (open f0-2f) (open f0-3f) (open f0-4f) (open f1-0f) (open f1-1f) (open f1-2f) (open f1-3f) (open f1-4f) (open f2-0f) (open f2-1f) (open f2-2f) (open f2-3f) (open f2-4f) (open f3-0f) (open f3-1f) (open f3-2f) (open f3-3f) (open f3-4f) (open f4-0f) (open f4-1f) (open f4-3f) (open f4-4f))\n (:goal (and (at key0-0 f2-0f) (at key0-1 f1-3f)))\n)" + - context: "There are several cities, each containing several locations, some of which are airports. There are also trucks, which can drive within a single city, and airplanes, which can fly between airports. The goal is to get some packages from various locations to various new locations. \nThere are 2 trucks and 1 airplane, as well as 4 packages. There are 4 locations across 2 cities. \nThe locations are in cities as follows: l0-1 and l0-0 are in c0; l1-0 and l1-1 are in c1. \nCurrently, a0 and t0 are at l0-0, t1 is at l1-1, p2, p1, and p3 are at l1-0, p0 is in t1. The goal is to reach a state where the following facts hold: p1 is at l1-0, p0 is at l0-0, p2 is at l1-0, and p3 is at l0-1. The available propositions are: (at ?obj ?loc) - ?obj is at ?loc and (in ?obj1 ?obj2) - ?obj1 is in ?obj2." + question: "Generate a non-trivial fact landmark, one that does not hold in the initial state or goal." + answer: "(in p3 t0)" + PDDL_domain: "(define (domain logistics-strips)\n (:requirements :strips :typing) \n\n (:types \n location locatable city - object \n package movable - locatable\n airport - location\n airplane truck - movable \n )\t\t\n \n (:predicates \t\n\t\t(at ?obj - locatable ?loc - location)\n\t\t(in ?obj1 - package ?obj2 - movable)\n\t\t(in-city ?obj - location ?city - city))\n\n\n(:action LOAD-TRUCK\n :parameters\n (?obj - package\n ?truck - truck\n ?loc - location)\n :precondition\n (and \n (at ?truck ?loc) (at ?obj ?loc))\n :effect\n (and (not (at ?obj ?loc)) (in ?obj ?truck)))\n\n(:action LOAD-AIRPLANE\n :parameters\n (?obj - package\n ?airplane - airplane\n ?loc - location)\n :precondition\n (and \n (at ?obj ?loc) (at ?airplane ?loc))\n :effect\n (and (not (at ?obj ?loc)) (in ?obj ?airplane)))\n\n\n\n(:action UNLOAD-TRUCK\n :parameters\n (?obj - package\n ?truck - truck\n ?loc - location)\n :precondition\n (and \n (at ?truck ?loc) (in ?obj ?truck))\n :effect\n (and (not (in ?obj ?truck)) (at ?obj ?loc)))\n\n(:action UNLOAD-AIRPLANE\n :parameters\n (?obj - package\n ?airplane - airplane\n ?loc - location)\n :precondition\n (and \n (in ?obj ?airplane) (at ?airplane ?loc))\n :effect\n (and (not (in ?obj ?airplane)) (at ?obj ?loc)))\n\n(:action DRIVE-TRUCK\n :parameters\n (?truck - truck\n ?loc-from - location\n ?loc-to - location\n ?city - city)\n :precondition\n (and \n (at ?truck ?loc-from)\n (in-city ?loc-from ?city)\n (in-city ?loc-to ?city))\n :effect\n (and (not (at ?truck ?loc-from)) (at ?truck ?loc-to)))\n\n(:action FLY-AIRPLANE\n :parameters\n (?airplane - airplane\n ?loc-from - airport\n ?loc-to - airport)\n :precondition\n (and \n\t(at ?airplane ?loc-from))\n :effect\n (and (not (at ?airplane ?loc-from)) (at ?airplane ?loc-to)))\n)" + PDDL_problem: "(define (problem logistics-c2-s2-p4-a1)\n (:domain logistics-strips)\n (:requirements :strips :typing)\n (:objects a0 - airplane l0-0 l1-0 - airport c0 c1 - city l0-1 l1-1 - location p0 p1 p2 p3 - package t0 t1 - truck)\n (:init (at a0 l0-0) (at p1 l1-0) (at p2 l1-0) (at p3 l1-0) (at t0 l0-0) (at t1 l1-1) (in p0 t1) (in-city l0-0 c0) (in-city l0-1 c0) (in-city l1-0 c1) (in-city l1-1 c1))\n (:goal (and (at p0 l0-0) (at p1 l1-0) (at p2 l1-0) (at p3 l0-1)))\n)" +doc_to_text: "# PDDL DOMAIN \n\n```\n{{PDDL_domain}}\n```\n\n# PDDL PROBLEM \n\n```\n{{PDDL_problem}}\n```\n\n**Question**: {{context}} {{question}} Provide only the ground proposition or None. **Final Answer**:" +filter_list: + - name: "acp_grammar_parse" + filter: + - function: "ACP_grammar_filter" + grammar_task: "act" + - function: "take_first" diff --git a/lm_eval/tasks/acpbench/gen_2shot_with_pddl/next_act.yaml b/lm_eval/tasks/acpbench/gen_2shot_with_pddl/next_act.yaml new file mode 100644 index 00000000..0f43ca61 --- /dev/null +++ b/lm_eval/tasks/acpbench/gen_2shot_with_pddl/next_act.yaml @@ -0,0 +1,23 @@ +task: acp_nexta_gen_with_pddl +dataset_name: acp_nexta_gen +include: _gen_yaml_2shot +fewshot_config: + sampler: first_n + samples: + - context: "A robot is in a grid and can only move to places that are connected to its current position. \nThe grid size is 5x5, and the locations are of the form fi-jf (e.g., f3-2f or f0-1f). The grid cells are connected to their neighbors (e.g., f1-2f is connected to the four neighbors f0-2f, f2-2f, f1-1f, and f1-3f). Some positions on the grid are locked and can be opened with a key of a matching shape. The robot has an arm that can pick up a key when the key is in same location as the robot and the arm is empty. \nThere are 2 keys in 0 different shapes: Key key0-0 is of shape shape0, Key key0-1 is of shape shape0. \nCurrently, the robot is at position f1-1f and its arm is empty. All the positions are open except the following: f4-2f has shape0 shaped lock. Key key0-0 is at position f1-0f. Key key0-1 is at position f1-3f. The goal is to reach a state where the following facts hold: Key key0-1 is at f1-3f location and Key key0-0 is at f2-0f location. The available actions are: (unlock ?curpos ?lockpos ?key ?shape) - use the key ?key of shape ?shape to unlock the place ?lockpos from the current position ?curpos, (move ?curpos ?nextpos) - move to place ?nextpos from place ?curpos, (pickup ?curpos ?key) - retrieve the key ?key from its current position ?curpos, (pickup-and-loose ?curpos ?newkey ?oldkey) - pick up key ?newkey at current position place ?curpos and loose key ?oldkey being held, and (putdown ?curpos ?key) - put the key ?key at the current position place ?curpos." + question: "What is the next action that takes us towards the goal?" + answer: "(move f1-1f f1-0f)" + PDDL_domain: "(define (domain grid)\n (:requirements :strips :typing)\n (:types key place shape - object)\n (:predicates (arm-empty) (at ?r - key ?x - place) (at-robot ?x - place) (conn ?x - place ?y - place) (holding ?k - key) (key-shape ?k - key ?s - shape) (lock-shape ?x - place ?s - shape) (locked ?x - place) (open ?x - place))\n (:action move\n :parameters (?curpos - place ?nextpos - place)\n :precondition (and (at-robot ?curpos) (conn ?curpos ?nextpos) (open ?nextpos))\n :effect (and (at-robot ?nextpos) (not (at-robot ?curpos)))\n )\n (:action pickup\n :parameters (?curpos - place ?key - key)\n :precondition (and (at-robot ?curpos) (at ?key ?curpos) (arm-empty))\n :effect (and (holding ?key) (not (at ?key ?curpos)) (not (arm-empty)))\n )\n (:action pickup-and-loose\n :parameters (?curpos - place ?newkey - key ?oldkey - key)\n :precondition (and (at-robot ?curpos) (holding ?oldkey) (at ?newkey ?curpos))\n :effect (and (holding ?newkey) (at ?oldkey ?curpos) (not (holding ?oldkey)) (not (at ?newkey ?curpos)))\n )\n (:action putdown\n :parameters (?curpos - place ?key - key)\n :precondition (and (at-robot ?curpos) (holding ?key))\n :effect (and (arm-empty) (at ?key ?curpos) (not (holding ?key)))\n )\n (:action unlock\n :parameters (?curpos - place ?lockpos - place ?key - key ?shape - shape)\n :precondition (and (conn ?curpos ?lockpos) (key-shape ?key ?shape) (lock-shape ?lockpos ?shape) (at-robot ?curpos) (locked ?lockpos) (holding ?key))\n :effect (and (open ?lockpos) (not (locked ?lockpos)))\n )\n)" + PDDL_problem: "(define (problem grid-x5-y5-t1-k2-l2-p100)\n (:domain grid)\n (:requirements :strips :typing)\n (:objects key0-0 key0-1 - key f0-0f f0-1f f0-2f f0-3f f0-4f f1-0f f1-1f f1-2f f1-3f f1-4f f2-0f f2-1f f2-2f f2-3f f2-4f f3-0f f3-1f f3-2f f3-3f f3-4f f4-0f f4-1f f4-2f f4-3f f4-4f - place shape0 - shape)\n (:init (arm-empty) (at key0-0 f1-0f) (at key0-1 f1-3f) (at-robot f1-1f) (conn f0-0f f0-1f) (conn f0-0f f1-0f) (conn f0-1f f0-0f) (conn f0-1f f0-2f) (conn f0-1f f1-1f) (conn f0-2f f0-1f) (conn f0-2f f0-3f) (conn f0-2f f1-2f) (conn f0-3f f0-2f) (conn f0-3f f0-4f) (conn f0-3f f1-3f) (conn f0-4f f0-3f) (conn f0-4f f1-4f) (conn f1-0f f0-0f) (conn f1-0f f1-1f) (conn f1-0f f2-0f) (conn f1-1f f0-1f) (conn f1-1f f1-0f) (conn f1-1f f1-2f) (conn f1-1f f2-1f) (conn f1-2f f0-2f) (conn f1-2f f1-1f) (conn f1-2f f1-3f) (conn f1-2f f2-2f) (conn f1-3f f0-3f) (conn f1-3f f1-2f) (conn f1-3f f1-4f) (conn f1-3f f2-3f) (conn f1-4f f0-4f) (conn f1-4f f1-3f) (conn f1-4f f2-4f) (conn f2-0f f1-0f) (conn f2-0f f2-1f) (conn f2-0f f3-0f) (conn f2-1f f1-1f) (conn f2-1f f2-0f) (conn f2-1f f2-2f) (conn f2-1f f3-1f) (conn f2-2f f1-2f) (conn f2-2f f2-1f) (conn f2-2f f2-3f) (conn f2-2f f3-2f) (conn f2-3f f1-3f) (conn f2-3f f2-2f) (conn f2-3f f2-4f) (conn f2-3f f3-3f) (conn f2-4f f1-4f) (conn f2-4f f2-3f) (conn f2-4f f3-4f) (conn f3-0f f2-0f) (conn f3-0f f3-1f) (conn f3-0f f4-0f) (conn f3-1f f2-1f) (conn f3-1f f3-0f) (conn f3-1f f3-2f) (conn f3-1f f4-1f) (conn f3-2f f2-2f) (conn f3-2f f3-1f) (conn f3-2f f3-3f) (conn f3-2f f4-2f) (conn f3-3f f2-3f) (conn f3-3f f3-2f) (conn f3-3f f3-4f) (conn f3-3f f4-3f) (conn f3-4f f2-4f) (conn f3-4f f3-3f) (conn f3-4f f4-4f) (conn f4-0f f3-0f) (conn f4-0f f4-1f) (conn f4-1f f3-1f) (conn f4-1f f4-0f) (conn f4-1f f4-2f) (conn f4-2f f3-2f) (conn f4-2f f4-1f) (conn f4-2f f4-3f) (conn f4-3f f3-3f) (conn f4-3f f4-2f) (conn f4-3f f4-4f) (conn f4-4f f3-4f) (conn f4-4f f4-3f) (key-shape key0-0 shape0) (key-shape key0-1 shape0) (lock-shape f2-0f shape0) (lock-shape f4-2f shape0) (locked f4-2f) (open f0-0f) (open f0-1f) (open f0-2f) (open f0-3f) (open f0-4f) (open f1-0f) (open f1-1f) (open f1-2f) (open f1-3f) (open f1-4f) (open f2-0f) (open f2-1f) (open f2-2f) (open f2-3f) (open f2-4f) (open f3-0f) (open f3-1f) (open f3-2f) (open f3-3f) (open f3-4f) (open f4-0f) (open f4-1f) (open f4-3f) (open f4-4f))\n (:goal (and (at key0-0 f2-0f) (at key0-1 f1-3f)))\n)" + - context: "There are several cities, each containing several locations, some of which are airports. There are also trucks, which can drive within a single city, and airplanes, which can fly between airports. The goal is to get some packages from various locations to various new locations. \nThere are 2 trucks and 1 airplane, as well as 4 packages. There are 4 locations across 2 cities. \nThe locations are in cities as follows: l1-1 and l1-0 are in c1; l0-0 and l0-1 are in c0. \nCurrently, p1, p3, t1, p2, and a0 are at l1-0, t0 is at l0-0, p0 is in a0. The goal is to reach a state where the following facts hold: p1 is at l1-0, p3 is at l0-1, p0 is at l0-0, and p2 is at l1-0. The available actions are: (load-truck ?obj ?truck ?loc) - load the object ?obj from location ?loc into the truck ?truck, (load-airplane ?obj ?airplane ?loc) - load the object ?obj from location ?loc onto the airplane ?airplane, (unload-truck ?obj ?truck ?loc) - unload the object ?obj from the truck ?truck at location ?loc, (unload-airplane ?obj ?airplane ?loc) - offload the object ?obj from the airplane ?airplane at location ?loc, (drive-truck ?truck ?loc-from ?loc-to ?city) - navigate the truck ?truck which is in location ?loc-from in city ?city to another location ?loc-to in the same city, and (fly-airplane ?airplane ?loc-from ?loc-to) - fly the airplane ?airplane from airport ?loc-from to airport ?loc-to." + question: "What is the next action that takes us towards the goal?" + answer: "(load-airplane p3 a0 l1-0)" + PDDL_domain: "(define (domain logistics-strips)\n (:requirements :strips :typing) \n\n (:types \n location locatable city - object \n package movable - locatable\n airport - location\n airplane truck - movable \n )\t\t\n \n (:predicates \t\n\t\t(at ?obj - locatable ?loc - location)\n\t\t(in ?obj1 - package ?obj2 - movable)\n\t\t(in-city ?obj - location ?city - city))\n\n\n(:action LOAD-TRUCK\n :parameters\n (?obj - package\n ?truck - truck\n ?loc - location)\n :precondition\n (and \n (at ?truck ?loc) (at ?obj ?loc))\n :effect\n (and (not (at ?obj ?loc)) (in ?obj ?truck)))\n\n(:action LOAD-AIRPLANE\n :parameters\n (?obj - package\n ?airplane - airplane\n ?loc - location)\n :precondition\n (and \n (at ?obj ?loc) (at ?airplane ?loc))\n :effect\n (and (not (at ?obj ?loc)) (in ?obj ?airplane)))\n\n\n\n(:action UNLOAD-TRUCK\n :parameters\n (?obj - package\n ?truck - truck\n ?loc - location)\n :precondition\n (and \n (at ?truck ?loc) (in ?obj ?truck))\n :effect\n (and (not (in ?obj ?truck)) (at ?obj ?loc)))\n\n(:action UNLOAD-AIRPLANE\n :parameters\n (?obj - package\n ?airplane - airplane\n ?loc - location)\n :precondition\n (and \n (in ?obj ?airplane) (at ?airplane ?loc))\n :effect\n (and (not (in ?obj ?airplane)) (at ?obj ?loc)))\n\n(:action DRIVE-TRUCK\n :parameters\n (?truck - truck\n ?loc-from - location\n ?loc-to - location\n ?city - city)\n :precondition\n (and \n (at ?truck ?loc-from)\n (in-city ?loc-from ?city)\n (in-city ?loc-to ?city))\n :effect\n (and (not (at ?truck ?loc-from)) (at ?truck ?loc-to)))\n\n(:action FLY-AIRPLANE\n :parameters\n (?airplane - airplane\n ?loc-from - airport\n ?loc-to - airport)\n :precondition\n (and \n\t(at ?airplane ?loc-from))\n :effect\n (and (not (at ?airplane ?loc-from)) (at ?airplane ?loc-to)))\n)" + PDDL_problem: "(define (problem logistics-c2-s2-p4-a1)\n (:domain logistics-strips)\n (:requirements :strips :typing)\n (:objects a0 - airplane l0-0 l1-0 - airport c0 c1 - city l0-1 l1-1 - location p0 p1 p2 p3 - package t0 t1 - truck)\n (:init (at a0 l1-0) (at p1 l1-0) (at p2 l1-0) (at p3 l1-0) (at t0 l0-0) (at t1 l1-0) (in p0 a0) (in-city l0-0 c0) (in-city l0-1 c0) (in-city l1-0 c1) (in-city l1-1 c1))\n (:goal (and (at p0 l0-0) (at p1 l1-0) (at p2 l1-0) (at p3 l0-1)))\n)" +doc_to_text: "# PDDL DOMAIN \n\n```\n{{PDDL_domain}}\n```\n\n# PDDL PROBLEM \n\n```\n{{PDDL_problem}}\n```\n\n**Question**: {{context}} {{question}} Each action starts with an opening parenthesis and ends with closing parenthesis. Provide only the action. **Final Answer**:" +filter_list: + - name: "acp_grammar_parse" + filter: + - function: "ACP_grammar_filter" + grammar_task: "action_name" + - function: "take_first" diff --git a/lm_eval/tasks/acpbench/gen_2shot_with_pddl/prog.yaml b/lm_eval/tasks/acpbench/gen_2shot_with_pddl/prog.yaml new file mode 100644 index 00000000..545c56ee --- /dev/null +++ b/lm_eval/tasks/acpbench/gen_2shot_with_pddl/prog.yaml @@ -0,0 +1,24 @@ +task: acp_prog_gen_with_pddl +dataset_name: acp_prog_gen +include: _gen_yaml_2shot +fewshot_config: + sampler: first_n + samples: + - context: "A robot is in a grid and can only move to places that are connected to its current position. \nThe grid size is 5x5, and the locations are of the form fi-jf (e.g., f3-2f or f0-1f). The grid cells are connected to their neighbors (e.g., f1-2f is connected to the four neighbors f0-2f, f2-2f, f1-1f, and f1-3f). Some positions on the grid are locked and can be opened with a key of a matching shape. The robot has an arm that can pick up a key when the key is in same location as the robot and the arm is empty. \nThere are 2 keys in 0 different shapes: Key key0-0 is of shape shape0, Key key0-1 is of shape shape0. \nCurrently, the robot is at position f2-2f and its arm is empty. All the positions are open except the following: f4-2f has shape0 shaped lock, f2-0f has shape0 shaped lock. Key key0-1 is at position f1-3f. Key key0-0 is at position f2-2f. The available propositions are: (at ?r ?x) - Key ?r is at ?x location, (at-robot ?x) - Robot is at ?x location, (locked ?x) - Location ?x is locked, (holding ?k) - Robot is holding ?k, (open ?x) - Location ?x is open, and (arm-empty) - Robot is not holding anything." + question: "Break down the outcomes of performing the action \"retrieve the key key0-0 from its current position f0-1f\" into two lists, positive effects and negative effects. Positive effects are the propositions that are false in the current state but will become true after performing the action. Negative effects are the propositions that are true in the current state and will become false after performing the action." + answer: "[(at-robot f1-2f)] [(at-robot f2-2f)]" + PDDL_domain: "(define (domain grid)\n (:requirements :strips :typing)\n (:types key place shape - object)\n (:predicates (arm-empty) (at ?r - key ?x - place) (at-robot ?x - place) (conn ?x - place ?y - place) (holding ?k - key) (key-shape ?k - key ?s - shape) (lock-shape ?x - place ?s - shape) (locked ?x - place) (open ?x - place))\n (:action move\n :parameters (?curpos - place ?nextpos - place)\n :precondition (and (at-robot ?curpos) (conn ?curpos ?nextpos) (open ?nextpos))\n :effect (and (at-robot ?nextpos) (not (at-robot ?curpos)))\n )\n (:action pickup\n :parameters (?curpos - place ?key - key)\n :precondition (and (at-robot ?curpos) (at ?key ?curpos) (arm-empty))\n :effect (and (holding ?key) (not (at ?key ?curpos)) (not (arm-empty)))\n )\n (:action pickup-and-loose\n :parameters (?curpos - place ?newkey - key ?oldkey - key)\n :precondition (and (at-robot ?curpos) (holding ?oldkey) (at ?newkey ?curpos))\n :effect (and (holding ?newkey) (at ?oldkey ?curpos) (not (holding ?oldkey)) (not (at ?newkey ?curpos)))\n )\n (:action putdown\n :parameters (?curpos - place ?key - key)\n :precondition (and (at-robot ?curpos) (holding ?key))\n :effect (and (arm-empty) (at ?key ?curpos) (not (holding ?key)))\n )\n (:action unlock\n :parameters (?curpos - place ?lockpos - place ?key - key ?shape - shape)\n :precondition (and (conn ?curpos ?lockpos) (key-shape ?key ?shape) (lock-shape ?lockpos ?shape) (at-robot ?curpos) (locked ?lockpos) (holding ?key))\n :effect (and (open ?lockpos) (not (locked ?lockpos)))\n )\n)" + PDDL_problem: "(define (problem grid-x5-y5-t1-k2-l2-p100)\n (:domain grid)\n (:requirements :strips :typing)\n (:objects key0-0 key0-1 - key f0-0f f0-1f f0-2f f0-3f f0-4f f1-0f f1-1f f1-2f f1-3f f1-4f f2-0f f2-1f f2-2f f2-3f f2-4f f3-0f f3-1f f3-2f f3-3f f3-4f f4-0f f4-1f f4-2f f4-3f f4-4f - place shape0 - shape)\n (:init (arm-empty) (at key0-0 f2-2f) (at key0-1 f1-3f) (at-robot f2-2f) (conn f0-0f f0-1f) (conn f0-0f f1-0f) (conn f0-1f f0-0f) (conn f0-1f f0-2f) (conn f0-1f f1-1f) (conn f0-2f f0-1f) (conn f0-2f f0-3f) (conn f0-2f f1-2f) (conn f0-3f f0-2f) (conn f0-3f f0-4f) (conn f0-3f f1-3f) (conn f0-4f f0-3f) (conn f0-4f f1-4f) (conn f1-0f f0-0f) (conn f1-0f f1-1f) (conn f1-0f f2-0f) (conn f1-1f f0-1f) (conn f1-1f f1-0f) (conn f1-1f f1-2f) (conn f1-1f f2-1f) (conn f1-2f f0-2f) (conn f1-2f f1-1f) (conn f1-2f f1-3f) (conn f1-2f f2-2f) (conn f1-3f f0-3f) (conn f1-3f f1-2f) (conn f1-3f f1-4f) (conn f1-3f f2-3f) (conn f1-4f f0-4f) (conn f1-4f f1-3f) (conn f1-4f f2-4f) (conn f2-0f f1-0f) (conn f2-0f f2-1f) (conn f2-0f f3-0f) (conn f2-1f f1-1f) (conn f2-1f f2-0f) (conn f2-1f f2-2f) (conn f2-1f f3-1f) (conn f2-2f f1-2f) (conn f2-2f f2-1f) (conn f2-2f f2-3f) (conn f2-2f f3-2f) (conn f2-3f f1-3f) (conn f2-3f f2-2f) (conn f2-3f f2-4f) (conn f2-3f f3-3f) (conn f2-4f f1-4f) (conn f2-4f f2-3f) (conn f2-4f f3-4f) (conn f3-0f f2-0f) (conn f3-0f f3-1f) (conn f3-0f f4-0f) (conn f3-1f f2-1f) (conn f3-1f f3-0f) (conn f3-1f f3-2f) (conn f3-1f f4-1f) (conn f3-2f f2-2f) (conn f3-2f f3-1f) (conn f3-2f f3-3f) (conn f3-2f f4-2f) (conn f3-3f f2-3f) (conn f3-3f f3-2f) (conn f3-3f f3-4f) (conn f3-3f f4-3f) (conn f3-4f f2-4f) (conn f3-4f f3-3f) (conn f3-4f f4-4f) (conn f4-0f f3-0f) (conn f4-0f f4-1f) (conn f4-1f f3-1f) (conn f4-1f f4-0f) (conn f4-1f f4-2f) (conn f4-2f f3-2f) (conn f4-2f f4-1f) (conn f4-2f f4-3f) (conn f4-3f f3-3f) (conn f4-3f f4-2f) (conn f4-3f f4-4f) (conn f4-4f f3-4f) (conn f4-4f f4-3f) (key-shape key0-0 shape0) (key-shape key0-1 shape0) (lock-shape f2-0f shape0) (lock-shape f4-2f shape0) (locked f2-0f) (locked f4-2f) (open f0-0f) (open f0-1f) (open f0-2f) (open f0-3f) (open f0-4f) (open f1-0f) (open f1-1f) (open f1-2f) (open f1-3f) (open f1-4f) (open f2-1f) (open f2-2f) (open f2-3f) (open f2-4f) (open f3-0f) (open f3-1f) (open f3-2f) (open f3-3f) (open f3-4f) (open f4-0f) (open f4-1f) (open f4-3f) (open f4-4f))\n (:goal (and (at key0-0 f2-0f) (at key0-1 f1-3f)))\n)" + - context: "There are several cities, each containing several locations, some of which are airports. There are also trucks, which can drive within a single city, and airplanes, which can fly between airports. The goal is to get some packages from various locations to various new locations. \nThere are 2 trucks and 1 airplane, as well as 4 packages. There are 4 locations across 2 cities. \nThe locations are in cities as follows: l0-0 and l0-1 are in c0; l1-0 and l1-1 are in c1. \nCurrently, p2 and t1 are at l1-0, a0 and t0 are at l0-0, p0 and p3 are in a0, p1 is in t1. The available propositions are: (at ?obj ?loc) - ?obj is at ?loc and (in ?obj1 ?obj2) - ?obj1 is in ?obj2." + question: "Break down the outcomes of performing the action \"navigate the truck t1 which is in location l1-0 in city c1 to another location l1-1 in the same city\" into two lists, positive effects and negative effects. Positive effects are the propositions that are false in the current state but will become true after performing the action. Negative effects are the propositions that are true in the current state and will become false after performing the action." + answer: "[(at t1 l1-1)] [(at t1 l1-0)]" + PDDL_domain: "(define (domain logistics-strips)\n (:requirements :strips :typing) \n\n (:types \n location locatable city - object \n package movable - locatable\n airport - location\n airplane truck - movable \n )\t\t\n \n (:predicates \t\n\t\t(at ?obj - locatable ?loc - location)\n\t\t(in ?obj1 - package ?obj2 - movable)\n\t\t(in-city ?obj - location ?city - city))\n\n\n(:action LOAD-TRUCK\n :parameters\n (?obj - package\n ?truck - truck\n ?loc - location)\n :precondition\n (and \n (at ?truck ?loc) (at ?obj ?loc))\n :effect\n (and (not (at ?obj ?loc)) (in ?obj ?truck)))\n\n(:action LOAD-AIRPLANE\n :parameters\n (?obj - package\n ?airplane - airplane\n ?loc - location)\n :precondition\n (and \n (at ?obj ?loc) (at ?airplane ?loc))\n :effect\n (and (not (at ?obj ?loc)) (in ?obj ?airplane)))\n\n\n\n(:action UNLOAD-TRUCK\n :parameters\n (?obj - package\n ?truck - truck\n ?loc - location)\n :precondition\n (and \n (at ?truck ?loc) (in ?obj ?truck))\n :effect\n (and (not (in ?obj ?truck)) (at ?obj ?loc)))\n\n(:action UNLOAD-AIRPLANE\n :parameters\n (?obj - package\n ?airplane - airplane\n ?loc - location)\n :precondition\n (and \n (in ?obj ?airplane) (at ?airplane ?loc))\n :effect\n (and (not (in ?obj ?airplane)) (at ?obj ?loc)))\n\n(:action DRIVE-TRUCK\n :parameters\n (?truck - truck\n ?loc-from - location\n ?loc-to - location\n ?city - city)\n :precondition\n (and \n (at ?truck ?loc-from)\n (in-city ?loc-from ?city)\n (in-city ?loc-to ?city))\n :effect\n (and (not (at ?truck ?loc-from)) (at ?truck ?loc-to)))\n\n(:action FLY-AIRPLANE\n :parameters\n (?airplane - airplane\n ?loc-from - airport\n ?loc-to - airport)\n :precondition\n (and \n\t(at ?airplane ?loc-from))\n :effect\n (and (not (at ?airplane ?loc-from)) (at ?airplane ?loc-to)))\n)" + PDDL_problem: "(define (problem logistics-c2-s2-p4-a1)\n (:domain logistics-strips)\n (:requirements :strips :typing)\n (:objects a0 - airplane l0-0 l1-0 - airport c0 c1 - city l0-1 l1-1 - location p0 p1 p2 p3 - package t0 t1 - truck)\n (:init (at a0 l0-0) (at p2 l1-0) (at t0 l0-0) (at t1 l1-0) (in p0 a0) (in p1 t1) (in p3 a0) (in-city l0-0 c0) (in-city l0-1 c0) (in-city l1-0 c1) (in-city l1-1 c1))\n (:goal (and (at p0 l0-0) (at p1 l1-0) (at p2 l1-0) (at p3 l0-1)))\n)" +doc_to_text: "# PDDL DOMAIN \n\n```\n{{PDDL_domain}}\n```\n\n# PDDL PROBLEM \n\n```\n{{PDDL_problem}}\n```\n\n**Question**: {{context}} {{question}} Provide only the two lists with the ground propositions. **Final Answer**:" +filter_list: + - name: "acp_grammar_parse" + filter: + - function: "ACP_grammar_filter" + grammar_task: "progression_list" + clean: "pos_neg" + - function: "take_first" diff --git a/lm_eval/tasks/acpbench/gen_2shot_with_pddl/reach.yaml b/lm_eval/tasks/acpbench/gen_2shot_with_pddl/reach.yaml new file mode 100644 index 00000000..6cb78bbd --- /dev/null +++ b/lm_eval/tasks/acpbench/gen_2shot_with_pddl/reach.yaml @@ -0,0 +1,23 @@ +task: acp_reach_gen_with_pddl +dataset_name: acp_reach_gen +include: _gen_yaml_2shot +fewshot_config: + sampler: first_n + samples: + - context: "A robot is in a grid and can only move to places that are connected to its current position. \nThe grid size is 5x5, and the locations are of the form fi-jf (e.g., f3-2f or f0-1f). The grid cells are connected to their neighbors (e.g., f1-2f is connected to the four neighbors f0-2f, f2-2f, f1-1f, and f1-3f). Some positions on the grid are locked and can be opened with a key of a matching shape. The robot has an arm that can pick up a key when the key is in same location as the robot and the arm is empty. \nThere are 2 keys in 0 different shapes: Key key0-1 is of shape shape0, Key key0-0 is of shape shape0. \nCurrently, the robot is at position f3-1f and its arm is empty. All the positions are open except the following: f2-0f has shape0 shaped lock. Key key0-1 is at position f1-3f. Key key0-0 is at position f3-1f. The available propositions are: (at ?r ?x) - Key ?r is at ?x location, (at-robot ?x) - Robot is at ?x location, (locked ?x) - Location ?x is locked, (holding ?k) - Robot is holding ?k, (open ?x) - Location ?x is open, and (arm-empty) - Robot's arm is empty." + question: "What proposition can never hold in any potentially reachable state?" + answer: "(locked f2-2f)" + PDDL_domain: "(define (domain grid)\n (:requirements :strips :typing)\n (:types key place shape - object)\n (:predicates (arm-empty) (at ?r - key ?x - place) (at-robot ?x - place) (conn ?x - place ?y - place) (holding ?k - key) (key-shape ?k - key ?s - shape) (lock-shape ?x - place ?s - shape) (locked ?x - place) (open ?x - place))\n (:action move\n :parameters (?curpos - place ?nextpos - place)\n :precondition (and (at-robot ?curpos) (conn ?curpos ?nextpos) (open ?nextpos))\n :effect (and (at-robot ?nextpos) (not (at-robot ?curpos)))\n )\n (:action pickup\n :parameters (?curpos - place ?key - key)\n :precondition (and (at-robot ?curpos) (at ?key ?curpos) (arm-empty))\n :effect (and (holding ?key) (not (at ?key ?curpos)) (not (arm-empty)))\n )\n (:action pickup-and-loose\n :parameters (?curpos - place ?newkey - key ?oldkey - key)\n :precondition (and (at-robot ?curpos) (holding ?oldkey) (at ?newkey ?curpos))\n :effect (and (holding ?newkey) (at ?oldkey ?curpos) (not (holding ?oldkey)) (not (at ?newkey ?curpos)))\n )\n (:action putdown\n :parameters (?curpos - place ?key - key)\n :precondition (and (at-robot ?curpos) (holding ?key))\n :effect (and (arm-empty) (at ?key ?curpos) (not (holding ?key)))\n )\n (:action unlock\n :parameters (?curpos - place ?lockpos - place ?key - key ?shape - shape)\n :precondition (and (conn ?curpos ?lockpos) (key-shape ?key ?shape) (lock-shape ?lockpos ?shape) (at-robot ?curpos) (locked ?lockpos) (holding ?key))\n :effect (and (open ?lockpos) (not (locked ?lockpos)))\n )\n)" + PDDL_problem: "(define (problem grid-x5-y5-t1-k2-l2-p100)\n (:domain grid)\n (:requirements :strips :typing)\n (:objects key0-0 key0-1 - key f0-0f f0-1f f0-2f f0-3f f0-4f f1-0f f1-1f f1-2f f1-3f f1-4f f2-0f f2-1f f2-2f f2-3f f2-4f f3-0f f3-1f f3-2f f3-3f f3-4f f4-0f f4-1f f4-2f f4-3f f4-4f - place shape0 - shape)\n (:init (arm-empty) (at key0-0 f3-1f) (at key0-1 f1-3f) (at-robot f3-1f) (conn f0-0f f0-1f) (conn f0-0f f1-0f) (conn f0-1f f0-0f) (conn f0-1f f0-2f) (conn f0-1f f1-1f) (conn f0-2f f0-1f) (conn f0-2f f0-3f) (conn f0-2f f1-2f) (conn f0-3f f0-2f) (conn f0-3f f0-4f) (conn f0-3f f1-3f) (conn f0-4f f0-3f) (conn f0-4f f1-4f) (conn f1-0f f0-0f) (conn f1-0f f1-1f) (conn f1-0f f2-0f) (conn f1-1f f0-1f) (conn f1-1f f1-0f) (conn f1-1f f1-2f) (conn f1-1f f2-1f) (conn f1-2f f0-2f) (conn f1-2f f1-1f) (conn f1-2f f1-3f) (conn f1-2f f2-2f) (conn f1-3f f0-3f) (conn f1-3f f1-2f) (conn f1-3f f1-4f) (conn f1-3f f2-3f) (conn f1-4f f0-4f) (conn f1-4f f1-3f) (conn f1-4f f2-4f) (conn f2-0f f1-0f) (conn f2-0f f2-1f) (conn f2-0f f3-0f) (conn f2-1f f1-1f) (conn f2-1f f2-0f) (conn f2-1f f2-2f) (conn f2-1f f3-1f) (conn f2-2f f1-2f) (conn f2-2f f2-1f) (conn f2-2f f2-3f) (conn f2-2f f3-2f) (conn f2-3f f1-3f) (conn f2-3f f2-2f) (conn f2-3f f2-4f) (conn f2-3f f3-3f) (conn f2-4f f1-4f) (conn f2-4f f2-3f) (conn f2-4f f3-4f) (conn f3-0f f2-0f) (conn f3-0f f3-1f) (conn f3-0f f4-0f) (conn f3-1f f2-1f) (conn f3-1f f3-0f) (conn f3-1f f3-2f) (conn f3-1f f4-1f) (conn f3-2f f2-2f) (conn f3-2f f3-1f) (conn f3-2f f3-3f) (conn f3-2f f4-2f) (conn f3-3f f2-3f) (conn f3-3f f3-2f) (conn f3-3f f3-4f) (conn f3-3f f4-3f) (conn f3-4f f2-4f) (conn f3-4f f3-3f) (conn f3-4f f4-4f) (conn f4-0f f3-0f) (conn f4-0f f4-1f) (conn f4-1f f3-1f) (conn f4-1f f4-0f) (conn f4-1f f4-2f) (conn f4-2f f3-2f) (conn f4-2f f4-1f) (conn f4-2f f4-3f) (conn f4-3f f3-3f) (conn f4-3f f4-2f) (conn f4-3f f4-4f) (conn f4-4f f3-4f) (conn f4-4f f4-3f) (key-shape key0-0 shape0) (key-shape key0-1 shape0) (lock-shape f2-0f shape0) (lock-shape f4-2f shape0) (locked f2-0f) (open f0-0f) (open f0-1f) (open f0-2f) (open f0-3f) (open f0-4f) (open f1-0f) (open f1-1f) (open f1-2f) (open f1-3f) (open f1-4f) (open f2-1f) (open f2-2f) (open f2-3f) (open f2-4f) (open f3-0f) (open f3-1f) (open f3-2f) (open f3-3f) (open f3-4f) (open f4-0f) (open f4-1f) (open f4-2f) (open f4-3f) (open f4-4f))\n (:goal (and (at key0-0 f2-0f) (at key0-1 f1-3f)))\n)" + - context: "There are several cities, each containing several locations, some of which are airports. There are also trucks, which can drive within a single city, and airplanes, which can fly between airports. The goal is to get some packages from various locations to various new locations. \nThere are 2 trucks and 1 airplane, as well as 4 packages. There are 4 locations across 2 cities. \nThe locations are in cities as follows: l1-0 and l1-1 are in c1; l0-1 and l0-0 are in c0. \nCurrently, t1, a0, and p2 are at l1-0, t0 and p0 are at l0-0, p1 is in t1, p3 is in t0. The available propositions are: (at ?obj ?loc) - ?obj is at ?loc and (in ?obj1 ?obj2) - ?obj1 is in ?obj2." + question: "What proposition can never hold in any potentially reachable state?" + answer: "(at t0 l1-0)" + PDDL_domain: "(define (domain logistics-strips)\n (:requirements :strips :typing) \n\n (:types \n location locatable city - object \n package movable - locatable\n airport - location\n airplane truck - movable \n )\t\t\n \n (:predicates \t\n\t\t(at ?obj - locatable ?loc - location)\n\t\t(in ?obj1 - package ?obj2 - movable)\n\t\t(in-city ?obj - location ?city - city))\n\n\n(:action LOAD-TRUCK\n :parameters\n (?obj - package\n ?truck - truck\n ?loc - location)\n :precondition\n (and \n (at ?truck ?loc) (at ?obj ?loc))\n :effect\n (and (not (at ?obj ?loc)) (in ?obj ?truck)))\n\n(:action LOAD-AIRPLANE\n :parameters\n (?obj - package\n ?airplane - airplane\n ?loc - location)\n :precondition\n (and \n (at ?obj ?loc) (at ?airplane ?loc))\n :effect\n (and (not (at ?obj ?loc)) (in ?obj ?airplane)))\n\n\n\n(:action UNLOAD-TRUCK\n :parameters\n (?obj - package\n ?truck - truck\n ?loc - location)\n :precondition\n (and \n (at ?truck ?loc) (in ?obj ?truck))\n :effect\n (and (not (in ?obj ?truck)) (at ?obj ?loc)))\n\n(:action UNLOAD-AIRPLANE\n :parameters\n (?obj - package\n ?airplane - airplane\n ?loc - location)\n :precondition\n (and \n (in ?obj ?airplane) (at ?airplane ?loc))\n :effect\n (and (not (in ?obj ?airplane)) (at ?obj ?loc)))\n\n(:action DRIVE-TRUCK\n :parameters\n (?truck - truck\n ?loc-from - location\n ?loc-to - location\n ?city - city)\n :precondition\n (and \n (at ?truck ?loc-from)\n (in-city ?loc-from ?city)\n (in-city ?loc-to ?city))\n :effect\n (and (not (at ?truck ?loc-from)) (at ?truck ?loc-to)))\n\n(:action FLY-AIRPLANE\n :parameters\n (?airplane - airplane\n ?loc-from - airport\n ?loc-to - airport)\n :precondition\n (and \n\t(at ?airplane ?loc-from))\n :effect\n (and (not (at ?airplane ?loc-from)) (at ?airplane ?loc-to)))\n)" + PDDL_problem: "(define (problem logistics-c2-s2-p4-a1)\n (:domain logistics-strips)\n (:requirements :strips :typing)\n (:objects a0 - airplane l0-0 l1-0 - airport c0 c1 - city l0-1 l1-1 - location p0 p1 p2 p3 - package t0 t1 - truck)\n (:init (at a0 l1-0) (at p0 l0-0) (at p2 l1-0) (at t0 l0-0) (at t1 l1-0) (in p1 t1) (in p3 t0) (in-city l0-0 c0) (in-city l0-1 c0) (in-city l1-0 c1) (in-city l1-1 c1))\n (:goal (and (at p0 l0-0) (at p1 l1-0) (at p2 l1-0) (at p3 l0-1)))\n)" +doc_to_text: "# PDDL DOMAIN \n\n```\n{{PDDL_domain}}\n```\n\n# PDDL PROBLEM \n\n```\n{{PDDL_problem}}\n```\n\n**Question**: {{context}} {{question}} Provide one proposition or None. **Final Answer**:" +filter_list: + - name: "acp_grammar_parse" + filter: + - function: "ACP_grammar_filter" + grammar_task: "act" + - function: "take_first" diff --git a/lm_eval/tasks/acpbench/gen_2shot_with_pddl/val.yaml b/lm_eval/tasks/acpbench/gen_2shot_with_pddl/val.yaml new file mode 100644 index 00000000..6012ebf9 --- /dev/null +++ b/lm_eval/tasks/acpbench/gen_2shot_with_pddl/val.yaml @@ -0,0 +1,23 @@ +task: acp_val_gen_with_pddl +dataset_name: acp_val_gen +include: _gen_yaml_2shot +fewshot_config: + sampler: first_n + samples: + - context: "A robot is in a grid and can only move to places that are connected to its current position. \nThe grid size is 5x5, and the locations are of the form fi-jf (e.g., f3-2f or f0-1f). The grid cells are connected to their neighbors (e.g., f1-2f is connected to the four neighbors f0-2f, f2-2f, f1-1f, and f1-3f). Some positions on the grid are locked and can be opened with a key of a matching shape. The robot has an arm that can pick up a key when the key is in same location as the robot and the arm is empty. \nThere are 2 keys in 0 different shapes: Key key0-1 is of shape shape0, Key key0-0 is of shape shape0. \nCurrently, the robot is at position f3-3f and its arm is empty. All the positions are open except the following: f2-0f has shape0 shaped lock, f4-2f has shape0 shaped lock. Key key0-1 is at position f1-3f. Key key0-0 is at position f2-2f. The goal is to reach a state where the following facts hold: Key key0-1 is at f1-3f location and Key key0-0 is at f2-0f location. The available actions are: (unlock ?curpos ?lockpos ?key ?shape) - unlock the place ?lockpos with the key ?key of the shape ?shape from the current position place ?curpos, (move ?curpos ?nextpos) - move from place ?curpos to place ?nextpos, (pickup ?curpos ?key) - retrieve the key ?key from its current position ?curpos, (pickup-and-loose ?curpos ?newkey ?oldkey) - pick up the key ?newkey at the current position place ?curpos and loose the key ?oldkey being held, and (putdown ?curpos ?key) - put the key ?key at the current position place ?curpos." + question: "What is the first inapplicable action in the next sequence of actions: \"(unlock f1-0f f2-0f key0-1 shape0) (move f2-3f f2-2f) (pickup f2-2f key0-0) (move f2-2f f2-1f) (unlock f2-1f f2-0f key0-0 shape0) (move f2-1f f2-0f) (putdown f2-0f key0-0)\"?" + answer: "0" + PDDL_domain: "(define (domain grid)\n (:requirements :strips :typing)\n (:types key place shape - object)\n (:predicates (arm-empty) (at ?r - key ?x - place) (at-robot ?x - place) (conn ?x - place ?y - place) (holding ?k - key) (key-shape ?k - key ?s - shape) (lock-shape ?x - place ?s - shape) (locked ?x - place) (open ?x - place))\n (:action move\n :parameters (?curpos - place ?nextpos - place)\n :precondition (and (at-robot ?curpos) (conn ?curpos ?nextpos) (open ?nextpos))\n :effect (and (at-robot ?nextpos) (not (at-robot ?curpos)))\n )\n (:action pickup\n :parameters (?curpos - place ?key - key)\n :precondition (and (at-robot ?curpos) (at ?key ?curpos) (arm-empty))\n :effect (and (holding ?key) (not (at ?key ?curpos)) (not (arm-empty)))\n )\n (:action pickup-and-loose\n :parameters (?curpos - place ?newkey - key ?oldkey - key)\n :precondition (and (at-robot ?curpos) (holding ?oldkey) (at ?newkey ?curpos))\n :effect (and (holding ?newkey) (at ?oldkey ?curpos) (not (holding ?oldkey)) (not (at ?newkey ?curpos)))\n )\n (:action putdown\n :parameters (?curpos - place ?key - key)\n :precondition (and (at-robot ?curpos) (holding ?key))\n :effect (and (arm-empty) (at ?key ?curpos) (not (holding ?key)))\n )\n (:action unlock\n :parameters (?curpos - place ?lockpos - place ?key - key ?shape - shape)\n :precondition (and (conn ?curpos ?lockpos) (key-shape ?key ?shape) (lock-shape ?lockpos ?shape) (at-robot ?curpos) (locked ?lockpos) (holding ?key))\n :effect (and (open ?lockpos) (not (locked ?lockpos)))\n )\n)" + PDDL_problem: "(define (problem grid-x5-y5-t1-k2-l2-p100)\n (:domain grid)\n (:requirements :strips :typing)\n (:objects key0-0 key0-1 - key f0-0f f0-1f f0-2f f0-3f f0-4f f1-0f f1-1f f1-2f f1-3f f1-4f f2-0f f2-1f f2-2f f2-3f f2-4f f3-0f f3-1f f3-2f f3-3f f3-4f f4-0f f4-1f f4-2f f4-3f f4-4f - place shape0 - shape)\n (:init (arm-empty) (at key0-0 f2-2f) (at key0-1 f1-3f) (at-robot f3-3f) (conn f0-0f f0-1f) (conn f0-0f f1-0f) (conn f0-1f f0-0f) (conn f0-1f f0-2f) (conn f0-1f f1-1f) (conn f0-2f f0-1f) (conn f0-2f f0-3f) (conn f0-2f f1-2f) (conn f0-3f f0-2f) (conn f0-3f f0-4f) (conn f0-3f f1-3f) (conn f0-4f f0-3f) (conn f0-4f f1-4f) (conn f1-0f f0-0f) (conn f1-0f f1-1f) (conn f1-0f f2-0f) (conn f1-1f f0-1f) (conn f1-1f f1-0f) (conn f1-1f f1-2f) (conn f1-1f f2-1f) (conn f1-2f f0-2f) (conn f1-2f f1-1f) (conn f1-2f f1-3f) (conn f1-2f f2-2f) (conn f1-3f f0-3f) (conn f1-3f f1-2f) (conn f1-3f f1-4f) (conn f1-3f f2-3f) (conn f1-4f f0-4f) (conn f1-4f f1-3f) (conn f1-4f f2-4f) (conn f2-0f f1-0f) (conn f2-0f f2-1f) (conn f2-0f f3-0f) (conn f2-1f f1-1f) (conn f2-1f f2-0f) (conn f2-1f f2-2f) (conn f2-1f f3-1f) (conn f2-2f f1-2f) (conn f2-2f f2-1f) (conn f2-2f f2-3f) (conn f2-2f f3-2f) (conn f2-3f f1-3f) (conn f2-3f f2-2f) (conn f2-3f f2-4f) (conn f2-3f f3-3f) (conn f2-4f f1-4f) (conn f2-4f f2-3f) (conn f2-4f f3-4f) (conn f3-0f f2-0f) (conn f3-0f f3-1f) (conn f3-0f f4-0f) (conn f3-1f f2-1f) (conn f3-1f f3-0f) (conn f3-1f f3-2f) (conn f3-1f f4-1f) (conn f3-2f f2-2f) (conn f3-2f f3-1f) (conn f3-2f f3-3f) (conn f3-2f f4-2f) (conn f3-3f f2-3f) (conn f3-3f f3-2f) (conn f3-3f f3-4f) (conn f3-3f f4-3f) (conn f3-4f f2-4f) (conn f3-4f f3-3f) (conn f3-4f f4-4f) (conn f4-0f f3-0f) (conn f4-0f f4-1f) (conn f4-1f f3-1f) (conn f4-1f f4-0f) (conn f4-1f f4-2f) (conn f4-2f f3-2f) (conn f4-2f f4-1f) (conn f4-2f f4-3f) (conn f4-3f f3-3f) (conn f4-3f f4-2f) (conn f4-3f f4-4f) (conn f4-4f f3-4f) (conn f4-4f f4-3f) (key-shape key0-0 shape0) (key-shape key0-1 shape0) (lock-shape f2-0f shape0) (lock-shape f4-2f shape0) (locked f2-0f) (locked f4-2f) (open f0-0f) (open f0-1f) (open f0-2f) (open f0-3f) (open f0-4f) (open f1-0f) (open f1-1f) (open f1-2f) (open f1-3f) (open f1-4f) (open f2-1f) (open f2-2f) (open f2-3f) (open f2-4f) (open f3-0f) (open f3-1f) (open f3-2f) (open f3-3f) (open f3-4f) (open f4-0f) (open f4-1f) (open f4-3f) (open f4-4f))\n (:goal (and (at key0-0 f2-0f) (at key0-1 f1-3f)))\n)" + - context: "There are several cities, each containing several locations, some of which are airports. There are also trucks, which can drive within a single city, and airplanes, which can fly between airports. The goal is to get some packages from various locations to various new locations. \nThere are 2 trucks and 1 airplane, as well as 4 packages. There are 4 locations across 2 cities. \nThe locations are in cities as follows: l1-1 and l1-0 are in c1; l0-1 and l0-0 are in c0. \nCurrently, t1 and p0 are at l1-1, p3, p2, and p1 are at l1-0, t0 is at l0-1, a0 is at l0-0. The goal is to reach a state where the following facts hold: p0 is at l0-0, p3 is at l0-1, p2 is at l1-0, and p1 is at l1-0. The available actions are: (load-truck ?obj ?truck ?loc) - load object ?obj into truck ?truck at location ?loc, (load-airplane ?obj ?airplane ?loc) - load the object ?obj from location ?loc into the airplane ?airplane, (unload-truck ?obj ?truck ?loc) - offload the object ?obj from the truck ?truck at location ?loc, (unload-airplane ?obj ?airplane ?loc) - remove the object ?obj from the airplane ?airplane and place it on the location ?loc, (drive-truck ?truck ?loc-from ?loc-to ?city) - navigate the truck ?truck which is in location ?loc-from in city ?city to another location ?loc-to in the same city, and (fly-airplane ?airplane ?loc-from ?loc-to) - fly airplane ?airplane from airport ?loc-from to airport ?loc-to." + question: "What is the first inapplicable action in the next sequence of actions: \"(drive-truck t0 l0-1 l0-0 c0) (fly-airplane a0 l0-0 l1-0) (load-airplane p3 a0 l1-0) (load-truck p0 t1 l1-1) (drive-truck t1 l1-1 l1-0 c1) (unload-truck p0 t1 l1-0) (load-airplane p0 a0 l1-0) (fly-airplane a0 l1-0 l0-0) (unload-airplane p0 a0 l0-0) (unload-airplane p3 a0 l0-0) (load-truck p3 t0 l0-0) (drive-truck t0 l0-0 l0-1 c0) (unload-airplane p3 a0 l0-0)\"?" + answer: "12" + PDDL_domain: "(define (domain logistics-strips)\n (:requirements :strips :typing) \n\n (:types \n location locatable city - object \n package movable - locatable\n airport - location\n airplane truck - movable \n )\t\t\n \n (:predicates \t\n\t\t(at ?obj - locatable ?loc - location)\n\t\t(in ?obj1 - package ?obj2 - movable)\n\t\t(in-city ?obj - location ?city - city))\n\n\n(:action LOAD-TRUCK\n :parameters\n (?obj - package\n ?truck - truck\n ?loc - location)\n :precondition\n (and \n (at ?truck ?loc) (at ?obj ?loc))\n :effect\n (and (not (at ?obj ?loc)) (in ?obj ?truck)))\n\n(:action LOAD-AIRPLANE\n :parameters\n (?obj - package\n ?airplane - airplane\n ?loc - location)\n :precondition\n (and \n (at ?obj ?loc) (at ?airplane ?loc))\n :effect\n (and (not (at ?obj ?loc)) (in ?obj ?airplane)))\n\n\n\n(:action UNLOAD-TRUCK\n :parameters\n (?obj - package\n ?truck - truck\n ?loc - location)\n :precondition\n (and \n (at ?truck ?loc) (in ?obj ?truck))\n :effect\n (and (not (in ?obj ?truck)) (at ?obj ?loc)))\n\n(:action UNLOAD-AIRPLANE\n :parameters\n (?obj - package\n ?airplane - airplane\n ?loc - location)\n :precondition\n (and \n (in ?obj ?airplane) (at ?airplane ?loc))\n :effect\n (and (not (in ?obj ?airplane)) (at ?obj ?loc)))\n\n(:action DRIVE-TRUCK\n :parameters\n (?truck - truck\n ?loc-from - location\n ?loc-to - location\n ?city - city)\n :precondition\n (and \n (at ?truck ?loc-from)\n (in-city ?loc-from ?city)\n (in-city ?loc-to ?city))\n :effect\n (and (not (at ?truck ?loc-from)) (at ?truck ?loc-to)))\n\n(:action FLY-AIRPLANE\n :parameters\n (?airplane - airplane\n ?loc-from - airport\n ?loc-to - airport)\n :precondition\n (and \n\t(at ?airplane ?loc-from))\n :effect\n (and (not (at ?airplane ?loc-from)) (at ?airplane ?loc-to)))\n)" + PDDL_problem: "(define (problem logistics-c2-s2-p4-a1)\n (:domain logistics-strips)\n (:requirements :strips :typing)\n (:objects a0 - airplane l0-0 l1-0 - airport c0 c1 - city l0-1 l1-1 - location p0 p1 p2 p3 - package t0 t1 - truck)\n (:init (at a0 l0-0) (at p0 l1-1) (at p1 l1-0) (at p2 l1-0) (at p3 l1-0) (at t0 l0-1) (at t1 l1-1) (in-city l0-0 c0) (in-city l0-1 c0) (in-city l1-0 c1) (in-city l1-1 c1))\n (:goal (and (at p0 l0-0) (at p1 l1-0) (at p2 l1-0) (at p3 l0-1)))\n)" +doc_to_text: "# PDDL DOMAIN \n\n```\n{{PDDL_domain}}\n```\n\n# PDDL PROBLEM \n\n```\n{{PDDL_problem}}\n```\n\n**Question**: {{context}} {{question}} Provide only the index of the action. **Final Answer**:" +filter_list: + - name: "acp_grammar_parse" + filter: + - function: "ACP_grammar_filter" + grammar_task: "index" + - function: "take_first" diff --git a/pyproject.toml b/pyproject.toml index 94e40063..45dd4418 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,7 @@ Homepage = "https://github.com/EleutherAI/lm-evaluation-harness" Repository = "https://github.com/EleutherAI/lm-evaluation-harness" [project.optional-dependencies] +acpbench = ["lark>=1.1.9", "tarski[clingo]==0.8.2", "pddl==0.4.2", "kstar-planner==1.4.2"] api = ["requests", "aiohttp", "tenacity", "tqdm", "tiktoken"] audiolm_qwen = ["librosa", "soundfile"] deepsparse = ["deepsparse-nightly[llm]>=1.8.0.20240404"] @@ -85,6 +86,7 @@ vllm = ["vllm>=0.4.2"] wandb = ["wandb>=0.16.3", "pandas", "numpy"] zeno = ["pandas", "zeno-client"] all = [ + "lm_eval[acpbench]", "lm_eval[api]", "lm_eval[audiolm_qwen]", "lm_eval[deepsparse]", -- GitLab From 53c653008182339e67b964a4cd3316f651611f38 Mon Sep 17 00:00:00 2001 From: Baber Abbasi <92168766+baberabb@users.noreply.github.com> Date: Mon, 19 May 2025 18:38:27 +0500 Subject: [PATCH 20/46] [SGLANG] Add the SGLANG generate API (#2997) * add `sglang-generate` * nit * nit * nit * pacify pre-commit --- lm_eval/api/task.py | 18 +++-- lm_eval/models/__init__.py | 1 + lm_eval/models/sglang_generate_API.py | 100 ++++++++++++++++++++++++++ lm_eval/models/vllm_causallms.py | 5 +- lm_eval/tasks/c4/README.md | 4 +- lm_eval/tasks/c4/c4.yaml | 2 +- lm_eval/tasks/mmlu_pro/utils.py | 7 +- 7 files changed, 128 insertions(+), 9 deletions(-) create mode 100644 lm_eval/models/sglang_generate_API.py diff --git a/lm_eval/api/task.py b/lm_eval/api/task.py index a0dc389b..893784e5 100644 --- a/lm_eval/api/task.py +++ b/lm_eval/api/task.py @@ -401,7 +401,7 @@ class Task(abc.ABC): fewshot_as_multiturn: bool = False, chat_template: Optional[Callable] = None, tokenizer_name: str = "", - question_suffix: str = "" + question_suffix: str = "", ) -> None: """Build a set of Instances for a task, and store them in task.instances""" @@ -1077,13 +1077,23 @@ class ConfigurableTask(Task): if not fewshot_as_multiturn: # if no messages or last message is system, append as new user entry if len(labeled_examples) == 0 or labeled_examples[-1]["role"] == "system": - labeled_examples.append({"role": "user", "content": question + question_suffix} if question_suffix else {"role": "user", "content": question} ) + labeled_examples.append( + {"role": "user", "content": question + question_suffix} + if question_suffix + else {"role": "user", "content": question} + ) # if last message is user, append to it to avoid two user messages in a row else: - labeled_examples[-1]["content"] += question + question_suffix if question_suffix else question + labeled_examples[-1]["content"] += ( + question + question_suffix if question_suffix else question + ) else: # if fewshot_as_multiturn is True, append as next user entry (last is always assistant) - labeled_examples.append({"role": "user", "content": question + question_suffix} if question_suffix else {"role": "user", "content": question} ) + labeled_examples.append( + {"role": "user", "content": question + question_suffix} + if question_suffix + else {"role": "user", "content": question} + ) if gen_prefix: labeled_examples.append({"role": "assistant", "content": gen_prefix}) diff --git a/lm_eval/models/__init__.py b/lm_eval/models/__init__.py index cf551559..8582f019 100644 --- a/lm_eval/models/__init__.py +++ b/lm_eval/models/__init__.py @@ -16,6 +16,7 @@ from . import ( optimum_ipex, optimum_lm, sglang_causallms, + sglang_generate_API, textsynth, vllm_causallms, vllm_vlms, diff --git a/lm_eval/models/sglang_generate_API.py b/lm_eval/models/sglang_generate_API.py new file mode 100644 index 00000000..2b6582a3 --- /dev/null +++ b/lm_eval/models/sglang_generate_API.py @@ -0,0 +1,100 @@ +from typing import Dict, List, Optional, Tuple, Union + +from lm_eval.api.registry import register_model +from lm_eval.models.openai_completions import LocalCompletionsAPI +from lm_eval.models.utils import handle_stop_sequences + + +@register_model("sglang-generate") +class SGLANGGENERATEAPI(LocalCompletionsAPI): + def __init__( + self, + base_url=None, + tokenizer_backend="huggingface", + **kwargs, + ): + super().__init__( + base_url=base_url, tokenizer_backend=tokenizer_backend, **kwargs + ) + + def _create_payload( + self, + messages: Union[List[List[int]], List[dict], List[str], str], + generate=False, + gen_kwargs: Optional[dict] = None, + seed: int = 1234, + eos=None, + **kwargs, + ) -> dict: + is_string = ( + True + if (isinstance(messages, str) or isinstance(messages[0], str)) + else False + ) + if generate: + gen_kwargs.pop("do_sample", False) + if "max_tokens" in gen_kwargs: + max_tokens = gen_kwargs.pop("max_tokens") + else: + max_tokens = gen_kwargs.pop("max_gen_toks", self._max_gen_toks) + temperature = gen_kwargs.pop("temperature", 0) + stop = handle_stop_sequences(gen_kwargs.pop("until", None), eos) + request = { + "sampling_params": { + "max_new_tokens": max_tokens, + "temperature": temperature, + "stop": stop, + **gen_kwargs, + }, + } + request.update({"text": messages}) if is_string else request.update( + {"input_ids": messages} + ) + return request + else: + assert not is_string, "Logprobs are only supported for tokenized inputs" + request = { + "input_ids": messages, + "sampling_params": {"max_new_tokens": 1, "temperature": 0}, + "logprob_start_len": 0, + "top_logprobs_num": 1, + "return_logprob": True, + } + return request + + @staticmethod + def parse_logprobs( + outputs: Union[Dict, List[Dict]], + tokens: List[List[int]] = None, + ctxlens: List[int] = None, + **kwargs, + ) -> List[Tuple[float, bool]]: + res = [] + if not isinstance(outputs, list): + outputs = [outputs] + for choice, ctxlen in zip(outputs, ctxlens): + choice = choice["meta_info"] + assert ctxlen > 0, "Context length must be greater than 0" + logprobs = sum(x[0] for x in choice["input_token_logprobs"][ctxlen:]) + is_greedy = all( + x[1] != y[0][1] + for x, y in zip( + choice["input_token_logprobs"][ctxlen:], + choice["input_top_logprobs"][ctxlen:], + ) + ) + res.append((logprobs, is_greedy)) + return res + + @staticmethod + def parse_generations(outputs: Union[Dict, List[Dict]], **kwargs) -> List[str]: + res = [] + if not isinstance(outputs, list): + outputs = [outputs] + for out in outputs: + res.append(out["text"]) + return res + + @property + def api_key(self): + return "" diff --git a/lm_eval/models/vllm_causallms.py b/lm_eval/models/vllm_causallms.py index 8d71a7b8..29587946 100644 --- a/lm_eval/models/vllm_causallms.py +++ b/lm_eval/models/vllm_causallms.py @@ -487,7 +487,10 @@ class VLLM(TemplateLM): inputs = [] ctxlens = [] for cache_key, context_enc, continuation_enc in chunk: - if full_length := len(context_enc + continuation_enc) >= self.max_length: + if ( + full_length := len(context_enc + continuation_enc) + >= self.max_length + ): eval_logger.warning( f"Context length {full_length} exceeds max length ({self.max_length}). Truncating context." ) diff --git a/lm_eval/tasks/c4/README.md b/lm_eval/tasks/c4/README.md index 5953415b..9ade56f6 100644 --- a/lm_eval/tasks/c4/README.md +++ b/lm_eval/tasks/c4/README.md @@ -14,13 +14,13 @@ This is the processed version of Google's C4 dataset. ```text @misc{raffel2023exploringlimitstransferlearning, - title={Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer}, + title={Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer}, author={Colin Raffel and Noam Shazeer and Adam Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu}, year={2023}, eprint={1910.10683}, archivePrefix={arXiv}, primaryClass={cs.LG}, - url={https://arxiv.org/abs/1910.10683}, + url={https://arxiv.org/abs/1910.10683}, } ``` diff --git a/lm_eval/tasks/c4/c4.yaml b/lm_eval/tasks/c4/c4.yaml index daea7dd1..bdbd70c6 100644 --- a/lm_eval/tasks/c4/c4.yaml +++ b/lm_eval/tasks/c4/c4.yaml @@ -21,4 +21,4 @@ dataset_kwargs: validation: en/c4-validation.00000-of-00008.json.gz # following the choice of https://arxiv.org/abs/2410.07461 trust_remote_code: true - verification_mode: "no_checks" \ No newline at end of file + verification_mode: "no_checks" diff --git a/lm_eval/tasks/mmlu_pro/utils.py b/lm_eval/tasks/mmlu_pro/utils.py index 94bd6f73..ca254a29 100644 --- a/lm_eval/tasks/mmlu_pro/utils.py +++ b/lm_eval/tasks/mmlu_pro/utils.py @@ -1,7 +1,9 @@ from functools import partial + choices = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J"] + def format_cot_example(example, including_answer=True): prompt = "Question:\n" question = example["question"] @@ -21,15 +23,18 @@ def format_cot_example(example, including_answer=True): prompt += cot_content + "\n\n" else: prompt += "Answer: Let's think step by step." - + return prompt + doc_to_text = partial(format_cot_example, including_answer=False) fewshot_to_text = partial(format_cot_example, including_answer=True) + def process_docs(dataset, subject): return dataset.filter(lambda x: x["category"] == subject) + process_biology = partial(process_docs, subject="biology") process_business = partial(process_docs, subject="business") process_chemistry = partial(process_docs, subject="chemistry") -- GitLab From 81fc0826817416c1c5a2d92f7d1e6caa7ae2bb2b Mon Sep 17 00:00:00 2001 From: Baber Abbasi <92168766+baberabb@users.noreply.github.com> Date: Mon, 19 May 2025 19:58:04 +0500 Subject: [PATCH 21/46] fix github parse error (#2998) --- examples/lm-eval-overview.ipynb | 740 ++++++++++++++++---------------- 1 file changed, 372 insertions(+), 368 deletions(-) diff --git a/examples/lm-eval-overview.ipynb b/examples/lm-eval-overview.ipynb index 3a06e96e..2a9053c1 100644 --- a/examples/lm-eval-overview.ipynb +++ b/examples/lm-eval-overview.ipynb @@ -79,48 +79,48 @@ " Switched to a new branch 'big-refactor'\n", " Branch 'big-refactor' set up to track remote branch 'big-refactor' from 'origin'.\n", " Resolved https://github.com/EleutherAI/lm-evaluation-harness.git to commit 42f486ee49b65926a444cb0620870a39a5b4b0a8\n", - " Installing build dependencies ... \u001B[?25l\u001B[?25hdone\n", - " Getting requirements to build wheel ... \u001B[?25l\u001B[?25hdone\n", - " Preparing metadata (pyproject.toml) ... \u001B[?25l\u001B[?25hdone\n", + " Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n", + " Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n", + " Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", "Collecting accelerate>=0.21.0 (from lm-eval==1.0.0)\n", " Downloading accelerate-0.24.1-py3-none-any.whl (261 kB)\n", - "\u001B[2K \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m261.4/261.4 kB\u001B[0m \u001B[31m4.1 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n", - "\u001B[?25hCollecting evaluate (from lm-eval==1.0.0)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m261.4/261.4 kB\u001b[0m \u001b[31m4.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting evaluate (from lm-eval==1.0.0)\n", " Downloading evaluate-0.4.1-py3-none-any.whl (84 kB)\n", - "\u001B[2K \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m84.1/84.1 kB\u001B[0m \u001B[31m5.9 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n", - "\u001B[?25hCollecting datasets>=2.0.0 (from lm-eval==1.0.0)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m84.1/84.1 kB\u001b[0m \u001b[31m5.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting datasets>=2.0.0 (from lm-eval==1.0.0)\n", " Downloading datasets-2.15.0-py3-none-any.whl (521 kB)\n", - "\u001B[2K \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m521.2/521.2 kB\u001B[0m \u001B[31m9.5 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n", - "\u001B[?25hCollecting jsonlines (from lm-eval==1.0.0)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m521.2/521.2 kB\u001b[0m \u001b[31m9.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting jsonlines (from lm-eval==1.0.0)\n", " Downloading jsonlines-4.0.0-py3-none-any.whl (8.7 kB)\n", "Requirement already satisfied: numexpr in /usr/local/lib/python3.10/dist-packages (from lm-eval==1.0.0) (2.8.7)\n", "Collecting peft>=0.2.0 (from lm-eval==1.0.0)\n", " Downloading peft-0.6.2-py3-none-any.whl (174 kB)\n", - "\u001B[2K \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m174.7/174.7 kB\u001B[0m \u001B[31m7.2 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n", - "\u001B[?25hCollecting pybind11>=2.6.2 (from lm-eval==1.0.0)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m174.7/174.7 kB\u001b[0m \u001b[31m7.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting pybind11>=2.6.2 (from lm-eval==1.0.0)\n", " Downloading pybind11-2.11.1-py3-none-any.whl (227 kB)\n", - "\u001B[2K \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m227.7/227.7 kB\u001B[0m \u001B[31m12.9 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n", - "\u001B[?25hCollecting pytablewriter (from lm-eval==1.0.0)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m227.7/227.7 kB\u001b[0m \u001b[31m12.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting pytablewriter (from lm-eval==1.0.0)\n", " Downloading pytablewriter-1.2.0-py3-none-any.whl (111 kB)\n", - "\u001B[2K \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m111.1/111.1 kB\u001B[0m \u001B[31m8.3 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n", - "\u001B[?25hCollecting rouge-score>=0.0.4 (from lm-eval==1.0.0)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m111.1/111.1 kB\u001b[0m \u001b[31m8.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting rouge-score>=0.0.4 (from lm-eval==1.0.0)\n", " Downloading rouge_score-0.1.2.tar.gz (17 kB)\n", - " Preparing metadata (setup.py) ... \u001B[?25l\u001B[?25hdone\n", + " Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", "Collecting sacrebleu>=1.5.0 (from lm-eval==1.0.0)\n", " Downloading sacrebleu-2.3.2-py3-none-any.whl (119 kB)\n", - "\u001B[2K \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m119.7/119.7 kB\u001B[0m \u001B[31m8.7 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n", - "\u001B[?25hRequirement already satisfied: scikit-learn>=0.24.1 in /usr/local/lib/python3.10/dist-packages (from lm-eval==1.0.0) (1.2.2)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m119.7/119.7 kB\u001b[0m \u001b[31m8.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: scikit-learn>=0.24.1 in /usr/local/lib/python3.10/dist-packages (from lm-eval==1.0.0) (1.2.2)\n", "Collecting sqlitedict (from lm-eval==1.0.0)\n", " Downloading sqlitedict-2.1.0.tar.gz (21 kB)\n", - " Preparing metadata (setup.py) ... \u001B[?25l\u001B[?25hdone\n", + " Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", "Requirement already satisfied: torch>=1.8 in /usr/local/lib/python3.10/dist-packages (from lm-eval==1.0.0) (2.1.0+cu118)\n", "Collecting tqdm-multiprocess (from lm-eval==1.0.0)\n", " Downloading tqdm_multiprocess-0.0.11-py3-none-any.whl (9.8 kB)\n", "Requirement already satisfied: transformers>=4.1 in /usr/local/lib/python3.10/dist-packages (from lm-eval==1.0.0) (4.35.2)\n", "Collecting zstandard (from lm-eval==1.0.0)\n", " Downloading zstandard-0.22.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (5.4 MB)\n", - "\u001B[2K \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m5.4/5.4 MB\u001B[0m \u001B[31m29.2 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n", - "\u001B[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.21.0->lm-eval==1.0.0) (1.23.5)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.4/5.4 MB\u001b[0m \u001b[31m29.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.21.0->lm-eval==1.0.0) (1.23.5)\n", "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.21.0->lm-eval==1.0.0) (23.2)\n", "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.21.0->lm-eval==1.0.0) (5.9.5)\n", "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.21.0->lm-eval==1.0.0) (6.0.1)\n", @@ -130,15 +130,15 @@ " Downloading pyarrow_hotfix-0.6-py3-none-any.whl (7.9 kB)\n", "Collecting dill<0.3.8,>=0.3.0 (from datasets>=2.0.0->lm-eval==1.0.0)\n", " Downloading dill-0.3.7-py3-none-any.whl (115 kB)\n", - "\u001B[2K \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m115.3/115.3 kB\u001B[0m \u001B[31m14.4 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n", - "\u001B[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->lm-eval==1.0.0) (1.5.3)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m115.3/115.3 kB\u001b[0m \u001b[31m14.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->lm-eval==1.0.0) (1.5.3)\n", "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->lm-eval==1.0.0) (2.31.0)\n", "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->lm-eval==1.0.0) (4.66.1)\n", "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->lm-eval==1.0.0) (3.4.1)\n", "Collecting multiprocess (from datasets>=2.0.0->lm-eval==1.0.0)\n", " Downloading multiprocess-0.70.15-py310-none-any.whl (134 kB)\n", - "\u001B[2K \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m134.8/134.8 kB\u001B[0m \u001B[31m19.9 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n", - "\u001B[?25hRequirement already satisfied: fsspec[http]<=2023.10.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->lm-eval==1.0.0) (2023.6.0)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m19.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: fsspec[http]<=2023.10.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->lm-eval==1.0.0) (2023.6.0)\n", "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->lm-eval==1.0.0) (3.8.6)\n", "Collecting responses<0.19 (from evaluate->lm-eval==1.0.0)\n", " Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n", @@ -193,13 +193,13 @@ "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk->rouge-score>=0.0.4->lm-eval==1.0.0) (8.1.7)\n", "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.8->lm-eval==1.0.0) (1.3.0)\n", "Building wheels for collected packages: lm-eval, rouge-score, sqlitedict\n", - " Building wheel for lm-eval (pyproject.toml) ... \u001B[?25l\u001B[?25hdone\n", + " Building wheel for lm-eval (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", " Created wheel for lm-eval: filename=lm_eval-1.0.0-py3-none-any.whl size=994254 sha256=88356155b19f2891981ecef948326ad6ce8ca40a6009378410ec20d0e225995a\n", " Stored in directory: /tmp/pip-ephem-wheel-cache-9v6ye7h3/wheels/17/01/26/599c0779e9858a70a73fa8a306699b5b9a868f820c225457b0\n", - " Building wheel for rouge-score (setup.py) ... \u001B[?25l\u001B[?25hdone\n", + " Building wheel for rouge-score (setup.py) ... \u001b[?25l\u001b[?25hdone\n", " Created wheel for rouge-score: filename=rouge_score-0.1.2-py3-none-any.whl size=24933 sha256=6bb0d44e4881972c43ce194e7cb65233d309758cb15f0dec54590d3d2efcfc36\n", " Stored in directory: /root/.cache/pip/wheels/5f/dd/89/461065a73be61a532ff8599a28e9beef17985c9e9c31e541b4\n", - " Building wheel for sqlitedict (setup.py) ... \u001B[?25l\u001B[?25hdone\n", + " Building wheel for sqlitedict (setup.py) ... \u001b[?25l\u001b[?25hdone\n", " Created wheel for sqlitedict: filename=sqlitedict-2.1.0-py3-none-any.whl size=16863 sha256=5747f7dd73ddf3d8fbcebf51b5e4f718fabe1e94bccdf16d2f22a2e65ee7fdf4\n", " Stored in directory: /root/.cache/pip/wheels/79/d6/e7/304e0e6cb2221022c26d8161f7c23cd4f259a9e41e8bbcfabd\n", "Successfully built lm-eval rouge-score sqlitedict\n", @@ -886,348 +886,352 @@ }, "widgets": { "application/vnd.jupyter.widget-state+json": { - "46f521b73fd943c081c648fd873ebc0a": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "48763b6233374554ae76035c0483066f": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "4986a21eb560448fa79f4b25cde48951": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "6b2d90209ec14230b3d58a74ac9b83bf": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "7c5689bc13684db8a22681f41863dddd": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "a1d3a8aa016544a78e8821c8f6199e06": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_f61ed33fad754146bdd2ac9db1ba1c48", - "IPY_MODEL_bfa0af6aeff344c6845e1080a878e92e", - "IPY_MODEL_fd1ad9e0367d4004aae853b91c3a7617" - ], - "layout": "IPY_MODEL_6b2d90209ec14230b3d58a74ac9b83bf" - } - }, - "a73f357065d34d7baf0453ae4a8d75e2": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "aed3acd2f2d74003b44079c333a0698e": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "bfa0af6aeff344c6845e1080a878e92e": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_7c5689bc13684db8a22681f41863dddd", - "max": 5669, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_48763b6233374554ae76035c0483066f", - "value": 5669 - } - }, - "f61ed33fad754146bdd2ac9db1ba1c48": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_a73f357065d34d7baf0453ae4a8d75e2", - "placeholder": "​", - "style": "IPY_MODEL_46f521b73fd943c081c648fd873ebc0a", - "value": "Downloading builder script: 100%" + "state": { + "46f521b73fd943c081c648fd873ebc0a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "48763b6233374554ae76035c0483066f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "4986a21eb560448fa79f4b25cde48951": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "6b2d90209ec14230b3d58a74ac9b83bf": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "7c5689bc13684db8a22681f41863dddd": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a1d3a8aa016544a78e8821c8f6199e06": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_f61ed33fad754146bdd2ac9db1ba1c48", + "IPY_MODEL_bfa0af6aeff344c6845e1080a878e92e", + "IPY_MODEL_fd1ad9e0367d4004aae853b91c3a7617" + ], + "layout": "IPY_MODEL_6b2d90209ec14230b3d58a74ac9b83bf" + } + }, + "a73f357065d34d7baf0453ae4a8d75e2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "aed3acd2f2d74003b44079c333a0698e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "bfa0af6aeff344c6845e1080a878e92e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_7c5689bc13684db8a22681f41863dddd", + "max": 5669, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_48763b6233374554ae76035c0483066f", + "value": 5669 + } + }, + "f61ed33fad754146bdd2ac9db1ba1c48": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a73f357065d34d7baf0453ae4a8d75e2", + "placeholder": "​", + "style": "IPY_MODEL_46f521b73fd943c081c648fd873ebc0a", + "value": "Downloading builder script: 100%" + } + }, + "fd1ad9e0367d4004aae853b91c3a7617": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4986a21eb560448fa79f4b25cde48951", + "placeholder": "​", + "style": "IPY_MODEL_aed3acd2f2d74003b44079c333a0698e", + "value": " 5.67k/5.67k [00:00<00:00, 205kB/s]" + } } }, - "fd1ad9e0367d4004aae853b91c3a7617": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_4986a21eb560448fa79f4b25cde48951", - "placeholder": "​", - "style": "IPY_MODEL_aed3acd2f2d74003b44079c333a0698e", - "value": " 5.67k/5.67k [00:00<00:00, 205kB/s]" - } - } + "version_major": 2, + "version_minor": 0 } } }, -- GitLab From 07e5348c94487464dcb3c3fbf70ba5f7b6914046 Mon Sep 17 00:00:00 2001 From: Rob Geada Date: Wed, 21 May 2025 08:04:27 +0100 Subject: [PATCH 22/46] Log tokenized request warning only once (#3002) * Log tokenized request warning only once * Fix logging for concurrent usecase as well --- lm_eval/models/api_models.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/lm_eval/models/api_models.py b/lm_eval/models/api_models.py index 0142af94..4b38ed0c 100644 --- a/lm_eval/models/api_models.py +++ b/lm_eval/models/api_models.py @@ -597,6 +597,10 @@ class TemplateAPI(TemplateLM): chunked = re_ord.get_batched( n=self._batch_size if self._concurrent <= 1 else 0, batch_fn=None ) + if not self.tokenized_requests: + eval_logger.info( + "Tokenized requests are disabled. Context + generation length is not checked." + ) if self._concurrent <= 1: pbar = tqdm(desc="Requesting API", total=len(requests)) for chunk in chunked: @@ -615,10 +619,7 @@ class TemplateAPI(TemplateLM): eval_logger.warning( f"Some contexts exceeded (max length: ({self.max_length}) - max_gen_toks: ({max_gen_toks}). They were left truncated." ) - else: - eval_logger.info( - "Tokenized requests are disabled. Context + generation length is not checked." - ) + req = encodings_list if self.tokenized_requests else contexts outputs = retry( stop=stop_after_attempt(self.max_retries), @@ -664,10 +665,7 @@ class TemplateAPI(TemplateLM): eval_logger.warning( f"Some contexts exceeded (max length: ({self.max_length}) - max_gen_toks ({max_gen_toks}). They were left truncated." ) - else: - eval_logger.info( - "Tokenized requests are disabled. Context + generation length is not checked." - ) + req = encodings_list if self.tokenized_requests else contexts results = itertools.chain.from_iterable( asyncio.run( -- GitLab From 8be417a8ae30f8e15bb1c165398ee61bf7665a30 Mon Sep 17 00:00:00 2001 From: Hongseok Oh <97136787+abzb1@users.noreply.github.com> Date: Wed, 21 May 2025 16:29:12 +0900 Subject: [PATCH 23/46] add kbl 2025 (#3000) --- .../tasks/kbl/bar_exam/civil/kbl_bar_exam_em_civil_2025.yaml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 lm_eval/tasks/kbl/bar_exam/civil/kbl_bar_exam_em_civil_2025.yaml diff --git a/lm_eval/tasks/kbl/bar_exam/civil/kbl_bar_exam_em_civil_2025.yaml b/lm_eval/tasks/kbl/bar_exam/civil/kbl_bar_exam_em_civil_2025.yaml new file mode 100644 index 00000000..be366cc3 --- /dev/null +++ b/lm_eval/tasks/kbl/bar_exam/civil/kbl_bar_exam_em_civil_2025.yaml @@ -0,0 +1,3 @@ +task: kbl_bar_exam_em_civil_2025 +dataset_name: bar_exam_civil_2025 +include: _base_em_yaml -- GitLab From 178fa84da05dd1050e309f8a9d8bcd67f34f42d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Ajroldi?= <61059403+Niccolo-Ajroldi@users.noreply.github.com> Date: Wed, 21 May 2025 10:32:58 +0200 Subject: [PATCH 24/46] Output path fix (#2993) * fix(output_path): support direct JSON file paths * fix linting * turn off external Lm tests for now * Update help text for `output_path` --------- Co-authored-by: Baber --- .github/workflows/unit_tests.yml | 66 +++++++++++++-------------- lm_eval/__main__.py | 2 +- lm_eval/loggers/evaluation_tracker.py | 31 +++++++++---- 3 files changed, 56 insertions(+), 43 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 002a3a87..b9a44864 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -79,36 +79,36 @@ jobs: path: | test_logs/* - testmodels: - name: External LM Tests - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - name: Checkout Code - uses: actions/checkout@v4 - - name: Set up Python 3.9 - uses: actions/setup-python@v5 - with: - python-version: 3.9 - cache: pip - cache-dependency-path: pyproject.toml - - # Cache HuggingFace cache directory for External LM tests - - name: Cache HuggingFace cache (External LM tests) - uses: actions/cache@v3 - id: cache-hf-lm - with: - path: ~/.cache/huggingface - key: ${{ runner.os }}-hf-cache-external-lm - restore-keys: | - ${{ runner.os }}-hf-cache-external-lm - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -e '.[dev,optimum,deepsparse,sparseml,api]' --extra-index-url https://download.pytorch.org/whl/cpu - pip install -U transformers peft accelerate - - - name: Test with pytest - run: python -m pytest tests/models --showlocals -s -vv - continue-on-error: true # Continue workflow even if tests fail +# testmodels: +# name: External LM Tests +# runs-on: ubuntu-latest +# timeout-minutes: 30 +# steps: +# - name: Checkout Code +# uses: actions/checkout@v4 +# - name: Set up Python 3.9 +# uses: actions/setup-python@v5 +# with: +# python-version: 3.9 +# cache: pip +# cache-dependency-path: pyproject.toml +# +# # Cache HuggingFace cache directory for External LM tests +# - name: Cache HuggingFace cache (External LM tests) +# uses: actions/cache@v3 +# id: cache-hf-lm +# with: +# path: ~/.cache/huggingface +# key: ${{ runner.os }}-hf-cache-external-lm +# restore-keys: | +# ${{ runner.os }}-hf-cache-external-lm +# +# - name: Install dependencies +# run: | +# python -m pip install --upgrade pip +# pip install -e '.[dev,optimum,deepsparse,sparseml,api]' --extra-index-url https://download.pytorch.org/whl/cpu +# pip install -U transformers peft accelerate +# +# - name: Test with pytest +# run: python -m pytest tests/models --showlocals -s -vv +# continue-on-error: true # Continue workflow even if tests fail diff --git a/lm_eval/__main__.py b/lm_eval/__main__.py index 406faddf..5a5c75ae 100644 --- a/lm_eval/__main__.py +++ b/lm_eval/__main__.py @@ -135,7 +135,7 @@ def setup_parser() -> argparse.ArgumentParser: default=None, type=str, metavar="DIR|DIR/file.json", - help="The path to the output file where the result metrics will be saved. If the path is a directory and log_samples is true, the results will be saved in the directory. Else the parent directory will be used.", + help="Path where result metrics will be saved. Can be either a directory or a .json file. If the path is a directory and log_samples is true, the results will be saved in the directory. Else the parent directory will be used.", ) parser.add_argument( "--limit", diff --git a/lm_eval/loggers/evaluation_tracker.py b/lm_eval/loggers/evaluation_tracker.py index ef56965d..634a6257 100644 --- a/lm_eval/loggers/evaluation_tracker.py +++ b/lm_eval/loggers/evaluation_tracker.py @@ -229,11 +229,21 @@ class EvaluationTracker: ) path = Path(self.output_path if self.output_path else Path.cwd()) - path = path.joinpath(self.general_config_tracker.model_name_sanitized) - path.mkdir(parents=True, exist_ok=True) - self.date_id = datetime.now().isoformat().replace(":", "-") - file_results_aggregated = path.joinpath(f"results_{self.date_id}.json") + if path.suffix == ".json": + path.parent.mkdir(parents=True, exist_ok=True) + file_results_aggregated = path.with_name( + f"{path.stem}_{self.date_id}.json" + ) + else: + path = path.joinpath( + self.general_config_tracker.model_name_sanitized + ) + path.mkdir(parents=True, exist_ok=True) + file_results_aggregated = path.joinpath( + f"results_{self.date_id}.json" + ) + file_results_aggregated.open("w", encoding="utf-8").write(dumped) if self.api and self.push_results_to_hub: @@ -250,12 +260,10 @@ class EvaluationTracker: ) self.api.upload_file( repo_id=repo_id, - path_or_fileobj=str( - path.joinpath(f"results_{self.date_id}.json") - ), + path_or_fileobj=str(file_results_aggregated), path_in_repo=os.path.join( self.general_config_tracker.model_name, - f"results_{self.date_id}.json", + file_results_aggregated.name, ), repo_type="dataset", commit_message=f"Adding aggregated results for {self.general_config_tracker.model_name}", @@ -290,7 +298,12 @@ class EvaluationTracker: eval_logger.info(f"Saving per-sample results for: {task_name}") path = Path(self.output_path if self.output_path else Path.cwd()) - path = path.joinpath(self.general_config_tracker.model_name_sanitized) + if path.suffix == ".json": + path = path.parent + else: + path = path.joinpath( + self.general_config_tracker.model_name_sanitized + ) path.mkdir(parents=True, exist_ok=True) file_results_samples = path.joinpath( -- GitLab From 2cfdd0a294214fe156caa1b8a69da17a29c39e63 Mon Sep 17 00:00:00 2001 From: Baber Abbasi <92168766+baberabb@users.noreply.github.com> Date: Wed, 21 May 2025 13:43:50 +0500 Subject: [PATCH 25/46] use images with api models (#2981) * use images with apis * pacify pre-commit --- lm_eval/models/api_models.py | 84 +++++++++++++++++++++++++++++++++++- 1 file changed, 82 insertions(+), 2 deletions(-) diff --git a/lm_eval/models/api_models.py b/lm_eval/models/api_models.py index 4b38ed0c..23d12203 100644 --- a/lm_eval/models/api_models.py +++ b/lm_eval/models/api_models.py @@ -6,6 +6,7 @@ import json import logging from functools import cached_property from typing import ( + TYPE_CHECKING, Any, Awaitable, Callable, @@ -30,7 +31,9 @@ except ModuleNotFoundError: pass +import base64 from importlib.util import find_spec +from io import BytesIO from lm_eval import utils from lm_eval.api.instance import Instance @@ -38,6 +41,10 @@ from lm_eval.api.model import TemplateLM from lm_eval.models.utils import Collator, chunks, configure_pad_token +if TYPE_CHECKING: + from PIL import Image + + eval_logger = logging.getLogger(__name__) LogLikelihoodInputs = Tuple[Tuple[str, str], List[int], List[int]] @@ -51,7 +58,52 @@ class JsonChatStr(NamedTuple): return self.prompt.encode(encoding) +def create_image_prompt( + imgs: list["Image.Image"], chat: dict, fmt: str = "PNG" +) -> dict: + """ + + Parameters + ---------- + img : list[PIL.Image.Image] + The list of images to encode to base64 + chat : dict + fmt : str, optional + Any format Pillow understands (e.g. "PNG", "JPEG"). + Defaults to "PNG". + + Returns + ------- + dict + """ + images = [] + for img in imgs: + buf = BytesIO() + img.save(buf, format=fmt) + img_b64 = base64.b64encode(buf.getvalue()).decode("utf-8") + img_dict = { + "type": "image_url", + "image_url": {"url": f"data:image/png;base64,{img_b64}", "detail": "auto"}, + } + images.append(img_dict) + + # chat is in format of list[dict["role": "user"/"system", "content": str, "type": "text"],...] + # with images, we need "content" to be a list of dicts with "type" and "text"/"image_url" + # currently we do not support few-shots so only one user message + # text content also has placeholders, which apparently is not necessary for API class (confirm) + + if isinstance(chat[-1]["content"], list): + chat[-1]["content"] = images + chat[-1]["content"] + else: + text_content = {"type": "text", "text": chat[-1]["content"]} + chat[-1]["content"] = images + [text_content] + chat[-1].pop("type") + return chat + + class TemplateAPI(TemplateLM): + MULTIMODAL = True + def __init__( self, model: str = None, @@ -83,6 +135,7 @@ class TemplateAPI(TemplateLM): eos_string: str = None, # timeout in seconds timeout: int = 300, + max_images: int = 1, **kwargs, ) -> None: super().__init__() @@ -129,6 +182,7 @@ class TemplateAPI(TemplateLM): self.verify_certificate = verify_certificate self._eos_string = eos_string self.timeout = int(timeout) + self.max_images = int(max_images) eval_logger.info(f"Using tokenizer {self.tokenizer_backend}") if self.tokenizer_backend is None: @@ -265,7 +319,12 @@ class TemplateAPI(TemplateLM): ) else: # bit of a hack. We'll load back before sending to the API - return JsonChatStr(json.dumps(chat_history, ensure_ascii=False)) + return JsonChatStr( + json.dumps( + [{**item, "type": "text"} for item in chat_history], + ensure_ascii=False, + ) + ) @cached_property def eot_token_id(self) -> Optional[int]: @@ -578,7 +637,28 @@ class TemplateAPI(TemplateLM): return -len(_requests[0]) # Let the API deal with tokenization - requests, all_gen_kwargs = zip(*(req.args for req in requests)) + if len(requests[0].args) > 2: + assert self.tokenizer is None, ( + "tokenizer is not supported for multimodal requests yet!" + ) + eval_logger.info( + f"Using max_images {self.max_images}. Set in the model args." + ) + requests, all_gen_kwargs, auxiliary_args = zip( + *(req.args for req in requests) + ) + requests = tuple( + JsonChatStr( + json.dumps( + create_image_prompt( + y["visual"][: self.max_images], json.loads(x.prompt) + ) + ) + ) + for x, y in zip(requests, auxiliary_args) + ) + else: + requests, all_gen_kwargs = zip(*(req.args for req in requests)) if self.tokenized_requests: encodings_list = self.tok_encode( requests, add_special_tokens=self.add_bos_token -- GitLab From 143a7fe0d4b5504fb411dbaf8d2df0c734e293cd Mon Sep 17 00:00:00 2001 From: achervyakov <77295913+artemorloff@users.noreply.github.com> Date: Wed, 21 May 2025 16:05:20 +0300 Subject: [PATCH 26/46] Adding resize images support (#2958) * first version of image resizing * fixed bug * clean up `resize_image` --------- Co-authored-by: Artem Safin Co-authored-by: Baber --- lm_eval/models/hf_vlms.py | 24 ++++++++- lm_eval/models/utils.py | 101 ++++++++++++++++++++++++++++++++++++ lm_eval/models/vllm_vlms.py | 23 +++++++- 3 files changed, 146 insertions(+), 2 deletions(-) diff --git a/lm_eval/models/hf_vlms.py b/lm_eval/models/hf_vlms.py index 073c0ba0..5422c2c5 100644 --- a/lm_eval/models/hf_vlms.py +++ b/lm_eval/models/hf_vlms.py @@ -17,6 +17,7 @@ from lm_eval.models.utils import ( handle_stop_sequences, pad_and_concat, replace_placeholders, + resize_image, stop_sequences_criteria, ) @@ -45,10 +46,23 @@ class HFMultimodalLM(HFLM): # TODO: handle whitespace in image placeholder (replacement) max_images: Optional[int] = 999, convert_img_format=False, + # For image resizing min_pixels: Optional[int] = None, max_pixels: Optional[int] = None, + image_width: Optional[int] = None, + image_height: Optional[int] = None, + image_max_side: Optional[int] = None, **kwargs, ): + self.image_width = image_width + self.image_height = image_height + self.image_max_side = image_max_side + if self.image_max_side and (self.image_width or self.image_height): + raise ValueError( + "Ambiguous config for image resize: you can not specify both " + "image_max_side and (image_width or image_height)" + ) + # init pixels before calling tokenizer creation to avoid errors self.pixels = ({"min_pixels": min_pixels} if min_pixels else {}) | ( {"max_pixels": max_pixels} if max_pixels else {} @@ -646,7 +660,15 @@ class HFMultimodalLM(HFLM): for chunk in chunks: contexts, all_gen_kwargs, aux_arguments = zip(*chunk) - visuals = [arg["visual"] for arg in aux_arguments] + visuals = [ + [ + resize_image( + img, self.image_width, self.image_height, self.image_max_side + ) + for img in arg["visual"] + ] + for arg in aux_arguments + ] if not isinstance(contexts, list): contexts = list( diff --git a/lm_eval/models/utils.py b/lm_eval/models/utils.py index 2878de6e..e56225dd 100644 --- a/lm_eval/models/utils.py +++ b/lm_eval/models/utils.py @@ -28,6 +28,7 @@ eval_logger = logging.getLogger(__name__) if TYPE_CHECKING: + from PIL import Image from transformers import PreTrainedTokenizerBase from transformers.configuration_utils import PretrainedConfig @@ -729,3 +730,103 @@ def handle_stop_sequences( if eos is not None and eos not in until: until.append(eos) return until + + +def resize_image( + image: "Image.Image", + width: Optional[int] = None, + height: Optional[int] = None, + max_dimension: Optional[int] = None, + keep_aspect_ratio: bool = True, + resample_filter: Union[int, str] = "Image.BICUBIC", + min_width: int = 1, + min_height: int = 1, +) -> "Image.Image": + """ + Resizes a PIL Image object with flexible options. + + Args: + image: The PIL Image object to resize. + width: Target width in pixels. + height: Target height in pixels. + max_dimension: Maximum size for the longer dimension of the image. + keep_aspect_ratio: If True (default) and both width and height are provided, + the image is resized to fit within these dimensions while + maintaining its aspect ratio. If False, the image is stretched + to the exact width and height. + resample_filter: The resampling filter to use for resizing. + Defaults to Image.BICUBIC. + min_width: Minimum width for the resized image. Defaults to 1. + min_height: Minimum height for the resized image. Defaults to 1. + + Returns: + The resized PIL Image object. If no resize parameters are provided + or if the image already meets the criteria, the original image is returned. + + Order of precedence for resizing: + 1. If width AND height are provided: + - If keep_aspect_ratio is True: Fits image within bounds, preserving aspect ratio. + - If keep_aspect_ratio is False: Resizes to exact dimensions (may distort). + 2. Else if only width is provided: Calculates height proportionally. + 3. Else if only height is provided: Calculates width proportionally. + 4. Else if max_dimension is provided: Resizes the longest side to max_dimension + and scales the other side proportionally. + 5. If none of the above are provided, returns the original image. + """ + original_width, original_height = image.size + + # If no arguments are provided, return the original image + if width is None and height is None and max_dimension is None: + return image + + new_width = original_width + new_height = original_height + + if width is not None and height is not None: + # No resize needed if image is already smaller than target dimensions + if original_width <= width and original_height <= height: + return image + + if keep_aspect_ratio: + # Calculate the ratio to fit within the target dimensions + ratio = min(width / original_width, height / original_height) + new_width = int(original_width * ratio) + new_height = int(original_height * ratio) + else: + # Stretch to exact dimensions + new_width = width + new_height = height + elif width is not None: + # No resize needed if width is already smaller + if original_width <= width: + return image + # Calculate height proportionally + new_width = width + new_height = int((original_height / original_width) * new_width) + elif height is not None: + # No resize needed if height is already smaller + if original_height <= height: + return image + # Calculate width proportionally + new_height = height + new_width = int((original_width / original_height) * new_height) + elif max_dimension is not None: + # No resize needed if both dimensions are smaller than max_dimension + if max(original_height, original_width) <= max_dimension: + return image + + if original_width > original_height: + # Width is the longer side + new_width = max_dimension + new_height = int((original_height / original_width) * new_width) + else: + # Height is the longer side or sides are equal + new_height = max_dimension + new_width = int((original_width / original_height) * new_height) + + # Ensure dimensions are at least minimum values + new_width = max(min_width, new_width) + new_height = max(min_height, new_height) + + # Perform the resize operation with the calculated dimensions + return image.resize((new_width, new_height), resample_filter) diff --git a/lm_eval/models/vllm_vlms.py b/lm_eval/models/vllm_vlms.py index 62c35592..15ebc764 100644 --- a/lm_eval/models/vllm_vlms.py +++ b/lm_eval/models/vllm_vlms.py @@ -12,6 +12,7 @@ from lm_eval.models.utils import ( Collator, handle_stop_sequences, replace_placeholders, + resize_image, undistribute, ) from lm_eval.models.vllm_causallms import VLLM @@ -44,8 +45,20 @@ class VLLM_VLM(VLLM): interleave: bool = True, # TODO: handle max_images and limit_mm_per_prompt better max_images: int = 999, + image_width: Optional[int] = None, + image_height: Optional[int] = None, + image_max_side: Optional[int] = None, **kwargs, ): + self.image_width = image_width + self.image_height = image_height + self.image_max_side = image_max_side + if self.image_max_side and (self.image_width or self.image_height): + raise ValueError( + "Ambiguous config for image resize: you can not specify both " + "image_max_side and (image_width or image_height)" + ) + if max_images != 999: kwargs["limit_mm_per_prompt"] = {"image": max_images} eval_logger.info(f"Setting limit_mm_per_prompt[image] to {max_images}") @@ -239,7 +252,15 @@ class VLLM_VLM(VLLM): for chunk in chunks: contexts, all_gen_kwargs, aux_arguments = zip(*chunk) - visuals = [arg["visual"] for arg in aux_arguments] + visuals = [ + [ + resize_image( + img, self.image_width, self.image_height, self.image_max_side + ) + for img in arg["visual"] + ] + for arg in aux_arguments + ] if not isinstance(contexts, list): contexts = list( -- GitLab From 29ea6832cd913b055ec1d6962180c773e8a7ac88 Mon Sep 17 00:00:00 2001 From: Baber Abbasi <92168766+baberabb@users.noreply.github.com> Date: Wed, 21 May 2025 18:50:09 +0500 Subject: [PATCH 27/46] Revert "feat: add question suffix (#2876)" (#3007) This reverts commit 4dbd5ec9 --- lm_eval/__main__.py | 6 ------ lm_eval/api/task.py | 24 +++--------------------- lm_eval/evaluator.py | 2 -- 3 files changed, 3 insertions(+), 29 deletions(-) diff --git a/lm_eval/__main__.py b/lm_eval/__main__.py index 5a5c75ae..f1faae48 100644 --- a/lm_eval/__main__.py +++ b/lm_eval/__main__.py @@ -261,12 +261,6 @@ def setup_parser() -> argparse.ArgumentParser: default="", help="Comma separated string arguments passed to Hugging Face Hub's log function, e.g. `hub_results_org=EleutherAI,hub_repo_name=lm-eval-results`", ) - parser.add_argument( - "--question_suffix", - type=str, - default=None, - help="Suffix to append to the target question before the <|assistant|>, e.g., Think for maximum 128 tokens", - ) parser.add_argument( "--predict_only", "-x", diff --git a/lm_eval/api/task.py b/lm_eval/api/task.py index 893784e5..c1bc967a 100644 --- a/lm_eval/api/task.py +++ b/lm_eval/api/task.py @@ -401,7 +401,6 @@ class Task(abc.ABC): fewshot_as_multiturn: bool = False, chat_template: Optional[Callable] = None, tokenizer_name: str = "", - question_suffix: str = "", ) -> None: """Build a set of Instances for a task, and store them in task.instances""" @@ -465,7 +464,6 @@ class Task(abc.ABC): fewshot_as_multiturn, chat_template, gen_prefix=self.doc_to_prefix(doc), - question_suffix=question_suffix, ) # TODO: we should override self.config.repeats if doing greedy gen so users don't waste time+compute @@ -1068,7 +1066,6 @@ class ConfigurableTask(Task): question: str, fewshot_as_multiturn: bool = False, gen_prefix: Optional[str] = None, - question_suffix: Optional[str] = None, ) -> None: """Adds a target question to the labeled examples list. If fewshot_as_multiturn is True, or labeled_examples is empty, or the last entry is a system turn, appends the question as a new user entry. @@ -1077,23 +1074,13 @@ class ConfigurableTask(Task): if not fewshot_as_multiturn: # if no messages or last message is system, append as new user entry if len(labeled_examples) == 0 or labeled_examples[-1]["role"] == "system": - labeled_examples.append( - {"role": "user", "content": question + question_suffix} - if question_suffix - else {"role": "user", "content": question} - ) + labeled_examples.append({"role": "user", "content": question}) # if last message is user, append to it to avoid two user messages in a row else: - labeled_examples[-1]["content"] += ( - question + question_suffix if question_suffix else question - ) + labeled_examples[-1]["content"] += question else: # if fewshot_as_multiturn is True, append as next user entry (last is always assistant) - labeled_examples.append( - {"role": "user", "content": question + question_suffix} - if question_suffix - else {"role": "user", "content": question} - ) + labeled_examples.append({"role": "user", "content": question}) if gen_prefix: labeled_examples.append({"role": "assistant", "content": gen_prefix}) @@ -1107,7 +1094,6 @@ class ConfigurableTask(Task): fewshot_as_multiturn: bool = False, chat_template: Optional[Callable] = None, gen_prefix: Optional[str] = None, - question_suffix: Optional[str] = None, ) -> Union[str, List[str]]: """Returns a fewshot context string that is made up of a prepended description (if provided), the `num_fewshot` number of examples, and an appended prompt example. @@ -1185,7 +1171,6 @@ class ConfigurableTask(Task): example, fewshot_as_multiturn, gen_prefix=gen_prefix, - question_suffix=question_suffix, ) # for loglikelihood create a list of questions with appended choices elif isinstance(example, list): @@ -1198,7 +1183,6 @@ class ConfigurableTask(Task): ex, fewshot_as_multiturn, gen_prefix=gen_prefix, - question_suffix=question_suffix, ) # TODO: append prefill? labeled_examples_list.append( @@ -1217,7 +1201,6 @@ class ConfigurableTask(Task): choices[example], fewshot_as_multiturn, gen_prefix=gen_prefix, - question_suffix=question_suffix, ) else: self.append_target_question( @@ -1225,7 +1208,6 @@ class ConfigurableTask(Task): str(example), fewshot_as_multiturn, gen_prefix=gen_prefix, - question_suffix=question_suffix, ) # return lm.apply_chat_template(labeled_examples) return chat_template( diff --git a/lm_eval/evaluator.py b/lm_eval/evaluator.py index f9875f3d..d1312b28 100644 --- a/lm_eval/evaluator.py +++ b/lm_eval/evaluator.py @@ -413,7 +413,6 @@ def evaluate( fewshot_as_multiturn: bool = False, verbosity: str = "INFO", confirm_run_unsafe_code: bool = False, - question_suffix: Optional[str] = None, ): """Instantiate and evaluate a model on a list of tasks. @@ -527,7 +526,6 @@ def evaluate( tokenizer_name=getattr(lm, "tokenizer_name", "") if apply_chat_template else "", - question_suffix=question_suffix, ) eval_logger.debug( f"Task: {task_output.task_name}; number of requests on this rank: {len(task.instances)}" -- GitLab From e1a7a39c7f08eeff38880cf9a3a07e1390f86d63 Mon Sep 17 00:00:00 2001 From: Baber Abbasi <92168766+baberabb@users.noreply.github.com> Date: Thu, 22 May 2025 22:24:01 +0500 Subject: [PATCH 28/46] change multimodal check in evaluate (#3013) changed multimodal check from strict equality --- lm_eval/evaluator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lm_eval/evaluator.py b/lm_eval/evaluator.py index d1312b28..54d4d0ea 100644 --- a/lm_eval/evaluator.py +++ b/lm_eval/evaluator.py @@ -483,7 +483,7 @@ def evaluate( for task_output in eval_tasks: task: Task = task_output.task - if getattr(lm, "MULTIMODAL", False) != getattr(task, "MULTIMODAL", False): + if getattr(task, "MULTIMODAL", False) and not getattr(lm, "MULTIMODAL", False): incompatible_tasks.append(task_output.task_name) elif getattr(task, "UNSAFE_CODE", False) and not confirm_run_unsafe_code: raise ValueError( -- GitLab From 357d4eaafdc77bd74ad029b2cef25aea99fa6f7a Mon Sep 17 00:00:00 2001 From: fxmarty-amd Date: Fri, 23 May 2025 08:11:01 +0200 Subject: [PATCH 29/46] [Fix] Update `resolve_hf_chat_template` arguments (#2992) * fix arguments * pacify pre-commit --------- Co-authored-by: Baber --- lm_eval/models/vllm_causallms.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/lm_eval/models/vllm_causallms.py b/lm_eval/models/vllm_causallms.py index 29587946..c5867a93 100644 --- a/lm_eval/models/vllm_causallms.py +++ b/lm_eval/models/vllm_causallms.py @@ -1,4 +1,5 @@ import copy +import inspect import logging from importlib.metadata import version from importlib.util import find_spec @@ -140,11 +141,28 @@ class VLLM(TemplateLM): ) if parse_version(version("vllm")) >= parse_version("0.8.3"): + kwargs_resolve_hf_chat_template = { + "tokenizer": self.tokenizer, + "chat_template": None, + "tools": None, + } + + if parse_version(version("vllm")) >= parse_version("0.9.0"): + kwargs_resolve_hf_chat_template["model_config"] = ( + self.model.llm_engine.model_config + ) + + # https://github.com/vllm-project/vllm/pull/18259 + if ( + "trsut_remote_code" + in inspect.signature(resolve_hf_chat_template).parameters + ): + kwargs_resolve_hf_chat_template["trsut_remote_code"] = trust_remote_code + else: + kwargs_resolve_hf_chat_template["trust_remote_code"] = trust_remote_code + self.hf_chat_template = resolve_hf_chat_template( - tokenizer=self.tokenizer, - chat_template=None, - tools=None, - trust_remote_code=trust_remote_code, + **kwargs_resolve_hf_chat_template ) else: self.hf_chat_template = None -- GitLab From 7aaceeec2e7b686d95d0e55b43af641dc2484b4a Mon Sep 17 00:00:00 2001 From: Ameya Godbole Date: Thu, 22 May 2025 23:12:58 -0700 Subject: [PATCH 30/46] Fix error due in Collating queries with different continuation lengths (fixes #2984) (#2987) * FIX error due to grouping queries with different continuation length Make Collator choose query with the longest continuation as the candidate for generation * use max for key selection * added comments explaining variable cont length (identical ctx+cont[:-1]) --------- Co-authored-by: Baber --- lm_eval/models/huggingface.py | 9 +++++++-- lm_eval/models/utils.py | 8 ++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/lm_eval/models/huggingface.py b/lm_eval/models/huggingface.py index 492923f0..c38397e6 100644 --- a/lm_eval/models/huggingface.py +++ b/lm_eval/models/huggingface.py @@ -1136,7 +1136,7 @@ class HFLM(TemplateLM): if self.backend == "causal": total_length = len(context_enc) + len(continuation_enc) if total_length > self.max_length + 1: - eval_logger.warn( + eval_logger.warning( f"Combined length of context ({len(context_enc)}) and continuation ({len(continuation_enc)}) " f"exceeds model's maximum length ({self.max_length}). " f"Truncating {total_length - self.max_length + 1} tokens from the left." @@ -1247,7 +1247,12 @@ class HFLM(TemplateLM): cont_toks = torch.tensor( cont_toks, dtype=torch.long, device=self.device ).unsqueeze(0) # [1, seq] - max_equal = (greedy_tokens == cont_toks).all() + # Use trailing slice [-cont_toks.shape[1]:] to handle variable length cont_len (but same ctx+cont[:-1]). + # i.e. continuations can be sliced at diff points. Collator ensures we have sufficient greedy_tokens + # by choosing key with longest cont if group_by="contexts". + max_equal = ( + greedy_tokens[:, -cont_toks.shape[1] :] == cont_toks + ).all() # Obtain log-probs at the corresponding continuation token indices # last_token_slice = logits[:, -1, :].squeeze(0).tolist() diff --git a/lm_eval/models/utils.py b/lm_eval/models/utils.py index e56225dd..daac5743 100644 --- a/lm_eval/models/utils.py +++ b/lm_eval/models/utils.py @@ -428,9 +428,13 @@ class Collator: batch = self.get_chunks(values, n=n, fn=batch_fn) yield from batch elif self._group_by == "contexts": - # Get one sample from each key + # Get one sample from each key. + # Select longest continuation per group to ensure sufficient context logits values = self._reorder( - [value[0] for value in self._arr_with_indices.values()] + [ + max(value, key=lambda x: len(x[1][-1])) + for value in self._arr_with_indices.values() + ] ) batch = self.get_chunks(values, n=n, fn=batch_fn) yield from batch -- GitLab From 5a481f4317ade9b12d2d9654f36a001f1a9192a6 Mon Sep 17 00:00:00 2001 From: Baber Abbasi <92168766+baberabb@users.noreply.github.com> Date: Mon, 26 May 2025 17:45:16 +0500 Subject: [PATCH 31/46] [vllm] data parallel for V1 (#3011) * add data_parallel for V1 * use Process instead of Queue * ray used if V0 DP * better error handling * fix truncation warning comparison --- lm_eval/models/vllm_causallms.py | 160 ++++++++++++++++++++++++++++--- 1 file changed, 148 insertions(+), 12 deletions(-) diff --git a/lm_eval/models/vllm_causallms.py b/lm_eval/models/vllm_causallms.py index c5867a93..b35dcb3b 100644 --- a/lm_eval/models/vllm_causallms.py +++ b/lm_eval/models/vllm_causallms.py @@ -1,8 +1,13 @@ import copy +import gc import inspect import logging +import os from importlib.metadata import version from importlib.util import find_spec +from multiprocessing import Process, Queue +from queue import Empty +from time import sleep from typing import TYPE_CHECKING, Dict, List, Literal, Optional, Tuple, Union from more_itertools import distribute @@ -29,6 +34,7 @@ try: from vllm import LLM, SamplingParams from vllm.lora.request import LoRARequest from vllm.transformers_utils.tokenizer import get_tokenizer + from vllm.utils import get_open_port if parse_version(version("vllm")) >= parse_version("0.8.3"): from vllm.entrypoints.chat_utils import resolve_hf_chat_template @@ -41,6 +47,63 @@ if TYPE_CHECKING: eval_logger = logging.getLogger(__name__) +def _vllm_mp_worker( + model_args: dict, + sampling_params: "SamplingParams", + requests: list[list[int]], + lora_request: "LoRARequest", + result_queue: "Queue", + dp_size: int, + local_dp_rank: int, + dp_master_port: int, + dp_master_ip: str = "127.0.0.1", +) -> None: + """ + Worker process for vLLM multiprocessing. + Initializes a vLLM engine, processes requests, and puts results or errors + onto the result_queue. + """ + + if not requests: + result_queue.put((local_dp_rank, [])) + return None + + os.environ["VLLM_DP_RANK"] = os.environ["VLLM_DP_RANK_LOCAL"] = str(local_dp_rank) + os.environ["VLLM_DP_SIZE"] = str(dp_size) + os.environ["VLLM_DP_MASTER_IP"] = str(dp_master_ip) + os.environ["VLLM_DP_MASTER_PORT"] = str(dp_master_port) + + llm = None + try: + llm = LLM(**model_args) + res = llm.generate( + prompt_token_ids=requests, + sampling_params=sampling_params, + lora_request=lora_request, + ) + # Give engines time to pause their processing loops before exiting." + sleep(1) + result_queue.put((local_dp_rank, res)) + + except Exception as e: + error_message = f"Worker {local_dp_rank} failed during generation: {type(e).__name__}: {str(e)}" + eval_logger.error(error_message, exc_info=True) + result_queue.put((local_dp_rank, {"error": error_message})) + + finally: + if llm is not None: + try: + del llm + gc.collect() + except Exception as e_cleanup: + eval_logger.warning( + f"Worker {local_dp_rank} encountered an error during LLM cleanup: {type(e_cleanup).__name__}: {str(e_cleanup)}", + exc_info=True, + ) + + return None + + @register_model("vllm") class VLLM(TemplateLM): _DEFAULT_MAX_LENGTH = 2048 @@ -83,7 +146,7 @@ class VLLM(TemplateLM): assert max_length is None or max_model_len is None, ( "Either max_length or max_model_len may be provided, but not both" ) - + self.V1 = os.environ.get("VLLM_USE_V1", "1") != "0" self._max_length = max_model_len if max_model_len is not None else max_length self.tensor_parallel_size = int(tensor_parallel_size) self.data_parallel_size = int(data_parallel_size) @@ -98,6 +161,7 @@ class VLLM(TemplateLM): "trust_remote_code": trust_remote_code, "tensor_parallel_size": int(tensor_parallel_size), "max_model_len": int(self._max_length) if self._max_length else None, + "max_num_seqs": kwargs.get("max_num_seqs", max_batch_size), "swap_space": int(swap_space), "quantization": quantization, "seed": int(seed), @@ -115,7 +179,11 @@ class VLLM(TemplateLM): eval_logger.warning( "You might experience occasional issues with model weight downloading when data_parallel is in use. To ensure stable performance, run with data_parallel_size=1 until the weights are downloaded and cached." ) - self.model_args["distributed_executor_backend"] = "ray" + self.model_args["distributed_executor_backend"] = ( + "ray" + if not self.V1 + else self.model_args.get("distributed_executor_backend", None) + ) self.batch_size = "auto" eval_logger.info("Manual batching is not compatible with data parallelism.") @@ -279,7 +347,7 @@ class VLLM(TemplateLM): sampling_params = SamplingParams( temperature=0, prompt_logprobs=1, max_tokens=1, detokenize=False ) - if self.data_parallel_size > 1: + if self.data_parallel_size > 1 and not self.V1: # vLLM hangs if resources are set in ray.remote # also seems to only work with decorator and not with ray.remote() fn # see https://github.com/vllm-project/vllm/issues/973 @@ -310,14 +378,83 @@ class VLLM(TemplateLM): ray.shutdown() # flatten results return undistribute(results) + elif self.data_parallel_size > 1: + # based on https://github.com/vllm-project/vllm/blob/a04720bc36401d831cb048c3917b9e58173d9c1d/examples/offline_inference/data_parallel.py + dp_size = self.data_parallel_size + dp_master_ip = os.environ.get("VLLM_DP_MASTER_IP", "127.0.0.1") + dp_master_port = os.environ.get("VLLM_DP_MASTER_PORT") or get_open_port() + + requests = (list(x) for x in distribute(self.data_parallel_size, requests)) + + procs, resq = [], Queue() + # We use Process as it is non-daemonic + try: + for rank, req in enumerate(requests): + proc = Process( + target=_vllm_mp_worker, + args=( + self.model_args.copy(), + sampling_params, + req, + self.lora_request, + resq, + dp_size, + rank, + dp_master_port, + dp_master_ip, + ), + ) + proc.start() + procs.append(proc) + + # Collect results + rank_res = {} + while len(rank_res) < len(procs): + try: + rank, result = resq.get(timeout=30) + if isinstance(result, dict) and "error" in result: + raise RuntimeError(result["error"]) + rank_res[rank] = result + except Empty: + dead_procs = [ + idx + for idx, p in enumerate(procs) + if not p.is_alive() and idx not in rank_res + ] + if dead_procs: + raise RuntimeError( + f"Worker processes {dead_procs} died unexpectedly" + ) + continue + + results = [rank_res[i] for i in range(len(procs))] + return undistribute(results) + + # cleanup + finally: + try: + resq.close() + resq.join_thread() + except Exception: + eval_logger.debug( + "Failed to close vllm DP results queue", exc_info=True + ) + for proc in procs: + proc.join(timeout=10) + if proc.is_alive(): + proc.terminate() + proc.join(timeout=5) + if proc.is_alive(): + proc.kill() - outputs = self.model.generate( - prompt_token_ids=requests, - sampling_params=sampling_params, - use_tqdm=True if self.batch_size == "auto" else False, - lora_request=self.lora_request, - ) - return outputs + else: + outputs = self.model.generate( + prompt_token_ids=requests, + sampling_params=sampling_params, + use_tqdm=True if self.batch_size == "auto" else False, + lora_request=self.lora_request, + ) + return outputs def loglikelihood_rolling( self, requests: List[Instance], disable_tqdm: bool = False @@ -507,8 +644,7 @@ class VLLM(TemplateLM): for cache_key, context_enc, continuation_enc in chunk: if ( full_length := len(context_enc + continuation_enc) - >= self.max_length - ): + ) > self.max_length: eval_logger.warning( f"Context length {full_length} exceeds max length ({self.max_length}). Truncating context." ) -- GitLab From 8bc4afff22e73995883de41018388428e39f8a92 Mon Sep 17 00:00:00 2001 From: Boda Sadallah Date: Mon, 26 May 2025 17:07:23 +0400 Subject: [PATCH 32/46] add arab_culture task (#3006) * add arab_culture tasks * add target_delimeter and remove debugging code --- lm_eval/tasks/README.md | 3 +- lm_eval/tasks/arab_culture/README.md | 70 ++++++++++ lm_eval/tasks/arab_culture/_arab_culture.yaml | 12 ++ .../arab_culture/_arab_culture_gulf.yaml | 10 ++ .../arab_culture/_arab_culture_levant.yaml | 10 ++ .../_arab_culture_nile_valley.yaml | 10 ++ .../_arab_culture_north_africa.yaml | 10 ++ .../_default_arab_culture_mcq_template_yaml | 19 +++ .../tasks/arab_culture/_generate_configs.py | 122 +++++++++++++++++ .../arab_culture/arab_culture_algeria.yaml | 5 + .../arab_culture/arab_culture_egypt.yaml | 5 + .../arab_culture/arab_culture_jordan.yaml | 5 + .../tasks/arab_culture/arab_culture_ksa.yaml | 5 + .../arab_culture/arab_culture_lebanon.yaml | 5 + .../arab_culture/arab_culture_libya.yaml | 5 + .../arab_culture/arab_culture_morocco.yaml | 5 + .../arab_culture/arab_culture_palestine.yaml | 5 + .../arab_culture/arab_culture_sudan.yaml | 5 + .../arab_culture/arab_culture_syria.yaml | 5 + .../arab_culture/arab_culture_tunisia.yaml | 5 + .../tasks/arab_culture/arab_culture_uae.yaml | 5 + .../arab_culture/arab_culture_yemen.yaml | 5 + lm_eval/tasks/arab_culture/prompts.py | 73 ++++++++++ lm_eval/tasks/arab_culture/utils_mcq.py | 112 ++++++++++++++++ .../tasks/arab_culture_completion/README.md | 70 ++++++++++ .../_arab_culture_completion.yaml | 12 ++ .../_arab_culture_completion_gulf.yaml | 10 ++ .../_arab_culture_completion_levant.yaml | 10 ++ .../_arab_culture_completion_nile_valley.yaml | 10 ++ ..._arab_culture_completion_north_africa.yaml | 10 ++ ...ault_arab_culture_completion_template_yaml | 19 +++ .../_generate_configs.py | 125 ++++++++++++++++++ .../arab_culture_completion_algeria.yaml | 5 + .../arab_culture_completion_egypt.yaml | 5 + .../arab_culture_completion_jordan.yaml | 5 + .../arab_culture_completion_ksa.yaml | 5 + .../arab_culture_completion_lebanon.yaml | 5 + .../arab_culture_completion_libya.yaml | 5 + .../arab_culture_completion_morocco.yaml | 5 + .../arab_culture_completion_palestine.yaml | 5 + .../arab_culture_completion_sudan.yaml | 5 + .../arab_culture_completion_syria.yaml | 5 + .../arab_culture_completion_tunisia.yaml | 5 + .../arab_culture_completion_uae.yaml | 5 + .../arab_culture_completion_yemen.yaml | 5 + .../tasks/arab_culture_completion/prompts.py | 56 ++++++++ .../utils_completion.py | 102 ++++++++++++++ 47 files changed, 1004 insertions(+), 1 deletion(-) create mode 100644 lm_eval/tasks/arab_culture/README.md create mode 100644 lm_eval/tasks/arab_culture/_arab_culture.yaml create mode 100644 lm_eval/tasks/arab_culture/_arab_culture_gulf.yaml create mode 100644 lm_eval/tasks/arab_culture/_arab_culture_levant.yaml create mode 100644 lm_eval/tasks/arab_culture/_arab_culture_nile_valley.yaml create mode 100644 lm_eval/tasks/arab_culture/_arab_culture_north_africa.yaml create mode 100644 lm_eval/tasks/arab_culture/_default_arab_culture_mcq_template_yaml create mode 100644 lm_eval/tasks/arab_culture/_generate_configs.py create mode 100644 lm_eval/tasks/arab_culture/arab_culture_algeria.yaml create mode 100644 lm_eval/tasks/arab_culture/arab_culture_egypt.yaml create mode 100644 lm_eval/tasks/arab_culture/arab_culture_jordan.yaml create mode 100644 lm_eval/tasks/arab_culture/arab_culture_ksa.yaml create mode 100644 lm_eval/tasks/arab_culture/arab_culture_lebanon.yaml create mode 100644 lm_eval/tasks/arab_culture/arab_culture_libya.yaml create mode 100644 lm_eval/tasks/arab_culture/arab_culture_morocco.yaml create mode 100644 lm_eval/tasks/arab_culture/arab_culture_palestine.yaml create mode 100644 lm_eval/tasks/arab_culture/arab_culture_sudan.yaml create mode 100644 lm_eval/tasks/arab_culture/arab_culture_syria.yaml create mode 100644 lm_eval/tasks/arab_culture/arab_culture_tunisia.yaml create mode 100644 lm_eval/tasks/arab_culture/arab_culture_uae.yaml create mode 100644 lm_eval/tasks/arab_culture/arab_culture_yemen.yaml create mode 100644 lm_eval/tasks/arab_culture/prompts.py create mode 100644 lm_eval/tasks/arab_culture/utils_mcq.py create mode 100644 lm_eval/tasks/arab_culture_completion/README.md create mode 100644 lm_eval/tasks/arab_culture_completion/_arab_culture_completion.yaml create mode 100644 lm_eval/tasks/arab_culture_completion/_arab_culture_completion_gulf.yaml create mode 100644 lm_eval/tasks/arab_culture_completion/_arab_culture_completion_levant.yaml create mode 100644 lm_eval/tasks/arab_culture_completion/_arab_culture_completion_nile_valley.yaml create mode 100644 lm_eval/tasks/arab_culture_completion/_arab_culture_completion_north_africa.yaml create mode 100644 lm_eval/tasks/arab_culture_completion/_default_arab_culture_completion_template_yaml create mode 100644 lm_eval/tasks/arab_culture_completion/_generate_configs.py create mode 100644 lm_eval/tasks/arab_culture_completion/arab_culture_completion_algeria.yaml create mode 100644 lm_eval/tasks/arab_culture_completion/arab_culture_completion_egypt.yaml create mode 100644 lm_eval/tasks/arab_culture_completion/arab_culture_completion_jordan.yaml create mode 100644 lm_eval/tasks/arab_culture_completion/arab_culture_completion_ksa.yaml create mode 100644 lm_eval/tasks/arab_culture_completion/arab_culture_completion_lebanon.yaml create mode 100644 lm_eval/tasks/arab_culture_completion/arab_culture_completion_libya.yaml create mode 100644 lm_eval/tasks/arab_culture_completion/arab_culture_completion_morocco.yaml create mode 100644 lm_eval/tasks/arab_culture_completion/arab_culture_completion_palestine.yaml create mode 100644 lm_eval/tasks/arab_culture_completion/arab_culture_completion_sudan.yaml create mode 100644 lm_eval/tasks/arab_culture_completion/arab_culture_completion_syria.yaml create mode 100644 lm_eval/tasks/arab_culture_completion/arab_culture_completion_tunisia.yaml create mode 100644 lm_eval/tasks/arab_culture_completion/arab_culture_completion_uae.yaml create mode 100644 lm_eval/tasks/arab_culture_completion/arab_culture_completion_yemen.yaml create mode 100644 lm_eval/tasks/arab_culture_completion/prompts.py create mode 100644 lm_eval/tasks/arab_culture_completion/utils_completion.py diff --git a/lm_eval/tasks/README.md b/lm_eval/tasks/README.md index 8578a8af..6f3ac175 100644 --- a/lm_eval/tasks/README.md +++ b/lm_eval/tasks/README.md @@ -16,7 +16,8 @@ | [arabic_leaderboard_complete](arabic_leaderboard_complete/README.md) | A full version of the tasks in the Open Arabic LLM Leaderboard, focusing on the evaluation of models that reflect the characteristics of Arabic language understanding and comprehension, culture, and heritage. Note that some of these tasks are machine-translated. | Arabic (Some MT) | | [arabic_leaderboard_light](arabic_leaderboard_light/README.md) | A light version of the tasks in the Open Arabic LLM Leaderboard (i.e., 10% samples of the test set in the original benchmarks), focusing on the evaluation of models that reflect the characteristics of Arabic language understanding and comprehension, culture, and heritage. Note that some of these tasks are machine-translated. | Arabic (Some MT) | | [arabicmmlu](arabicmmlu/README.md) | Localized Arabic version of MMLU with multiple-choice questions from 40 subjects. | Arabic | -| [AraDICE](aradice/README.md) | A collection of multiple tasks carefully designed to evaluate dialectal and cultural capabilities in large language models (LLMs). | Arabic | +| [ArabCulture](arab_culture/README.md) | Benchmark for evaluating modeles' commonsense cultural knowledge across different 13 different Arab Countries. | Arabic | +[AraDICE](aradice/README.md) | A collection of multiple tasks carefully designed to evaluate dialectal and cultural capabilities in large language models (LLMs). | Arabic | | [arc](arc/README.md) | Tasks involving complex reasoning over a diverse set of questions. | English | | [arithmetic](arithmetic/README.md) | Tasks involving numerical computations and arithmetic reasoning. | English | | [asdiv](asdiv/README.md) | Tasks involving arithmetic and mathematical reasoning challenges. | English | diff --git a/lm_eval/tasks/arab_culture/README.md b/lm_eval/tasks/arab_culture/README.md new file mode 100644 index 00000000..f8bc5a8c --- /dev/null +++ b/lm_eval/tasks/arab_culture/README.md @@ -0,0 +1,70 @@ +# Arab Culture + +### Paper + +Title: Commonsense Reasoning in Arab Culture + + +Abstract: https://arxiv.org/abs/2502.12788 + +Despite progress in Arabic large language models, such as Jais and AceGPT, their evaluation on commonsense reasoning has largely relied on machine-translated datasets, which lack cultural depth and may introduce Anglocentric biases. Commonsense reasoning is shaped by geographical and cultural contexts, and existing English datasets fail to capture the diversity of the Arab world. To address this, we introduce \datasetname, a commonsense reasoning dataset in Modern Standard Arabic (MSA), covering cultures of 13 countries across the Gulf, Levant, North Africa, and the Nile Valley. The dataset was built from scratch by engaging native speakers to write and validate culturally relevant questions for their respective countries. \datasetname spans 12 daily life domains with 54 fine-grained subtopics, reflecting various aspects of social norms, traditions, and everyday experiences. Zero-shot evaluations show that open-weight language models with up to 32B parameters struggle to comprehend diverse Arab cultures, with performance varying across regions. These findings highlight the need for more culturally aware models and datasets tailored to the Arabic-speaking world. + +Homepage: https://github.com/fajri91/ArabicCulture + + +### Citation + +``` +@misc{sadallah2025commonsensereasoningarabculture, + title={Commonsense Reasoning in Arab Culture}, + author={Abdelrahman Sadallah and Junior Cedric Tonga and Khalid Almubarak and Saeed Almheiri and Farah Atif and Chatrine Qwaider and Karima Kadaoui and Sara Shatnawi and Yaser Alesh and Fajri Koto}, + year={2025}, + eprint={2502.12788}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2502.12788}, +} +``` + +### There are two variant of this task: `arab_culture`, and `arab_culture_completion` + +- The `arab_culture` is the normal MCQ evaluation type, which appends the answers to the question, and then measure the likelihood of the different choices markers (A,B,C or "أ","ب","ج"). For more info, follow the MMLU style [tempelate](https://github.com/EleutherAI/lm-evaluation-harness/blob/main/lm_eval/tasks/mmlu/default/_default_template_yaml#L7-L8) +- The `arab_culture_completion` do the evaluation in a sentence completion manner, by appending each asnwer to the question separetley and chooses the answer with the higher likelihood. See [this](https://github.com/EleutherAI/lm-evaluation-harness/blob/1f9bc88fe61f6bfa36f74e91ce3d59ab5685e4f1/lm_eval/tasks/arc/arc_easy.yaml#L10-L12) for more information + +### Groups and Tasks + +#### Groups + +* `arabculture`: evaluates all ArabCulture tasks. + +* `arab_culture_gulf`: evaluates Gulf countires ArabCulture tasks. +* `arab_culture_levant`: evaluates Levant countires ArabCulture tasks. +* `arab_culture_nile_valley`: evaluates Nile Valley countires ArabCulture tasks. +* `arab_culture_north_africa`: evaluates North Africa ArabCulture tasks. + +### Evaluation modes +This bechmark allows for different evaluation settings by allowing to adding more extra context for the model: + +We have three settings: +* without any information +``` +COUNTRY=False +REGION=False +``` +* with only region information +``` +COUNTRY=False +REGION=True +``` +* with region and country information +``` +COUNTRY=True +REGION=True +``` + +**Please add these flags add environment variables.** + + +* We also allow for prompting in English, which we found to acheive higher results on most of the evaluated models (please refer to our paper). + +* To change the language of the prompt, Define the `ARABIC` environment variable. diff --git a/lm_eval/tasks/arab_culture/_arab_culture.yaml b/lm_eval/tasks/arab_culture/_arab_culture.yaml new file mode 100644 index 00000000..8da809e6 --- /dev/null +++ b/lm_eval/tasks/arab_culture/_arab_culture.yaml @@ -0,0 +1,12 @@ +aggregate_metric_list: + metric: acc + weight_by_size: true +group: arab_culture +metadata: + description: Arab Culture tasks + version: 0 +task: +- arab_culture_gulf +- arab_culture_levant +- arab_culture_north_africa +- arab_culture_nile_valley diff --git a/lm_eval/tasks/arab_culture/_arab_culture_gulf.yaml b/lm_eval/tasks/arab_culture/_arab_culture_gulf.yaml new file mode 100644 index 00000000..ca0ca4d8 --- /dev/null +++ b/lm_eval/tasks/arab_culture/_arab_culture_gulf.yaml @@ -0,0 +1,10 @@ +aggregate_metric_list: + metric: acc + weight_by_size: true +group: arab_culture_gulf +group_alias: Gulf +metadata: + description: arab Culture tasks + version: 0 +task: +- arab_culture_gulf_tasks diff --git a/lm_eval/tasks/arab_culture/_arab_culture_levant.yaml b/lm_eval/tasks/arab_culture/_arab_culture_levant.yaml new file mode 100644 index 00000000..b3344d37 --- /dev/null +++ b/lm_eval/tasks/arab_culture/_arab_culture_levant.yaml @@ -0,0 +1,10 @@ +aggregate_metric_list: + metric: acc + weight_by_size: true +group: arab_culture_levant +group_alias: Levant +metadata: + description: arab Culture tasks + version: 0 +task: +- arab_culture_levant_tasks diff --git a/lm_eval/tasks/arab_culture/_arab_culture_nile_valley.yaml b/lm_eval/tasks/arab_culture/_arab_culture_nile_valley.yaml new file mode 100644 index 00000000..e858409a --- /dev/null +++ b/lm_eval/tasks/arab_culture/_arab_culture_nile_valley.yaml @@ -0,0 +1,10 @@ +aggregate_metric_list: + metric: acc + weight_by_size: true +group: arab_culture_nile_valley +group_alias: Nile Valley +metadata: + description: arab Culture tasks + version: 0 +task: +- arab_culture_nile_valley_tasks diff --git a/lm_eval/tasks/arab_culture/_arab_culture_north_africa.yaml b/lm_eval/tasks/arab_culture/_arab_culture_north_africa.yaml new file mode 100644 index 00000000..30f31ffa --- /dev/null +++ b/lm_eval/tasks/arab_culture/_arab_culture_north_africa.yaml @@ -0,0 +1,10 @@ +aggregate_metric_list: + metric: acc + weight_by_size: true +group: arab_culture_north_africa +group_alias: North Africa +metadata: + description: arab Culture tasks + version: 0 +task: +- arab_culture_north_africa_tasks diff --git a/lm_eval/tasks/arab_culture/_default_arab_culture_mcq_template_yaml b/lm_eval/tasks/arab_culture/_default_arab_culture_mcq_template_yaml new file mode 100644 index 00000000..30b78fe6 --- /dev/null +++ b/lm_eval/tasks/arab_culture/_default_arab_culture_mcq_template_yaml @@ -0,0 +1,19 @@ +dataset_path: MBZUAI/ArabCulture +test_split: test +fewshot_split: test +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: !function utils_mcq.doc_to_text +doc_to_choice: !function utils_mcq.doc_to_choice +doc_to_target: !function utils_mcq.doc_to_target +target_delimiter: "" +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm_eval/tasks/arab_culture/_generate_configs.py b/lm_eval/tasks/arab_culture/_generate_configs.py new file mode 100644 index 00000000..81ee1c61 --- /dev/null +++ b/lm_eval/tasks/arab_culture/_generate_configs.py @@ -0,0 +1,122 @@ +""" +Take in a YAML, and output all "other" splits with this YAML +""" + +import argparse +import logging +import os + +import yaml +from tqdm import tqdm + + +eval_logger = logging.getLogger("lm-eval") + +countries = { + "KSA": "Gulf", + "UAE": "Gulf", + "Yemen": "Gulf", + "Lebanon": "Levant", + "Syria": "Levant", + "Palestine": "Levant", + "Jordan": "Levant", + "Tunisia": "North Africa", + "Algeria": "North Africa", + "Morocco": "North Africa", + "Libya": "North Africa", + "Egypt": "Nile Valley", + "Sudan": "Nile Valley", +} + +VERSION = 0 + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--base_yaml_path", default="_default_arab_culture_mcq_template_yaml" + ) + parser.add_argument("--save_prefix_path", default="arab_culture") + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + + # get filename of base_yaml so we can `"include": ` it in our "other" YAMLs. + base_yaml_name = os.path.split(args.base_yaml_path)[-1] + # with open(args.base_yaml_path, encoding="utf-8") as f: + # base_yaml = yaml.full_load(f) + + ALL_REGIONS = [] + for country, region in tqdm(countries.items()): + if region not in ALL_REGIONS: + ALL_REGIONS.append(region) + + # description = f"The following are multiple choice questions (with answers) about {' '.join(subject.split('_'))}.\n\n" + + yaml_dict = { + "include": base_yaml_name, + "tag": f"arab_culture_{region.lower().replace(' ', '_')}_tasks", + "task": f"arab_culture_{country.lower().replace(' ', '_')}", + "task_alias": country, + "dataset_name": country, + # "description": description, + } + + file_save_path = ( + args.save_prefix_path + + f"_{country.lower().replace(' ', '_').replace('(', '').replace(')', '')}.yaml" + ) + eval_logger.info(f"Saving yaml for subset {country} to {file_save_path}") + with open(file_save_path, "w", encoding="utf-8") as yaml_file: + yaml.dump( + yaml_dict, + yaml_file, + allow_unicode=True, + default_style='"', + ) + + arab_culture_mcq_regions = [ + f"arab_culture_{region.lower().replace(' ', '_')}" for region in ALL_REGIONS + ] + + file_save_path = args.save_prefix_path + ".yaml" + + eval_logger.info(f"Saving benchmark config to {file_save_path}") + + for region in ALL_REGIONS: + file_save_path = ( + args.save_prefix_path + f"_{region.lower().replace(' ', '_')}.yaml" + ) + eval_logger.info(f"Saving yaml for subset {region} to {file_save_path}") + with open("_" + file_save_path, "w", encoding="utf-8") as yaml_file: + yaml.dump( + { + "group": f"arab_culture_{region.lower().replace(' ', '_')}", + "group_alias": region, + "task": [f"arab_culture_{region.lower().replace(' ', '_')}_tasks"], + "aggregate_metric_list": {"metric": "acc", "weight_by_size": True}, + "metadata": { + "description": "arab Culture tasks", + "version": VERSION, + }, + }, + yaml_file, + indent=4, + default_flow_style=False, + ) + + file_save_path = args.save_prefix_path + ".yaml" + with open("_" + file_save_path, "w", encoding="utf-8") as yaml_file: + yaml.dump( + { + "group": "arab_culture", + "task": arab_culture_mcq_regions, + "aggregate_metric_list": {"metric": "acc", "weight_by_size": True}, + "metadata": {"description": "Arab Culture tasks", "version": VERSION}, + }, + yaml_file, + indent=4, + default_flow_style=False, + ) diff --git a/lm_eval/tasks/arab_culture/arab_culture_algeria.yaml b/lm_eval/tasks/arab_culture/arab_culture_algeria.yaml new file mode 100644 index 00000000..705606b8 --- /dev/null +++ b/lm_eval/tasks/arab_culture/arab_culture_algeria.yaml @@ -0,0 +1,5 @@ +"dataset_name": "Algeria" +"include": "_default_arab_culture_mcq_template_yaml" +"tag": "arab_culture_north_africa_tasks" +"task": "arab_culture_algeria" +"task_alias": "Algeria" diff --git a/lm_eval/tasks/arab_culture/arab_culture_egypt.yaml b/lm_eval/tasks/arab_culture/arab_culture_egypt.yaml new file mode 100644 index 00000000..f7186314 --- /dev/null +++ b/lm_eval/tasks/arab_culture/arab_culture_egypt.yaml @@ -0,0 +1,5 @@ +"dataset_name": "Egypt" +"include": "_default_arab_culture_mcq_template_yaml" +"tag": "arab_culture_nile_valley_tasks" +"task": "arab_culture_egypt" +"task_alias": "Egypt" diff --git a/lm_eval/tasks/arab_culture/arab_culture_jordan.yaml b/lm_eval/tasks/arab_culture/arab_culture_jordan.yaml new file mode 100644 index 00000000..c6587d29 --- /dev/null +++ b/lm_eval/tasks/arab_culture/arab_culture_jordan.yaml @@ -0,0 +1,5 @@ +"dataset_name": "Jordan" +"include": "_default_arab_culture_mcq_template_yaml" +"tag": "arab_culture_levant_tasks" +"task": "arab_culture_jordan" +"task_alias": "Jordan" diff --git a/lm_eval/tasks/arab_culture/arab_culture_ksa.yaml b/lm_eval/tasks/arab_culture/arab_culture_ksa.yaml new file mode 100644 index 00000000..07d87cb8 --- /dev/null +++ b/lm_eval/tasks/arab_culture/arab_culture_ksa.yaml @@ -0,0 +1,5 @@ +"dataset_name": "KSA" +"include": "_default_arab_culture_mcq_template_yaml" +"tag": "arab_culture_gulf_tasks" +"task": "arab_culture_ksa" +"task_alias": "KSA" diff --git a/lm_eval/tasks/arab_culture/arab_culture_lebanon.yaml b/lm_eval/tasks/arab_culture/arab_culture_lebanon.yaml new file mode 100644 index 00000000..41c2b53c --- /dev/null +++ b/lm_eval/tasks/arab_culture/arab_culture_lebanon.yaml @@ -0,0 +1,5 @@ +"dataset_name": "Lebanon" +"include": "_default_arab_culture_mcq_template_yaml" +"tag": "arab_culture_levant_tasks" +"task": "arab_culture_lebanon" +"task_alias": "Lebanon" diff --git a/lm_eval/tasks/arab_culture/arab_culture_libya.yaml b/lm_eval/tasks/arab_culture/arab_culture_libya.yaml new file mode 100644 index 00000000..e82c5598 --- /dev/null +++ b/lm_eval/tasks/arab_culture/arab_culture_libya.yaml @@ -0,0 +1,5 @@ +"dataset_name": "Libya" +"include": "_default_arab_culture_mcq_template_yaml" +"tag": "arab_culture_north_africa_tasks" +"task": "arab_culture_libya" +"task_alias": "Libya" diff --git a/lm_eval/tasks/arab_culture/arab_culture_morocco.yaml b/lm_eval/tasks/arab_culture/arab_culture_morocco.yaml new file mode 100644 index 00000000..847a86f5 --- /dev/null +++ b/lm_eval/tasks/arab_culture/arab_culture_morocco.yaml @@ -0,0 +1,5 @@ +"dataset_name": "Morocco" +"include": "_default_arab_culture_mcq_template_yaml" +"tag": "arab_culture_north_africa_tasks" +"task": "arab_culture_morocco" +"task_alias": "Morocco" diff --git a/lm_eval/tasks/arab_culture/arab_culture_palestine.yaml b/lm_eval/tasks/arab_culture/arab_culture_palestine.yaml new file mode 100644 index 00000000..dcbe183b --- /dev/null +++ b/lm_eval/tasks/arab_culture/arab_culture_palestine.yaml @@ -0,0 +1,5 @@ +"dataset_name": "Palestine" +"include": "_default_arab_culture_mcq_template_yaml" +"tag": "arab_culture_levant_tasks" +"task": "arab_culture_palestine" +"task_alias": "Palestine" diff --git a/lm_eval/tasks/arab_culture/arab_culture_sudan.yaml b/lm_eval/tasks/arab_culture/arab_culture_sudan.yaml new file mode 100644 index 00000000..9920d5ba --- /dev/null +++ b/lm_eval/tasks/arab_culture/arab_culture_sudan.yaml @@ -0,0 +1,5 @@ +"dataset_name": "Sudan" +"include": "_default_arab_culture_mcq_template_yaml" +"tag": "arab_culture_nile_valley_tasks" +"task": "arab_culture_sudan" +"task_alias": "Sudan" diff --git a/lm_eval/tasks/arab_culture/arab_culture_syria.yaml b/lm_eval/tasks/arab_culture/arab_culture_syria.yaml new file mode 100644 index 00000000..0ed6f767 --- /dev/null +++ b/lm_eval/tasks/arab_culture/arab_culture_syria.yaml @@ -0,0 +1,5 @@ +"dataset_name": "Syria" +"include": "_default_arab_culture_mcq_template_yaml" +"tag": "arab_culture_levant_tasks" +"task": "arab_culture_syria" +"task_alias": "Syria" diff --git a/lm_eval/tasks/arab_culture/arab_culture_tunisia.yaml b/lm_eval/tasks/arab_culture/arab_culture_tunisia.yaml new file mode 100644 index 00000000..de8d78a8 --- /dev/null +++ b/lm_eval/tasks/arab_culture/arab_culture_tunisia.yaml @@ -0,0 +1,5 @@ +"dataset_name": "Tunisia" +"include": "_default_arab_culture_mcq_template_yaml" +"tag": "arab_culture_north_africa_tasks" +"task": "arab_culture_tunisia" +"task_alias": "Tunisia" diff --git a/lm_eval/tasks/arab_culture/arab_culture_uae.yaml b/lm_eval/tasks/arab_culture/arab_culture_uae.yaml new file mode 100644 index 00000000..5bcbeb83 --- /dev/null +++ b/lm_eval/tasks/arab_culture/arab_culture_uae.yaml @@ -0,0 +1,5 @@ +"dataset_name": "UAE" +"include": "_default_arab_culture_mcq_template_yaml" +"tag": "arab_culture_gulf_tasks" +"task": "arab_culture_uae" +"task_alias": "UAE" diff --git a/lm_eval/tasks/arab_culture/arab_culture_yemen.yaml b/lm_eval/tasks/arab_culture/arab_culture_yemen.yaml new file mode 100644 index 00000000..caa32073 --- /dev/null +++ b/lm_eval/tasks/arab_culture/arab_culture_yemen.yaml @@ -0,0 +1,5 @@ +"dataset_name": "Yemen" +"include": "_default_arab_culture_mcq_template_yaml" +"tag": "arab_culture_gulf_tasks" +"task": "arab_culture_yemen" +"task_alias": "Yemen" diff --git a/lm_eval/tasks/arab_culture/prompts.py b/lm_eval/tasks/arab_culture/prompts.py new file mode 100644 index 00000000..983bd8b2 --- /dev/null +++ b/lm_eval/tasks/arab_culture/prompts.py @@ -0,0 +1,73 @@ +REGION_COUNTRY_PROMPT_AR = """ +مهمتك هي اختيار الخيار الأنسب ثقافياً بناءً على السياق المقدم أدناه. + +الموقع: {country}, {region} +الجملة: {first_statement} + +يرجى مراعاة الفروق الثقافية للموقع المحدد واختيار الإجابة الأكثر ملاءمة من الخيارات المتاحة. + +الخيارات: +{choices} +""" + +REGION_PROMPT_AR = """ +مهمتك هي اختيار الخيار الأنسب ثقافياً بناءً على السياق المقدم أدناه. + +الموقع: {region} +الجملة: {first_statement} + +يرجى مراعاة الفروق الثقافية للموقع المحدد واختيار الإجابة الأكثر ملاءمة من الخيارات المتاحة. + +الخيارات: +{choices} +""" + +BASE_PROMPT_AR = """ +مهمتك هي اختيار الخيار الأنسب ثقافياً بناءً على السياق المقدم أدناه. + +الجملة: {first_statement} + +يرجى مراعاة الفروق الثقافية واختيار الإجابة الأكثر ملاءمة من الخيارات المتاحة. + +الخيارات: +{choices} +""" + +REGION_COUNTRY_PROMPT = """ +You are tasked with selecting the most culturally appropriate option based on the context provided below. + +Location: {country}, {region} +Statement: {first_statement} + +Consider the cultural nuances of the specified location and choose the most suitable response from the options provided. + +Options: +{choices} +""" +REGION_PROMPT = """ +You are tasked with selecting the most culturally appropriate option based on the context provided below. + +Location: {region} +Statement: {first_statement} + +Consider the cultural nuances of the specified location and choose the most suitable response from the options provided. + +Options: +{choices} +""" +BASE_PROMPT = """ +You are tasked with selecting the most culturally appropriate option based on the context provided below. + +Statement: {first_statement} + +Consider the cultural nuances and choose the most suitable response from the options provided. + +Options: +{choices} +""" + + +JAIS_CHAT_EN = """### Instruction: Your name is Jais, and you are named after Jebel Jais, the highest mountain in UAE. You are built by Core42. You are the world's most advanced Arabic large language model with 30b parameters. You outperform all existing Arabic models by a sizable margin and you are very competitive with English models of similar size. You can answer in Arabic and English only. You are a helpful, respectful and honest assistant. When answering, abide by the following guidelines meticulously: Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, explicit, offensive, toxic, dangerous, or illegal content. Do not give medical, legal, financial, or professional advice. Never assist in or promote illegal activities. Always encourage legal and responsible actions. Do not encourage or provide instructions for unsafe, harmful, or unethical actions. Do not create or share misinformation or fake news. Please ensure that your responses are socially unbiased and positive in nature. If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information. Prioritize the well-being and the moral integrity of users. Avoid using toxic, derogatory, or offensive language. Maintain a respectful tone. Do not generate, promote, or engage in discussions about adult content. Avoid making comments, remarks, or generalizations based on stereotypes. Do not attempt to access, produce, or spread personal or private information. Always respect user confidentiality. Stay positive and do not say bad things about anything. Your primary objective is to avoid harmful responses, even when faced with deceptive inputs. Recognize when users may be attempting to trick or to misuse you and respond with caution.\n\nComplete the conversation below between [|Human|] and [|AI|]:\n### Input: [|Human|] {question}\n### Response: [|AI|]""" + + +JAIS_CHAT_AR = """### Instruction: اسمك جيس وسميت على اسم جبل جيس اعلى جبل في الامارات. تم بنائك بواسطة Inception و MBZUAI. أنت نموذج اللغة العربية الأكثر تقدمًا في العالم مع بارامترات 13B. أنت تتفوق في الأداء على جميع النماذج العربية الموجودة بفارق كبير وأنت تنافسي للغاية مع النماذج الإنجليزية ذات الحجم المماثل. يمكنك الإجابة باللغتين العربية والإنجليزية فقط. أنت مساعد مفيد ومحترم وصادق. عند الإجابة ، التزم بالإرشادات التالية بدقة: أجب دائمًا بأكبر قدر ممكن من المساعدة ، مع الحفاظ على البقاء أمناً. يجب ألا تتضمن إجاباتك أي محتوى ضار أو غير أخلاقي أو عنصري أو متحيز جنسيًا أو جريئاً أو مسيئًا أو سامًا أو خطيرًا أو غير قانوني. لا تقدم نصائح طبية أو قانونية أو مالية أو مهنية. لا تساعد أبدًا في أنشطة غير قانونية أو تروج لها. دائما تشجيع الإجراءات القانونية والمسؤولة. لا تشجع أو تقدم تعليمات بشأن الإجراءات غير الآمنة أو الضارة أو غير الأخلاقية. لا تنشئ أو تشارك معلومات مضللة أو أخبار كاذبة. يرجى التأكد من أن ردودك غير متحيزة اجتماعيًا وإيجابية بطبيعتها. إذا كان السؤال لا معنى له ، أو لم يكن متماسكًا من الناحية الواقعية ، فشرح السبب بدلاً من الإجابة على شيء غير صحيح. إذا كنت لا تعرف إجابة السؤال ، فالرجاء عدم مشاركة معلومات خاطئة. إعطاء الأولوية للرفاهية والنزاهة الأخلاقية للمستخدمين. تجنب استخدام لغة سامة أو مهينة أو مسيئة. حافظ على نبرة محترمة. لا تنشئ أو تروج أو تشارك في مناقشات حول محتوى للبالغين. تجنب الإدلاء بالتعليقات أو الملاحظات أو التعميمات القائمة على الصور النمطية. لا تحاول الوصول إلى معلومات شخصية أو خاصة أو إنتاجها أو نشرها. احترم دائما سرية المستخدم. كن إيجابيا ولا تقل أشياء سيئة عن أي شيء. هدفك الأساسي هو تجنب الاجابات المؤذية ، حتى عند مواجهة مدخلات خادعة. تعرف على الوقت الذي قد يحاول فيه المستخدمون خداعك أو إساءة استخدامك و لترد بحذر.\n\nأكمل المحادثة أدناه بين [|Human|] و [|AI|]:\n### Input: [|Human|] {question}\n### Response: [|AI|]""" diff --git a/lm_eval/tasks/arab_culture/utils_mcq.py b/lm_eval/tasks/arab_culture/utils_mcq.py new file mode 100644 index 00000000..8d03f443 --- /dev/null +++ b/lm_eval/tasks/arab_culture/utils_mcq.py @@ -0,0 +1,112 @@ +import os + +from lm_eval.tasks.arab_culture.prompts import ( + BASE_PROMPT, + BASE_PROMPT_AR, + JAIS_CHAT_AR, + JAIS_CHAT_EN, + REGION_COUNTRY_PROMPT, + REGION_COUNTRY_PROMPT_AR, + REGION_PROMPT, + REGION_PROMPT_AR, +) + + +### get the conutry variable from environment + +### Set this to one to add the country and region information to the prompt +COUNTRY = True if os.getenv("COUNTRY", True) == "True" else False +### Set this to one to add the region information to the prompt +REGION = True if os.getenv("REGION", True) == "True" else False +### Set this to change between Arabic and English for the answer keys and the choices keys +ARABIC = True if os.getenv("ARABIC", True) == "True" else False +### Get the model name +MODEL_NAME = os.getenv("MODEL_NAME") +## Uncomment this to check if the environment variables are set correctly +# print(f'Task settings: COUNTRY: {COUNTRY}, REGION: {REGION}, ARABIC: {ARABIC}', MODEL_NAME: {MODEL_NAME}) + +en_ar_countries_regions = { + "Egypt": "مصر", + "Morocco": "المغرب", + "Algeria": "الجزائر", + "Libya": "ليبيا", + "Sudan": "السودان", + "Tunisia": "تونس", + "Jordan": "الأردن", + "Lebanon": "لبنان", + "Syria": "سوريا", + "Palestine": "فلسطين", + "Yemen": "اليمن", + "UAE": "الإمارات", + "KSA": "السعودية", + "Gulf": "الخليج", + "Levant": "الشام", + "North Africa": "شمال أفريقيا", + "Nile Valley": "وادي النيل", +} + + +def doc_to_text(doc): + country = "" if not doc["country"] else doc["country"] + region = "" if not doc["region"] else doc["region"] + first_statement = doc["first_statement"].strip() + + ## We don't have a setting for only information about the country without the region + if COUNTRY: + assert REGION, ( + "If you want to add the country information, you must also add the region information" + ) + + ## convert contry and region name to arabic if the language is arabic + if ARABIC: + country = en_ar_countries_regions[country] + region = en_ar_countries_regions[region] + + choices = doc["options"] + choices_str = "" + for i in range(3): + key = choices["arabic_keys"][i] if ARABIC else choices["english_keys"][i] + choice_str = key + ". " + choices["text"][i].strip() + "\n" + choices_str += choice_str + + if COUNTRY and REGION: + cur_prompt = REGION_COUNTRY_PROMPT_AR if ARABIC else REGION_COUNTRY_PROMPT + doc_text = cur_prompt.format( + country=country, + region=region, + first_statement=first_statement, + choices=choices_str, + ) + elif REGION: + cur_prompt = REGION_PROMPT_AR if ARABIC else REGION_PROMPT + doc_text = cur_prompt.format( + region=region, first_statement=first_statement, choices=choices_str + ) + else: + cur_prompt = BASE_PROMPT_AR if ARABIC else BASE_PROMPT + doc_text = cur_prompt.format( + first_statement=first_statement, choices=choices_str + ) + + ### apply jais chat tempelate + if MODEL_NAME and "jais" in MODEL_NAME and "chat" in MODEL_NAME: + if ARABIC: + doc_text = JAIS_CHAT_AR.format(question=doc_text) + else: + doc_text = JAIS_CHAT_EN.format(question=doc_text) + + return doc_text + + +def doc_to_choice(doc): + return doc["options"]["arabic_keys"] if ARABIC else doc["options"]["english_keys"] + + +def doc_to_target(doc): + ans = ( + doc["answer_key"]["arabic_answer_key"] + if ARABIC + else doc["answer_key"]["english_answer_key"] + ) + ans = ans.strip() + return ans diff --git a/lm_eval/tasks/arab_culture_completion/README.md b/lm_eval/tasks/arab_culture_completion/README.md new file mode 100644 index 00000000..f8bc5a8c --- /dev/null +++ b/lm_eval/tasks/arab_culture_completion/README.md @@ -0,0 +1,70 @@ +# Arab Culture + +### Paper + +Title: Commonsense Reasoning in Arab Culture + + +Abstract: https://arxiv.org/abs/2502.12788 + +Despite progress in Arabic large language models, such as Jais and AceGPT, their evaluation on commonsense reasoning has largely relied on machine-translated datasets, which lack cultural depth and may introduce Anglocentric biases. Commonsense reasoning is shaped by geographical and cultural contexts, and existing English datasets fail to capture the diversity of the Arab world. To address this, we introduce \datasetname, a commonsense reasoning dataset in Modern Standard Arabic (MSA), covering cultures of 13 countries across the Gulf, Levant, North Africa, and the Nile Valley. The dataset was built from scratch by engaging native speakers to write and validate culturally relevant questions for their respective countries. \datasetname spans 12 daily life domains with 54 fine-grained subtopics, reflecting various aspects of social norms, traditions, and everyday experiences. Zero-shot evaluations show that open-weight language models with up to 32B parameters struggle to comprehend diverse Arab cultures, with performance varying across regions. These findings highlight the need for more culturally aware models and datasets tailored to the Arabic-speaking world. + +Homepage: https://github.com/fajri91/ArabicCulture + + +### Citation + +``` +@misc{sadallah2025commonsensereasoningarabculture, + title={Commonsense Reasoning in Arab Culture}, + author={Abdelrahman Sadallah and Junior Cedric Tonga and Khalid Almubarak and Saeed Almheiri and Farah Atif and Chatrine Qwaider and Karima Kadaoui and Sara Shatnawi and Yaser Alesh and Fajri Koto}, + year={2025}, + eprint={2502.12788}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2502.12788}, +} +``` + +### There are two variant of this task: `arab_culture`, and `arab_culture_completion` + +- The `arab_culture` is the normal MCQ evaluation type, which appends the answers to the question, and then measure the likelihood of the different choices markers (A,B,C or "أ","ب","ج"). For more info, follow the MMLU style [tempelate](https://github.com/EleutherAI/lm-evaluation-harness/blob/main/lm_eval/tasks/mmlu/default/_default_template_yaml#L7-L8) +- The `arab_culture_completion` do the evaluation in a sentence completion manner, by appending each asnwer to the question separetley and chooses the answer with the higher likelihood. See [this](https://github.com/EleutherAI/lm-evaluation-harness/blob/1f9bc88fe61f6bfa36f74e91ce3d59ab5685e4f1/lm_eval/tasks/arc/arc_easy.yaml#L10-L12) for more information + +### Groups and Tasks + +#### Groups + +* `arabculture`: evaluates all ArabCulture tasks. + +* `arab_culture_gulf`: evaluates Gulf countires ArabCulture tasks. +* `arab_culture_levant`: evaluates Levant countires ArabCulture tasks. +* `arab_culture_nile_valley`: evaluates Nile Valley countires ArabCulture tasks. +* `arab_culture_north_africa`: evaluates North Africa ArabCulture tasks. + +### Evaluation modes +This bechmark allows for different evaluation settings by allowing to adding more extra context for the model: + +We have three settings: +* without any information +``` +COUNTRY=False +REGION=False +``` +* with only region information +``` +COUNTRY=False +REGION=True +``` +* with region and country information +``` +COUNTRY=True +REGION=True +``` + +**Please add these flags add environment variables.** + + +* We also allow for prompting in English, which we found to acheive higher results on most of the evaluated models (please refer to our paper). + +* To change the language of the prompt, Define the `ARABIC` environment variable. diff --git a/lm_eval/tasks/arab_culture_completion/_arab_culture_completion.yaml b/lm_eval/tasks/arab_culture_completion/_arab_culture_completion.yaml new file mode 100644 index 00000000..814f366e --- /dev/null +++ b/lm_eval/tasks/arab_culture_completion/_arab_culture_completion.yaml @@ -0,0 +1,12 @@ +aggregate_metric_list: + metric: acc + weight_by_size: true +group: arab_culture_completion +metadata: + description: Arab Culture tasks + version: 0 +task: +- arab_culture_completion_gulf +- arab_culture_completion_levant +- arab_culture_completion_north_africa +- arab_culture_completion_nile_valley diff --git a/lm_eval/tasks/arab_culture_completion/_arab_culture_completion_gulf.yaml b/lm_eval/tasks/arab_culture_completion/_arab_culture_completion_gulf.yaml new file mode 100644 index 00000000..b342b428 --- /dev/null +++ b/lm_eval/tasks/arab_culture_completion/_arab_culture_completion_gulf.yaml @@ -0,0 +1,10 @@ +aggregate_metric_list: + metric: acc + weight_by_size: true +group: arab_culture_completion_gulf +group_alias: Gulf +metadata: + description: arab Culture tasks + version: 0 +task: +- arab_culture_completion_gulf_tasks diff --git a/lm_eval/tasks/arab_culture_completion/_arab_culture_completion_levant.yaml b/lm_eval/tasks/arab_culture_completion/_arab_culture_completion_levant.yaml new file mode 100644 index 00000000..199f68dc --- /dev/null +++ b/lm_eval/tasks/arab_culture_completion/_arab_culture_completion_levant.yaml @@ -0,0 +1,10 @@ +aggregate_metric_list: + metric: acc + weight_by_size: true +group: arab_culture_completion_levant +group_alias: Levant +metadata: + description: arab Culture tasks + version: 0 +task: +- arab_culture_completion_levant_tasks diff --git a/lm_eval/tasks/arab_culture_completion/_arab_culture_completion_nile_valley.yaml b/lm_eval/tasks/arab_culture_completion/_arab_culture_completion_nile_valley.yaml new file mode 100644 index 00000000..284711ee --- /dev/null +++ b/lm_eval/tasks/arab_culture_completion/_arab_culture_completion_nile_valley.yaml @@ -0,0 +1,10 @@ +aggregate_metric_list: + metric: acc + weight_by_size: true +group: arab_culture_completion_nile_valley +group_alias: Nile Valley +metadata: + description: arab Culture tasks + version: 0 +task: +- arab_culture_completion_nile_valley_tasks diff --git a/lm_eval/tasks/arab_culture_completion/_arab_culture_completion_north_africa.yaml b/lm_eval/tasks/arab_culture_completion/_arab_culture_completion_north_africa.yaml new file mode 100644 index 00000000..10c32d9f --- /dev/null +++ b/lm_eval/tasks/arab_culture_completion/_arab_culture_completion_north_africa.yaml @@ -0,0 +1,10 @@ +aggregate_metric_list: + metric: acc + weight_by_size: true +group: arab_culture_completion_north_africa +group_alias: North Africa +metadata: + description: arab Culture tasks + version: 0 +task: +- arab_culture_completion_north_africa_tasks diff --git a/lm_eval/tasks/arab_culture_completion/_default_arab_culture_completion_template_yaml b/lm_eval/tasks/arab_culture_completion/_default_arab_culture_completion_template_yaml new file mode 100644 index 00000000..6d5961ac --- /dev/null +++ b/lm_eval/tasks/arab_culture_completion/_default_arab_culture_completion_template_yaml @@ -0,0 +1,19 @@ +dataset_path: boda/arabic_cluture +test_split: test +fewshot_split: test +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: !function utils_completion.doc_to_text +doc_to_choice: !function utils_completion.doc_to_choice +doc_to_target: !function utils_completion.doc_to_target +target_delimiter: "" +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm_eval/tasks/arab_culture_completion/_generate_configs.py b/lm_eval/tasks/arab_culture_completion/_generate_configs.py new file mode 100644 index 00000000..d5c530f5 --- /dev/null +++ b/lm_eval/tasks/arab_culture_completion/_generate_configs.py @@ -0,0 +1,125 @@ +""" +Take in a YAML, and output all "other" splits with this YAML +""" + +import argparse +import logging +import os + +import yaml +from tqdm import tqdm + + +eval_logger = logging.getLogger("lm-eval") + +countries = { + "KSA": "Gulf", + "UAE": "Gulf", + "Yemen": "Gulf", + "Lebanon": "Levant", + "Syria": "Levant", + "Palestine": "Levant", + "Jordan": "Levant", + "Tunisia": "North Africa", + "Algeria": "North Africa", + "Morocco": "North Africa", + "Libya": "North Africa", + "Egypt": "Nile Valley", + "Sudan": "Nile Valley", +} + +VERSION = 0 + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--base_yaml_path", default="_default_arab_culture_completion_template_yaml" + ) + parser.add_argument("--save_prefix_path", default="arab_culture_completion") + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + + # get filename of base_yaml so we can `"include": ` it in our "other" YAMLs. + base_yaml_name = os.path.split(args.base_yaml_path)[-1] + # with open(args.base_yaml_path, encoding="utf-8") as f: + # base_yaml = yaml.full_load(f) + + ALL_REGIONS = [] + for country, region in tqdm(countries.items()): + if region not in ALL_REGIONS: + ALL_REGIONS.append(region) + + # description = f"The following are multiple choice questions (with answers) about {' '.join(subject.split('_'))}.\n\n" + + yaml_dict = { + "include": base_yaml_name, + "tag": f"arab_culture_completion_{region.lower().replace(' ', '_')}_tasks", + "task": f"arab_culture_completion_{country.lower().replace(' ', '_')}", + "task_alias": country, + "dataset_name": country, + # "description": description, + } + + file_save_path = ( + args.save_prefix_path + + f"_{country.lower().replace(' ', '_').replace('(', '').replace(')', '')}.yaml" + ) + eval_logger.info(f"Saving yaml for subset {country} to {file_save_path}") + with open(file_save_path, "w", encoding="utf-8") as yaml_file: + yaml.dump( + yaml_dict, + yaml_file, + allow_unicode=True, + default_style='"', + ) + + arab_culture_completion_regions = [ + f"arab_culture_completion_{region.lower().replace(' ', '_')}" + for region in ALL_REGIONS + ] + + file_save_path = args.save_prefix_path + ".yaml" + + eval_logger.info(f"Saving benchmark config to {file_save_path}") + + for region in ALL_REGIONS: + file_save_path = ( + args.save_prefix_path + f"_{region.lower().replace(' ', '_')}.yaml" + ) + eval_logger.info(f"Saving yaml for subset {region} to {file_save_path}") + with open("_" + file_save_path, "w", encoding="utf-8") as yaml_file: + yaml.dump( + { + "group": f"arab_culture_completion_{region.lower().replace(' ', '_')}", + "group_alias": region, + "task": [ + f"arab_culture_completion_{region.lower().replace(' ', '_')}_tasks" + ], + "aggregate_metric_list": {"metric": "acc", "weight_by_size": True}, + "metadata": { + "description": "arab Culture tasks", + "version": VERSION, + }, + }, + yaml_file, + indent=4, + default_flow_style=False, + ) + + file_save_path = args.save_prefix_path + ".yaml" + with open("_" + file_save_path, "w", encoding="utf-8") as yaml_file: + yaml.dump( + { + "group": "arab_culture_completion", + "task": arab_culture_completion_regions, + "aggregate_metric_list": {"metric": "acc", "weight_by_size": True}, + "metadata": {"description": "Arab Culture tasks", "version": VERSION}, + }, + yaml_file, + indent=4, + default_flow_style=False, + ) diff --git a/lm_eval/tasks/arab_culture_completion/arab_culture_completion_algeria.yaml b/lm_eval/tasks/arab_culture_completion/arab_culture_completion_algeria.yaml new file mode 100644 index 00000000..e3cb7d8b --- /dev/null +++ b/lm_eval/tasks/arab_culture_completion/arab_culture_completion_algeria.yaml @@ -0,0 +1,5 @@ +"dataset_name": "Algeria" +"include": "_default_arab_culture_completion_template_yaml" +"tag": "arab_culture_completion_north_africa_tasks" +"task": "arab_culture_completion_algeria" +"task_alias": "Algeria" diff --git a/lm_eval/tasks/arab_culture_completion/arab_culture_completion_egypt.yaml b/lm_eval/tasks/arab_culture_completion/arab_culture_completion_egypt.yaml new file mode 100644 index 00000000..f740d4b8 --- /dev/null +++ b/lm_eval/tasks/arab_culture_completion/arab_culture_completion_egypt.yaml @@ -0,0 +1,5 @@ +"dataset_name": "Egypt" +"include": "_default_arab_culture_completion_template_yaml" +"tag": "arab_culture_completion_nile_valley_tasks" +"task": "arab_culture_completion_egypt" +"task_alias": "Egypt" diff --git a/lm_eval/tasks/arab_culture_completion/arab_culture_completion_jordan.yaml b/lm_eval/tasks/arab_culture_completion/arab_culture_completion_jordan.yaml new file mode 100644 index 00000000..dec15211 --- /dev/null +++ b/lm_eval/tasks/arab_culture_completion/arab_culture_completion_jordan.yaml @@ -0,0 +1,5 @@ +"dataset_name": "Jordan" +"include": "_default_arab_culture_completion_template_yaml" +"tag": "arab_culture_completion_levant_tasks" +"task": "arab_culture_completion_jordan" +"task_alias": "Jordan" diff --git a/lm_eval/tasks/arab_culture_completion/arab_culture_completion_ksa.yaml b/lm_eval/tasks/arab_culture_completion/arab_culture_completion_ksa.yaml new file mode 100644 index 00000000..ec1ea890 --- /dev/null +++ b/lm_eval/tasks/arab_culture_completion/arab_culture_completion_ksa.yaml @@ -0,0 +1,5 @@ +"dataset_name": "KSA" +"include": "_default_arab_culture_completion_template_yaml" +"tag": "arab_culture_completion_gulf_tasks" +"task": "arab_culture_completion_ksa" +"task_alias": "KSA" diff --git a/lm_eval/tasks/arab_culture_completion/arab_culture_completion_lebanon.yaml b/lm_eval/tasks/arab_culture_completion/arab_culture_completion_lebanon.yaml new file mode 100644 index 00000000..5f31061f --- /dev/null +++ b/lm_eval/tasks/arab_culture_completion/arab_culture_completion_lebanon.yaml @@ -0,0 +1,5 @@ +"dataset_name": "Lebanon" +"include": "_default_arab_culture_completion_template_yaml" +"tag": "arab_culture_completion_levant_tasks" +"task": "arab_culture_completion_lebanon" +"task_alias": "Lebanon" diff --git a/lm_eval/tasks/arab_culture_completion/arab_culture_completion_libya.yaml b/lm_eval/tasks/arab_culture_completion/arab_culture_completion_libya.yaml new file mode 100644 index 00000000..2541f87c --- /dev/null +++ b/lm_eval/tasks/arab_culture_completion/arab_culture_completion_libya.yaml @@ -0,0 +1,5 @@ +"dataset_name": "Libya" +"include": "_default_arab_culture_completion_template_yaml" +"tag": "arab_culture_completion_north_africa_tasks" +"task": "arab_culture_completion_libya" +"task_alias": "Libya" diff --git a/lm_eval/tasks/arab_culture_completion/arab_culture_completion_morocco.yaml b/lm_eval/tasks/arab_culture_completion/arab_culture_completion_morocco.yaml new file mode 100644 index 00000000..86e1cc65 --- /dev/null +++ b/lm_eval/tasks/arab_culture_completion/arab_culture_completion_morocco.yaml @@ -0,0 +1,5 @@ +"dataset_name": "Morocco" +"include": "_default_arab_culture_completion_template_yaml" +"tag": "arab_culture_completion_north_africa_tasks" +"task": "arab_culture_completion_morocco" +"task_alias": "Morocco" diff --git a/lm_eval/tasks/arab_culture_completion/arab_culture_completion_palestine.yaml b/lm_eval/tasks/arab_culture_completion/arab_culture_completion_palestine.yaml new file mode 100644 index 00000000..44731f7b --- /dev/null +++ b/lm_eval/tasks/arab_culture_completion/arab_culture_completion_palestine.yaml @@ -0,0 +1,5 @@ +"dataset_name": "Palestine" +"include": "_default_arab_culture_completion_template_yaml" +"tag": "arab_culture_completion_levant_tasks" +"task": "arab_culture_completion_palestine" +"task_alias": "Palestine" diff --git a/lm_eval/tasks/arab_culture_completion/arab_culture_completion_sudan.yaml b/lm_eval/tasks/arab_culture_completion/arab_culture_completion_sudan.yaml new file mode 100644 index 00000000..76282e98 --- /dev/null +++ b/lm_eval/tasks/arab_culture_completion/arab_culture_completion_sudan.yaml @@ -0,0 +1,5 @@ +"dataset_name": "Sudan" +"include": "_default_arab_culture_completion_template_yaml" +"tag": "arab_culture_completion_nile_valley_tasks" +"task": "arab_culture_completion_sudan" +"task_alias": "Sudan" diff --git a/lm_eval/tasks/arab_culture_completion/arab_culture_completion_syria.yaml b/lm_eval/tasks/arab_culture_completion/arab_culture_completion_syria.yaml new file mode 100644 index 00000000..33a07dd9 --- /dev/null +++ b/lm_eval/tasks/arab_culture_completion/arab_culture_completion_syria.yaml @@ -0,0 +1,5 @@ +"dataset_name": "Syria" +"include": "_default_arab_culture_completion_template_yaml" +"tag": "arab_culture_completion_levant_tasks" +"task": "arab_culture_completion_syria" +"task_alias": "Syria" diff --git a/lm_eval/tasks/arab_culture_completion/arab_culture_completion_tunisia.yaml b/lm_eval/tasks/arab_culture_completion/arab_culture_completion_tunisia.yaml new file mode 100644 index 00000000..41c53529 --- /dev/null +++ b/lm_eval/tasks/arab_culture_completion/arab_culture_completion_tunisia.yaml @@ -0,0 +1,5 @@ +"dataset_name": "Tunisia" +"include": "_default_arab_culture_completion_template_yaml" +"tag": "arab_culture_completion_north_africa_tasks" +"task": "arab_culture_completion_tunisia" +"task_alias": "Tunisia" diff --git a/lm_eval/tasks/arab_culture_completion/arab_culture_completion_uae.yaml b/lm_eval/tasks/arab_culture_completion/arab_culture_completion_uae.yaml new file mode 100644 index 00000000..47944053 --- /dev/null +++ b/lm_eval/tasks/arab_culture_completion/arab_culture_completion_uae.yaml @@ -0,0 +1,5 @@ +"dataset_name": "UAE" +"include": "_default_arab_culture_completion_template_yaml" +"tag": "arab_culture_completion_gulf_tasks" +"task": "arab_culture_completion_uae" +"task_alias": "UAE" diff --git a/lm_eval/tasks/arab_culture_completion/arab_culture_completion_yemen.yaml b/lm_eval/tasks/arab_culture_completion/arab_culture_completion_yemen.yaml new file mode 100644 index 00000000..88dd6dd5 --- /dev/null +++ b/lm_eval/tasks/arab_culture_completion/arab_culture_completion_yemen.yaml @@ -0,0 +1,5 @@ +"dataset_name": "Yemen" +"include": "_default_arab_culture_completion_template_yaml" +"tag": "arab_culture_completion_gulf_tasks" +"task": "arab_culture_completion_yemen" +"task_alias": "Yemen" diff --git a/lm_eval/tasks/arab_culture_completion/prompts.py b/lm_eval/tasks/arab_culture_completion/prompts.py new file mode 100644 index 00000000..b042c056 --- /dev/null +++ b/lm_eval/tasks/arab_culture_completion/prompts.py @@ -0,0 +1,56 @@ +REGION_COUNTRY_PROMPT_AR = """ +مهمتك هي اختيار الخيار الأنسب ثقافياً بناءً على السياق المقدم أدناه. + +الموقع: {country}, {region} +الجملة: {first_statement} + +يرجى مراعاة الفروق الثقافية للموقع المحدد واختيار الإجابة الأكثر ملاءمة من الخيارات المتاحة. +""" + +REGION_PROMPT_AR = """ +مهمتك هي اختيار الخيار الأنسب ثقافياً بناءً على السياق المقدم أدناه. + +الموقع: {region} +الجملة: {first_statement} + +يرجى مراعاة الفروق الثقافية للموقع المحدد واختيار الإجابة الأكثر ملاءمة من الخيارات المتاحة. +""" + +BASE_PROMPT_AR = """ +مهمتك هي اختيار الخيار الأنسب ثقافياً بناءً على السياق المقدم أدناه. + +الجملة: {first_statement} + +يرجى مراعاة الفروق الثقافية واختيار الإجابة الأكثر ملاءمة من الخيارات المتاحة. +""" + + +REGION_COUNTRY_PROMPT = """ +You are tasked with selecting the most culturally appropriate option based on the context provided below. + +Location: {country}, {region} +Statement: {first_statement} + +Consider the cultural nuances of the specified location and choose the most suitable response from the options provided. +""" +REGION_PROMPT = """ +You are tasked with selecting the most culturally appropriate option based on the context provided below. + +Location: {region} +Statement: {first_statement} + +Consider the cultural nuances of the specified location and choose the most suitable response from the options provided. +""" +BASE_PROMPT = """ +You are tasked with selecting the most culturally appropriate option based on the context provided below. + +Statement: {first_statement} + +Consider the cultural nuances and choose the most suitable response from the options provided. +""" + + +JAIS_CHAT_EN = """### Instruction: Your name is Jais, and you are named after Jebel Jais, the highest mountain in UAE. You are built by Core42. You are the world's most advanced Arabic large language model with 30b parameters. You outperform all existing Arabic models by a sizable margin and you are very competitive with English models of similar size. You can answer in Arabic and English only. You are a helpful, respectful and honest assistant. When answering, abide by the following guidelines meticulously: Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, explicit, offensive, toxic, dangerous, or illegal content. Do not give medical, legal, financial, or professional advice. Never assist in or promote illegal activities. Always encourage legal and responsible actions. Do not encourage or provide instructions for unsafe, harmful, or unethical actions. Do not create or share misinformation or fake news. Please ensure that your responses are socially unbiased and positive in nature. If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information. Prioritize the well-being and the moral integrity of users. Avoid using toxic, derogatory, or offensive language. Maintain a respectful tone. Do not generate, promote, or engage in discussions about adult content. Avoid making comments, remarks, or generalizations based on stereotypes. Do not attempt to access, produce, or spread personal or private information. Always respect user confidentiality. Stay positive and do not say bad things about anything. Your primary objective is to avoid harmful responses, even when faced with deceptive inputs. Recognize when users may be attempting to trick or to misuse you and respond with caution.\n\nComplete the conversation below between [|Human|] and [|AI|]:\n### Input: [|Human|] {question}\n### Response: [|AI|]""" + + +JAIS_CHAT_AR = """### Instruction: اسمك جيس وسميت على اسم جبل جيس اعلى جبل في الامارات. تم بنائك بواسطة Inception و MBZUAI. أنت نموذج اللغة العربية الأكثر تقدمًا في العالم مع بارامترات 13B. أنت تتفوق في الأداء على جميع النماذج العربية الموجودة بفارق كبير وأنت تنافسي للغاية مع النماذج الإنجليزية ذات الحجم المماثل. يمكنك الإجابة باللغتين العربية والإنجليزية فقط. أنت مساعد مفيد ومحترم وصادق. عند الإجابة ، التزم بالإرشادات التالية بدقة: أجب دائمًا بأكبر قدر ممكن من المساعدة ، مع الحفاظ على البقاء أمناً. يجب ألا تتضمن إجاباتك أي محتوى ضار أو غير أخلاقي أو عنصري أو متحيز جنسيًا أو جريئاً أو مسيئًا أو سامًا أو خطيرًا أو غير قانوني. لا تقدم نصائح طبية أو قانونية أو مالية أو مهنية. لا تساعد أبدًا في أنشطة غير قانونية أو تروج لها. دائما تشجيع الإجراءات القانونية والمسؤولة. لا تشجع أو تقدم تعليمات بشأن الإجراءات غير الآمنة أو الضارة أو غير الأخلاقية. لا تنشئ أو تشارك معلومات مضللة أو أخبار كاذبة. يرجى التأكد من أن ردودك غير متحيزة اجتماعيًا وإيجابية بطبيعتها. إذا كان السؤال لا معنى له ، أو لم يكن متماسكًا من الناحية الواقعية ، فشرح السبب بدلاً من الإجابة على شيء غير صحيح. إذا كنت لا تعرف إجابة السؤال ، فالرجاء عدم مشاركة معلومات خاطئة. إعطاء الأولوية للرفاهية والنزاهة الأخلاقية للمستخدمين. تجنب استخدام لغة سامة أو مهينة أو مسيئة. حافظ على نبرة محترمة. لا تنشئ أو تروج أو تشارك في مناقشات حول محتوى للبالغين. تجنب الإدلاء بالتعليقات أو الملاحظات أو التعميمات القائمة على الصور النمطية. لا تحاول الوصول إلى معلومات شخصية أو خاصة أو إنتاجها أو نشرها. احترم دائما سرية المستخدم. كن إيجابيا ولا تقل أشياء سيئة عن أي شيء. هدفك الأساسي هو تجنب الاجابات المؤذية ، حتى عند مواجهة مدخلات خادعة. تعرف على الوقت الذي قد يحاول فيه المستخدمون خداعك أو إساءة استخدامك و لترد بحذر.\n\nأكمل المحادثة أدناه بين [|Human|] و [|AI|]:\n### Input: [|Human|] {question}\n### Response: [|AI|]""" diff --git a/lm_eval/tasks/arab_culture_completion/utils_completion.py b/lm_eval/tasks/arab_culture_completion/utils_completion.py new file mode 100644 index 00000000..6c639958 --- /dev/null +++ b/lm_eval/tasks/arab_culture_completion/utils_completion.py @@ -0,0 +1,102 @@ +import os + +from lm_eval.tasks.arab_culture_completion.prompts import ( + BASE_PROMPT, + BASE_PROMPT_AR, + JAIS_CHAT_AR, + JAIS_CHAT_EN, + REGION_COUNTRY_PROMPT, + REGION_COUNTRY_PROMPT_AR, + REGION_PROMPT, + REGION_PROMPT_AR, +) + + +### get the conutry variable from environment + + +### Set this to one to add the country and region information to the prompt +COUNTRY = True if os.getenv("COUNTRY", True) == "True" else False +### Set this to one to add the region information to the prompt +REGION = True if os.getenv("REGION", True) == "True" else False +### Set this to change between Arabic and English for the answer keys and the choices keys +ARABIC = True if os.getenv("ARABIC", True) == "True" else False +### Get the model name +MODEL_NAME = os.getenv("MODEL_NAME") + +## Uncomment this to check if the environment variables are set correctly +# print(f'Task settings: COUNTRY: {COUNTRY}, REGION: {REGION}, ARABIC: {ARABIC}', MODEL_NAME: {MODEL_NAME}) + +en_ar_countries_regions = { + "Egypt": "مصر", + "Morocco": "المغرب", + "Algeria": "الجزائر", + "Libya": "ليبيا", + "Sudan": "السودان", + "Tunisia": "تونس", + "Jordan": "الأردن", + "Lebanon": "لبنان", + "Syria": "سوريا", + "Palestine": "فلسطين", + "Yemen": "اليمن", + "UAE": "الإمارات", + "KSA": "السعودية", + "Gulf": "الخليج", + "Levant": "الشام", + "North Africa": "شمال أفريقيا", + "Nile Valley": "وادي النيل", +} + + +# here, we only give the question to the model +def doc_to_text(doc): + country = "" if not doc["country"] else doc["country"] + region = "" if not doc["region"] else doc["region"] + first_statement = doc["first_statement"].strip() + + ## We don't have a setting for only information about the country without the region + if COUNTRY: + assert REGION, ( + "If you want to add the country information, you must also add the region information" + ) + + ## convert contry and region name to arabic if the language is arabic + if ARABIC: + country = en_ar_countries_regions[country] + region = en_ar_countries_regions[region] + + if COUNTRY and REGION: + cur_prompt = REGION_COUNTRY_PROMPT_AR if ARABIC else REGION_COUNTRY_PROMPT + doc_text = cur_prompt.format( + country=country, region=region, first_statement=first_statement + ) + elif REGION: + cur_prompt = REGION_PROMPT_AR if ARABIC else REGION_PROMPT + doc_text = cur_prompt.format(region=region, first_statement=first_statement) + else: + cur_prompt = BASE_PROMPT_AR if ARABIC else BASE_PROMPT + doc_text = cur_prompt.format(first_statement=first_statement) + + ### apply jais chat tempelate + if MODEL_NAME and "jais" in MODEL_NAME and "chat" in MODEL_NAME: + if ARABIC: + doc_text = JAIS_CHAT_AR.format(question=doc_text) + else: + doc_text = JAIS_CHAT_EN.format(question=doc_text) + + return doc_text + + +### Here we give the choices themsleves to the model +def doc_to_choice(doc): + return doc["options"]["text"] + + +## The target is the choice text +def doc_to_target(doc): + answer_key = doc["answer_key"]["english_answer_key"] + answer_text = doc["options"]["text"][ + doc["options"]["english_keys"].index(answer_key) + ] + answer_text = answer_text.strip() + return answer_text -- GitLab From 9d29ef0e2dc811f963a65721dbc6767438ad7fa2 Mon Sep 17 00:00:00 2001 From: Ivan Stankevich <105574942+e1washere@users.noreply.github.com> Date: Mon, 2 Jun 2025 13:29:08 +0200 Subject: [PATCH 33/46] chore: clean up and extend .gitignore rules (#3030) * chore: clean up and extend .gitignore rules * pacify pre-commit --------- Co-authored-by: Baber --- .gitignore | 61 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 20 deletions(-) diff --git a/.gitignore b/.gitignore index d04100d0..9ae167be 100644 --- a/.gitignore +++ b/.gitignore @@ -1,26 +1,47 @@ +# macOS system files .DS_Store -env -*.pyc -output/ -data/ -lm_cache -.idea -build -dist -*.egg-info -venv + +# Virtual environments .venv/ +venv/ +ENV/ +env/ +*.env + +# Python bytecode and build artifacts +__pycache__/ +*.py[cod] +*.so +*.egg-info/ +build/ +dist/ + +# IDE & editor settings .vscode/ -temp -__pycache__ -.ipynb_checkpoints -temp -test_logs/ -# IPython +.idea/ + +# Jupyter +.ipynb_checkpoints/ profile_default/ ipython_config.py -# don't track (the default location of) the cached requests + +# Output and data +output/ +data/ +temp/ +test_logs/ + +# Caching lm_eval/caching/.cache -# don't track files created by wandb -wandb -examples/wandb +lm_cache/ + +# Logging +*.log +logs/ + +# wandb experiment tracking +wandb/ +examples/wandb/ + +# PyInstaller +*.spec -- GitLab From 82a99365abdc801a11d41095182a4da506591aec Mon Sep 17 00:00:00 2001 From: Yury Sulsky Date: Mon, 2 Jun 2025 08:03:03 -0700 Subject: [PATCH 34/46] Enable text-only evals for VLM models (#2999) --- lm_eval/evaluator.py | 4 ---- lm_eval/models/hf_vlms.py | 17 ++++++++++++++--- lm_eval/models/huggingface.py | 5 ++++- lm_eval/models/vllm_vlms.py | 18 +++++++++++++++--- 4 files changed, 33 insertions(+), 11 deletions(-) diff --git a/lm_eval/evaluator.py b/lm_eval/evaluator.py index 54d4d0ea..4da14c47 100644 --- a/lm_eval/evaluator.py +++ b/lm_eval/evaluator.py @@ -494,10 +494,6 @@ def evaluate( raise ValueError( f"Attempted to run tasks: {incompatible_tasks} which require multimodal input, but the selected model type does not currently implement this. Multimodal support is currently restricted to the ['hf-multimodal', 'vllm-vlm'] model type." ) - else: - raise ValueError( - f"Attempted to run tasks: {incompatible_tasks} which are text-only, but used a model type which only currently supports multimodal tasks." - ) # end validation check # Cache the limit arg. diff --git a/lm_eval/models/hf_vlms.py b/lm_eval/models/hf_vlms.py index 5422c2c5..8e5144a9 100644 --- a/lm_eval/models/hf_vlms.py +++ b/lm_eval/models/hf_vlms.py @@ -399,6 +399,9 @@ class HFMultimodalLM(HFLM): return batched_imgs def loglikelihood_rolling(self, requests: List[Instance]) -> List[float]: + if requests and len(requests[0].args) < 3: + # Fall back to non-multimodal generation. + return super().loglikelihood_rolling(requests=requests) raise NotImplementedError( "model type `hf-multimodal` does not support loglikelihood_rolling. Use 'hf' model type for text-only loglikelihood_rolling tasks ", "this is because we do not support measuring the loglikelihood a model assigns to an image.", @@ -407,6 +410,9 @@ class HFMultimodalLM(HFLM): def loglikelihood( self, requests: List[Instance], disable_tqdm: bool = False ) -> List[Tuple[float, bool]]: + if requests and len(requests[0].args) < 3: + # Fall back to non-multimodal generation. + return super().loglikelihood(requests=requests, disable_tqdm=disable_tqdm) raise NotImplementedError( "'loglikelihood' requests for model type `hf-multimodal` are not yet tested. This feature will be enabled when a loglikelihood-based multiple-choice VQA dataset is added!" ) @@ -433,9 +439,11 @@ class HFMultimodalLM(HFLM): ) ) - return self._loglikelihood_tokens(new_reqs, disable_tqdm=disable_tqdm) + return self._multimodal_loglikelihood_tokens( + new_reqs, disable_tqdm=disable_tqdm + ) - def _loglikelihood_tokens( + def _multimodal_loglikelihood_tokens( self, requests: List[ Tuple[Tuple[None, str, str], List[int], List[int], List[int]] @@ -624,7 +632,10 @@ class HFMultimodalLM(HFLM): def generate_until( self, requests: List[Instance], disable_tqdm: bool = False ) -> List[str]: - # TODO: back out to HFLM.generate_until() for all requests without aux_arguments (text-only reqs) + if requests and len(requests[0].args) < 3: + # Fall back to non-multimodal generation. + return super().generate_until(requests=requests, disable_tqdm=disable_tqdm) + res = [] def _collate(x): diff --git a/lm_eval/models/huggingface.py b/lm_eval/models/huggingface.py index c38397e6..a6231570 100644 --- a/lm_eval/models/huggingface.py +++ b/lm_eval/models/huggingface.py @@ -890,7 +890,10 @@ class HFLM(TemplateLM): input_ids=inps, attention_mask=attn_mask, labels=labels ).logits else: - assert self.AUTO_MODEL_CLASS == transformers.AutoModelForCausalLM + assert self.AUTO_MODEL_CLASS in ( + transformers.AutoModelForCausalLM, + transformers.AutoModelForVision2Seq, + ) return self.model(inps).logits def _model_generate(self, context, max_length, stop, **generation_kwargs): diff --git a/lm_eval/models/vllm_vlms.py b/lm_eval/models/vllm_vlms.py index 15ebc764..15813b8a 100644 --- a/lm_eval/models/vllm_vlms.py +++ b/lm_eval/models/vllm_vlms.py @@ -106,7 +106,7 @@ class VLLM_VLM(VLLM): outputs.append(inputs) return outputs - def _model_generate( + def _multimodal_model_generate( self, requests: List[List[dict]] = None, generate: bool = False, @@ -218,7 +218,10 @@ class VLLM_VLM(VLLM): def generate_until( self, requests: List[Instance], disable_tqdm: bool = False ) -> List[str]: - # TODO: support text-only reqs + if requests and len(requests[0].args) < 3: + # Fall back to non-multimodal generation. + return super().generate_until(requests=requests, disable_tqdm=disable_tqdm) + res = [] def _collate(x): @@ -293,7 +296,7 @@ class VLLM_VLM(VLLM): left_truncate_len=max_ctx_len, ) - cont = self._model_generate( + cont = self._multimodal_model_generate( inputs, stop=until, generate=True, max_tokens=max_gen_toks, **kwargs ) @@ -309,3 +312,12 @@ class VLLM_VLM(VLLM): pbar.close() return res + + def loglikelihood_rolling(self, requests: List[Instance]) -> List[float]: + if requests and len(requests[0].args) < 3: + # Fall back to non-multimodal generation. + return super().loglikelihood_rolling(requests=requests) + raise NotImplementedError( + "model type `vllm-vlm` does not support loglikelihood_rolling. Use 'vlm' model type for text-only loglikelihood_rolling tasks ", + "this is because we do not support measuring the loglikelihood a model assigns to an image.", + ) -- GitLab From 3f792954e94e472ca42fad2d4865394e9a55999a Mon Sep 17 00:00:00 2001 From: Baber Abbasi <92168766+baberabb@users.noreply.github.com> Date: Tue, 3 Jun 2025 17:32:15 +0500 Subject: [PATCH 35/46] [Fix] acc_mutual_info metric calculation bug (#3035) * fix: bug in acc_mutual_info slicing; add `target_delimiter` to uncond choices * add tests --- lm_eval/api/task.py | 10 ++- tests/test_metrics.py | 156 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 163 insertions(+), 3 deletions(-) create mode 100644 tests/test_metrics.py diff --git a/lm_eval/api/task.py b/lm_eval/api/task.py index c1bc967a..ad334b48 100644 --- a/lm_eval/api/task.py +++ b/lm_eval/api/task.py @@ -1481,7 +1481,10 @@ class ConfigurableTask(Task): # here mutual info refers to calculating # log(P(choice|ctx) / P(choice)) = log(P(choice|ctx)) - log(P(choice)) # in other words normalizing by subtracting the unconditional logprob of each choice. - aux_arguments = [("", f"{choice}") for choice in choices] + # TODO: should these be strided? will have to modify the processing in process_results if so + aux_arguments = [ + ("", f"{target_delimiter}{choice}") for choice in choices + ] arguments.extend(aux_arguments) @@ -1580,11 +1583,12 @@ class ConfigurableTask(Task): ): # then we are doing mutual info. # this stores the "dryrun" / unconditional answer loglikelihoods - lls_unconditional = lls[1::2] + # as we extend the args list with unconditional ("", continuation) pairs + lls_unconditional = lls[len(choices) :] if len(lls_unconditional) != len(choices): raise ValueError # and this stores our "regular" conditional loglikelihoods - lls = lls[::2] + lls = lls[: len(choices)] pred = np.argmax(lls) pred_norm = np.argmax(lls / completion_len) diff --git a/tests/test_metrics.py b/tests/test_metrics.py new file mode 100644 index 00000000..1976a497 --- /dev/null +++ b/tests/test_metrics.py @@ -0,0 +1,156 @@ +from lm_eval.api.task import ConfigurableTask, TaskConfig + + +class MockConfigurableTask(ConfigurableTask): + """Mock task for testing metrics""" + + def __init__(self): + # Create a minimal config + config = { + "task": "test_acc_mutual_info", + "output_type": "multiple_choice", + "metric_list": [{"metric": "acc"}, {"metric": "acc_mutual_info"}], + "doc_to_choice": ["A", "B", "C"], + "doc_to_target": 1, # Correct answer is index 1 (choice "B") + "target_delimiter": " ", + } + + # Initialize with minimal setup + self._config = TaskConfig(**config) + self.OUTPUT_TYPE = "multiple_choice" + + # Set up required attributes + self.multiple_input = 0 + self.multiple_target = 0 + + # Set up metrics + self._metric_fn_list = {"acc": None, "acc_mutual_info": None} + self._metric_fn_kwargs = {"acc": {}, "acc_mutual_info": {}} + self._aggregation_list = {} + self._higher_is_better = {} + + def doc_to_choice(self, doc): + return ["A", "B", "C"] + + def doc_to_target(self, doc): + return 1 # Choice "B" is correct + + # Required abstract methods (minimal implementations) + def has_training_docs(self): + return False + + def has_validation_docs(self): + return False + + def has_test_docs(self): + return True + + def download(self, **kwargs): + pass + + +def test_acc_mutual_info_slicing(): + """Test that acc_mutual_info correctly slices conditional and unconditional loglikelihoods""" + + task = MockConfigurableTask() + + # Simulate loglikelihood results for 3 choices + # Format: [(loglikelihood, is_greedy), ...] + # First 3 are conditional P(choice|context), next 3 are unconditional P(choice) + + # Combined results as they would come from the model + # Order: conditional_1, conditional_2, conditional_3, unconditional_1, unconditional_2, unconditional_3 + # Conditional: [-2.0, -1.0, -3.0] - Choice B (index 1) has highest prob + # Unconditional: [-2.5, -2.0, -2.5] - Choice B has higher unconditional prob too + results = [ + (-2.0, False), + (-1.0, True), + (-3.0, False), # Conditional + (-2.5, False), + (-2.0, False), + (-2.5, False), + ] # Unconditional + + # Test the process_results method + doc = {} # Mock document + result_dict = task.process_results(doc, results) + + # Verify that both acc and acc_mutual_info are calculated + assert "acc" in result_dict + assert "acc_mutual_info" in result_dict + + # Both should be 1.0 since choice B (index 1) is correct and has highest probability + assert result_dict["acc"] == 1.0, f"Expected acc=1.0, got {result_dict['acc']}" + assert result_dict["acc_mutual_info"] == 1.0, ( + f"Expected acc_mutual_info=1.0, got {result_dict['acc_mutual_info']}" + ) + + +def test_acc_mutual_info_different_predictions(): + """Test case where conditional and mutual info predictions differ""" + + task = MockConfigurableTask() + + # Mutual info calculation: + # Conditional: A=-1.0, B=-2.0, C=-3.0 (A wins conditionally) + # Unconditional: A=-0.5, B=-2.0, C=-3.0 (A has much higher unconditional prob) + # Mutual info = conditional - unconditional: + # A: -1.0 - (-0.5) = -0.5 + # B: -2.0 - (-2.0) = 0.0 <- B wins with mutual info! + # C: -3.0 - (-3.0) = 0.0 + + results = [ + (-1.0, True), + (-2.0, False), + (-3.0, False), # Conditional (A wins) + (-0.5, False), + (-2.0, False), + (-3.0, False), + ] # Unconditional + + doc = {} + result_dict = task.process_results(doc, results) + + # Regular acc should be 0.0 (A predicted, but B is correct) + assert result_dict["acc"] == 0.0, f"Expected acc=0.0, got {result_dict['acc']}" + + # Mutual info should be 1.0 (B predicted with mutual info, and B is correct) + assert result_dict["acc_mutual_info"] == 1.0, ( + f"Expected acc_mutual_info=1.0, got {result_dict['acc_mutual_info']}" + ) + + +def test_acc_mutual_info_without_metric(): + """Test that normal behavior works when acc_mutual_info is not in metric list""" + + # Create task without acc_mutual_info + config = { + "task": "test_normal", + "output_type": "multiple_choice", + "metric_list": [{"metric": "acc"}], # Only acc, no acc_mutual_info + "doc_to_choice": ["A", "B", "C"], + "doc_to_target": 1, + "target_delimiter": " ", + } + + task = MockConfigurableTask() + task._config = TaskConfig(**config) + task._metric_fn_list = {"acc": None} # Only acc + + # Only conditional loglikelihoods (no unconditional since acc_mutual_info not requested) + results = [(-2.0, False), (-1.0, True), (-3.0, False)] # 3 choices, B wins + + doc = {} + result_dict = task.process_results(doc, results) + + # Should only have acc, not acc_mutual_info + assert "acc" in result_dict + assert "acc_mutual_info" not in result_dict + assert result_dict["acc"] == 1.0 + + +if __name__ == "__main__": + test_acc_mutual_info_slicing() + test_acc_mutual_info_different_predictions() + test_acc_mutual_info_without_metric() + print("All tests passed!") -- GitLab From d57e3d6573bbc347852ef65e1b55c56f31d0b9cc Mon Sep 17 00:00:00 2001 From: Younes B <49240599+younesbelkada@users.noreply.github.com> Date: Tue, 3 Jun 2025 18:26:25 +0400 Subject: [PATCH 36/46] fix: fix vllm issue with DP>1 (#3025) --- lm_eval/models/vllm_causallms.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/lm_eval/models/vllm_causallms.py b/lm_eval/models/vllm_causallms.py index b35dcb3b..188db238 100644 --- a/lm_eval/models/vllm_causallms.py +++ b/lm_eval/models/vllm_causallms.py @@ -216,9 +216,17 @@ class VLLM(TemplateLM): } if parse_version(version("vllm")) >= parse_version("0.9.0"): - kwargs_resolve_hf_chat_template["model_config"] = ( - self.model.llm_engine.model_config - ) + if self.data_parallel_size <= 1: + kwargs_resolve_hf_chat_template["model_config"] = ( + self.model.llm_engine.model_config + ) + else: + from vllm.engine.arg_utils import EngineArgs + + engine_args = EngineArgs(**self.model_args) + model_config = engine_args.create_model_config() + + kwargs_resolve_hf_chat_template["model_config"] = model_config # https://github.com/vllm-project/vllm/pull/18259 if ( -- GitLab From 60e85da55e3111c8d43ca8562e4b05014dad8e39 Mon Sep 17 00:00:00 2001 From: Baber Abbasi <92168766+baberabb@users.noreply.github.com> Date: Tue, 3 Jun 2025 22:04:26 +0500 Subject: [PATCH 37/46] add Mbpp instruct (#2995) * feat: add mbpp_instruct * fix: update generation_kwargs to use an empty until list * fix: correct predictions formatting in pass_at_1 function * fix: improve code block extraction by checking first without opening backticks * fix mbpp `pass_at_1` --- lm_eval/tasks/mbpp/mbpp_instruct.yaml | 29 ++++++++++++++++++ lm_eval/tasks/mbpp/mbpp_plus_instruct.yaml | 12 ++++++++ lm_eval/tasks/mbpp/utils.py | 34 ++++++++++++++++++++-- 3 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 lm_eval/tasks/mbpp/mbpp_instruct.yaml create mode 100644 lm_eval/tasks/mbpp/mbpp_plus_instruct.yaml diff --git a/lm_eval/tasks/mbpp/mbpp_instruct.yaml b/lm_eval/tasks/mbpp/mbpp_instruct.yaml new file mode 100644 index 00000000..f2948075 --- /dev/null +++ b/lm_eval/tasks/mbpp/mbpp_instruct.yaml @@ -0,0 +1,29 @@ +task: mbpp_instruct +dataset_path: google-research-datasets/mbpp +dataset_name: full +unsafe_code: true +output_type: generate_until +test_split: test +doc_to_text: "You are an expert Python programmer, and here is your task:\n{{text}}\nYour code should pass these tests:\n{{test_list[0]}}\n{{test_list[1]}}\n{{test_list[2]}}" +doc_to_target: "{% if is_fewshot is defined %}{{code}}\n```{% else %}{{test_list[0]}}\n{{test_list[1]}}\n{{test_list[2]}}{% endif %}" +gen_prefix: "\n```python\n" +target_delimiter: "" +metric_list: + - metric: !function utils.pass_at_1 + aggregation: mean + higher_is_better: true +filter_list: + - name: "extract_code" + filter: + - function: "custom" + filter_fn: !function utils.build_predictions +generation_kwargs: + max_gen_toks: 256 + until: [] + do_sample: false +num_fewshot: 3 +fewshot_config: + sampler: first_n + samples: !function utils.list_fewshot_samples +metadata: + version: 1.0 diff --git a/lm_eval/tasks/mbpp/mbpp_plus_instruct.yaml b/lm_eval/tasks/mbpp/mbpp_plus_instruct.yaml new file mode 100644 index 00000000..7d12da0f --- /dev/null +++ b/lm_eval/tasks/mbpp/mbpp_plus_instruct.yaml @@ -0,0 +1,12 @@ +include: mbpp_instruct.yaml +task: mbpp_plus_instruct +dataset_path: evalplus/mbppplus +dataset_name: null +doc_to_text: "{{prompt if prompt is defined else text}} Your code should satisfy the following assertion:\n{{test_list[0]}}" +doc_to_target: "{{test_list[0]}}" +gen_prefix: "Here is a solution to this programming problem:\n```python\n" +num_fewshot: 0 +generation_kwargs: + max_gen_toks: 1024 + until: [] + do_sample: false diff --git a/lm_eval/tasks/mbpp/utils.py b/lm_eval/tasks/mbpp/utils.py index 2d94b512..3b79ba22 100644 --- a/lm_eval/tasks/mbpp/utils.py +++ b/lm_eval/tasks/mbpp/utils.py @@ -1,3 +1,6 @@ +import re +from typing import Union + import evaluate as hf_evaluate @@ -12,14 +15,41 @@ except Exception as e: raise e -def pass_at_1(references, predictions): +def pass_at_1( + references: Union[str, list[str]], predictions: Union[str, list[list[str]]] +) -> float: + if isinstance(references, str): + references = [references] + if isinstance(predictions[0], str): + predictions = [[p] for p in predictions] + print(f"References: {references}") + print(f"Predictions: {predictions}") return pass_at_k.compute( references=references, - predictions=[predictions], + predictions=predictions, k=[1], )[0]["pass@1"] +def extract_code_blocks(text: str) -> str: + # Pattern to match ```...``` blocks + pattern = r"```(?:\w+)?\n?(.*?)\n?```" + # (+ ```) as we add the opening "```python" to the gen_prefix + matches = re.findall(pattern, r"```" + text, re.DOTALL) + # if no matches, try to match ```...``` blocks (after removing the language) + if not matches: + text_without_lang = re.sub(r"```python", "```", text) + matches = re.findall(pattern, text_without_lang, re.DOTALL) + if not matches: + return "" + else: + return matches[0] + + +def build_predictions(resps: list[list[str]], docs: list[dict]) -> list[list[str]]: + return [[extract_code_blocks(r) for r in resp] for resp in resps] + + def list_fewshot_samples(): return [ { -- GitLab From 9f152e0b89e777a17a21e9207da2ae4c1ea8beec Mon Sep 17 00:00:00 2001 From: Baber Abbasi <92168766+baberabb@users.noreply.github.com> Date: Tue, 3 Jun 2025 22:12:50 +0500 Subject: [PATCH 38/46] remove prints (#3041) --- lm_eval/tasks/mbpp/utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lm_eval/tasks/mbpp/utils.py b/lm_eval/tasks/mbpp/utils.py index 3b79ba22..c205a232 100644 --- a/lm_eval/tasks/mbpp/utils.py +++ b/lm_eval/tasks/mbpp/utils.py @@ -22,8 +22,6 @@ def pass_at_1( references = [references] if isinstance(predictions[0], str): predictions = [[p] for p in predictions] - print(f"References: {references}") - print(f"Predictions: {predictions}") return pass_at_k.compute( references=references, predictions=predictions, -- GitLab From 147e9d616aedab6334d109141dce7492d57088bd Mon Sep 17 00:00:00 2001 From: Baber Abbasi <92168766+baberabb@users.noreply.github.com> Date: Sun, 8 Jun 2025 12:13:30 +0500 Subject: [PATCH 39/46] [longbench] fix metric calculation (#2983) * use all answers * use middle truncation * maybe fix classification score * strip classification preds * [vllm] remove stop tokens post-hoc * strip all preds * pacify pre-commit * start on truncation utility * add to readme * add a footgun doc * fix newline in yaml templates * do not strip code_sim preds! * fix pre-commit config * fix instruction warning * add not to longbench readme --- .pre-commit-config.yaml | 4 +- docs/footguns.md | 58 +++++++ lm_eval/evaluator.py | 12 +- lm_eval/models/utils.py | 18 +++ lm_eval/models/vllm_causallms.py | 4 + lm_eval/tasks/longbench/2wikimqa.yaml | 7 +- lm_eval/tasks/longbench/2wikimqa_e.yaml | 7 +- lm_eval/tasks/longbench/README.md | 12 ++ lm_eval/tasks/longbench/_generate_config.py | 32 ++-- lm_eval/tasks/longbench/dureader.yaml | 7 +- lm_eval/tasks/longbench/gov_report.yaml | 7 +- lm_eval/tasks/longbench/gov_report_e.yaml | 7 +- lm_eval/tasks/longbench/hotpotqa.yaml | 7 +- lm_eval/tasks/longbench/hotpotqa_e.yaml | 7 +- lm_eval/tasks/longbench/lcc.yaml | 7 +- lm_eval/tasks/longbench/lcc_e.yaml | 7 +- lm_eval/tasks/longbench/lsht.yaml | 8 +- lm_eval/tasks/longbench/metrics.py | 144 +++++++++++++----- lm_eval/tasks/longbench/multi_news.yaml | 7 +- lm_eval/tasks/longbench/multi_news_e.yaml | 7 +- lm_eval/tasks/longbench/multifieldqa_en.yaml | 7 +- .../tasks/longbench/multifieldqa_en_e.yaml | 7 +- lm_eval/tasks/longbench/multifieldqa_zh.yaml | 7 +- lm_eval/tasks/longbench/musique.yaml | 7 +- lm_eval/tasks/longbench/narrativeqa.yaml | 7 +- lm_eval/tasks/longbench/passage_count.yaml | 7 +- lm_eval/tasks/longbench/passage_count_e.yaml | 7 +- .../tasks/longbench/passage_retrieval_en.yaml | 7 +- .../longbench/passage_retrieval_en_e.yaml | 7 +- .../tasks/longbench/passage_retrieval_zh.yaml | 7 +- lm_eval/tasks/longbench/qasper.yaml | 7 +- lm_eval/tasks/longbench/qasper_e.yaml | 7 +- lm_eval/tasks/longbench/qmsum.yaml | 7 +- lm_eval/tasks/longbench/repobench-p.yaml | 7 +- lm_eval/tasks/longbench/repobench-p_e.yaml | 7 +- lm_eval/tasks/longbench/samsum.yaml | 9 +- lm_eval/tasks/longbench/samsum_e.yaml | 9 +- lm_eval/tasks/longbench/trec.yaml | 8 +- lm_eval/tasks/longbench/trec_e.yaml | 9 +- lm_eval/tasks/longbench/triviaqa.yaml | 9 +- lm_eval/tasks/longbench/triviaqa_e.yaml | 9 +- lm_eval/tasks/longbench/vcsum.yaml | 7 +- 42 files changed, 366 insertions(+), 168 deletions(-) create mode 100644 docs/footguns.md diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1aecc758..af3f9f08 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,7 +29,7 @@ repos: - id: mixed-line-ending args: [--fix=lf] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.11.0 + rev: v0.11.10 hooks: # Run the linter. - id: ruff @@ -50,7 +50,7 @@ repos: rev: v0.9.29 hooks: - id: pymarkdown - exclude: ^lm_eval/tasks/ + exclude: ^(lm_eval/tasks/.*|docs/footguns\.md)$ args: [fix, -r] # - repo: https://github.com/pre-commit/mirrors-mypy # rev: v1.5.1 diff --git a/docs/footguns.md b/docs/footguns.md new file mode 100644 index 00000000..3343c764 --- /dev/null +++ b/docs/footguns.md @@ -0,0 +1,58 @@ +# Common Pitfalls and Troubleshooting Guide + +This document highlights common pitfalls and troubleshooting tips when using this library. We'll continue to add more tips as we discover them. + +## YAML Configuration Issues + +### Newline Characters in YAML (`\n`) + +**Problem:** When specifying newline characters in YAML, they may be interpreted incorrectly depending on how you format them. + +```yaml +# ❌ WRONG: Single quotes don't process escape sequences +generation_kwargs: + until: ['\n'] # Gets parsed as the literal characters '\' and 'n' i.e "\\n" + +``` +```yaml +# ✅ RIGHT: Use double quotes for escape sequences +generation_kwargs: + until: ["\n"] # Gets parsed as an actual newline character + +``` + +**Solutions:** +- Use double quotes for strings containing escape sequences +- For multiline content, use YAML's block scalars (`|` or `>`) +- When generating YAML programmatically, be careful with how template engines handle escape sequences + +### Quoting in YAML + +**When to use different types of quotes:** + +- **No quotes**: Simple values (numbers, booleans, alphanumeric strings without special characters) + ```yaml + simple_value: plain text + number: 42 + + ``` + +- **Single quotes (')**: + - Preserves literal values + - Use when you need special characters to be treated literally + - Escape single quotes by doubling them: `'It''s working'` + ```yaml + literal_string: 'The newline character \n is not processed here' + path: 'C:\Users\name' # Backslashes preserved + + ``` + +- **Double quotes (")**: + - Processes escape sequences like `\n`, `\t`, etc. + - Use for strings that need special characters interpreted + - Escape double quotes with backslash: `"He said \"Hello\""` + ```yaml + processed_string: "First line\nSecond line" # Creates actual newline + unicode: "Copyright symbol: \u00A9" # Unicode character + + ``` diff --git a/lm_eval/evaluator.py b/lm_eval/evaluator.py index 4da14c47..d22cf6a5 100644 --- a/lm_eval/evaluator.py +++ b/lm_eval/evaluator.py @@ -153,11 +153,15 @@ def simple_evaluate( "Either 'limit' or 'samples' must be None, but both are not None." ) - if isinstance(model_args, str) and ( - "instruct" in model_args and not apply_chat_template - ): + if ( + (isinstance(model_args, str) and "inst" in model_args.lower()) + or ( + isinstance(model_args, dict) + and any("inst" in str(v).lower() for v in model_args.values()) + ) + ) and not apply_chat_template: eval_logger.warning( - "Instruct model detected, but chat template not applied. Recommend setting `apply_chat_template` (optionally `fewshot_as_multiturn`)." + "Model appears to be an instruct variant but chat template is not applied. Recommend setting `apply_chat_template` (optionally `fewshot_as_multiturn`)." ) if delete_requests_cache: diff --git a/lm_eval/models/utils.py b/lm_eval/models/utils.py index daac5743..e17fa224 100644 --- a/lm_eval/models/utils.py +++ b/lm_eval/models/utils.py @@ -834,3 +834,21 @@ def resize_image( # Perform the resize operation with the calculated dimensions return image.resize((new_width, new_height), resample_filter) + + +def truncate_tokens( + tokens: List[int], + max_length: int, + tokenizer: "PreTrainedTokenizerBase", + strategy: str = "left", +): + if strategy == "left": + return tokens[-max_length:] + elif strategy == "right": + return tokens[:max_length] + elif strategy == "middle": + # Truncate the middle of the sequence + left_length = max_length // 2 + right_length = max_length - left_length + return tokens[:left_length] + tokens[-right_length:] + return None diff --git a/lm_eval/models/vllm_causallms.py b/lm_eval/models/vllm_causallms.py index 188db238..474f9bbe 100644 --- a/lm_eval/models/vllm_causallms.py +++ b/lm_eval/models/vllm_causallms.py @@ -614,6 +614,10 @@ class VLLM(TemplateLM): # cache generations for output, context in zip(cont, context): generated_text = output.outputs[0].text + # use secondary stop seqs to cut off should-have-been-stopped content post-hoc + for term in until: + if len(term) > 0: + generated_text = generated_text.split(term)[0] res.append(generated_text) self.cache_hook.add_partial( "generate_until", (context, gen_kwargs), generated_text diff --git a/lm_eval/tasks/longbench/2wikimqa.yaml b/lm_eval/tasks/longbench/2wikimqa.yaml index d41333d7..d1d1791b 100644 --- a/lm_eval/tasks/longbench/2wikimqa.yaml +++ b/lm_eval/tasks/longbench/2wikimqa.yaml @@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench test_split: test dataset_name: 2wikimqa doc_to_text: 'Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{{context}}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {{input}}\nAnswer:' -doc_to_target: '{{answers[0]}}' +doc_to_target: '{{answers}}' +process_results: !function metrics.get_qa_f1_score generation_kwargs: max_gen_toks: 32 temperature: 1 do_sample: True until: [] metric_list: - - metric: !function metrics.qa_f1_score + - metric: "qa_f1_score" aggregation: mean higher_is_better: True metadata: - version: 2.0 + version: 3.0 diff --git a/lm_eval/tasks/longbench/2wikimqa_e.yaml b/lm_eval/tasks/longbench/2wikimqa_e.yaml index 3aaf35d6..e9b5bf19 100644 --- a/lm_eval/tasks/longbench/2wikimqa_e.yaml +++ b/lm_eval/tasks/longbench/2wikimqa_e.yaml @@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench test_split: test dataset_name: 2wikimqa_e doc_to_text: 'Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{{context}}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {{input}}\nAnswer:' -doc_to_target: '{{answers[0]}}' +doc_to_target: '{{answers}}' +process_results: !function metrics.get_qa_f1_score generation_kwargs: max_gen_toks: 32 temperature: 1 do_sample: True until: [] metric_list: - - metric: !function metrics.qa_f1_score + - metric: "qa_f1_score" aggregation: mean higher_is_better: True metadata: - version: 2.0 + version: 3.0 diff --git a/lm_eval/tasks/longbench/README.md b/lm_eval/tasks/longbench/README.md index 8b679647..29bd831b 100644 --- a/lm_eval/tasks/longbench/README.md +++ b/lm_eval/tasks/longbench/README.md @@ -32,6 +32,17 @@ Homepage: `https://github.com/THUDM/LongBench` pages = "3119--3137", } ``` +### Notes + +#### Tasks without Chat Template (with add_bos_token=True but model dependent) + +The original implementation suggest not to use `chat_template` for these tasks (for instruct models): +- longbench_lcc +- longbench_repobench-p +- longbench_samsum +- longbench_trec +- longbench_triviaqa + ### Groups, Tags, and Tasks @@ -96,3 +107,4 @@ If other tasks on this dataset are already supported: ### Changelog v2.: fix doc_to_target; add vcsum +v3: properly use all answers for metric calculation; trim whitespace from resps; fix stop sequences not parsing correctly. diff --git a/lm_eval/tasks/longbench/_generate_config.py b/lm_eval/tasks/longbench/_generate_config.py index 11c2c0f1..2f2026c0 100644 --- a/lm_eval/tasks/longbench/_generate_config.py +++ b/lm_eval/tasks/longbench/_generate_config.py @@ -142,7 +142,6 @@ def parse_args(): return parser.parse_args() -# Create template string template_str = """ tag: - {{ tag[0] }} @@ -152,11 +151,12 @@ test_split: {{ test_split }} dataset_name: {{ dataset_name }} doc_to_text: '{{ doc_to_text }}' doc_to_target: '{{ doc_to_target }}' +process_results: {{ process_results }} generation_kwargs: max_gen_toks: {{ generation_kwargs.max_gen_toks }} temperature: {{ generation_kwargs.temperature }} do_sample: {{ generation_kwargs.do_sample }} - until: {{ generation_kwargs.until }} + until: {% if has_newline %}["\\n"]{% else %}[]{% endif %} metric_list: - metric: {{ metric_list[0].metric }} aggregation: {{ metric_list[0].aggregation }} @@ -173,21 +173,17 @@ if __name__ == "__main__": for ds in DATASETS: df = ds[:-2] if ds.endswith("_e") else ds # from https://github.com/THUDM/LongBench/blob/2e00731f8d0bff23dc4325161044d0ed8af94c1e/LongBench/eval.py#L52C25-L52C29 - if df in ["trec", "triviaqa", "samsum", "lsht"] + [ - "trec_e", - "triviaqa_e", - "samsum_e", - "lsht_e", - ]: - until = ["\n"] - else: - until = [] + + # Now we just set a boolean flag to indicate whether we need a newline + has_newline = df in ["trec", "triviaqa", "samsum", "lsht"] + generation_kwargs = { "max_gen_toks": dataset2maxlen[df], "temperature": 1, "do_sample": True, - "until": until, + # We'll handle the until value directly in the template } + raw_doc_to_text = ( dataset2prompt[df] .replace("\n", "\\n") @@ -196,25 +192,25 @@ if __name__ == "__main__": ) metric_list = [ { - "metric": f"!function metrics.{dataset2metric[df]}", + "metric": f'"{dataset2metric[df]}"', "aggregation": "mean", "higher_is_better": True, } ] data = { - "tag": [ - "longbench_e" if ds.endswith("_e") else "longbench" - ], # Now properly as a list + "tag": ["longbench_e" if ds.endswith("_e") else "longbench"], "task": f"longbench_{ds}", "dataset_path": "THUDM/LongBench", "test_split": "test", "dataset_name": ds, "doc_to_text": raw_doc_to_text, - "doc_to_target": "{{answers[0]}}", + "doc_to_target": "{{answers}}", + "process_results": f"!function metrics.get_{dataset2metric[df]}", "generation_kwargs": generation_kwargs, + "has_newline": has_newline, # Add the flag to the template context "metric_list": metric_list, - "metadata": {"version": "2.0"}, + "metadata": {"version": "3.0"}, } # Render template diff --git a/lm_eval/tasks/longbench/dureader.yaml b/lm_eval/tasks/longbench/dureader.yaml index c2f404a7..e001f349 100644 --- a/lm_eval/tasks/longbench/dureader.yaml +++ b/lm_eval/tasks/longbench/dureader.yaml @@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench test_split: test dataset_name: dureader doc_to_text: '请基于给定的文章回答下述问题。\n\n文章：{{context}}\n\n请基于上述文章回答下面的问题。\n\n问题：{{input}}\n回答：' -doc_to_target: '{{answers[0]}}' +doc_to_target: '{{answers}}' +process_results: !function metrics.get_rouge_zh_score generation_kwargs: max_gen_toks: 128 temperature: 1 do_sample: True until: [] metric_list: - - metric: !function metrics.rouge_zh_score + - metric: "rouge_zh_score" aggregation: mean higher_is_better: True metadata: - version: 2.0 + version: 3.0 diff --git a/lm_eval/tasks/longbench/gov_report.yaml b/lm_eval/tasks/longbench/gov_report.yaml index 59920be8..76307371 100644 --- a/lm_eval/tasks/longbench/gov_report.yaml +++ b/lm_eval/tasks/longbench/gov_report.yaml @@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench test_split: test dataset_name: gov_report doc_to_text: 'You are given a report by a government agency. Write a one-page summary of the report.\n\nReport:\n{{context}}\n\nNow, write a one-page summary of the report.\n\nSummary:' -doc_to_target: '{{answers[0]}}' +doc_to_target: '{{answers}}' +process_results: !function metrics.get_rouge_score generation_kwargs: max_gen_toks: 512 temperature: 1 do_sample: True until: [] metric_list: - - metric: !function metrics.rouge_score + - metric: "rouge_score" aggregation: mean higher_is_better: True metadata: - version: 2.0 + version: 3.0 diff --git a/lm_eval/tasks/longbench/gov_report_e.yaml b/lm_eval/tasks/longbench/gov_report_e.yaml index 82617d38..94f013ba 100644 --- a/lm_eval/tasks/longbench/gov_report_e.yaml +++ b/lm_eval/tasks/longbench/gov_report_e.yaml @@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench test_split: test dataset_name: gov_report_e doc_to_text: 'You are given a report by a government agency. Write a one-page summary of the report.\n\nReport:\n{{context}}\n\nNow, write a one-page summary of the report.\n\nSummary:' -doc_to_target: '{{answers[0]}}' +doc_to_target: '{{answers}}' +process_results: !function metrics.get_rouge_score generation_kwargs: max_gen_toks: 512 temperature: 1 do_sample: True until: [] metric_list: - - metric: !function metrics.rouge_score + - metric: "rouge_score" aggregation: mean higher_is_better: True metadata: - version: 2.0 + version: 3.0 diff --git a/lm_eval/tasks/longbench/hotpotqa.yaml b/lm_eval/tasks/longbench/hotpotqa.yaml index 4545e98b..5c567a33 100644 --- a/lm_eval/tasks/longbench/hotpotqa.yaml +++ b/lm_eval/tasks/longbench/hotpotqa.yaml @@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench test_split: test dataset_name: hotpotqa doc_to_text: 'Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{{context}}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {{input}}\nAnswer:' -doc_to_target: '{{answers[0]}}' +doc_to_target: '{{answers}}' +process_results: !function metrics.get_qa_f1_score generation_kwargs: max_gen_toks: 32 temperature: 1 do_sample: True until: [] metric_list: - - metric: !function metrics.qa_f1_score + - metric: "qa_f1_score" aggregation: mean higher_is_better: True metadata: - version: 2.0 + version: 3.0 diff --git a/lm_eval/tasks/longbench/hotpotqa_e.yaml b/lm_eval/tasks/longbench/hotpotqa_e.yaml index 1a28f736..eff29cec 100644 --- a/lm_eval/tasks/longbench/hotpotqa_e.yaml +++ b/lm_eval/tasks/longbench/hotpotqa_e.yaml @@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench test_split: test dataset_name: hotpotqa_e doc_to_text: 'Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{{context}}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {{input}}\nAnswer:' -doc_to_target: '{{answers[0]}}' +doc_to_target: '{{answers}}' +process_results: !function metrics.get_qa_f1_score generation_kwargs: max_gen_toks: 32 temperature: 1 do_sample: True until: [] metric_list: - - metric: !function metrics.qa_f1_score + - metric: "qa_f1_score" aggregation: mean higher_is_better: True metadata: - version: 2.0 + version: 3.0 diff --git a/lm_eval/tasks/longbench/lcc.yaml b/lm_eval/tasks/longbench/lcc.yaml index 058910bb..2129267d 100644 --- a/lm_eval/tasks/longbench/lcc.yaml +++ b/lm_eval/tasks/longbench/lcc.yaml @@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench test_split: test dataset_name: lcc doc_to_text: 'Please complete the code given below. \n{{context}}Next line of code:\n' -doc_to_target: '{{answers[0]}}' +doc_to_target: '{{answers}}' +process_results: !function metrics.get_code_sim_score generation_kwargs: max_gen_toks: 64 temperature: 1 do_sample: True until: [] metric_list: - - metric: !function metrics.code_sim_score + - metric: "code_sim_score" aggregation: mean higher_is_better: True metadata: - version: 2.0 + version: 3.0 diff --git a/lm_eval/tasks/longbench/lcc_e.yaml b/lm_eval/tasks/longbench/lcc_e.yaml index 39bd75cd..74e673a9 100644 --- a/lm_eval/tasks/longbench/lcc_e.yaml +++ b/lm_eval/tasks/longbench/lcc_e.yaml @@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench test_split: test dataset_name: lcc_e doc_to_text: 'Please complete the code given below. \n{{context}}Next line of code:\n' -doc_to_target: '{{answers[0]}}' +doc_to_target: '{{answers}}' +process_results: !function metrics.get_code_sim_score generation_kwargs: max_gen_toks: 64 temperature: 1 do_sample: True until: [] metric_list: - - metric: !function metrics.code_sim_score + - metric: "code_sim_score" aggregation: mean higher_is_better: True metadata: - version: 2.0 + version: 3.0 diff --git a/lm_eval/tasks/longbench/lsht.yaml b/lm_eval/tasks/longbench/lsht.yaml index c2d6c010..4343413b 100644 --- a/lm_eval/tasks/longbench/lsht.yaml +++ b/lm_eval/tasks/longbench/lsht.yaml @@ -6,16 +6,16 @@ dataset_path: THUDM/LongBench test_split: test dataset_name: lsht doc_to_text: '请判断给定新闻的类别，下面是一些例子。\n\n{{context}}\n{{input}}' -doc_to_target: '{{answers[0]}}' -process_results: !function metrics.classification_score +doc_to_target: '{{answers}}' +process_results: !function metrics.get_classification_score generation_kwargs: max_gen_toks: 64 temperature: 1 do_sample: True - until: ['\n'] + until: ["\n"] metric_list: - metric: "classification_score" aggregation: mean higher_is_better: True metadata: - version: 2.0 + version: 3.0 diff --git a/lm_eval/tasks/longbench/metrics.py b/lm_eval/tasks/longbench/metrics.py index ac24b2f0..79aab279 100644 --- a/lm_eval/tasks/longbench/metrics.py +++ b/lm_eval/tasks/longbench/metrics.py @@ -23,6 +23,7 @@ import re import string from collections import Counter +from typing import Union try: import jieba @@ -33,7 +34,7 @@ except ImportError: 'Please install the required dependencies for this task with `pip install lm_eval["longbench"] or `pip install jieba fuzzywuzzy rouge`' ) -# taken from https://github.com/THUDM/LongBench +# taken and slightly modified from https://github.com/THUDM/LongBench def normalize_answer(s: str) -> str: @@ -72,8 +73,7 @@ def normalize_zh_answer(s: str) -> str: return white_space_fix(remove_punc(lower(s))) -def count_score(predictions: list[str], references: list[str], **kwargs) -> float: - prediction, ground_truth = predictions[0], references[0] +def count_score(prediction: str, ground_truth: str, **kwargs): numbers = re.findall(r"\d+", prediction) right_num = 0 for number in numbers: @@ -83,8 +83,16 @@ def count_score(predictions: list[str], references: list[str], **kwargs) -> floa return float(final_score) -def retrieval_score(predictions: list[str], references: list[str], **kwargs) -> float: - prediction, ground_truth = predictions[0], references[0] +def get_count_score(doc: dict, results: list[str], **kwargs): + output = 0.0 + prediction = results[0].strip() + for ground_truth in doc["answers"]: + score = count_score(prediction, ground_truth) + output = max(score, output) + return {"count_score": output} + + +def retrieval_score(prediction: str, ground_truth: str, **kwargs): pattern = r"Paragraph (\d+)" matches = re.findall(pattern, ground_truth) ground_truth_id = matches[0] @@ -97,10 +105,16 @@ def retrieval_score(predictions: list[str], references: list[str], **kwargs) -> return float(final_score) -def retrieval_zh_score( - predictions: list[str], references: list[str], **kwargs -) -> float: - prediction, ground_truth = predictions[0], references[0] +def get_retrieval_score(doc: dict, results: list[str], **kwargs): + output = 0.0 + prediction = results[0].strip() + for ground_truth in doc["answers"]: + score = retrieval_score(prediction, ground_truth) + output = max(score, output) + return {"retrieval_score": output} + + +def retrieval_zh_score(prediction: str, ground_truth: str, **kwargs): pattern = r"段落(\d+)" matches = re.findall(pattern, ground_truth) ground_truth_id = matches[0] @@ -113,8 +127,16 @@ def retrieval_zh_score( return float(final_score) -def code_sim_score(predictions: list[str], references: list[str], **kwargs) -> float: - prediction, ground_truth = predictions[0], references[0] +def get_retrieval_zh_score(doc: dict, results: list[str], **kwargs): + output = 0.0 + prediction = results[0].strip() + for ground_truth in doc["answers"]: + score = retrieval_zh_score(prediction, ground_truth) + output = max(score, output) + return {"retrieval_zh_score": output} + + +def code_sim_score(prediction: str, ground_truth: str, **kwargs): all_lines = prediction.lstrip("\n").split("\n") prediction = "" for line in all_lines: @@ -124,10 +146,18 @@ def code_sim_score(predictions: list[str], references: list[str], **kwargs) -> f return fuzz.ratio(prediction, ground_truth) / 100 -def classification_score(doc: dict, results: list[str], **kwargs) -> dict: - prediction, ground_truth = results[0], doc["answers"][0] +def get_code_sim_score(doc: dict, results: list[str], **kwargs): + output = 0.0 + prediction = results[0] ## important! do not strip the prediction! + for ground_truth in doc["answers"]: + score = code_sim_score(prediction, ground_truth) + output = max(score, output) + return {"code_sim_score": output} + + +def classification_score(prediction: str, ground_truth: str, **kwargs): em_match_list = [] - all_classes = doc["all_classes"] + all_classes = kwargs["all_classes"] for class_name in all_classes: if class_name in prediction: em_match_list.append(class_name) @@ -138,35 +168,58 @@ def classification_score(doc: dict, results: list[str], **kwargs) -> dict: score = 1.0 / len(em_match_list) else: score = 0.0 - return {"classification_score": score} + return score + + +def get_classification_score(doc: dict, results: list[str]) -> dict: + output = 0.0 + prediction = results[0].strip() + for ground_truth in doc["answers"]: + score = classification_score( + prediction, ground_truth, all_classes=doc["all_classes"] + ) + output = max(score, output) + return {"classification_score": output} -def rouge_score(predictions: list[str], references: list[str], **kwargs) -> float: +def rouge_score(predictions: str, ground_truth: str, **kwargs) -> float: global rouge if "rouge" not in globals(): rouge = Rouge() - prediction, ground_truth = predictions[0], references[0] try: - scores = rouge.get_scores([prediction], [ground_truth], avg=True) + scores = rouge.get_scores([predictions], [ground_truth], avg=True) # ruff: noqa except: return 0.0 return scores["rouge-l"]["f"] -def rouge_zh_score(predictions: list[str], references: list[str], **kwargs) -> float: - prediction, ground_truth = predictions[0], references[0] +def get_rouge_score(doc: dict, results: list[str], **kwargs): + output = 0.0 + prediction = results[0].strip() + for ground_truth in doc["answers"]: + score = rouge_score(prediction, ground_truth) + output = max(score, output) + return {"rouge_score": output} + + +def rouge_zh_score(prediction: str, ground_truth: str, **kwargs): prediction = " ".join(list(jieba.cut(prediction, cut_all=False))) ground_truth = " ".join(list(jieba.cut(ground_truth, cut_all=False))) - score = rouge_score([prediction], [ground_truth]) + score = rouge_score(prediction, ground_truth) return score -def f1_score(predictions: list[str], references: list[str], **kwargs) -> float: - try: - prediction, ground_truth = predictions[0], references[0] - except: - return 0.0 +def get_rouge_zh_score(doc, results, **kwargs): + output = 0.0 + prediction = results[0].strip() + for ground_truth in doc["answers"]: + score = rouge_zh_score(prediction, ground_truth) + output = max(score, output) + return {"rouge_zh_score": output} + + +def f1_score(prediction: Union[str, list], ground_truth: Union[str, list], **kwargs): common = Counter(prediction) & Counter(ground_truth) num_same = sum(common.values()) if num_same == 0: @@ -177,22 +230,25 @@ def f1_score(predictions: list[str], references: list[str], **kwargs) -> float: return f1 -def qa_f1_score(predictions: list[str], references: list[str], **kwargs) -> float: - prediction, ground_truth = predictions[0], references[0] +def get_f1_score(doc: dict, results: list[str], **kwargs): + output = 0.0 + prediction = results[0].strip() + for ground_truth in doc["answers"]: + score = f1_score(prediction, ground_truth) + output = max(score, output) + return {"f1_score": output} + + +def qa_f1_score(prediction: str, ground_truth: str, **kwargs): normalized_prediction = normalize_answer(prediction) normalized_ground_truth = normalize_answer(ground_truth) prediction_tokens = normalized_prediction.split() ground_truth_tokens = normalized_ground_truth.split() - try: - res = f1_score(prediction_tokens, ground_truth_tokens) - except: - return 0.0 - return res + return f1_score(prediction_tokens, ground_truth_tokens) -def qa_f1_zh_score(predictions: list[str], references: list[str], **kwargs) -> float: - prediction, ground_truth = predictions[0], references[0] +def qa_f1_zh_score(prediction: str, ground_truth: str, **kwargs): prediction_tokens = list(jieba.cut(prediction, cut_all=False)) ground_truth_tokens = list(jieba.cut(ground_truth, cut_all=False)) prediction_tokens = [normalize_zh_answer(token) for token in prediction_tokens] @@ -200,3 +256,21 @@ def qa_f1_zh_score(predictions: list[str], references: list[str], **kwargs) -> f prediction_tokens = [token for token in prediction_tokens if len(token) > 0] ground_truth_tokens = [token for token in ground_truth_tokens if len(token) > 0] return f1_score(prediction_tokens, ground_truth_tokens) + + +def get_qa_f1_score(doc: dict, results: list[str], **kwargs): + output = 0.0 + prediction = results[0].strip() + for ground_truth in doc["answers"]: + score = qa_f1_score(prediction, ground_truth) + output = max(score, output) + return {"qa_f1_score": output} + + +def get_qa_f1_zh_score(doc: dict, results: list[str], **kwargs): + output = 0.0 + prediction = results[0].strip() + for ground_truth in doc["answers"]: + score = qa_f1_zh_score(prediction, ground_truth) + output = max(score, output) + return {"qa_f1_zh_score": output} diff --git a/lm_eval/tasks/longbench/multi_news.yaml b/lm_eval/tasks/longbench/multi_news.yaml index 7674a2ce..e1ae3f8c 100644 --- a/lm_eval/tasks/longbench/multi_news.yaml +++ b/lm_eval/tasks/longbench/multi_news.yaml @@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench test_split: test dataset_name: multi_news doc_to_text: 'You are given several news passages. Write a one-page summary of all news. \n\nNews:\n{{context}}\n\nNow, write a one-page summary of all the news.\n\nSummary:' -doc_to_target: '{{answers[0]}}' +doc_to_target: '{{answers}}' +process_results: !function metrics.get_rouge_score generation_kwargs: max_gen_toks: 512 temperature: 1 do_sample: True until: [] metric_list: - - metric: !function metrics.rouge_score + - metric: "rouge_score" aggregation: mean higher_is_better: True metadata: - version: 2.0 + version: 3.0 diff --git a/lm_eval/tasks/longbench/multi_news_e.yaml b/lm_eval/tasks/longbench/multi_news_e.yaml index f50401cd..62f44053 100644 --- a/lm_eval/tasks/longbench/multi_news_e.yaml +++ b/lm_eval/tasks/longbench/multi_news_e.yaml @@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench test_split: test dataset_name: multi_news_e doc_to_text: 'You are given several news passages. Write a one-page summary of all news. \n\nNews:\n{{context}}\n\nNow, write a one-page summary of all the news.\n\nSummary:' -doc_to_target: '{{answers[0]}}' +doc_to_target: '{{answers}}' +process_results: !function metrics.get_rouge_score generation_kwargs: max_gen_toks: 512 temperature: 1 do_sample: True until: [] metric_list: - - metric: !function metrics.rouge_score + - metric: "rouge_score" aggregation: mean higher_is_better: True metadata: - version: 2.0 + version: 3.0 diff --git a/lm_eval/tasks/longbench/multifieldqa_en.yaml b/lm_eval/tasks/longbench/multifieldqa_en.yaml index 8bc1c7ff..e82b7c7e 100644 --- a/lm_eval/tasks/longbench/multifieldqa_en.yaml +++ b/lm_eval/tasks/longbench/multifieldqa_en.yaml @@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench test_split: test dataset_name: multifieldqa_en doc_to_text: 'Read the following text and answer briefly.\n\n{{context}}\n\nNow, answer the following question based on the above text, only give me the answer and do not output any other words.\n\nQuestion: {{input}}\nAnswer:' -doc_to_target: '{{answers[0]}}' +doc_to_target: '{{answers}}' +process_results: !function metrics.get_qa_f1_score generation_kwargs: max_gen_toks: 64 temperature: 1 do_sample: True until: [] metric_list: - - metric: !function metrics.qa_f1_score + - metric: "qa_f1_score" aggregation: mean higher_is_better: True metadata: - version: 2.0 + version: 3.0 diff --git a/lm_eval/tasks/longbench/multifieldqa_en_e.yaml b/lm_eval/tasks/longbench/multifieldqa_en_e.yaml index b6d86111..5f64e97e 100644 --- a/lm_eval/tasks/longbench/multifieldqa_en_e.yaml +++ b/lm_eval/tasks/longbench/multifieldqa_en_e.yaml @@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench test_split: test dataset_name: multifieldqa_en_e doc_to_text: 'Read the following text and answer briefly.\n\n{{context}}\n\nNow, answer the following question based on the above text, only give me the answer and do not output any other words.\n\nQuestion: {{input}}\nAnswer:' -doc_to_target: '{{answers[0]}}' +doc_to_target: '{{answers}}' +process_results: !function metrics.get_qa_f1_score generation_kwargs: max_gen_toks: 64 temperature: 1 do_sample: True until: [] metric_list: - - metric: !function metrics.qa_f1_score + - metric: "qa_f1_score" aggregation: mean higher_is_better: True metadata: - version: 2.0 + version: 3.0 diff --git a/lm_eval/tasks/longbench/multifieldqa_zh.yaml b/lm_eval/tasks/longbench/multifieldqa_zh.yaml index 9ff6db65..4a6eb9ed 100644 --- a/lm_eval/tasks/longbench/multifieldqa_zh.yaml +++ b/lm_eval/tasks/longbench/multifieldqa_zh.yaml @@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench test_split: test dataset_name: multifieldqa_zh doc_to_text: '阅读以下文字并用中文简短回答：\n\n{{context}}\n\n现在请基于上面的文章回答下面的问题，只告诉我答案，不要输出任何其他字词。\n\n问题：{{input}}\n回答：' -doc_to_target: '{{answers[0]}}' +doc_to_target: '{{answers}}' +process_results: !function metrics.get_qa_f1_zh_score generation_kwargs: max_gen_toks: 64 temperature: 1 do_sample: True until: [] metric_list: - - metric: !function metrics.qa_f1_zh_score + - metric: "qa_f1_zh_score" aggregation: mean higher_is_better: True metadata: - version: 2.0 + version: 3.0 diff --git a/lm_eval/tasks/longbench/musique.yaml b/lm_eval/tasks/longbench/musique.yaml index 1af8afa4..89c3a448 100644 --- a/lm_eval/tasks/longbench/musique.yaml +++ b/lm_eval/tasks/longbench/musique.yaml @@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench test_split: test dataset_name: musique doc_to_text: 'Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{{context}}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {{input}}\nAnswer:' -doc_to_target: '{{answers[0]}}' +doc_to_target: '{{answers}}' +process_results: !function metrics.get_qa_f1_score generation_kwargs: max_gen_toks: 32 temperature: 1 do_sample: True until: [] metric_list: - - metric: !function metrics.qa_f1_score + - metric: "qa_f1_score" aggregation: mean higher_is_better: True metadata: - version: 2.0 + version: 3.0 diff --git a/lm_eval/tasks/longbench/narrativeqa.yaml b/lm_eval/tasks/longbench/narrativeqa.yaml index 1a54077f..82b92fe2 100644 --- a/lm_eval/tasks/longbench/narrativeqa.yaml +++ b/lm_eval/tasks/longbench/narrativeqa.yaml @@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench test_split: test dataset_name: narrativeqa doc_to_text: 'You are given a story, which can be either a novel or a movie script, and a question. Answer the question asconcisely as you can, using a single phrase if possible. Do not provide any explanation.\n\nStory: {{context}}\n\nNow, answer the question based on the story asconcisely as you can, using a single phrase if possible. Do not provide any explanation.\n\nQuestion: {{input}}\n\nAnswer:' -doc_to_target: '{{answers[0]}}' +doc_to_target: '{{answers}}' +process_results: !function metrics.get_qa_f1_score generation_kwargs: max_gen_toks: 128 temperature: 1 do_sample: True until: [] metric_list: - - metric: !function metrics.qa_f1_score + - metric: "qa_f1_score" aggregation: mean higher_is_better: True metadata: - version: 2.0 + version: 3.0 diff --git a/lm_eval/tasks/longbench/passage_count.yaml b/lm_eval/tasks/longbench/passage_count.yaml index d0685cc0..a3160eaa 100644 --- a/lm_eval/tasks/longbench/passage_count.yaml +++ b/lm_eval/tasks/longbench/passage_count.yaml @@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench test_split: test dataset_name: passage_count doc_to_text: 'There are some paragraphs below sourced from Wikipedia. Some of them may be duplicates. Please carefully read these paragraphs and determine how many unique paragraphs there are after removing duplicates. In other words, how many non-repeating paragraphs are there in total?\n\n{{context}}\n\nPlease enter the final count of unique paragraphs after removing duplicates. The output format should only contain the number, such as 1, 2, 3, and so on.\n\nThe final answer is: ' -doc_to_target: '{{answers[0]}}' +doc_to_target: '{{answers}}' +process_results: !function metrics.get_count_score generation_kwargs: max_gen_toks: 32 temperature: 1 do_sample: True until: [] metric_list: - - metric: !function metrics.count_score + - metric: "count_score" aggregation: mean higher_is_better: True metadata: - version: 2.0 + version: 3.0 diff --git a/lm_eval/tasks/longbench/passage_count_e.yaml b/lm_eval/tasks/longbench/passage_count_e.yaml index d15dd727..602ab400 100644 --- a/lm_eval/tasks/longbench/passage_count_e.yaml +++ b/lm_eval/tasks/longbench/passage_count_e.yaml @@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench test_split: test dataset_name: passage_count_e doc_to_text: 'There are some paragraphs below sourced from Wikipedia. Some of them may be duplicates. Please carefully read these paragraphs and determine how many unique paragraphs there are after removing duplicates. In other words, how many non-repeating paragraphs are there in total?\n\n{{context}}\n\nPlease enter the final count of unique paragraphs after removing duplicates. The output format should only contain the number, such as 1, 2, 3, and so on.\n\nThe final answer is: ' -doc_to_target: '{{answers[0]}}' +doc_to_target: '{{answers}}' +process_results: !function metrics.get_count_score generation_kwargs: max_gen_toks: 32 temperature: 1 do_sample: True until: [] metric_list: - - metric: !function metrics.count_score + - metric: "count_score" aggregation: mean higher_is_better: True metadata: - version: 2.0 + version: 3.0 diff --git a/lm_eval/tasks/longbench/passage_retrieval_en.yaml b/lm_eval/tasks/longbench/passage_retrieval_en.yaml index a7e521b5..b4e69378 100644 --- a/lm_eval/tasks/longbench/passage_retrieval_en.yaml +++ b/lm_eval/tasks/longbench/passage_retrieval_en.yaml @@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench test_split: test dataset_name: passage_retrieval_en doc_to_text: 'Here are 30 paragraphs from Wikipedia, along with an abstract. Please determine which paragraph the abstract is from.\n\n{{context}}\n\nThe following is an abstract.\n\n{{input}}\n\nPlease enter the number of the paragraph that the abstract is from. The answer format must be like "Paragraph 1", "Paragraph 2", etc.\n\nThe answer is: ' -doc_to_target: '{{answers[0]}}' +doc_to_target: '{{answers}}' +process_results: !function metrics.get_retrieval_score generation_kwargs: max_gen_toks: 32 temperature: 1 do_sample: True until: [] metric_list: - - metric: !function metrics.retrieval_score + - metric: "retrieval_score" aggregation: mean higher_is_better: True metadata: - version: 2.0 + version: 3.0 diff --git a/lm_eval/tasks/longbench/passage_retrieval_en_e.yaml b/lm_eval/tasks/longbench/passage_retrieval_en_e.yaml index 1ca0b608..19811548 100644 --- a/lm_eval/tasks/longbench/passage_retrieval_en_e.yaml +++ b/lm_eval/tasks/longbench/passage_retrieval_en_e.yaml @@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench test_split: test dataset_name: passage_retrieval_en_e doc_to_text: 'Here are 30 paragraphs from Wikipedia, along with an abstract. Please determine which paragraph the abstract is from.\n\n{{context}}\n\nThe following is an abstract.\n\n{{input}}\n\nPlease enter the number of the paragraph that the abstract is from. The answer format must be like "Paragraph 1", "Paragraph 2", etc.\n\nThe answer is: ' -doc_to_target: '{{answers[0]}}' +doc_to_target: '{{answers}}' +process_results: !function metrics.get_retrieval_score generation_kwargs: max_gen_toks: 32 temperature: 1 do_sample: True until: [] metric_list: - - metric: !function metrics.retrieval_score + - metric: "retrieval_score" aggregation: mean higher_is_better: True metadata: - version: 2.0 + version: 3.0 diff --git a/lm_eval/tasks/longbench/passage_retrieval_zh.yaml b/lm_eval/tasks/longbench/passage_retrieval_zh.yaml index 2556cc2f..36bf8295 100644 --- a/lm_eval/tasks/longbench/passage_retrieval_zh.yaml +++ b/lm_eval/tasks/longbench/passage_retrieval_zh.yaml @@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench test_split: test dataset_name: passage_retrieval_zh doc_to_text: '以下是若干段落文字，以及其中一个段落的摘要。请确定给定的摘要出自哪一段。\n\n{{context}}\n\n下面是一个摘要\n\n{{input}}\n\n请输入摘要所属段落的编号。答案格式必须是"段落1"，"段落2"等格式\n\n答案是：' -doc_to_target: '{{answers[0]}}' +doc_to_target: '{{answers}}' +process_results: !function metrics.get_retrieval_zh_score generation_kwargs: max_gen_toks: 32 temperature: 1 do_sample: True until: [] metric_list: - - metric: !function metrics.retrieval_zh_score + - metric: "retrieval_zh_score" aggregation: mean higher_is_better: True metadata: - version: 2.0 + version: 3.0 diff --git a/lm_eval/tasks/longbench/qasper.yaml b/lm_eval/tasks/longbench/qasper.yaml index 21dd8c58..44b40590 100644 --- a/lm_eval/tasks/longbench/qasper.yaml +++ b/lm_eval/tasks/longbench/qasper.yaml @@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench test_split: test dataset_name: qasper doc_to_text: 'You are given a scientific article and a question. Answer the question as concisely as you can, using a single phrase or sentence if possible. If the question cannot be answered based on the information in the article, write "unanswerable". If the question is a yes/no question, answer "yes", "no", or "unanswerable". Do not provide any explanation.\n\nArticle: {{context}}\n\n Answer the question based on the above article as concisely as you can, using a single phrase or sentence if possible. If the question cannot be answered based on the information in the article, write "unanswerable". If the question is a yes/no question, answer "yes", "no", or "unanswerable". Do not provide any explanation.\n\nQuestion: {{input}}\n\nAnswer:' -doc_to_target: '{{answers[0]}}' +doc_to_target: '{{answers}}' +process_results: !function metrics.get_qa_f1_score generation_kwargs: max_gen_toks: 128 temperature: 1 do_sample: True until: [] metric_list: - - metric: !function metrics.qa_f1_score + - metric: "qa_f1_score" aggregation: mean higher_is_better: True metadata: - version: 2.0 + version: 3.0 diff --git a/lm_eval/tasks/longbench/qasper_e.yaml b/lm_eval/tasks/longbench/qasper_e.yaml index 986101f0..e3808433 100644 --- a/lm_eval/tasks/longbench/qasper_e.yaml +++ b/lm_eval/tasks/longbench/qasper_e.yaml @@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench test_split: test dataset_name: qasper_e doc_to_text: 'You are given a scientific article and a question. Answer the question as concisely as you can, using a single phrase or sentence if possible. If the question cannot be answered based on the information in the article, write "unanswerable". If the question is a yes/no question, answer "yes", "no", or "unanswerable". Do not provide any explanation.\n\nArticle: {{context}}\n\n Answer the question based on the above article as concisely as you can, using a single phrase or sentence if possible. If the question cannot be answered based on the information in the article, write "unanswerable". If the question is a yes/no question, answer "yes", "no", or "unanswerable". Do not provide any explanation.\n\nQuestion: {{input}}\n\nAnswer:' -doc_to_target: '{{answers[0]}}' +doc_to_target: '{{answers}}' +process_results: !function metrics.get_qa_f1_score generation_kwargs: max_gen_toks: 128 temperature: 1 do_sample: True until: [] metric_list: - - metric: !function metrics.qa_f1_score + - metric: "qa_f1_score" aggregation: mean higher_is_better: True metadata: - version: 2.0 + version: 3.0 diff --git a/lm_eval/tasks/longbench/qmsum.yaml b/lm_eval/tasks/longbench/qmsum.yaml index 9c1d225e..8c922985 100644 --- a/lm_eval/tasks/longbench/qmsum.yaml +++ b/lm_eval/tasks/longbench/qmsum.yaml @@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench test_split: test dataset_name: qmsum doc_to_text: 'You are given a meeting transcript and a query containing a question or instruction. Answer the query in one or more sentences.\n\nTranscript:\n{{context}}\n\nNow, answer the query based on the above meeting transcript in one or more sentences.\n\nQuery: {{input}}\nAnswer:' -doc_to_target: '{{answers[0]}}' +doc_to_target: '{{answers}}' +process_results: !function metrics.get_rouge_score generation_kwargs: max_gen_toks: 512 temperature: 1 do_sample: True until: [] metric_list: - - metric: !function metrics.rouge_score + - metric: "rouge_score" aggregation: mean higher_is_better: True metadata: - version: 2.0 + version: 3.0 diff --git a/lm_eval/tasks/longbench/repobench-p.yaml b/lm_eval/tasks/longbench/repobench-p.yaml index 1e1af77e..8413e1e6 100644 --- a/lm_eval/tasks/longbench/repobench-p.yaml +++ b/lm_eval/tasks/longbench/repobench-p.yaml @@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench test_split: test dataset_name: repobench-p doc_to_text: 'Please complete the code given below. \n{{context}}{{input}}Next line of code:\n' -doc_to_target: '{{answers[0]}}' +doc_to_target: '{{answers}}' +process_results: !function metrics.get_code_sim_score generation_kwargs: max_gen_toks: 64 temperature: 1 do_sample: True until: [] metric_list: - - metric: !function metrics.code_sim_score + - metric: "code_sim_score" aggregation: mean higher_is_better: True metadata: - version: 2.0 + version: 3.0 diff --git a/lm_eval/tasks/longbench/repobench-p_e.yaml b/lm_eval/tasks/longbench/repobench-p_e.yaml index ee71b137..2c0a55e0 100644 --- a/lm_eval/tasks/longbench/repobench-p_e.yaml +++ b/lm_eval/tasks/longbench/repobench-p_e.yaml @@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench test_split: test dataset_name: repobench-p_e doc_to_text: 'Please complete the code given below. \n{{context}}{{input}}Next line of code:\n' -doc_to_target: '{{answers[0]}}' +doc_to_target: '{{answers}}' +process_results: !function metrics.get_code_sim_score generation_kwargs: max_gen_toks: 64 temperature: 1 do_sample: True until: [] metric_list: - - metric: !function metrics.code_sim_score + - metric: "code_sim_score" aggregation: mean higher_is_better: True metadata: - version: 2.0 + version: 3.0 diff --git a/lm_eval/tasks/longbench/samsum.yaml b/lm_eval/tasks/longbench/samsum.yaml index 102e9062..1e94d274 100644 --- a/lm_eval/tasks/longbench/samsum.yaml +++ b/lm_eval/tasks/longbench/samsum.yaml @@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench test_split: test dataset_name: samsum doc_to_text: 'Summarize the dialogue into a few short sentences. The following are some examples.\n\n{{context}}\n\n{{input}}' -doc_to_target: '{{answers[0]}}' +doc_to_target: '{{answers}}' +process_results: !function metrics.get_rouge_score generation_kwargs: max_gen_toks: 128 temperature: 1 do_sample: True - until: ['\n'] + until: ["\n"] metric_list: - - metric: !function metrics.rouge_score + - metric: "rouge_score" aggregation: mean higher_is_better: True metadata: - version: 2.0 + version: 3.0 diff --git a/lm_eval/tasks/longbench/samsum_e.yaml b/lm_eval/tasks/longbench/samsum_e.yaml index 8d886415..9b3b1d5e 100644 --- a/lm_eval/tasks/longbench/samsum_e.yaml +++ b/lm_eval/tasks/longbench/samsum_e.yaml @@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench test_split: test dataset_name: samsum_e doc_to_text: 'Summarize the dialogue into a few short sentences. The following are some examples.\n\n{{context}}\n\n{{input}}' -doc_to_target: '{{answers[0]}}' +doc_to_target: '{{answers}}' +process_results: !function metrics.get_rouge_score generation_kwargs: max_gen_toks: 128 temperature: 1 do_sample: True - until: ['\n'] + until: ["\n"] metric_list: - - metric: !function metrics.rouge_score + - metric: "rouge_score" aggregation: mean higher_is_better: True metadata: - version: 2.0 + version: 3.0 diff --git a/lm_eval/tasks/longbench/trec.yaml b/lm_eval/tasks/longbench/trec.yaml index 00de0c2a..525a1f4d 100644 --- a/lm_eval/tasks/longbench/trec.yaml +++ b/lm_eval/tasks/longbench/trec.yaml @@ -6,16 +6,16 @@ dataset_path: THUDM/LongBench test_split: test dataset_name: trec doc_to_text: 'Please determine the type of the question below. Here are some examples of questions.\n\n{{context}}\n{{input}}' -doc_to_target: '{{answers[0]}}' -process_results: !function metrics.classification_score +doc_to_target: '{{answers}}' +process_results: !function metrics.get_classification_score generation_kwargs: max_gen_toks: 64 temperature: 1 do_sample: True - until: ['\n'] + until: ["\n"] metric_list: - metric: "classification_score" aggregation: mean higher_is_better: True metadata: - version: 2.0 + version: 3.0 diff --git a/lm_eval/tasks/longbench/trec_e.yaml b/lm_eval/tasks/longbench/trec_e.yaml index 87ffa4c0..ff6595b9 100644 --- a/lm_eval/tasks/longbench/trec_e.yaml +++ b/lm_eval/tasks/longbench/trec_e.yaml @@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench test_split: test dataset_name: trec_e doc_to_text: 'Please determine the type of the question below. Here are some examples of questions.\n\n{{context}}\n{{input}}' -doc_to_target: '{{answers[0]}}' +doc_to_target: '{{answers}}' +process_results: !function metrics.get_classification_score generation_kwargs: max_gen_toks: 64 temperature: 1 do_sample: True - until: ['\n'] + until: ["\n"] metric_list: - - metric: !function metrics.classification_score + - metric: "classification_score" aggregation: mean higher_is_better: True metadata: - version: 2.0 + version: 3.0 diff --git a/lm_eval/tasks/longbench/triviaqa.yaml b/lm_eval/tasks/longbench/triviaqa.yaml index 501c63ab..d54cbab7 100644 --- a/lm_eval/tasks/longbench/triviaqa.yaml +++ b/lm_eval/tasks/longbench/triviaqa.yaml @@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench test_split: test dataset_name: triviaqa doc_to_text: 'Answer the question based on the given passage. Only give me the answer and do not output any other words. The following are some examples.\n\n{{context}}\n\n{{input}}' -doc_to_target: '{{answers[0]}}' +doc_to_target: '{{answers}}' +process_results: !function metrics.get_qa_f1_score generation_kwargs: max_gen_toks: 32 temperature: 1 do_sample: True - until: ['\n'] + until: ["\n"] metric_list: - - metric: !function metrics.qa_f1_score + - metric: "qa_f1_score" aggregation: mean higher_is_better: True metadata: - version: 2.0 + version: 3.0 diff --git a/lm_eval/tasks/longbench/triviaqa_e.yaml b/lm_eval/tasks/longbench/triviaqa_e.yaml index b475efe5..ceac823f 100644 --- a/lm_eval/tasks/longbench/triviaqa_e.yaml +++ b/lm_eval/tasks/longbench/triviaqa_e.yaml @@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench test_split: test dataset_name: triviaqa_e doc_to_text: 'Answer the question based on the given passage. Only give me the answer and do not output any other words. The following are some examples.\n\n{{context}}\n\n{{input}}' -doc_to_target: '{{answers[0]}}' +doc_to_target: '{{answers}}' +process_results: !function metrics.get_qa_f1_score generation_kwargs: max_gen_toks: 32 temperature: 1 do_sample: True - until: ['\n'] + until: ["\n"] metric_list: - - metric: !function metrics.qa_f1_score + - metric: "qa_f1_score" aggregation: mean higher_is_better: True metadata: - version: 2.0 + version: 3.0 diff --git a/lm_eval/tasks/longbench/vcsum.yaml b/lm_eval/tasks/longbench/vcsum.yaml index c642954d..ba590f5b 100644 --- a/lm_eval/tasks/longbench/vcsum.yaml +++ b/lm_eval/tasks/longbench/vcsum.yaml @@ -6,15 +6,16 @@ dataset_path: THUDM/LongBench test_split: test dataset_name: vcsum doc_to_text: '下面有一段会议记录，请你阅读后，写一段总结，总结会议的内容。\n会议记录：\n{{context}}\n\n会议总结：' -doc_to_target: '{{answers[0]}}' +doc_to_target: '{{answers}}' +process_results: !function metrics.get_rouge_zh_score generation_kwargs: max_gen_toks: 512 temperature: 1 do_sample: True until: [] metric_list: - - metric: !function metrics.rouge_zh_score + - metric: "rouge_zh_score" aggregation: mean higher_is_better: True metadata: - version: 2.0 + version: 3.0 -- GitLab From d09e03dd14e94a967d3411390961651ffb0dd2f2 Mon Sep 17 00:00:00 2001 From: Kiersten Stokes Date: Thu, 12 Jun 2025 04:12:42 -0500 Subject: [PATCH 40/46] Fallback to super impl in fewshot_context for Unitxt tasks (#3023) Signed-off-by: kiersten-stokes --- lm_eval/tasks/unitxt/task.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/lm_eval/tasks/unitxt/task.py b/lm_eval/tasks/unitxt/task.py index d8e7edfc..5c75cf57 100644 --- a/lm_eval/tasks/unitxt/task.py +++ b/lm_eval/tasks/unitxt/task.py @@ -105,7 +105,7 @@ class Unitxt(ConfigurableTask): return False def doc_to_target(self, doc): - doc["target"] + return doc["target"] def get_arguments(self, doc, ctx): return (ctx, {"until": ["\n"]}) @@ -120,8 +120,7 @@ class Unitxt(ConfigurableTask): chat_template: Optional[Callable] = None, gen_prefix: Optional[str] = None, ) -> str: - source = self.doc_to_text(doc) - if isinstance(source, list): + if isinstance(self.doc_to_text(doc), list): if apply_chat_template: formated_source = chat_template(self.doc_to_text(doc)) return formated_source @@ -130,7 +129,15 @@ class Unitxt(ConfigurableTask): "Got chat template format from Unitxt, but apply_chat_template is false. Add '--apply_chat_template' to command line." ) else: - return source + return super().fewshot_context( + doc=doc, + num_fewshot=num_fewshot, + system_instruction=system_instruction, + apply_chat_template=apply_chat_template, + fewshot_as_multiturn=fewshot_as_multiturn, + chat_template=chat_template, + gen_prefix=gen_prefix, + ) def construct_requests(self, doc, ctx, **kwargs): """Uses RequestFactory to construct Requests and returns an iterable of -- GitLab From e20ef72e451ca78efdf4b3f981173e2cf2e87bac Mon Sep 17 00:00:00 2001 From: "fuder.eth" <139509124+vtjl10@users.noreply.github.com> Date: Mon, 16 Jun 2025 11:42:08 +0300 Subject: [PATCH 41/46] Fix Typo in README and Comment in utils_mcq.py (#3057) * Update README.md * Update utils_mcq.py --- lm_eval/tasks/arab_culture/README.md | 2 +- lm_eval/tasks/arab_culture/utils_mcq.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lm_eval/tasks/arab_culture/README.md b/lm_eval/tasks/arab_culture/README.md index f8bc5a8c..cf6f8cf7 100644 --- a/lm_eval/tasks/arab_culture/README.md +++ b/lm_eval/tasks/arab_culture/README.md @@ -65,6 +65,6 @@ REGION=True **Please add these flags add environment variables.** -* We also allow for prompting in English, which we found to acheive higher results on most of the evaluated models (please refer to our paper). +* We also allow for prompting in English, which we found to achieve higher results on most of the evaluated models (please refer to our paper). * To change the language of the prompt, Define the `ARABIC` environment variable. diff --git a/lm_eval/tasks/arab_culture/utils_mcq.py b/lm_eval/tasks/arab_culture/utils_mcq.py index 8d03f443..315c50f5 100644 --- a/lm_eval/tasks/arab_culture/utils_mcq.py +++ b/lm_eval/tasks/arab_culture/utils_mcq.py @@ -88,7 +88,7 @@ def doc_to_text(doc): first_statement=first_statement, choices=choices_str ) - ### apply jais chat tempelate + ### apply jais chat template if MODEL_NAME and "jais" in MODEL_NAME and "chat" in MODEL_NAME: if ARABIC: doc_text = JAIS_CHAT_AR.format(question=doc_text) -- GitLab From 9fbe48c230c2649d9430c133290d6882b55105ea Mon Sep 17 00:00:00 2001 From: Baber Abbasi <92168766+baberabb@users.noreply.github.com> Date: Mon, 16 Jun 2025 14:28:03 +0500 Subject: [PATCH 42/46] fix longbech citation (#3061) * fix longbech citation --- lm_eval/tasks/longbench/README.md | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/lm_eval/tasks/longbench/README.md b/lm_eval/tasks/longbench/README.md index 29bd831b..bef2dfc1 100644 --- a/lm_eval/tasks/longbench/README.md +++ b/lm_eval/tasks/longbench/README.md @@ -1,10 +1,10 @@ -# Task-name +# LongBench ### Paper -Title: `LongBench v2: Towards Deeper Understanding and Reasoning on Realistic Long-context Multitasks` +Title: `LongBench: A Bilingual, Multitask Benchmark for Long Context Understanding` -Abstract: `This paper introduces LongBench v2, a benchmark designed to assess the ability of LLMs to handle long-context problems requiring deep understanding and reasoning across real-world multitasks. LongBench v2 consists of 503 challenging multiple-choice questions, with contexts ranging from 8k to 2M words, across six major task categories: single-document QA, multi-document QA, long in-context learning, long-dialogue history understanding, code repository understanding, and long structured data understanding.` +Abstract: `In this paper, we introduce LongBench, the first bilingual, multi-task benchmark for long context understanding, enabling a more rigorous evaluation of long context understanding. LongBench comprises 21 datasets across 6 task categories in both English and Chinese, with an average length of 6,711 words (English) and 13,386 characters (Chinese). These tasks cover key long-text application areas including single-doc QA, multi-doc QA, summarization, few-shot learning, synthetic tasks, and code completion. All datasets in LongBench are standardized into a unified format, allowing for effortless automatic evaluation of LLMs` Homepage: `https://github.com/THUDM/LongBench` @@ -12,12 +12,6 @@ Homepage: `https://github.com/THUDM/LongBench` ### Citation ``` -@article{bai2024longbench2, - title={LongBench v2: Towards Deeper Understanding and Reasoning on Realistic Long-context Multitasks}, - author={Yushi Bai and Shangqing Tu and Jiajie Zhang and Hao Peng and Xiaozhi Wang and Xin Lv and Shulin Cao and Jiazheng Xu and Lei Hou and Yuxiao Dong and Jie Tang and Juanzi Li}, - journal={arXiv preprint arXiv:2412.15204}, - year={2024} -} @inproceedings{bai2024longbench, title = "{L}ong{B}ench: A Bilingual, Multitask Benchmark for Long Context Understanding", author = "Bai, Yushi and Lv, Xin and Zhang, Jiajie and Lyu, Hongchang and -- GitLab From 5a15058ea7820fc995f88ea2b8aaecc695679b47 Mon Sep 17 00:00:00 2001 From: Anna Fontana <101867173+annafontanaa@users.noreply.github.com> Date: Thu, 19 Jun 2025 11:14:41 +0200 Subject: [PATCH 43/46] Update README.md (#3070) Wrong task name: mmlu_generation doesn't non exist -> mmlu_generative is the correct one --- lm_eval/tasks/mmlu/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lm_eval/tasks/mmlu/README.md b/lm_eval/tasks/mmlu/README.md index a3425d51..5924a1d2 100644 --- a/lm_eval/tasks/mmlu/README.md +++ b/lm_eval/tasks/mmlu/README.md @@ -36,11 +36,11 @@ Note: The `Flan` variants are derived from [here](https://github.com/jasonwei20/ * `mmlu`: `Original multiple-choice MMLU benchmark` * `mmlu_continuation`: `MMLU but with continuation prompts` -* `mmlu_generation`: `MMLU generation` +* `mmlu_generative`: `MMLU generation` MMLU is the original benchmark as implemented by Hendrycks et al. with the choices in context and the answer letters (e.g `A`, `B`, `C`, `D`) in the continuation. `mmlu_continuation` is a cloze-style variant without the choices in context and the full answer choice in the continuation. -`mmlu_generation` is a generation variant, similar to the original but the LLM is asked to generate the correct answer letter. +`mmlu_generative` is a generation variant, similar to the original but the LLM is asked to generate the correct answer letter. #### Subgroups -- GitLab From 37357004a2373f6cf8a06fd498fae66a3ff6ea7f Mon Sep 17 00:00:00 2001 From: Maxim Evtush <154841002+maximevtush@users.noreply.github.com> Date: Thu, 19 Jun 2025 12:15:42 +0300 Subject: [PATCH 44/46] Update instructions.py (#3060) --- lm_eval/tasks/ifeval/instructions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lm_eval/tasks/ifeval/instructions.py b/lm_eval/tasks/ifeval/instructions.py index 9a7bcce1..8a23d9e7 100644 --- a/lm_eval/tasks/ifeval/instructions.py +++ b/lm_eval/tasks/ifeval/instructions.py @@ -740,7 +740,7 @@ class RephraseChecker(Instruction): class KeywordChecker(Instruction): - """Check the exisitence of certain keywords.""" + """Check the existence of certain keywords.""" def build_description(self, *, keywords=None): """Build the instruction description. @@ -1161,7 +1161,7 @@ class RephraseParagraph(Instruction): Args: original_paragraph: A string presenting the original paragraph. The - rephrases response should have betweeb low-high words in common. + rephrases response should have between low-high words in common. low: An integer presenting the lower bound of similar words. high: An integer representing the upper bound of similar words. -- GitLab From 452749513f817315042df9286241a61051392470 Mon Sep 17 00:00:00 2001 From: Baber Abbasi <92168766+baberabb@users.noreply.github.com> Date: Thu, 19 Jun 2025 18:57:26 +0500 Subject: [PATCH 45/46] bump version to `0.4.9` (#3073) --- lm_eval/__init__.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lm_eval/__init__.py b/lm_eval/__init__.py index fece9162..c50ad3ed 100644 --- a/lm_eval/__init__.py +++ b/lm_eval/__init__.py @@ -4,4 +4,4 @@ import os from .evaluator import evaluate, simple_evaluate -__version__ = "0.4.8" +__version__ = "0.4.9" diff --git a/pyproject.toml b/pyproject.toml index 45dd4418..6c8b4403 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "lm_eval" -version = "0.4.8" +version = "0.4.9" authors = [ {name="EleutherAI", email="contact@eleuther.ai"} ] -- GitLab From 68c3a811715ca86101f88c0044665bb70ad447f6 Mon Sep 17 00:00:00 2001 From: Anna Fontana <101867173+annafontanaa@users.noreply.github.com> Date: Fri, 20 Jun 2025 11:45:33 +0200 Subject: [PATCH 46/46] llama3 task: update README.md (#3074) "arc_chalenge_chat" doesn't exist: I think it should be "arc_challenge_chat", but this task is not implemented here (see arc task folder). --- lm_eval/tasks/llama3/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/lm_eval/tasks/llama3/README.md b/lm_eval/tasks/llama3/README.md index 1b0b762b..1c84ec5e 100644 --- a/lm_eval/tasks/llama3/README.md +++ b/lm_eval/tasks/llama3/README.md @@ -39,7 +39,6 @@ BibTeX-formatted citation goes here * `mmlu_hi_llama`: `Hindi version of generation MMLU` * `mmlu_es_llama`: `Spanish version of generation MMLU` * `mmlu_de_llama`: `German version of generation MMLU` -* `arc_chalenge_chat`: `generation variant of ARC-Challenge using MMLU format` * `arc_challenge_llama`: `generation variant of ARC-Challenge following Meta pre-processing` * `gsm8k_llama`: `Chain-of-though variant of GSM8k` -- GitLab