Merge branch 'main' into feature/eval_from_config

lm_eval/tasks/afrobench/adr/prompt_2/afridiacritics_ibo.yaml

+# Generated by utils.py
+dataset_name: ibo
+doc_to_text: "Given a sentence without diacritics, add the appropriate diacritics\
+  \ to make it grammatically and semantically correct. \nSentence: {{text}}. Return\
+  \ output sentence only"
+include: afridiacritics_yaml
+task: afridiacritics_ibo_prompt_2

lm_eval/tasks/afrobench/adr/prompt_2/afridiacritics_wol.yaml

+# Generated by utils.py
+dataset_name: wol
+doc_to_text: "Given a sentence without diacritics, add the appropriate diacritics\
+  \ to make it grammatically and semantically correct. \nSentence: {{text}}. Return\
+  \ output sentence only"
+include: afridiacritics_yaml
+task: afridiacritics_wol_prompt_2

lm_eval/tasks/afrobench/adr/prompt_2/afridiacritics_yaml

+tag:
+- adr_tasks
+- adr_prompt_2
+dataset_path: masakhane/diacritics-restoration
+dataset_kwargs: {trust_remote_code: True}
+doc_to_target: target
+output_type: generate_until
+fewshot_split: dev
+test_split: test
+training_split: train
+metric_list:
+  - metric: bleu
+    aggregation: bleu
+    higher_is_better: true
+  - metric: chrf
+    aggregation: chrf
+    higher_is_better: true
+generation_kwargs:
+  do_sample: false
+  until:
+  - '<eos>'
+  - </s>
+  - <|im_end|>
+metadata:
+  version: 1.0

lm_eval/tasks/afrobench/adr/prompt_2/afridiacritics_yor.yaml

+# Generated by utils.py
+dataset_name: yor
+doc_to_text: "Given a sentence without diacritics, add the appropriate diacritics\
+  \ to make it grammatically and semantically correct. \nSentence: {{text}}. Return\
+  \ output sentence only"
+include: afridiacritics_yaml
+task: afridiacritics_yor_prompt_2

lm_eval/tasks/afrobench/adr/prompt_3/afridiacritics_bbj.yaml

+# Generated by utils.py
+dataset_name: bbj
+doc_to_text: 'This text is in Gbomala. Restore all diacritical marks to their proper
+  places in the following sentence: {{text}}. Return output sentence only'
+include: afridiacritics_yaml
+task: afridiacritics_bbj_prompt_3

lm_eval/tasks/afrobench/adr/prompt_3/afridiacritics_fon.yaml

+# Generated by utils.py
+dataset_name: fon
+doc_to_text: 'This text is in Fon. Restore all diacritical marks to their proper places
+  in the following sentence: {{text}}. Return output sentence only'
+include: afridiacritics_yaml
+task: afridiacritics_fon_prompt_3

lm_eval/tasks/afrobench/adr/prompt_3/afridiacritics_ibo.yaml

+# Generated by utils.py
+dataset_name: ibo
+doc_to_text: 'This text is in Igbo. Restore all diacritical marks to their proper
+  places in the following sentence: {{text}}. Return output sentence only'
+include: afridiacritics_yaml
+task: afridiacritics_ibo_prompt_3

lm_eval/tasks/afrobench/adr/prompt_3/afridiacritics_wol.yaml

+# Generated by utils.py
+dataset_name: wol
+doc_to_text: 'This text is in Wolof. Restore all diacritical marks to their proper
+  places in the following sentence: {{text}}. Return output sentence only'
+include: afridiacritics_yaml
+task: afridiacritics_wol_prompt_3

lm_eval/tasks/afrobench/adr/prompt_3/afridiacritics_yaml

+tag:
+- adr_tasks
+- adr_prompt_3
+dataset_path: masakhane/diacritics-restoration
+dataset_kwargs: {trust_remote_code: True}
+doc_to_target: target
+output_type: generate_until
+fewshot_split: dev
+test_split: test
+training_split: train
+metric_list:
+  - metric: bleu
+    aggregation: bleu
+    higher_is_better: true
+  - metric: chrf
+    aggregation: chrf
+    higher_is_better: true
+generation_kwargs:
+  do_sample: false
+  until:
+  - '<eos>'
+  - </s>
+  - <|im_end|>
+metadata:
+  version: 1.0

lm_eval/tasks/afrobench/adr/prompt_3/afridiacritics_yor.yaml

+# Generated by utils.py
+dataset_name: yor
+doc_to_text: 'This text is in Yoruba. Restore all diacritical marks to their proper
+  places in the following sentence: {{text}}. Return output sentence only'
+include: afridiacritics_yaml
+task: afridiacritics_yor_prompt_3

lm_eval/tasks/afrobench/adr/prompt_4/afridiacritics_bbj.yaml

+# Generated by utils.py
+dataset_name: bbj
+doc_to_text: 'You are a linguist specializing in diacritical marks for Gbomala. Add
+  the appropriate diacritics to this Gbomala sentence: {{text}}. Return output sentence
+  only'
+include: afridiacritics_yaml
+task: afridiacritics_bbj_prompt_4

lm_eval/tasks/afrobench/adr/prompt_4/afridiacritics_fon.yaml

+# Generated by utils.py
+dataset_name: fon
+doc_to_text: 'You are a linguist specializing in diacritical marks for Fon. Add the
+  appropriate diacritics to this Fon sentence: {{text}}. Return output sentence only'
+include: afridiacritics_yaml
+task: afridiacritics_fon_prompt_4

lm_eval/tasks/afrobench/adr/prompt_4/afridiacritics_ibo.yaml

+# Generated by utils.py
+dataset_name: ibo
+doc_to_text: 'You are a linguist specializing in diacritical marks for Igbo. Add the
+  appropriate diacritics to this Igbo sentence: {{text}}. Return output sentence only'
+include: afridiacritics_yaml
+task: afridiacritics_ibo_prompt_4

lm_eval/tasks/afrobench/adr/prompt_4/afridiacritics_wol.yaml

+# Generated by utils.py
+dataset_name: wol
+doc_to_text: 'You are a linguist specializing in diacritical marks for Wolof. Add
+  the appropriate diacritics to this Wolof sentence: {{text}}. Return output sentence
+  only'
+include: afridiacritics_yaml
+task: afridiacritics_wol_prompt_4

lm_eval/tasks/afrobench/adr/prompt_4/afridiacritics_yaml

+tag:
+- adr_tasks
+- adr_prompt_4
+dataset_path: masakhane/diacritics-restoration
+dataset_kwargs: {trust_remote_code: True}
+doc_to_target: target
+output_type: generate_until
+fewshot_split: dev
+test_split: test
+training_split: train
+metric_list:
+  - metric: bleu
+    aggregation: bleu
+    higher_is_better: true
+  - metric: chrf
+    aggregation: chrf
+    higher_is_better: true
+generation_kwargs:
+  do_sample: false
+  until:
+  - '<eos>'
+  - </s>
+  - <|im_end|>
+metadata:
+  version: 1.0

lm_eval/tasks/afrobench/adr/prompt_4/afridiacritics_yor.yaml

+# Generated by utils.py
+dataset_name: yor
+doc_to_text: 'You are a linguist specializing in diacritical marks for Yoruba. Add
+  the appropriate diacritics to this Yoruba sentence: {{text}}. Return output sentence
+  only'
+include: afridiacritics_yaml
+task: afridiacritics_yor_prompt_4

lm_eval/tasks/afrobench/adr/prompt_5/afridiacritics_bbj.yaml

+# Generated by utils.py
+dataset_name: bbj
+doc_to_text: 'You are a linguist specializing in diacritical marks for Gbomala. Diacritics
+  are essential for proper pronunciation and meaning in Gbomala. You are tasked with
+  converting Gbomala sentences  without diacritics into their correctly accented forms.
+  Here''s the input: {{text}}. Return output sentence only'
+include: afridiacritics_yaml
+task: afridiacritics_bbj_prompt_5

lm_eval/tasks/afrobench/adr/prompt_5/afridiacritics_fon.yaml

+# Generated by utils.py
+dataset_name: fon
+doc_to_text: 'You are a linguist specializing in diacritical marks for Fon. Diacritics
+  are essential for proper pronunciation and meaning in Fon. You are tasked with converting
+  Fon sentences  without diacritics into their correctly accented forms. Here''s the
+  input: {{text}}. Return output sentence only'
+include: afridiacritics_yaml
+task: afridiacritics_fon_prompt_5

lm_eval/tasks/afrobench/adr/prompt_5/afridiacritics_ibo.yaml

+# Generated by utils.py
+dataset_name: ibo
+doc_to_text: 'You are a linguist specializing in diacritical marks for Igbo. Diacritics
+  are essential for proper pronunciation and meaning in Igbo. You are tasked with
+  converting Igbo sentences  without diacritics into their correctly accented forms.
+  Here''s the input: {{text}}. Return output sentence only'
+include: afridiacritics_yaml
+task: afridiacritics_ibo_prompt_5

lm_eval/tasks/afrobench/adr/prompt_5/afridiacritics_wol.yaml

+# Generated by utils.py
+dataset_name: wol
+doc_to_text: 'You are a linguist specializing in diacritical marks for Wolof. Diacritics
+  are essential for proper pronunciation and meaning in Wolof. You are tasked with
+  converting Wolof sentences  without diacritics into their correctly accented forms.
+  Here''s the input: {{text}}. Return output sentence only'
+include: afridiacritics_yaml
+task: afridiacritics_wol_prompt_5