add Darija (Moroccan dialects) tasks including darijammlu. darijahellaswag and darija_bench (#2521)

* add Darija tasks

* fix multiple groups issue in darijammlu

* add MT to the description of the Darija tasks

* Update README.md

nit

* fix the recursion error caused by the darija_summarization task

* use a custom filter instead of the decorator for the strip function

---------
Co-authored-by: Baber Abbasi <92168766+baberabb@users.noreply.github.com>

lm_eval/tasks/darija_bench/darija_translation/doda_translation_dr_en.yaml

+"process_docs": !function utils.dr_en
+include:
+  - translation_common_yaml
+  - doda_common_yaml
+"tag":
+- "darija_translation_tasks_doda"
+"task": "trasnlation_dr_en_doda"
+"task_alias": "dr_en_doda"
+metric_list:
+  - metric: !function utils.bert
+    aggregation: !function utils.bertbase
+    higher_is_better: true
\ No newline at end of file

lm_eval/tasks/darija_bench/darija_translation/doda_translation_dr_fr.yaml

+"process_docs": !function utils.dr_fr
+include:
+  - translation_common_yaml
+  - doda_common_yaml
+"tag":
+- "darija_translation_tasks_doda"
+"task": "trasnlation_dr_fr_doda"
+"task_alias": "dr_fr_doda"
+metric_list:
+  - metric: !function utils.bert
+    aggregation: !function utils.camembert
+    higher_is_better: true
\ No newline at end of file

lm_eval/tasks/darija_bench/darija_translation/doda_translation_dr_msa.yaml

+"process_docs": !function utils.dr_msa
+include:
+  - translation_common_yaml
+  - doda_common_yaml
+"tag":
+- "darija_translation_tasks_doda"
+"task": "trasnlation_dr_msa_doda"
+"task_alias": "dr_msa_doda"
+metric_list:
+  - metric: !function utils.bert
+    aggregation: !function utils.arabert
+    higher_is_better: true
\ No newline at end of file

lm_eval/tasks/darija_bench/darija_translation/doda_translation_en_dr.yaml

+"process_docs": !function utils.en_dr
+include:
+  - translation_common_yaml
+  - doda_common_yaml
+"tag":
+  - "darija_translation_tasks_doda"
+"task": "trasnlation_en_dr_doda"
+"task_alias": "en_dr_doda"
+metric_list:
+  - metric: !function utils.bert
+    aggregation: !function utils.darijabert
+    higher_is_better: true

lm_eval/tasks/darija_bench/darija_translation/doda_translation_fr_dr.yaml

+"process_docs": !function utils.fr_dr
+include:
+  - translation_common_yaml
+  - doda_common_yaml
+"tag":
+- "darija_translation_tasks_doda"
+"task": "trasnlation_fr_dr_doda"
+"task_alias": "fr_dr_doda"
+metric_list:
+  - metric: !function utils.bert
+    aggregation: !function utils.darijabert
+    higher_is_better: true
\ No newline at end of file

lm_eval/tasks/darija_bench/darija_translation/doda_translation_msa_dr.yaml

+"process_docs": !function utils.msa_dr
+include:
+  - translation_common_yaml
+  - doda_common_yaml
+"tag":
+- "darija_translation_tasks_doda"
+"task": "trasnlation_msa_dr_doda"
+"task_alias": "msa_dr_doda"
+metric_list:
+  - metric: !function utils.bert
+    aggregation: !function utils.darijabert
+    higher_is_better: true
\ No newline at end of file

lm_eval/tasks/darija_bench/darija_translation/flores_common_yaml

+test_split: flores_plus
\ No newline at end of file

lm_eval/tasks/darija_bench/darija_translation/flores_translation_all.yaml

+include:
+  - translation_common_yaml
+  - flores_common_yaml
+"tag":
+- "darija_translation_tasks_flores"
+"task": "trasnlation_all_flores"
+"task_alias": "all_flores"
+metric_list:
+  - metric: !function utils.bert
+    aggregation: !function utils.mbert
+    higher_is_better: true

lm_eval/tasks/darija_bench/darija_translation/flores_translation_darija.yaml

+group: darija_translation_flores
+group_alias: translation_flores
+task:
+  - darija_translation_tasks_flores
+metric_list:
+  - metric: bleu
+    aggregation: bleu
+    higher_is_better: true
+  - metric: chrf
+    aggregation: chrf
+    higher_is_better: true
+  - metric: ter
+    aggregation: ter
+    higher_is_better: false
+  - metric: !function utils.bert
+    aggregation: !function utils.mbert
+    higher_is_better: true
+metadata:
+  version: 1.0

lm_eval/tasks/darija_bench/darija_translation/flores_translation_dr_en.yaml

+"process_docs": !function utils.dr_en
+include:
+  - translation_common_yaml
+  - flores_common_yaml
+"tag":
+- "darija_translation_tasks_flores"
+"task": "trasnlation_dr_en_flores"
+"task_alias": "dr_en_flores"
+metric_list:
+  - metric: !function utils.bert
+    aggregation: !function utils.bertbase
+    higher_is_better: true
\ No newline at end of file

lm_eval/tasks/darija_bench/darija_translation/flores_translation_dr_fr.yaml

+"process_docs": !function utils.dr_fr
+include:
+  - translation_common_yaml
+  - flores_common_yaml
+"tag":
+- "darija_translation_tasks_flores"
+"task": "trasnlation_dr_fr_flores"
+"task_alias": "dr_fr_flores"
+metric_list:
+  - metric: !function utils.bert
+    aggregation: !function utils.camembert
+    higher_is_better: true
\ No newline at end of file

lm_eval/tasks/darija_bench/darija_translation/flores_translation_dr_msa.yaml

+"process_docs": !function utils.dr_msa
+include:
+  - translation_common_yaml
+  - flores_common_yaml
+"tag":
+- "darija_translation_tasks_flores"
+"task": "trasnlation_dr_msa_flores"
+"task_alias": "dr_msa_flores"
+metric_list:
+  - metric: !function utils.bert
+    aggregation: !function utils.arabert
+    higher_is_better: true
\ No newline at end of file

lm_eval/tasks/darija_bench/darija_translation/flores_translation_en_dr.yaml

+"process_docs": !function utils.en_dr
+include:
+  - translation_common_yaml
+  - flores_common_yaml
+"tag":
+  - "darija_translation_tasks_flores"
+"task": "trasnlation_en_dr_flores"
+"task_alias": "en_dr_flores"
+metric_list:
+  - metric: !function utils.bert
+    aggregation: !function utils.darijabert
+    higher_is_better: true

lm_eval/tasks/darija_bench/darija_translation/flores_translation_fr_dr.yaml

+"process_docs": !function utils.fr_dr
+include:
+  - translation_common_yaml
+  - flores_common_yaml
+"tag":
+- "darija_translation_tasks_flores"
+"task": "trasnlation_fr_dr_flores"
+"task_alias": "fr_dr_flores"
+metric_list:
+  - metric: !function utils.bert
+    aggregation: !function utils.darijabert
+    higher_is_better: true
\ No newline at end of file

lm_eval/tasks/darija_bench/darija_translation/flores_translation_msa_dr.yaml

+"process_docs": !function utils.msa_dr
+include:
+  - translation_common_yaml
+  - flores_common_yaml
+"tag":
+- "darija_translation_tasks_flores"
+"task": "trasnlation_msa_dr_flores"
+"task_alias": "msa_dr_flores"
+metric_list:
+  - metric: !function utils.bert
+    aggregation: !function utils.darijabert
+    higher_is_better: true
\ No newline at end of file

lm_eval/tasks/darija_bench/darija_translation/madar_common_yaml

+test_split: madar
\ No newline at end of file

lm_eval/tasks/darija_bench/darija_translation/madar_translation_all.yaml

+include:
+  - translation_common_yaml
+  - madar_common_yaml
+"tag":
+- "darija_translation_tasks_madar"
+"task": "trasnlation_all_madar"
+"task_alias": "all_madar"
+metric_list:
+  - metric: !function utils.bert
+    aggregation: !function utils.mbert
+    higher_is_better: true

lm_eval/tasks/darija_bench/darija_translation/madar_translation_darija.yaml

+group: darija_translation_madar
+group_alias: translation_madar
+task:
+  - darija_translation_tasks_madar
+metric_list:
+  - metric: bleu
+    aggregation: bleu
+    higher_is_better: true
+  - metric: chrf
+    aggregation: chrf
+    higher_is_better: true
+  - metric: ter
+    aggregation: ter
+    higher_is_better: false
+  - metric: !function utils.bert
+    aggregation: !function utils.mbert
+    higher_is_better: true
+metadata:
+  version: 1.0

lm_eval/tasks/darija_bench/darija_translation/madar_translation_dr_msa.yaml

+"process_docs": !function utils.dr_msa
+include:
+  - translation_common_yaml
+  - madar_common_yaml
+"tag":
+- "darija_translation_tasks_madar"
+"task": "trasnlation_dr_msa_madar"
+"task_alias": "dr_msa_madar"
+metric_list:
+  - metric: !function utils.bert
+    aggregation: !function utils.arabert
+    higher_is_better: true
\ No newline at end of file

lm_eval/tasks/darija_bench/darija_translation/madar_translation_msa_dr.yaml

+"process_docs": !function utils.msa_dr
+include:
+  - translation_common_yaml
+  - madar_common_yaml
+"tag":
+- "darija_translation_tasks_madar"
+"task": "trasnlation_msa_dr_madar"
+"task_alias": "msa_dr_madar"
+metric_list:
+  - metric: !function utils.bert
+    aggregation: !function utils.darijabert
+    higher_is_better: true
\ No newline at end of file