Merge branch 'main' into mathvista

# Conflicts:
#	lm_eval/models/hf_vlms.py

lm_eval/tasks/catalan_bench/flores_ca/flores_ca-en.yaml

+# File generated by `create-yamls.py`
+include: _flores_common_yaml
+task: flores_ca-en
+doc_to_text: 'Catalan sentence: {{sentence_cat_Latn}}
+
+  English sentence:'
+doc_to_target: '{{sentence_eng_Latn}}'

lm_eval/tasks/catalan_bench/flores_ca/flores_ca-es.yaml

+# File generated by `create-yamls.py`
+include: _flores_common_yaml
+task: flores_ca-es
+doc_to_text: 'Catalan sentence: {{sentence_cat_Latn}}
+
+  Spanish sentence:'
+doc_to_target: '{{sentence_spa_Latn}}'

lm_eval/tasks/catalan_bench/flores_ca/flores_ca-eu.yaml

+# File generated by `create-yamls.py`
+include: _flores_common_yaml
+task: flores_ca-eu
+doc_to_text: 'Catalan sentence: {{sentence_cat_Latn}}
+
+  Basque sentence:'
+doc_to_target: '{{sentence_eus_Latn}}'

lm_eval/tasks/catalan_bench/flores_ca/flores_ca-fr.yaml

+# File generated by `create-yamls.py`
+include: _flores_common_yaml
+task: flores_ca-fr
+doc_to_text: 'Catalan sentence: {{sentence_cat_Latn}}
+
+  French sentence:'
+doc_to_target: '{{sentence_fra_Latn}}'

lm_eval/tasks/catalan_bench/flores_ca/flores_ca-gl.yaml

+# File generated by `create-yamls.py`
+include: _flores_common_yaml
+task: flores_ca-gl
+doc_to_text: 'Catalan sentence: {{sentence_cat_Latn}}
+
+  Galician sentence:'
+doc_to_target: '{{sentence_glg_Latn}}'

lm_eval/tasks/catalan_bench/flores_ca/flores_ca-it.yaml

+# File generated by `create-yamls.py`
+include: _flores_common_yaml
+task: flores_ca-it
+doc_to_text: 'Catalan sentence: {{sentence_cat_Latn}}
+
+  Italian sentence:'
+doc_to_target: '{{sentence_ita_Latn}}'

lm_eval/tasks/catalan_bench/flores_ca/flores_ca-pt.yaml

+# File generated by `create-yamls.py`
+include: _flores_common_yaml
+task: flores_ca-pt
+doc_to_text: 'Catalan sentence: {{sentence_cat_Latn}}
+
+  Portuguese sentence:'
+doc_to_target: '{{sentence_por_Latn}}'

lm_eval/tasks/catalan_bench/flores_ca/flores_ca.yaml

+group: flores_ca
+task:
+  - flores_es-ca
+  - flores_ca-es
+  - flores_en-ca
+  - flores_ca-en
+  - flores_eu-ca
+  - flores_ca-eu
+  - flores_pt-ca
+  - flores_ca-pt
+  - flores_it-ca
+  - flores_ca-it
+  - flores_fr-ca
+  - flores_ca-fr
+  - flores_ca-gl
+  - flores_gl-ca
+  - flores_ca-de
+  - flores_de-ca
+aggregate_metric_list:
+  - metric: bleu
+    aggregation: mean
+    weight_by_size: false
+metadata:
+  version: 1.0

lm_eval/tasks/catalan_bench/flores_ca/flores_de-ca.yaml

+# File generated by `create-yamls.py`
+include: _flores_common_yaml
+task: flores_de-ca
+doc_to_text: 'German sentence: {{sentence_deu_Latn}}
+
+  Catalan sentence:'
+doc_to_target: '{{sentence_cat_Latn}}'

lm_eval/tasks/catalan_bench/flores_ca/flores_en-ca.yaml

+# File generated by `create-yamls.py`
+include: _flores_common_yaml
+task: flores_en-ca
+doc_to_text: 'English sentence: {{sentence_eng_Latn}}
+
+  Catalan sentence:'
+doc_to_target: '{{sentence_cat_Latn}}'

lm_eval/tasks/catalan_bench/flores_ca/flores_es-ca.yaml

+# File generated by `create-yamls.py`
+include: _flores_common_yaml
+task: flores_es-ca
+doc_to_text: 'Spanish sentence: {{sentence_spa_Latn}}
+
+  Catalan sentence:'
+doc_to_target: '{{sentence_cat_Latn}}'

lm_eval/tasks/catalan_bench/flores_ca/flores_eu-ca.yaml

+# File generated by `create-yamls.py`
+include: _flores_common_yaml
+task: flores_eu-ca
+doc_to_text: 'Basque sentence: {{sentence_eus_Latn}}
+
+  Catalan sentence:'
+doc_to_target: '{{sentence_cat_Latn}}'

lm_eval/tasks/catalan_bench/flores_ca/flores_fr-ca.yaml

+# File generated by `create-yamls.py`
+include: _flores_common_yaml
+task: flores_fr-ca
+doc_to_text: 'French sentence: {{sentence_fra_Latn}}
+
+  Catalan sentence:'
+doc_to_target: '{{sentence_cat_Latn}}'

lm_eval/tasks/catalan_bench/flores_ca/flores_gl-ca.yaml

+# File generated by `create-yamls.py`
+include: _flores_common_yaml
+task: flores_gl-ca
+doc_to_text: 'Galician sentence: {{sentence_glg_Latn}}
+
+  Catalan sentence:'
+doc_to_target: '{{sentence_cat_Latn}}'

lm_eval/tasks/catalan_bench/flores_ca/flores_it-ca.yaml

+# File generated by `create-yamls.py`
+include: _flores_common_yaml
+task: flores_it-ca
+doc_to_text: 'Italian sentence: {{sentence_ita_Latn}}
+
+  Catalan sentence:'
+doc_to_target: '{{sentence_cat_Latn}}'

lm_eval/tasks/catalan_bench/flores_ca/flores_pt-ca.yaml

+# File generated by `create-yamls.py`
+include: _flores_common_yaml
+task: flores_pt-ca
+doc_to_text: 'Portuguese sentence: {{sentence_por_Latn}}
+
+  Catalan sentence:'
+doc_to_target: '{{sentence_cat_Latn}}'

lm_eval/tasks/catalan_bench/mgsm_direct_ca.yaml

+task: mgsm_direct_ca
+dataset_path: projecte-aina/mgsm_ca
+doc_to_target: '{{answer_number|string}}'
+doc_to_text: '{% if answer != None %}{{question + "\nResposta: "}}{% else %}{{"Pregunta: " + question + "\nResposta: "}}{% endif %}'
+output_type: generate_until
+training_split: train
+test_split: test
+target_delimiter: ""
+generation_kwargs:
+  until:
+    - "\n\n"
+    - "\n"
+filter_list:
+  - name: remove_whitespace
+    filter:
+      - function: remove_whitespace
+      - function: take_first
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+metadata:
+  version: 1.0

lm_eval/tasks/catalan_bench/openbookqa_ca.yaml

+task: openbookqa_ca
+dataset_path: projecte-aina/openbookqa_ca
+output_type: multiple_choice
+training_split: null
+validation_split: validation
+test_split: test
+doc_to_text: question_stem
+doc_to_target: "{{choices.label.index(answerKey.lstrip())}}"
+doc_to_choice: "{{choices.text}}"
+should_decontaminate: true
+doc_to_decontamination_query: question_stem
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1.0

lm_eval/tasks/catalan_bench/parafraseja.yaml

+task: parafraseja
+dataset_path: projecte-aina/Parafraseja
+output_type: multiple_choice
+dataset_name: null
+test_split: test
+training_split: train
+validation_split: validation
+doc_to_choice: '{{[sentence1+", veritat? No, "+sentence2, sentence1+", veritat? Sí, "+sentence2]}}'
+process_docs: !function utils.process_docs_paraphrases
+doc_to_text: ''
+doc_to_target: label
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1.0

lm_eval/tasks/catalan_bench/paws_ca.yaml

+task: paws_ca
+dataset_path: projecte-aina/PAWS-ca
+dataset_name: null
+output_type: multiple_choice
+training_split: train
+validation_split: validation
+test_split: test
+process_docs: !function utils.process_docs_paraphrases
+doc_to_text: ''
+doc_to_target: label
+doc_to_choice: '{{[sentence1+", veritat? No, "+sentence2, sentence1+", veritat? Sí, "+sentence2]}}'
+target_delimiter: ''
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1.0