Merge pull request #10 from JessicaOjo/africamgsm

Africamgsm - add f1 score

lm_eval/tasks/afrimgsm/direct/afrimgsm_common_yaml

-group: mgsm_direct
-task: afrimgsm_direct
+group:
+ - mgsm_direct
+ - afrimgsm
 dataset_path: masakhane/afrimgsm
 output_type: generate_until
 training_split: train
@@ -9,11 +10,11 @@ target_delimiter: ""
 doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|string}}{% endif %}'
 doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
 generation_kwargs:
-  until:
-    - "\n\n"
-    - "\n"
   do_sample: false
-  temperature: 0.0
+  until:
+  - 'Question:'
+  - </s>
+  - <|im_end|>
 filter_list:
   - name: remove_whitespace
     filter:
@@ -31,5 +32,11 @@ metric_list:
     higher_is_better: true
     ignore_case: true
     ignore_punctuation: true
+  - metric: f1
+    aggregation: !function utils.weighted_f1_score
+    average: weighted
+    higher_is_better: True
+    ignore_case: true
+    ignore_punctuation: true
 metadata:
-  version: 1.0
+  version: 2.0

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_amh.yaml

 # Generated by utils.py
 dataset_name: amh
-doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|int}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
-generation_kwargs:
-  do_sample: false
-  until:
-  - 'Question:'
-  - </s>
-  - <|im_end|>
 include: afrimgsm_common_yaml
 task: afrimgsm_direct_amh

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_ewe.yaml

 # Generated by utils.py
 dataset_name: ewe
-doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|int}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
-generation_kwargs:
-  do_sample: false
-  until:
-  - 'Question:'
-  - </s>
-  - <|im_end|>
 include: afrimgsm_common_yaml
 task: afrimgsm_direct_ewe

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_fra.yaml

 # Generated by utils.py
 dataset_name: fra
-doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|int}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
-generation_kwargs:
-  do_sample: false
-  until:
-  - 'Question:'
-  - </s>
-  - <|im_end|>
 include: afrimgsm_common_yaml
 task: afrimgsm_direct_fra

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_hau.yaml

 # Generated by utils.py
 dataset_name: hau
-doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|int}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
-generation_kwargs:
-  do_sample: false
-  until:
-  - 'Question:'
-  - </s>
-  - <|im_end|>
 include: afrimgsm_common_yaml
 task: afrimgsm_direct_hau

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_ibo.yaml

 # Generated by utils.py
 dataset_name: ibo
-doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|int}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
-generation_kwargs:
-  do_sample: false
-  until:
-  - 'Question:'
-  - </s>
-  - <|im_end|>
 include: afrimgsm_common_yaml
 task: afrimgsm_direct_ibo

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_kin.yaml

 # Generated by utils.py
 dataset_name: kin
-doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|int}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
-generation_kwargs:
-  do_sample: false
-  until:
-  - 'Question:'
-  - </s>
-  - <|im_end|>
 include: afrimgsm_common_yaml
 task: afrimgsm_direct_kin

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_lin.yaml

 # Generated by utils.py
 dataset_name: lin
-doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|int}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
-generation_kwargs:
-  do_sample: false
-  until:
-  - 'Question:'
-  - </s>
-  - <|im_end|>
 include: afrimgsm_common_yaml
 task: afrimgsm_direct_lin

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_lug.yaml

 # Generated by utils.py
 dataset_name: lug
-doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|int}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
-generation_kwargs:
-  do_sample: false
-  until:
-  - 'Question:'
-  - </s>
-  - <|im_end|>
 include: afrimgsm_common_yaml
 task: afrimgsm_direct_lug

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_orm.yaml

 # Generated by utils.py
 dataset_name: orm
-doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|int}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
-generation_kwargs:
-  do_sample: false
-  until:
-  - 'Question:'
-  - </s>
-  - <|im_end|>
 include: afrimgsm_common_yaml
 task: afrimgsm_direct_orm

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_sna.yaml

 # Generated by utils.py
 dataset_name: sna
-doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|int}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
-generation_kwargs:
-  do_sample: false
-  until:
-  - 'Question:'
-  - </s>
-  - <|im_end|>
 include: afrimgsm_common_yaml
 task: afrimgsm_direct_sna

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_sot.yaml

 # Generated by utils.py
 dataset_name: sot
-doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|int}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
-generation_kwargs:
-  do_sample: false
-  until:
-  - 'Question:'
-  - </s>
-  - <|im_end|>
 include: afrimgsm_common_yaml
 task: afrimgsm_direct_sot

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_swa.yaml

 # Generated by utils.py
 dataset_name: swa
-doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|int}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
-generation_kwargs:
-  do_sample: false
-  until:
-  - 'Question:'
-  - </s>
-  - <|im_end|>
 include: afrimgsm_common_yaml
 task: afrimgsm_direct_swa

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_twi.yaml

 # Generated by utils.py
 dataset_name: twi
-doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|int}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
-generation_kwargs:
-  do_sample: false
-  until:
-  - 'Question:'
-  - </s>
-  - <|im_end|>
 include: afrimgsm_common_yaml
 task: afrimgsm_direct_twi

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_wol.yaml

 # Generated by utils.py
 dataset_name: wol
-doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|int}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
-generation_kwargs:
-  do_sample: false
-  until:
-  - 'Question:'
-  - </s>
-  - <|im_end|>
 include: afrimgsm_common_yaml
 task: afrimgsm_direct_wol

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_xho.yaml

 # Generated by utils.py
 dataset_name: xho
-doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|int}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
-generation_kwargs:
-  do_sample: false
-  until:
-  - 'Question:'
-  - </s>
-  - <|im_end|>
 include: afrimgsm_common_yaml
 task: afrimgsm_direct_xho

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_yor.yaml

 # Generated by utils.py
 dataset_name: yor
-doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|int}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
-generation_kwargs:
-  do_sample: false
-  until:
-  - 'Question:'
-  - </s>
-  - <|im_end|>
 include: afrimgsm_common_yaml
 task: afrimgsm_direct_yor

lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_zul.yaml

 # Generated by utils.py
 dataset_name: zul
-doc_to_target: '{% if answer is not none %}{{answer}}{% else %}{{answer_number|int}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
-generation_kwargs:
-  do_sample: false
-  until:
-  - 'Question:'
-  - </s>
-  - <|im_end|>
 include: afrimgsm_common_yaml
 task: afrimgsm_direct_zul

lm_eval/tasks/afrimgsm/direct/utils.py

+from sklearn.metrics import f1_score
+
+
+def weighted_f1_score(items):
+    unzipped_list = list(zip(*items))
+    golds = unzipped_list[0]
+    preds = unzipped_list[1]
+    fscore = f1_score(golds, preds, average="weighted")
+    return fscore
\ No newline at end of file

lm_eval/tasks/afrimmlu/utils.py

 from sklearn.metrics import f1_score
 
+
 def doc_to_choice(doc):
     choices = eval(doc["choices"])
     return choices
 
+
 def doc_to_text(doc):
     output = """You are a highly knowledgeable and intelligent artificial intelligence 
                 model answers multiple-choice questions about '{subject}'
@@ -27,6 +29,7 @@ def doc_to_text(doc):
                          choice4=choices[3])
     return text
 
+
 def weighted_f1_score(items):
     unzipped_list = list(zip(*items))
     golds = unzipped_list[0]