formatting (#2104)

lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_twi.yaml

 dataset_name: twi
 include: afrimmlu_common_yaml
-task: afrimmlu_direct_twi
\ No newline at end of file
+task: afrimmlu_direct_twi

lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_wol.yaml

 dataset_name: wol
 include: afrimmlu_common_yaml
-task: afrimmlu_direct_wol
\ No newline at end of file
+task: afrimmlu_direct_wol

lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_xho.yaml

 dataset_name: xho
 include: afrimmlu_common_yaml
-task: afrimmlu_direct_xho
\ No newline at end of file
+task: afrimmlu_direct_xho

lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_yor.yaml

 dataset_name: yor
 include: afrimmlu_common_yaml
-task: afrimmlu_direct_yor
\ No newline at end of file
+task: afrimmlu_direct_yor

lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_zul.yaml

 dataset_name: zul
 include: afrimmlu_common_yaml
-task: afrimmlu_direct_zul
\ No newline at end of file
+task: afrimmlu_direct_zul

lm_eval/tasks/afrimmlu/direct/utils.py

-import re
-import sys
-import unicodedata
-
 from sklearn.metrics import f1_score
-from lm_eval.filters.extraction import RegexFilter
 
 
 def doc_to_choice(doc):
@@ -12,9 +7,9 @@ def doc_to_choice(doc):
 
 
 def doc_to_text(doc):
-    output = """You are a highly knowledgeable and intelligent artificial intelligence 
+    output = """You are a highly knowledgeable and intelligent artificial intelligence
                 model answers multiple-choice questions about {subject}
-                
+
                 Question: {question}
 
                 Choices:
@@ -22,16 +17,18 @@ def doc_to_text(doc):
                         B: {choice2}
                         C: {choice3}
                         D: {choice4}
-                       
+
                 Answer:  """
-    
+
     choices = eval(doc["choices"])
-    text = output.format(subject=doc['subject'],
-                         question=doc['question'],
-                         choice1=choices[0],
-                         choice2=choices[1],
-                         choice3=choices[2],
-                         choice4=choices[3])
+    text = output.format(
+        subject=doc["subject"],
+        question=doc["question"],
+        choice1=choices[0],
+        choice2=choices[1],
+        choice3=choices[2],
+        choice4=choices[3],
+    )
     return text
 
 
@@ -40,4 +37,4 @@ def weighted_f1_score(items):
     golds = unzipped_list[0]
     preds = unzipped_list[1]
     fscore = f1_score(golds, preds, average="weighted")
-    return fscore
\ No newline at end of file
+    return fscore

lm_eval/tasks/afrimmlu/fewshot.sh

@@ -5,4 +5,4 @@ lm_eval --model hf \
         --batch_size 1 \
         --num_fewshot 0 \
         --verbosity DEBUG \
-        --wandb_args project=afrimmlu
\ No newline at end of file
+        --wandb_args project=afrimmlu

lm_eval/tasks/afrimmlu/translate/afrimmlu_common_translate_yaml

@@ -6,18 +6,18 @@ dataset_path: masakhane/afrimmlu-translate-test
 dataset_name: null
 output_type: multiple_choice
 test_split: test
-doc_to_text: !function utils.doc_to_text 
+doc_to_text: !function utils.doc_to_text
 doc_to_target: "{{['A', 'B', 'C', 'D'].index(answer)}}"
 doc_to_choice: !function utils.doc_to_choice
 should_decontaminate: true
 doc_to_decontamination_query: "Question: {{question}}\nAnswer:"
 metric_list:
-  - metric: f1 
-    aggregation: !function utils.weighted_f1_score 
+  - metric: f1
+    aggregation: !function utils.weighted_f1_score
     # aggregation: mean
-    average: weighted 
-    hf_evaluate: true 
-    higher_is_better: True 
+    average: weighted
+    hf_evaluate: true
+    higher_is_better: True
     ignore_case: true
     ignore_punctuation: true
     regexes_to_ignore:

lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_eng.yaml

 dataset_name: eng
 include: afrimmlu_common_translate_yaml
 task: afrimmlu_translate_eng
-

lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_ewe.yaml

 dataset_name: ewe
 include: afrimmlu_common_translate_yaml
 task: afrimmlu_translate_ewe
-

lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_fra.yaml

 dataset_name: fra
 include: afrimmlu_common_translate_yaml
-task: afrimmlu_translate_fra
\ No newline at end of file
+task: afrimmlu_translate_fra

lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_hau.yaml

 dataset_name: hau
 include: afrimmlu_common_translate_yaml
-task: afrimmlu_translate_hau
\ No newline at end of file
+task: afrimmlu_translate_hau

lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_ibo.yaml

 dataset_name: ibo
 include: afrimmlu_common_translate_yaml
-task: afrimmlu_translate_ibo
\ No newline at end of file
+task: afrimmlu_translate_ibo

lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_kin.yaml

 dataset_name: kin
 include: afrimmlu_common_translate_yaml
-task: afrimmlu_translate_kin
\ No newline at end of file
+task: afrimmlu_translate_kin

lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_lin.yaml

 dataset_name: lin
 include: afrimmlu_common_translate_yaml
-task: afrimmlu_translate_lin
\ No newline at end of file
+task: afrimmlu_translate_lin

lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_lug.yaml

 dataset_name: lug
 include: afrimmlu_common_translate_yaml
-task: afrimmlu_translate_lug
\ No newline at end of file
+task: afrimmlu_translate_lug

lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_orm.yaml

 dataset_name: orm
 include: afrimmlu_common_translate_yaml
-task: afrimmlu_translate_orm
\ No newline at end of file
+task: afrimmlu_translate_orm

lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_sna.yaml

 dataset_name: sna
 include: afrimmlu_common_translate_yaml
-task: afrimmlu_translate_sna
\ No newline at end of file
+task: afrimmlu_translate_sna

lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_sot.yaml

 dataset_name: sot
 include: afrimmlu_common_translate_yaml
-task: afrimmlu_translate_sot
\ No newline at end of file
+task: afrimmlu_translate_sot

lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_swa.yaml

 dataset_name: swa
 include: afrimmlu_common_translate_yaml
-task: afrimmlu_translate_swa
\ No newline at end of file
+task: afrimmlu_translate_swa