updated prompt

lm_eval/tasks/afrimmlu/afrimmlu_common_yaml

@@ -8,7 +8,7 @@ output_type: multiple_choice
 validation_split: validation
 test_split: test
 fewshot_split: validation
-doc_to_text: "Question: {{question}}\nAnswer:"
+doc_to_text: !function utils.doc_to_text 
 doc_to_target: "{{['A', 'B', 'C', 'D'].index(answer)}}"
 doc_to_choice: !function utils.doc_to_choice
 should_decontaminate: true

lm_eval/tasks/afrimmlu/fewshot.sh

-# NUMBER OF SHOT IS SET HERE 
-num_fewshot=0
-
-
-
 lm_eval --model hf \
         --model_args pretrained=masakhane/African-ultrachat-alpaca  \
         --tasks afrimmlu_amh,afrimmlu_eng,afrimmlu_ewe,afrimmlu_fra,afrimmlu_hau,afrimmlu_ibo,afrimmlu_kin,afrimmlu_lin,afrimmlu_lug,afrimmlu_orm,afrimmlu_sna,afrimmlu_sot,afrimmlu_twi,afrimmlu_wol,afrimmlu_xho,afrimmlu_yor,afrimmlu_zul   \
         --device cuda:0     \
         --batch_size 1 \
-        --num_fewshot $num_fewshot \
+        --num_fewshot 0 \
         --verbosity DEBUG
\ No newline at end of file

lm_eval/tasks/afrimmlu/utils.py

@@ -4,6 +4,29 @@ def doc_to_choice(doc):
     choices = eval(doc["choices"])
     return choices
 
+def doc_to_text(doc):
+    output = """You are a highly knowledgeable and intelligent artificial intelligence 
+                model answers multiple-choice questions about '{subject}'
+                
+                Question: '''{question}'''
+
+                Choices:
+                        A: ''{choice1}'''
+                        B: ''{choice2}'''
+                        C: ''{choice3}'''
+                        D: ''{choice4}'''
+                       
+                
+                Answer:  """
+    choices = eval(doc["choices"])
+    text = output.format(subject=doc['subject'],
+                         question=doc['question'],
+                         choice1=choices[0],
+                         choice2=choices[1],
+                         choice3=choices[2],
+                         choice4=choices[3])
+    print(text)
+    return text
 
 def weighted_f1_score(items):
     unzipped_list = list(zip(*items))