Commit 452d2024 authored by Israel Abebe Azime's avatar Israel Abebe Azime
Browse files

added transate test

parent 901ad392
lm_eval --model hf \
--model_args pretrained=masakhane/African-ultrachat-alpaca \
--tasks afrimmlu_amh,afrimmlu_eng,afrimmlu_ewe,afrimmlu_fra,afrimmlu_hau,afrimmlu_ibo,afrimmlu_kin,afrimmlu_lin,afrimmlu_lug,afrimmlu_orm,afrimmlu_sna,afrimmlu_sot,afrimmlu_twi,afrimmlu_wol,afrimmlu_xho,afrimmlu_yor,afrimmlu_zul \
--device cuda:0 \
--batch_size 1 \
--num_fewshot 0 \
--verbosity DEBUG \
--wandb_args project=afrimmlu
\ No newline at end of file
from sklearn.metrics import f1_score
def doc_to_choice(doc):
choices = eval(doc["choices"])
return choices
def doc_to_text(doc):
output = """You are a highly knowledgeable and intelligent artificial intelligence
model answers multiple-choice questions about '{subject}'
Question: '''{question}'''
Choices:
A: ''{choice1}'''
B: ''{choice2}'''
C: ''{choice3}'''
D: ''{choice4}'''
Answer: """
choices = eval(doc["choices"])
text = output.format(subject=doc['subject'],
question=doc['question'],
choice1=choices[0],
choice2=choices[1],
choice3=choices[2],
choice4=choices[3])
return text
def weighted_f1_score(items):
unzipped_list = list(zip(*items))
golds = unzipped_list[0]
preds = unzipped_list[1]
fscore = f1_score(golds, preds, average="weighted")
return fscore
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment