fewshot.sh

lm_eval --model hf \
        --model_args pretrained=masakhane/African-ultrachat-alpaca  \
        --tasks afrimmlu_direct_amh,afrimmlu_direct_eng,afrimmlu_direct_ewe,afrimmlu_direct_fra,afrimmlu_direct_hau,afrimmlu_direct_ibo,afrimmlu_direct_kin,afrimmlu_direct_lin,afrimmlu_direct_lug,afrimmlu_direct_orm,afrimmlu_direct_sna,afrimmlu_direct_sot,afrimmlu_direct_twi,afrimmlu_direct_wol,afrimmlu_direct_xho,afrimmlu_direct_yor,afrimmlu_direct_zul   \
        --device cuda:0     \
        --batch_size 1 \
        --num_fewshot 0 \
        --verbosity DEBUG \
        --wandb_args project=afrimmlu