Add few_shot specifications

lm_eval/tasks/tinyBenchmarks/tinyArc.yaml

@@ -7,6 +7,7 @@ output_type: multiple_choice
 training_split: train
 validation_split: validation
 test_split: test
+num_fewshot: 25
 doc_to_text: "Question: {{question}}\nAnswer:"
 doc_to_target: "{{choices.label.index(answerKey)}}"
 doc_to_choice: "{{choices.text}}"

lm_eval/tasks/tinyBenchmarks/tinyGSM8k.yaml

@@ -8,6 +8,7 @@ output_type: generate_until
 training_split: train
 fewshot_split: train
 test_split: test
+num_fewshot: 5
 doc_to_text: "Question: {{question}}\nAnswer:"
 doc_to_target: "{{answer}}" #" {{answer.split('### ')[-1].rstrip()}}"
 metric_list:

lm_eval/tasks/tinyBenchmarks/tinyHellaswag.yaml

@@ -6,6 +6,7 @@ dataset_name: null
 output_type: multiple_choice
 training_split: train
 validation_split: validation
+num_fewshot: 10
 test_split: null
 process_docs: !function utils_hellaswag.process_docs
 doc_to_text: "{{query}}"

lm_eval/tasks/tinyBenchmarks/tinyMMLU.yaml

@@ -3,6 +3,7 @@ dataset_path: tinyBenchmarks/tinyMMLU
 dataset_name: all
 test_split: test
 fewshot_split: dev
+num_fewshot: 0
 fewshot_config:
   sampler: first_n
 output_type: multiple_choice

lm_eval/tasks/tinyBenchmarks/tinyWinogrande.yaml

@@ -4,6 +4,7 @@ dataset_name: winogrande_xl
 output_type: multiple_choice
 training_split: train
 validation_split: validation
+num_fewshot: 5
 doc_to_text: !function utils_winogrande.doc_to_text
 doc_to_target: !function utils_winogrande.doc_to_target
 doc_to_choice: !function utils_winogrande.doc_to_choice