temp

lm_eval/tasks/super_glue/record/t5-prompt.yaml

@@ -6,6 +6,7 @@ dataset_name: record
 training_split: train
 validation_split: validation
 output_type: greedy_until
+process_doc: !function t5_utils.process_doc
 doc_to_text: "record query: {{query}} entities: {{entities|join(\", \")}} passage: {{passage}}"
 doc_to_target: "{{answers}}"
 metric_list:

lm_eval/tasks/super_glue/wsc/t5-prompt.yaml

@@ -7,7 +7,7 @@ training_split: train
 validation_split: validation
 output_type: greedy_until
 doc_to_text: !function "t5_utils.t5_prompt_doc_to_text"
-doc_to_target: span1_text
+doc_to_target: "{{[span1_text, span2_text][label]}}"
 metric_list:
   - metric: exact_match
     aggregation: mean