Merge pull request #890 from tmabraham/big-refactor

[Refactor] Fix PubMedQA

lm_eval/tasks/pubmedqa/preprocess_pubmedqa.py

 def doc_to_text(doc) -> str:
-    ctxs = "\n".join(doc["context"]["contexts"])
+    ctxs = "\n".join(doc["CONTEXTS"])
     return "Abstract: {}\nQuestion: {}\nAnswer:".format(
-        ctxs, doc["question"], doc["final_decision"]
+        ctxs, doc["QUESTION"], doc["final_decision"]
     )
 
 

lm_eval/tasks/pubmedqa/pubmedqa.yaml

-task: pubmed_qa
-dataset_path: pubmed_qa
-dataset_name: pqa_labeled
+task: pubmedqa
+dataset_path: bigbio/pubmed_qa
+dataset_name: pubmed_qa_labeled_fold0_source
 output_type: multiple_choice
-training_split: null
-validation_split: null
-test_split: train
+training_split: train
+validation_split: validation
+test_split: test
 doc_to_text: !function preprocess_pubmedqa.doc_to_text
 doc_to_target: final_decision
 doc_to_choice: ["yes", "no", "maybe"]