Omit unimplemented tasks from coverage

.coveragerc

+[run]
+
+# tasks that aren't wired up.
+omit = 
+    lm_eval/tasks/quac.py
+    lm_eval/tasks/storycloze.py
+    lm_eval/tasks/cbt.py
+    lm_eval/tasks/sat.py
+    lm_eval/tasks/triviaqa.py
+    lm_eval/tasks/naturalqs.py
\ No newline at end of file

download_all.sh

-# NLP generally do not require separately downloading data
-
-#coqa
-mkdir -p data/coqa
-wget http://downloads.cs.stanford.edu/nlp/data/coqa/coqa-train-v1.0.json -O data/coqa/coqa-train-v1.0.json
-wget http://downloads.cs.stanford.edu/nlp/data/coqa/coqa-dev-v1.0.json -O data/coqa/coqa-dev-v1.0.json
-
-#drop
-mkdir -p data/drop
-wget https://s3-us-west-2.amazonaws.com/allennlp/datasets/drop/drop_dataset.zip -O data/drop.zip
-unzip data/drop.zip -d data/drop
-rm data/drop.zip
-mv data/drop/drop_dataset/* data/drop
-rm -rf data/drop/drop_dataset

lm_eval/tasks/__init__.py

@@ -103,7 +103,7 @@ TASK_REGISTRY = {
     # Science related
     "pubmedqa" : pubmedqa.Pubmed_QA,
     "sciq" : sciq.SciQ,
-    #"qa4mre" : qa4mre.QA4MRE,
+
     "qa4mre_2011" : qa4mre.QA4MRE_2011,
     "qa4mre_2012" : qa4mre.QA4MRE_2012,
     "qa4mre_2013" : qa4mre.QA4MRE_2013,