temporary scroll implementation

lm_eval/tasks/scrolls/scrolls.yaml

 group: scrolls
 task:
-  - scrolls_qasper
-  - scrolls_quality
-  - scrolls_narrativeqa
-  - scrolls_contractnli
-  - scrolls_govreport
-  - scrolls_summscreenfd
-  - scrolls_qmsum
+  # - task: scrolls_qasper
+  #   class: !function task.Qasper
+  - task: scrolls_quality
+    class: !function task.QuALITY
+  # - scrolls_narrativeqa
+  #   class: !function task.NarrativeQA
+  # - scrolls_contractnli
+  #   class: !function task.ContractNLI
+  # - scrolls_govreport
+  #   class: !function task.GovReport
+  # - scrolls_summscreenfd
+  #   class: !function task.SummScreenFD
+  # - scrolls_qmsum
+  #   class: !function task.QMSum

lm_eval/tasks/scrolls/task.py

@@ -279,7 +279,7 @@ class _SCROLLSSummaryTask(_SCROLLSTask):
         return f"{doc['input']}\n\nQuestion: What is a summary of the preceding text?\nAnswer:"
 
 
-@register_task("scrolls_qasper")
+# @register_task("scrolls_qasper")
 class Qasper(_SCROLLSTask):
     """A Dataset of Information-Seeking Questions and Answers Anchored in Research Papers
     https://arxiv.org/abs/2105.03011