testing returning to python task list

lm_eval/tasks/__init__.py

@@ -13,15 +13,17 @@ from lm_eval.api.task import TaskConfig, Task, ConfigurableTask
 import logging
 
 # import python tasks
-from .squadv2.task import SQuAD2
-from .scrolls.task import (
-    QuALITY,
-    NarrativeQA,
-    ContractNLI,
-    GovReport,
-    SummScreenFD,
-    QMSum,
-)
+import squadv2
+import scrolls
+python_tasks = {
+    "squadv2": squadv2.task.SQuAD2,
+    "scrolls_quality": scrolls.task.QuALITY,
+    "scrolls_narrativeqa": scrolls.task.NarrativeQA,
+    "scrolls_contractnli": scrolls.task.ContractNLI,
+    "scrolls_govreport": scrolls.task.GovReport,
+    "scrolls_summscreenfd": scrolls.task.SummScreenFD,
+    "scrolls_qmsum": scrolls.task.QMSum,
+}
 
 eval_logger = utils.eval_logger
 
@@ -41,6 +43,7 @@ class TaskManager(abc.ABC):
         self.ALL_TASKS = self.initialize_tasks(
             include_path=include_path
             )
+            #  + {k:v, "type":"task" for k,v in python_tasks.items()}
 
     def initialize_tasks(self, include_path=None):
 

lm_eval/tasks/scrolls/task.py

@@ -337,7 +337,7 @@ class Qasper(_SCROLLSTask):
             )
 
 
-@register_task("scrolls_quality")
+# @register_task("scrolls_quality")
 class QuALITY(_SCROLLSMultipleChoiceTask):
     """QuALITY: Question Answering with Long Input Texts, Yes!
     https://arxiv.org/abs/2112.08608
@@ -366,7 +366,7 @@ class QuALITY(_SCROLLSMultipleChoiceTask):
         return [doc]
 
 
-@register_task("scrolls_narrativeqa")
+# @register_task("scrolls_narrativeqa")
 class NarrativeQA(_SCROLLSTask):
     """The NarrativeQA Reading Comprehension Challenge
     https://arxiv.org/abs/1712.07040
@@ -400,7 +400,7 @@ class NarrativeQA(_SCROLLSTask):
         )
 
 
-@register_task("scrolls_contractnli")
+# @register_task("scrolls_contractnli")
 class ContractNLI(_SCROLLSMultipleChoiceTask):
     """ContractNLI: A Dataset for Document-level Natural Language Inference for Contracts
     https://arxiv.org/abs/1712.07040
@@ -419,7 +419,7 @@ class ContractNLI(_SCROLLSMultipleChoiceTask):
         return f"{doc['text']}\n\nHypothesis: {doc['question']}\nConclusion:"
 
 
-@register_task("scrolls_govreport")
+# @register_task("scrolls_govreport")
 class GovReport(_SCROLLSSummaryTask):
     """Efficient Attentions for Long Document Summarization
     https://arxiv.org/abs/2104.02112
@@ -433,7 +433,7 @@ class GovReport(_SCROLLSSummaryTask):
     DATASET_NAME = "gov_report"
 
 
-@register_task("scrolls_summscreenfd")
+# @register_task("scrolls_summscreenfd")
 class SummScreenFD(_SCROLLSSummaryTask):
     """SummScreen: A Dataset for Abstractive Screenplay Summarization
     https://arxiv.org/abs/2104.07091
@@ -442,7 +442,7 @@ class SummScreenFD(_SCROLLSSummaryTask):
     DATASET_NAME = "summ_screen_fd"
 
 
-@register_task("scrolls_qmsum")
+# @register_task("scrolls_qmsum")
 class QMSum(_SCROLLSSummaryTask):
     """QMSum: A New Benchmark for Query-based Multi-domain
     Meeting Summarization

lm_eval/tasks/squadv2/task.py

@@ -47,7 +47,7 @@ def _squad_agg(key, items):
     return _squad_metric(predictions=predictions, references=references).get(key, 0)
 
 
-@register_task("squadv2")
+# @register_task("squadv2")
 class SQuAD2(Task):
     VERSION = 3
     DATASET_PATH = "squad_v2"