Disable triviaqa and cbt tasks

ab0e4d26 · Leo Gao · 8f4d1b49 · ab0e4d26
Commit ab0e4d26 authored Apr 20, 2021 by Leo Gao
Hide whitespace changes
Inline Side-by-side

Showing with 3 additions and 3 deletions

lm_eval/tasks/__init__.py lm_eval/tasks/__init__.py +3 -3

No files found.
--- a/lm_eval/tasks/__init__.py
+++ b/lm_eval/tasks/__init__.py
@@ -94,8 +94,8 @@ TASK_REGISTRY = {
    "drop": drop.DROP,
    "lambada": lambada.LAMBADA,
    "lambada_cloze": lambada_cloze.LAMBADA_cloze,
-    "cbt-cn": cbt.CBTCN,
-    "cbt-ne": cbt.CBTNE,
+    # "cbt-cn": cbt.CBTCN, # disabled pending context length fix
+    # "cbt-ne": cbt.CBTNE, # disabled pending context length fix

    "piqa": piqa.PiQA,

@@ -107,7 +107,7 @@ TASK_REGISTRY = {
    "qa4mre_2012" : qa4mre.QA4MRE_2012,
    "qa4mre_2013" : qa4mre.QA4MRE_2013,

-    "triviaqa": triviaqa.TriviaQA,
+    #"triviaqa": triviaqa.TriviaQA, # disabled pending memory fix
    "arc_easy": arc.ARCEasy,
    "arc_challenge": arc.ARCChallenge,
    # "quac": quac.QuAC, # not implemented yet