Comment out truthfulqa before tests are fixed

134150a2 · Leo Gao · GitHub · ff8de120 · 134150a2
Unverified Commit 134150a2 authored Nov 15, 2021 by Leo Gao Committed by GitHub Nov 15, 2021
Hide whitespace changes
Inline Side-by-side

Showing with 3 additions and 3 deletions

lm_eval/tasks/__init__.py lm_eval/tasks/__init__.py +3 -3

No files found.
--- a/lm_eval/tasks/__init__.py
+++ b/lm_eval/tasks/__init__.py
@@ -43,7 +43,7 @@ from . import pile
 from . import wikitext
 from . import lambada_multilingual
 from . import mutual
-from . import truthfulqa
+# from . import truthfulqa

 ########################################
 # Translation tasks
@@ -148,8 +148,8 @@ TASK_REGISTRY = {
    "ethics_utilitarianism": hendrycks_ethics.EthicsUtilitarianism,
    "ethics_virtue": hendrycks_ethics.EthicsVirtue,

-    "truthfulqa_mc": truthfulqa.TruthfulQAMultipleChoice,
-    "truthfulqa_gen": truthfulqa.TruthfulQAGeneration,
+#     "truthfulqa_mc": truthfulqa.TruthfulQAMultipleChoice,
+#     "truthfulqa_gen": truthfulqa.TruthfulQAGeneration,

    # dialogue
    "mutual": mutual.MuTual,