Add reminder to rewrite

lm_eval/models/dummy.py

+# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
+
 from lm_eval.base import LM
 from . import MODEL_REGISTRY
 

lm_eval/models/gpt3.py

+# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
+
 import os
 import transformers
 from lm_eval.base import LM

lm_eval/tasks/anli.py

+# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
+
 from . common import HFTask
 
 class ANLIBase(HFTask):

lm_eval/tasks/arc.py

+# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
+
 from . common import HFTask
 
 class ARCEasy(HFTask):

lm_eval/tasks/coqa.py

+# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
+
 import json
 import random
 from lm_eval.base import Dataset

lm_eval/tasks/drop.py

+# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
+
 import numpy as np
 import json
 from scipy.stats import pearsonr, spearmanr

lm_eval/tasks/glue.py

+# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
+
 import numpy as np
 from scipy.stats import pearsonr, spearmanr
 from sklearn.metrics import f1_score, matthews_corrcoef

lm_eval/tasks/hellaswag.py

+# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
+
 import numpy as np
 from scipy.stats import pearsonr, spearmanr
 from sklearn.metrics import f1_score, matthews_corrcoef

lm_eval/tasks/lambada.py

+# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
+
 from lm_eval.base import Dataset
 from lm_eval.utils import sh
 import json

lm_eval/tasks/naturalqs.py

+# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
+
 from . common import HFTask
 from itertools import islice
 

lm_eval/tasks/openbookqa.py

+# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
+
 import numpy as np
 from scipy.stats import pearsonr, spearmanr
 from sklearn.metrics import f1_score, matthews_corrcoef

lm_eval/tasks/piqa.py

+# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
+
 import json
 import random
 from lm_eval.base import Dataset

lm_eval/tasks/quac.py

+# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
+
 import json
 import random
 import os

lm_eval/tasks/race.py

+# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
+
 from . common import HFTask
 from ..utils_stream import X, each, apply, join, filt, one
 import collections

lm_eval/tasks/sat.py

+# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
+
 import json
 import random
 import os

lm_eval/tasks/squad.py

+# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
+
 import numpy as np
 from scipy.stats import pearsonr, spearmanr
 from sklearn.metrics import f1_score, matthews_corrcoef
@@ -26,6 +28,7 @@ class SQuAD(HFTask):
             return self.data["validation"]
 
     def fewshot_description(self):
+        # TODO: redo description
         return "Title: The_Title_of_It\n\nBackground: A text passage as background to answer the question with.\n\nQ: Question about the passage.\n\nA: Answer."
 
     def doc_to_text(self, doc, include_target=True):

lm_eval/tasks/storycloze.py

+# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
+
 import json
 import random
 from lm_eval.base import Dataset

lm_eval/tasks/superglue.py

+# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
+
 import numpy as np
 from tqdm import auto as tqdm_lib
 from . common import HFTask, simple_accuracy_metric, yesno

lm_eval/tasks/triviaqa.py

+# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
+
 import json
 import random
 from lm_eval.base import Dataset

lm_eval/tasks/webqs.py

+# REMINDER: this code needs to be rewritten for the new framework. Remove this comment when the code is fully converted.
+
 from . common import HFTask
 
 class WebQs(HFTask):