Rename NLP_TASK

lm_eval/tasks/common.py

@@ -4,7 +4,7 @@ import random
 from ..base import Dataset
 
 
-class NLP_TASK(Dataset):
+class HFNLPTask(Dataset):
     NLP_PATH = None
     NLP_NAME = None
 

lm_eval/tasks/drop.py

@@ -3,7 +3,7 @@ import json
 from scipy.stats import pearsonr, spearmanr
 from sklearn.metrics import f1_score, matthews_corrcoef
 from tqdm import auto as tqdm_lib
-from . common import NLP_TASK, simple_accuracy_metric, yesno
+from . common import HFNLPTask, simple_accuracy_metric, yesno
 from pathlib import Path
 from ..base import Dataset
 

lm_eval/tasks/glue.py

@@ -2,7 +2,7 @@ import numpy as np
 from scipy.stats import pearsonr, spearmanr
 from sklearn.metrics import f1_score, matthews_corrcoef
 from tqdm import auto as tqdm_lib
-from . common import NLP_TASK, simple_accuracy_metric, yesno
+from . common import HFNLPTask, simple_accuracy_metric, yesno
 
 
 def get_accuracy_and_f1(preds, golds):
@@ -22,7 +22,7 @@ def get_accuracy_and_f1(preds, golds):
     }
 
 
-class CoLA(NLP_TASK):
+class CoLA(HFNLPTask):
     NLP_PATH = "glue"
     NLP_NAME = "cola"
 
@@ -64,7 +64,7 @@ class CoLA(NLP_TASK):
         }
 
 
-class MNLI(NLP_TASK):
+class MNLI(HFNLPTask):
     NLP_PATH = "glue"
     NLP_NAME = "mnli"
 
@@ -115,7 +115,7 @@ class MNLI(NLP_TASK):
         return simple_accuracy_metric(preds=preds, golds=golds)
 
 
-class MRPC(NLP_TASK):
+class MRPC(HFNLPTask):
     NLP_PATH = "glue"
     NLP_NAME = "mrpc"
 
@@ -153,7 +153,7 @@ class MRPC(NLP_TASK):
         return get_accuracy_and_f1(preds=preds, golds=golds)
 
 
-class RTE(NLP_TASK):
+class RTE(HFNLPTask):
     NLP_PATH = "glue"
     NLP_NAME = "rte"
 
@@ -190,7 +190,7 @@ class RTE(NLP_TASK):
         return simple_accuracy_metric(preds=preds, golds=golds)
 
 
-class QNLI(NLP_TASK):
+class QNLI(HFNLPTask):
     NLP_PATH = "glue"
     NLP_NAME = "qnli"
 
@@ -227,7 +227,7 @@ class QNLI(NLP_TASK):
         return simple_accuracy_metric(preds=preds, golds=golds)
 
 
-class QQP(NLP_TASK):
+class QQP(HFNLPTask):
     NLP_PATH = "glue"
     NLP_NAME = "qqp"
 
@@ -265,7 +265,7 @@ class QQP(NLP_TASK):
         return get_accuracy_and_f1(preds=preds, golds=golds)
 
 
-class STSB(NLP_TASK):
+class STSB(HFNLPTask):
     NLP_PATH = "glue"
     NLP_NAME = "stsb"
 
@@ -322,7 +322,7 @@ class STSB(NLP_TASK):
         }
 
 
-class SST(NLP_TASK):
+class SST(HFNLPTask):
     NLP_PATH = "glue"
     NLP_NAME = "sst2"
 
@@ -359,7 +359,7 @@ class SST(NLP_TASK):
         return simple_accuracy_metric(preds=preds, golds=golds)
 
 
-class WNLI(NLP_TASK):
+class WNLI(HFNLPTask):
     NLP_PATH = "glue"
     NLP_NAME = "wnli"
 

lm_eval/tasks/superglue.py

 import numpy as np
 from tqdm import auto as tqdm_lib
-from . common import NLP_TASK, simple_accuracy_metric, yesno
+from . common import HFNLPTask, simple_accuracy_metric, yesno
 
 
-class BoolQ(NLP_TASK):
+class BoolQ(HFNLPTask):
     NLP_PATH = "super_glue"
     NLP_NAME = "boolq"
 
@@ -36,7 +36,7 @@ class BoolQ(NLP_TASK):
         return simple_accuracy_metric(preds=preds, golds=golds)
 
 
-class CommitmentBank(NLP_TASK):
+class CommitmentBank(HFNLPTask):
     NLP_PATH = "super_glue"
     NLP_NAME = "cb"
 
@@ -79,7 +79,7 @@ class CommitmentBank(NLP_TASK):
         return simple_accuracy_metric(preds=preds, golds=golds)
 
 
-class Copa(NLP_TASK):
+class Copa(HFNLPTask):
     NLP_PATH = "super_glue"
     NLP_NAME = "copa"
 
@@ -120,7 +120,7 @@ class Copa(NLP_TASK):
         return choice[0].lower() + choice[1:]
 
 
-class WordsInContext(NLP_TASK):
+class WordsInContext(HFNLPTask):
     NLP_PATH = "super_glue"
     NLP_NAME = "wic"
 
@@ -157,7 +157,7 @@ class WordsInContext(NLP_TASK):
         return simple_accuracy_metric(preds=preds, golds=golds)
 
 
-class WinogradSchemaChallenge(NLP_TASK):
+class WinogradSchemaChallenge(HFNLPTask):
     NLP_PATH = "super_glue"
     NLP_NAME = "wsc"