"tools/overall.json_2024-04-10_17-06-39.json" did not exist on "2dcf477d9eafa89414c792a59027319040bdce11"
nsmc.py 1.58 KB
Newer Older
1
2
3
4
5
"""
NSMC:
"""
import numpy as np
from lm_eval.base import rf, Task
6
from lm_eval.metrics import mean
7
8
9
from lm_eval.utils import general_detokenize

_CITATION = """
10
11
12
13
14
@InProceedings{Park:2016,
  title        = "Naver Sentiment Movie Corpus",
  author       = "Lucy Park",
  year         = "2016",
  howpublished = {\\url{https://github.com/e9t/nsmc}}
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
}
"""


class NSMC(Task):
    VERSION = 0
    DATASET_PATH = "nsmc"
    DATASET_NAME = None

    def has_training_docs(self):
        return True

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
        return False

    def training_docs(self):
        if self._training_docs is None:
            self._training_docs = list(self.dataset["train"])
        return self._training_docs

    def validation_docs(self):
        return self.dataset["test"]

    def doc_to_text(self, doc):
42
        return "{}".format(general_detokenize(doc["document"]))
43
44

    def doc_to_target(self, doc):
45
        return " ({})".format({1: "긍정", 0: "부정"}[doc["label"]])
46
47

    def construct_requests(self, doc, ctx):
48
49
        ll_positive, _ = rf.loglikelihood(ctx, " (긍정)")
        ll_negative, _ = rf.loglikelihood(ctx, " (부정)")
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
        return ll_positive, ll_negative

    def process_results(self, doc, results):
        ll_positive, ll_negative = results
        pred = ll_positive > ll_negative
        gold = doc["label"]
        return {
            "acc": pred == gold
        }

    def higher_is_better(self):
        return {
            "acc": True
        }

    def aggregation(self):
        return {
            "acc": mean
        }