Merge pull request #446 from cardy20/polyglot

Add Korean Hate Speech evaluation, modify a key error in datasets/nsmc.py

lm_eval/datasets/kohatespeech/kohatespeech.py

+# coding=utf-8
+# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Korean corpus for toxic speech detection and the large unlabeled corpus. """
+
+import csv
+import datasets
+
+
+_CITATION = """\
+@inproceedings{moon-etal-2020-beep,
+    title = "{BEEP}! {K}orean Corpus of Online News Comments for Toxic Speech Detection",
+    author = "Moon, Jihyung  and
+      Cho, Won Ik  and
+      Lee, Junbum",
+    booktitle = "Proceedings of the Eighth International Workshop on Natural Language Processing for Social Media",
+    month = jul,
+    year = "2020",
+    address = "Online",
+    publisher = "Association for Computational Linguistics",
+    url = "https://www.aclweb.org/anthology/2020.socialnlp-1.4",
+    pages = "25--31",
+    abstract = "Toxic comments in online platforms are an unavoidable social issue under the cloak of anonymity. Hate speech detection has been actively done for languages such as English, German, or Italian, where manually labeled corpus has been released. In this work, we first present 9.4K manually labeled entertainment news comments for identifying Korean toxic speech, collected from a widely used online news platform in Korea. The comments are annotated regarding social bias and hate speech since both aspects are correlated. The inter-annotator agreement Krippendorff{'}s alpha score is 0.492 and 0.496, respectively. We provide benchmarks using CharCNN, BiLSTM, and BERT, where BERT achieves the highest score on all tasks. The models generally display better performance on bias identification, since the hate speech detection is a more subjective issue. Additionally, when BERT is trained with bias label for hate speech detection, the prediction score increases, implying that bias and hate are intertwined. We make our dataset publicly available and open competitions with the corpus and benchmarks.",
+}
+"""
+
+_DESCRIPTION = """\
+They provide the first human-annotated Korean corpus for toxic speech detection and the large unlabeled corpus.
+The data is comments from the Korean entertainment news aggregation platform.
+"""
+
+
+_LICENSE = "(CC BY-SA 4.0)"
+
+
+_URL = "https://raw.githubusercontent.com/kocohub/korean-hate-speech/master/labeled/"
+_URLs = {
+    "train": _URL + "train.tsv",
+    "valid": _URL + "dev.tsv",
+}
+
+
+# TODO: Name of the dataset usually match the script name with CamelCase instead of snake_case
+class HateSpeech(datasets.GeneratorBasedBuilder):
+    """Korean Hate speech dataset."""
+
+    VERSION = datasets.Version("1.1.0")
+
+    def _info(self):
+        return datasets.DatasetInfo(
+            description=_DESCRIPTION,
+            features=datasets.Features(
+                {
+                    "comments": datasets.Value("string"),
+                    "contain_gender_bias": datasets.ClassLabel(names=["False", "True"]),
+                    "bias": datasets.ClassLabel(names=["none", "others", "gender"]),
+                    "hate": datasets.ClassLabel(names=["none", "offensive", "hate"])
+                }
+            ),
+            supervised_keys=None,
+            license=_LICENSE,
+            citation=_CITATION,
+        )
+
+    def _split_generators(self, dl_manager):
+        downloaded_files = dl_manager.download_and_extract(_URLs)
+        print(downloaded_files)
+        
+        return [
+            datasets.SplitGenerator(
+                name=datasets.Split.TRAIN,
+                gen_kwargs={
+                    "filepath": downloaded_files["train"],
+                    "split": "train",
+                },
+            ),
+            datasets.SplitGenerator(
+                name=datasets.Split.TEST,
+                gen_kwargs={
+                    "filepath": downloaded_files["valid"],
+                    "split": "valid",
+                },
+            ),
+        ]
+
+    def _generate_examples(self, filepath, split):
+        with open(filepath, encoding="utf-8") as f:
+            next(f)
+            reader = csv.reader(f, delimiter="\t")
+            for id_, row in enumerate(reader):
+                yield id_, {
+                    "comments": row[0],
+                    "contain_gender_bias": row[1],
+                    "bias": row[2],
+                    "hate": row[3]
+                }

lm_eval/datasets/nsmc/nsmc.py

@@ -80,7 +80,7 @@ class NSMC(datasets.GeneratorBasedBuilder):
             datasets.SplitGenerator(
                 name=datasets.Split.TEST,
                 gen_kwargs={
-                    "filepath": downloaded_files["validation"],
+                    "filepath": downloaded_files["test"],
                     "split": "test",
                 },
             ),

lm_eval/tasks/__init__.py

@@ -56,6 +56,7 @@ from . import nsmc
 from . import klue
 from . import ko_translation
 from . import korquad
+from . import kohatespeech
 
 ########################################
 # Translation tasks
@@ -68,7 +69,6 @@ gpt3_translation_benchmarks = {
     "wmt16": ['en-ro', 'ro-en', 'de-en', 'en-de'],  # German, Romanian
 }
 
-
 # 28 total
 selected_translation_benchmarks = {
     **gpt3_translation_benchmarks,
@@ -87,7 +87,7 @@ all_translation_benchmarks = {
 # All tasks
 ########################################
 
-
+# task 레지스트리에 클래스 추가 
 TASK_REGISTRY = {
     # GLUE
     "cola": glue.CoLA,
@@ -319,10 +319,12 @@ TASK_REGISTRY = {
     "kobest_hellaswag": kobest.HellaSwag,
     "kobest_sentineg": kobest.SentiNeg,
     "ko_en_translation": ko_translation.KoEnTranslation,
-    "en_ko_translation": ko_translation.EnKoTranslation
+    "en_ko_translation": ko_translation.EnKoTranslation,
+    "kohatespeech":kohatespeech.HateSpeech,
+    "kohatespeech_gen_bias":kohatespeech.GenderBias,
+    "kohatespeech_apeach":kohatespeech.Apeach
 }
 
-
 ALL_TASKS = sorted(list(TASK_REGISTRY))
 
 

lm_eval/tasks/kohatespeech.py

+"""
+For hate speech, they introduce hate, offensive, and none labels.
+They also added binary label whether a comment contains gender bias or not.
+https://aclanthology.org/2020.socialnlp-1.4.pdf
+
+Updated on May 06 2023
+APEACH is the first crowd-generated Korean evaluation dataset for hate speech detection. 
+"""
+
+import numpy as np
+from lm_eval.base import Task, MultipleChoiceTask, rf
+from lm_eval.metrics import macro_f1_score, mean, matthews_corrcoef, f1_score, yesno
+from lm_eval.utils import general_detokenize
+
+_CITATION1 ="""
+@inproceedings{moon-etal-2020-beep,
+    title = "{BEEP}! {K}orean Corpus of Online News Comments for Toxic Speech Detection",
+    author = "Moon, Jihyung  and
+      Cho, Won Ik  and
+      Lee, Junbum",
+    booktitle = "Proceedings of the Eighth International Workshop on Natural Language Processing for Social Media",
+    month = jul,
+    year = "2020",
+    address = "Online",
+    publisher = "Association for Computational Linguistics",
+    url = "https://www.aclweb.org/anthology/2020.socialnlp-1.4",
+    pages = "25--31",
+    abstract = "Toxic comments in online platforms are an unavoidable social issue under the cloak of anonymity. Hate speech detection has been actively done for languages such as English, German, or Italian, where manually labeled corpus has been released. In this work, we first present 9.4K manually labeled entertainment news comments for identifying Korean toxic speech, collected from a widely used online news platform in Korea. The comments are annotated regarding social bias and hate speech since both aspects are correlated. The inter-annotator agreement Krippendorff{'}s alpha score is 0.492 and 0.496, respectively. We provide benchmarks using CharCNN, BiLSTM, and BERT, where BERT achieves the highest score on all tasks. The models generally display better performance on bias identification, since the hate speech detection is a more subjective issue. Additionally, when BERT is trained with bias label for hate speech detection, the prediction score increases, implying that bias and hate are intertwined. We make our dataset publicly available and open competitions with the corpus and benchmarks.",
+}
+"""
+
+class HateSpeech(MultipleChoiceTask):
+    VERSION = 0
+    DATASET_PATH = "cardy/kohatespeech"
+    DATASET_NAME = "hate_speech"
+    
+    def has_training_docs(self):
+        return True
+
+    def has_validation_docs(self):
+        return True
+
+    def has_test_docs(self):
+        return False
+
+    def training_docs(self):
+        if self._training_docs is None:
+            self._training_docs = list(map(self._process_doc, self.dataset["train"]))
+        return self._training_docs
+
+    def validation_docs(self):
+        return map(self._process_doc, self.dataset["test"])
+
+    def doc_to_text(self, doc):
+        return doc["query"]
+
+    def _process_doc(self, doc):
+        out_doc = {
+            "query": "문장: {}".format(doc["comments"]),
+            "choices": ["없음", "공격적", "혐오"], # ["none", "offensive", "hate"]
+            "gold": doc['hate']
+        }
+        return out_doc
+
+    def process_results(self, doc, results):
+        pred = np.argmax(results)
+        gold = doc["gold"]
+        return {
+            "acc": pred == gold,
+            "macro_f1": (gold, pred)
+        }
+
+    def higher_is_better(self):
+        return {
+            "acc": True,
+            "macro_f1": True
+        }
+
+    def aggregation(self):
+        return {
+            "acc": mean,
+            "macro_f1": macro_f1_score
+        }
+
+
+class GenderBias(Task):
+    VERSION = 0
+    DATASET_PATH = "cardy/kohatespeech"
+    DATASET_NAME = "hate_speech"
+    
+    def has_training_docs(self):
+        return True
+
+    def has_validation_docs(self):
+        return True
+
+    def has_test_docs(self):
+        return False
+
+    def training_docs(self):
+        if self._training_docs is None:
+            self._training_docs = list(map(self._process_doc, self.dataset["train"]))
+        return self._training_docs
+
+    def validation_docs(self):
+        return map(self._process_doc, self.dataset["test"])
+
+    def doc_to_text(self, doc):
+        return "문장: {} 성적 편향이 있습니까?".format(doc["comments"])
+
+    def doc_to_target(self, doc):
+        return " {}".format({0: "아니오", 1: "예"}[doc["contain_gender_bias"]])
+
+    def construct_requests(self, doc, ctx):
+        ll_no, _ = rf.loglikelihood(ctx, " 아니오")
+        ll_yes, _ = rf.loglikelihood(ctx, " 예")
+        return ll_no, ll_yes
+    
+    def process_results(self, doc, results):
+        pred = np.argmax(results)
+        gold = doc["contain_gender_bias"]
+        return {
+            "acc": pred == gold,
+            "macro_f1": (gold, pred)
+        }
+
+    def higher_is_better(self):
+        return {
+            "acc": True,
+            "macro_f1": True
+        }
+
+    def aggregation(self):
+        return {
+            "acc": mean,
+            "macro_f1": macro_f1_score
+        }
+
+_CITATION2 = """
+@inproceedings{yang-etal-2022-apeach,
+    title = "{APEACH}: Attacking Pejorative Expressions with Analysis on Crowd-Generated Hate Speech Evaluation Datasets",
+    author = "Yang, Kichang  and
+      Jang, Wonjun  and
+      Cho, Won Ik",
+    booktitle = "Findings of the Association for Computational Linguistics: EMNLP 2022",
+    month = dec,
+    year = "2022",
+    address = "Abu Dhabi, United Arab Emirates",
+    publisher = "Association for Computational Linguistics",
+    url = "https://aclanthology.org/2022.findings-emnlp.525",
+    pages = "7076--7086",
+    abstract = "In hate speech detection, developing training and evaluation datasets across various domains is the critical issue. Whereas, major approaches crawl social media texts and hire crowd-workers to annotate the data. Following this convention often restricts the scope of pejorative expressions to a single domain lacking generalization. Sometimes domain overlap between training corpus and evaluation set overestimate the prediction performance when pretraining language models on low-data language. To alleviate these problems in Korean, we propose APEACH that asks unspecified users to generate hate speech examples followed by minimal post-labeling. We find that APEACH can collect useful datasets that are less sensitive to the lexical overlaps between the pretraining corpus and the evaluation set, thereby properly measuring the model performance.",
+}
+"""
+
+
+class Apeach(Task):
+    VERSION = 0
+    DATASET_PATH = "jason9693/APEACH" 
+    DATASET_NAME = "apeach"
+    
+    def has_training_docs(self):
+        return False
+
+    def has_validation_docs(self):
+        return False
+
+    def has_test_docs(self):
+        return True
+
+    def test_docs(self):
+        return map(self._process_doc, self.dataset["test"])
+
+    def doc_to_text(self, doc):
+        return "문장: {} 혐오 발화가 있습니까?".format(doc["text"])
+
+    def doc_to_target(self, doc):
+        return " {}".format({0: "아니오", 1: "예"}[doc["class"]])
+
+    def construct_requests(self, doc, ctx):
+        ll_no, _ = rf.loglikelihood(ctx, " 아니오")
+        ll_yes, _ = rf.loglikelihood(ctx, " 예")
+        return ll_no, ll_yes
+    
+    def process_results(self, doc, results):
+        pred = np.argmax(results)
+        gold = doc["class"]
+        return {
+            "acc": pred == gold,
+            "macro_f1": (gold, pred)
+        }
+
+    def higher_is_better(self):
+        return {
+            "acc": True,
+            "macro_f1": True
+        }
+
+    def aggregation(self):
+        return {
+            "acc": mean,
+            "macro_f1": macro_f1_score
+        }
+    
\ No newline at end of file