Merge pull request #751 from EleutherAI/crows-pair

[Refactor] CrowS-Pairs

lm_eval/tasks/crows_pairs/crows_pairs_french_physical_appearance.yaml

+include: crows_pairs_english.yaml
+task: crows_pairs_french_physical_appearance
+dataset_name: french
+process_docs: !function utils.filter_appearance

lm_eval/tasks/crows_pairs/crows_pairs_french_race_color.yaml

+include: crows_pairs_english.yaml
+task: crows_pairs_french_race_color
+dataset_name: french
+process_docs: !function utils.filter_race_color

lm_eval/tasks/crows_pairs/crows_pairs_french_religion.yaml

+include: crows_pairs_english.yaml
+task: crows_pairs_french_religion
+dataset_name: french
+process_docs: !function utils.filter_religion

lm_eval/tasks/crows_pairs/crows_pairs_french_sexual_orientation.yaml

+include: crows_pairs_english.yaml
+task: crows_pairs_french_sexual_orientation
+dataset_name: french
+process_docs: !function utils.filter_orientation

lm_eval/tasks/crows_pairs/crows_pairs_french_socioeconomic.yaml

+include: crows_pairs_english.yaml
+task: crows_pairs_french_socioeconomic
+dataset_name: french
+process_docs: !function utils.filter_socio

lm_eval/tasks/crows_pairs/utils.py

+import datasets
+
+
+def process_results(doc, results):
+    lls, _ = zip(*results)
+
+    likelihood1, likelihood2 = lls
+
+    # Calculate the absolute difference in loglikelihoods
+    diff = abs(likelihood1 - likelihood2)
+
+    # if stereotypical sentence more likely (loglikelihood higher)
+    # then treat this as predicting stereotyped sentence
+    acc = 1.0 if likelihood1 > likelihood2 else 0.0
+
+    return {"likelihood_diff": diff, "acc": acc}
+
+
+def doc_to_choice(doc):
+    return [doc["sent_more"], doc["sent_less"]]
+
+
+def filter_dataset(dataset: datasets.Dataset, bias_type: str) -> datasets.Dataset:
+    return dataset.filter(lambda example: example["bias_type"].startswith(bias_type))
+
+
+def filter_race_color(dataset: datasets.Dataset) -> datasets.Dataset:
+    return filter_dataset(dataset, "race-color")
+
+
+def filter_socio(dataset: datasets.Dataset) -> datasets.Dataset:
+    return filter_dataset(dataset, "socioeconomic")
+
+
+def filter_gender(dataset: datasets.Dataset) -> datasets.Dataset:
+    return filter_dataset(dataset, "gender")
+
+
+def filter_age(dataset: datasets.Dataset) -> datasets.Dataset:
+    return filter_dataset(dataset, "age")
+
+
+def filter_religion(dataset: datasets.Dataset) -> datasets.Dataset:
+    return filter_dataset(dataset, "religion")
+
+
+def filter_disability(dataset: datasets.Dataset) -> datasets.Dataset:
+    return filter_dataset(dataset, "disability")
+
+
+def filter_orientation(dataset: datasets.Dataset) -> datasets.Dataset:
+    return filter_dataset(dataset, "sexual-orientation")
+
+
+def filter_nationality(dataset: datasets.Dataset) -> datasets.Dataset:
+    return filter_dataset(dataset, "nationality")
+
+
+def filter_appearance(dataset: datasets.Dataset) -> datasets.Dataset:
+    return filter_dataset(dataset, "physical-appearance")
+
+
+def filter_autre(dataset: datasets.Dataset) -> datasets.Dataset:
+    return filter_dataset(dataset, "autre")