Merge branch 'main' into inverse-scaling-tasks

lm_eval/tasks/tmmluplus/default/tmmluplus_physics.yaml

+"dataset_name": "physics"
+"description": "以下為物理的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_STEM"
+"group_alias": "STEM"
+"include": "_default_template_yaml"
+"task": "tmmluplus_physics"
+"task_alias": "physics"

lm_eval/tasks/tmmluplus/default/tmmluplus_politic_science.yaml

+"dataset_name": "politic_science"
+"description": "以下為政治的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_social_sciences"
+"group_alias": "social sciences"
+"include": "_default_template_yaml"
+"task": "tmmluplus_politic_science"
+"task_alias": "politic science"

lm_eval/tasks/tmmluplus/default/tmmluplus_real_estate.yaml

+"dataset_name": "real_estate"
+"description": "以下為房地產的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_other"
+"group_alias": "other"
+"include": "_default_template_yaml"
+"task": "tmmluplus_real_estate"
+"task_alias": "real estate"

lm_eval/tasks/tmmluplus/default/tmmluplus_secondary_physics.yaml

+"dataset_name": "secondary_physics"
+"description": "以下為高中物理的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_STEM"
+"group_alias": "STEM"
+"include": "_default_template_yaml"
+"task": "tmmluplus_secondary_physics"
+"task_alias": "secondary physics"

lm_eval/tasks/tmmluplus/default/tmmluplus_statistics_and_machine_learning.yaml

+"dataset_name": "statistics_and_machine_learning"
+"description": "以下為統計與機器學習的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_STEM"
+"group_alias": "STEM"
+"include": "_default_template_yaml"
+"task": "tmmluplus_statistics_and_machine_learning"
+"task_alias": "statistics and machine learning"

lm_eval/tasks/tmmluplus/default/tmmluplus_taiwanese_hokkien.yaml

+"dataset_name": "taiwanese_hokkien"
+"description": "以下為閩南語的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_social_sciences"
+"group_alias": "social sciences"
+"include": "_default_template_yaml"
+"task": "tmmluplus_taiwanese_hokkien"
+"task_alias": "taiwanese hokkien"

lm_eval/tasks/tmmluplus/default/tmmluplus_taxation.yaml

+"dataset_name": "taxation"
+"description": "以下為稅務的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_humanities"
+"group_alias": "humanities"
+"include": "_default_template_yaml"
+"task": "tmmluplus_taxation"
+"task_alias": "taxation"

lm_eval/tasks/tmmluplus/default/tmmluplus_technical.yaml

+"dataset_name": "technical"
+"description": "以下為技術工相關的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_other"
+"group_alias": "other"
+"include": "_default_template_yaml"
+"task": "tmmluplus_technical"
+"task_alias": "technical"

lm_eval/tasks/tmmluplus/default/tmmluplus_three_principles_of_people.yaml

+"dataset_name": "three_principles_of_people"
+"description": "以下為三民主義的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_social_sciences"
+"group_alias": "social sciences"
+"include": "_default_template_yaml"
+"task": "tmmluplus_three_principles_of_people"
+"task_alias": "three principles of people"

lm_eval/tasks/tmmluplus/default/tmmluplus_trade.yaml

+"dataset_name": "trade"
+"description": "以下為貿易的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_other"
+"group_alias": "other"
+"include": "_default_template_yaml"
+"task": "tmmluplus_trade"
+"task_alias": "trade"

lm_eval/tasks/tmmluplus/default/tmmluplus_traditional_chinese_medicine_clinical_medicine.yaml

+"dataset_name": "traditional_chinese_medicine_clinical_medicine"
+"description": "以下為中醫臨床醫學的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_other"
+"group_alias": "other"
+"include": "_default_template_yaml"
+"task": "tmmluplus_traditional_chinese_medicine_clinical_medicine"
+"task_alias": "traditional chinese medicine clinical medicine"

lm_eval/tasks/tmmluplus/default/tmmluplus_trust_practice.yaml

+"dataset_name": "trust_practice"
+"description": "以下為信託實務的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_humanities"
+"group_alias": "humanities"
+"include": "_default_template_yaml"
+"task": "tmmluplus_trust_practice"
+"task_alias": "trust practice"

lm_eval/tasks/tmmluplus/default/tmmluplus_ttqav2.yaml

+"dataset_name": "ttqav2"
+"description": "以下為台灣在地用語的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_social_sciences"
+"group_alias": "social sciences"
+"include": "_default_template_yaml"
+"task": "tmmluplus_ttqav2"
+"task_alias": "ttqav2"

lm_eval/tasks/tmmluplus/default/tmmluplus_tve_chinese_language.yaml

+"dataset_name": "tve_chinese_language"
+"description": "以下為統測國文的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_social_sciences"
+"group_alias": "social sciences"
+"include": "_default_template_yaml"
+"task": "tmmluplus_tve_chinese_language"
+"task_alias": "tve chinese language"

lm_eval/tasks/tmmluplus/default/tmmluplus_tve_design.yaml

+"dataset_name": "tve_design"
+"description": "以下為統測 設計的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_other"
+"group_alias": "other"
+"include": "_default_template_yaml"
+"task": "tmmluplus_tve_design"
+"task_alias": "tve design"

lm_eval/tasks/tmmluplus/default/tmmluplus_tve_mathematics.yaml

+"dataset_name": "tve_mathematics"
+"description": "以下為統測數學的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_STEM"
+"group_alias": "STEM"
+"include": "_default_template_yaml"
+"task": "tmmluplus_tve_mathematics"
+"task_alias": "tve mathematics"

lm_eval/tasks/tmmluplus/default/tmmluplus_tve_natural_sciences.yaml

+"dataset_name": "tve_natural_sciences"
+"description": "以下為統測自然科的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_STEM"
+"group_alias": "STEM"
+"include": "_default_template_yaml"
+"task": "tmmluplus_tve_natural_sciences"
+"task_alias": "tve natural sciences"

lm_eval/tasks/tmmluplus/default/tmmluplus_veterinary_pathology.yaml

+"dataset_name": "veterinary_pathology"
+"description": "以下為獸醫病理學的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_other"
+"group_alias": "other"
+"include": "_default_template_yaml"
+"task": "tmmluplus_veterinary_pathology"
+"task_alias": "veterinary pathology"

lm_eval/tasks/tmmluplus/default/tmmluplus_veterinary_pharmacology.yaml

+"dataset_name": "veterinary_pharmacology"
+"description": "以下為獸醫藥理學的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_other"
+"group_alias": "other"
+"include": "_default_template_yaml"
+"task": "tmmluplus_veterinary_pharmacology"
+"task_alias": "veterinary pharmacology"

lm_eval/tasks/tmmluplus/default/utils.py

+import datasets
+
+
+def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:
+    def _helper(doc):
+        # modifies the contents of a single
+        # document in our dataset.
+        answer_list = ["A", "B", "C", "D"]
+        out_doc = {
+            "questions": doc["question"],
+            "choices": [doc["A"], doc["B"], doc["C"], doc["D"]],
+            "goal": answer_list.index(doc["answer"]),
+        }
+        return out_doc
+
+    return dataset.map(_helper)  # returns back a datasets.Dataset object