merged from main

lm_eval/tasks/tmmluplus/default/tmmluplus_tve_chinese_language.yaml

+"dataset_name": "tve_chinese_language"
+"description": "以下為統測國文的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_social_sciences"
+"group_alias": "social sciences"
+"include": "_default_template_yaml"
+"task": "tmmluplus_tve_chinese_language"
+"task_alias": "tve chinese language"

lm_eval/tasks/tmmluplus/default/tmmluplus_tve_design.yaml

+"dataset_name": "tve_design"
+"description": "以下為統測 設計的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_other"
+"group_alias": "other"
+"include": "_default_template_yaml"
+"task": "tmmluplus_tve_design"
+"task_alias": "tve design"

lm_eval/tasks/tmmluplus/default/tmmluplus_tve_mathematics.yaml

+"dataset_name": "tve_mathematics"
+"description": "以下為統測數學的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_STEM"
+"group_alias": "STEM"
+"include": "_default_template_yaml"
+"task": "tmmluplus_tve_mathematics"
+"task_alias": "tve mathematics"

lm_eval/tasks/tmmluplus/default/tmmluplus_tve_natural_sciences.yaml

+"dataset_name": "tve_natural_sciences"
+"description": "以下為統測自然科的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_STEM"
+"group_alias": "STEM"
+"include": "_default_template_yaml"
+"task": "tmmluplus_tve_natural_sciences"
+"task_alias": "tve natural sciences"

lm_eval/tasks/tmmluplus/default/tmmluplus_veterinary_pathology.yaml

+"dataset_name": "veterinary_pathology"
+"description": "以下為獸醫病理學的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_other"
+"group_alias": "other"
+"include": "_default_template_yaml"
+"task": "tmmluplus_veterinary_pathology"
+"task_alias": "veterinary pathology"

lm_eval/tasks/tmmluplus/default/tmmluplus_veterinary_pharmacology.yaml

+"dataset_name": "veterinary_pharmacology"
+"description": "以下為獸醫藥理學的單選題，請提供正確答案的選項。\n\n"
+"group": "tmmluplus_other"
+"group_alias": "other"
+"include": "_default_template_yaml"
+"task": "tmmluplus_veterinary_pharmacology"
+"task_alias": "veterinary pharmacology"

lm_eval/tasks/tmmluplus/default/utils.py

+import datasets
+
+
+def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:
+    def _helper(doc):
+        # modifies the contents of a single
+        # document in our dataset.
+        answer_list = ["A", "B", "C", "D"]
+        out_doc = {
+            "questions": doc["question"],
+            "choices": [doc["A"], doc["B"], doc["C"], doc["D"]],
+            "goal": answer_list.index(doc["answer"]),
+        }
+        return out_doc
+
+    return dataset.map(_helper)  # returns back a datasets.Dataset object

lm_eval/tasks/tmmluplus/subject.tsv

+subject	name	category
+dentistry	牙醫學	health
+traditional_chinese_medicine_clinical_medicine	中醫臨床醫學	health
+clinical_psychology	臨床心理學	psychology
+technical	技術工相關	other
+culinary_skills	餐旅	other
+mechanical	機械與機電概論	other
+logic_reasoning	邏輯思維	other
+real_estate	房地產	other
+general_principles_of_law	法學大意	law
+finance_banking	金融與法規	business
+anti_money_laundering	洗錢防制	law
+ttqav2	台灣在地用語	culture
+marketing_management	行銷管理	other
+business_management	企業管理	other
+organic_chemistry	有機化學	chemistry
+advance_chemistry	化學	chemistry
+physics	物理	physics
+secondary_physics	高中物理	physics
+human_behavior	人類行為與社會	psychology
+national_protection	軍事	politics
+jce_humanities	指考人文科目	philosophy
+linear_algebra	線代	math
+politic_science	政治	politics
+agriculture	農業	other
+official_document_management	機關文書	other
+financial_analysis	財務分析	business
+pharmacy	藥劑學	biology
+educational_psychology	教育心理	psychology
+statistics_and_machine_learning	統計與機器學習	engineering
+management_accounting	管理會計	business
+introduction_to_law	法律概論	law
+computer_science	資訊工程	computer science
+veterinary_pathology	獸醫病理學	health
+accounting	會計學	business
+fire_science	火災學	other
+optometry	視光學	other
+insurance_studies	保險學	other
+pharmacology	藥理學	health
+taxation	稅務	law
+education_(profession_level)	教育專業	education
+economics	經濟學	economics
+veterinary_pharmacology	獸醫藥理學	health
+nautical_science	航海	other
+occupational_therapy_for_psychological_disorders	心理障礙職能治療學	psychology
+trust_practice	信託實務	law
+geography_of_taiwan	台灣地理	geography
+physical_education	體育	education
+auditing	審計學	business
+administrative_law	行政法	law
+basic_medical_science	基礎醫學	biology
+macroeconomics	總經	economics
+trade	貿易	business
+chinese_language_and_literature	國文	culture
+tve_design	統測＿設計	other
+junior_science_exam	國中會考基測自然科	biology
+junior_math_exam	國中會考基測數學科	math
+junior_chinese_exam	國中會考基測國文	culture
+junior_social_studies	國中會考基測社會科	other
+tve_mathematics	統測數學	math
+tve_chinese_language	統測國文	culture
+tve_natural_sciences	統測自然科	biology
+junior_chemistry	國中理化	chemistry
+music	音樂科	other
+education	教育常識	education
+three_principles_of_people	三民主義	culture
+taiwanese_hokkien	閩南語	culture
+engineering_math	工程數學	math

scripts/zeno_visualize.py

@@ -67,7 +67,7 @@ def main():
         # Upload data for all models
         for model_index, model in enumerate(models):
             model_args = re.sub(
-                "/|=",
+                r"[\"<>:/\|\\?\*\[\]]+",
                 "__",
                 json.load(
                     open(Path(args.data_path, model, "results.json"), encoding="utf-8")