added deepseekv2

LM-Evaluation-Harness-240310/lm_eval/tasks/ceval/ceval-valid_middle_school_mathematics.yaml

+"dataset_name": "middle_school_mathematics"
+"description": "以下是中国关于初中数学的单项选择题，请选出其中的正确答案。\n\n"
+"include": "_default_ceval_yaml"
+"task": "ceval-valid_middle_school_mathematics"

LM-Evaluation-Harness-240310/lm_eval/tasks/ceval/ceval-valid_middle_school_physics.yaml

+"dataset_name": "middle_school_physics"
+"description": "以下是中国关于初中物理的单项选择题，请选出其中的正确答案。\n\n"
+"include": "_default_ceval_yaml"
+"task": "ceval-valid_middle_school_physics"

LM-Evaluation-Harness-240310/lm_eval/tasks/ceval/ceval-valid_middle_school_politics.yaml

+"dataset_name": "middle_school_politics"
+"description": "以下是中国关于初中政治的单项选择题，请选出其中的正确答案。\n\n"
+"include": "_default_ceval_yaml"
+"task": "ceval-valid_middle_school_politics"

LM-Evaluation-Harness-240310/lm_eval/tasks/ceval/ceval-valid_modern_chinese_history.yaml

+"dataset_name": "modern_chinese_history"
+"description": "以下是中国关于近代史纲要的单项选择题，请选出其中的正确答案。\n\n"
+"include": "_default_ceval_yaml"
+"task": "ceval-valid_modern_chinese_history"

LM-Evaluation-Harness-240310/lm_eval/tasks/ceval/ceval-valid_operating_system.yaml

+"dataset_name": "operating_system"
+"description": "以下是中国关于操作系统的单项选择题，请选出其中的正确答案。\n\n"
+"include": "_default_ceval_yaml"
+"task": "ceval-valid_operating_system"

LM-Evaluation-Harness-240310/lm_eval/tasks/ceval/ceval-valid_physician.yaml

+"dataset_name": "physician"
+"description": "以下是中国关于医师资格的单项选择题，请选出其中的正确答案。\n\n"
+"include": "_default_ceval_yaml"
+"task": "ceval-valid_physician"

LM-Evaluation-Harness-240310/lm_eval/tasks/ceval/ceval-valid_plant_protection.yaml

+"dataset_name": "plant_protection"
+"description": "以下是中国关于植物保护的单项选择题，请选出其中的正确答案。\n\n"
+"include": "_default_ceval_yaml"
+"task": "ceval-valid_plant_protection"

LM-Evaluation-Harness-240310/lm_eval/tasks/ceval/ceval-valid_probability_and_statistics.yaml

+"dataset_name": "probability_and_statistics"
+"description": "以下是中国关于概率统计的单项选择题，请选出其中的正确答案。\n\n"
+"include": "_default_ceval_yaml"
+"task": "ceval-valid_probability_and_statistics"

LM-Evaluation-Harness-240310/lm_eval/tasks/ceval/ceval-valid_professional_tour_guide.yaml

+"dataset_name": "professional_tour_guide"
+"description": "以下是中国关于导游资格的单项选择题，请选出其中的正确答案。\n\n"
+"include": "_default_ceval_yaml"
+"task": "ceval-valid_professional_tour_guide"

LM-Evaluation-Harness-240310/lm_eval/tasks/ceval/ceval-valid_sports_science.yaml

+"dataset_name": "sports_science"
+"description": "以下是中国关于体育学的单项选择题，请选出其中的正确答案。\n\n"
+"include": "_default_ceval_yaml"
+"task": "ceval-valid_sports_science"

LM-Evaluation-Harness-240310/lm_eval/tasks/ceval/ceval-valid_tax_accountant.yaml

+"dataset_name": "tax_accountant"
+"description": "以下是中国关于税务师的单项选择题，请选出其中的正确答案。\n\n"
+"include": "_default_ceval_yaml"
+"task": "ceval-valid_tax_accountant"

LM-Evaluation-Harness-240310/lm_eval/tasks/ceval/ceval-valid_teacher_qualification.yaml

+"dataset_name": "teacher_qualification"
+"description": "以下是中国关于教师资格的单项选择题，请选出其中的正确答案。\n\n"
+"include": "_default_ceval_yaml"
+"task": "ceval-valid_teacher_qualification"

LM-Evaluation-Harness-240310/lm_eval/tasks/ceval/ceval-valid_urban_and_rural_planner.yaml

+"dataset_name": "urban_and_rural_planner"
+"description": "以下是中国关于注册城乡规划师的单项选择题，请选出其中的正确答案。\n\n"
+"include": "_default_ceval_yaml"
+"task": "ceval-valid_urban_and_rural_planner"

LM-Evaluation-Harness-240310/lm_eval/tasks/ceval/ceval-valid_veterinary_medicine.yaml

+"dataset_name": "veterinary_medicine"
+"description": "以下是中国关于兽医学的单项选择题，请选出其中的正确答案。\n\n"
+"include": "_default_ceval_yaml"
+"task": "ceval-valid_veterinary_medicine"

LM-Evaluation-Harness-240310/lm_eval/tasks/cmmlu/README.md

+# CMMLU
+
+### Paper
+
+CMMLU: Measuring massive multitask language understanding in Chinese
+https://arxiv.org/abs/2306.09212
+
+CMMLU is a comprehensive evaluation benchmark specifically designed to evaluate the knowledge and reasoning abilities of LLMs within the context of Chinese language and culture.
+CMMLU covers a wide range of subjects, comprising 67 topics that span from elementary to advanced professional levels.
+
+Homepage: https://github.com/haonan-li/CMMLU
+
+### Citation
+
+```bibtex
+@misc{li2023cmmlu,
+      title={CMMLU: Measuring massive multitask language understanding in Chinese},
+      author={Haonan Li and Yixuan Zhang and Fajri Koto and Yifei Yang and Hai Zhao and Yeyun Gong and Nan Duan and Timothy Baldwin},
+      year={2023},
+      eprint={2306.09212},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+```
+
+### Groups and Tasks
+
+#### Groups
+
+- `cmmlu`: All 67 subjects of the CMMLU dataset, evaluated following the methodology in MMLU's original implementation.
+
+#### Tasks
+
+
+The following tasks evaluate subjects in the CMMLU dataset using loglikelihood-based multiple-choice scoring:
+- `cmmlu_{subject_english}`
+
+### Checklist
+
+* [x] Is the task an existing benchmark in the literature?
+  * [x] Have you referenced the original paper that introduced the task?
+  * [x] If yes, does the original paper provide a reference implementation?
+    * [x] Yes, original implementation contributed by author of the benchmark
+
+If other tasks on this dataset are already supported:
+* [x] Is the "Main" variant of this task clearly denoted?
+* [x] Have you provided a short sentence in a README on what each new variant adds / evaluates?
+* [x] Have you noted which, if any, published evaluation setups are matched by this variant?

LM-Evaluation-Harness-240310/lm_eval/tasks/cmmlu/_default_template_yaml

+group: cmmlu
+dataset_path: /workspace/cmmlu
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{Question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n答案："
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: "{{['A', 'B', 'C', 'D'].index(Answer)}}"
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 0.0

LM-Evaluation-Harness-240310/lm_eval/tasks/cmmlu/_generate_configs.py

+"""
+Take in a YAML, and output all other splits with this YAML
+"""
+import argparse
+import os
+
+import yaml
+from tqdm import tqdm
+
+from lm_eval.logger import eval_logger
+
+
+SUBJECTS = {
+    "agronomy": "农学",
+    "anatomy": "解剖学",
+    "ancient_chinese": "古汉语",
+    "arts": "艺术学",
+    "astronomy": "天文学",
+    "business_ethics": "商业伦理",
+    "chinese_civil_service_exam": "中国公务员考试",
+    "chinese_driving_rule": "中国驾驶规则",
+    "chinese_food_culture": "中国饮食文化",
+    "chinese_foreign_policy": "中国外交政策",
+    "chinese_history": "中国历史",
+    "chinese_literature": "中国文学",
+    "chinese_teacher_qualification": "中国教师资格",
+    "clinical_knowledge": "临床知识",
+    "college_actuarial_science": "大学精算学",
+    "college_education": "大学教育学",
+    "college_engineering_hydrology": "大学工程水文学",
+    "college_law": "大学法律",
+    "college_mathematics": "大学数学",
+    "college_medical_statistics": "大学医学统计",
+    "college_medicine": "大学医学",
+    "computer_science": "计算机科学",
+    "computer_security": "计算机安全",
+    "conceptual_physics": "概念物理学",
+    "construction_project_management": "建设工程管理",
+    "economics": "经济学",
+    "education": "教育学",
+    "electrical_engineering": "电气工程",
+    "elementary_chinese": "小学语文",
+    "elementary_commonsense": "小学常识",
+    "elementary_information_and_technology": "小学信息技术",
+    "elementary_mathematics": "初等数学",
+    "ethnology": "民族学",
+    "food_science": "食品科学",
+    "genetics": "遗传学",
+    "global_facts": "全球事实",
+    "high_school_biology": "高中生物",
+    "high_school_chemistry": "高中化学",
+    "high_school_geography": "高中地理",
+    "high_school_mathematics": "高中数学",
+    "high_school_physics": "高中物理学",
+    "high_school_politics": "高中政治",
+    "human_sexuality": "人类性行为",
+    "international_law": "国际法学",
+    "journalism": "新闻学",
+    "jurisprudence": "法理学",
+    "legal_and_moral_basis": "法律与道德基础",
+    "logical": "逻辑学",
+    "machine_learning": "机器学习",
+    "management": "管理学",
+    "marketing": "市场营销",
+    "marxist_theory": "马克思主义理论",
+    "modern_chinese": "现代汉语",
+    "nutrition": "营养学",
+    "philosophy": "哲学",
+    "professional_accounting": "专业会计",
+    "professional_law": "专业法学",
+    "professional_medicine": "专业医学",
+    "professional_psychology": "专业心理学",
+    "public_relations": "公共关系",
+    "security_study": "安全研究",
+    "sociology": "社会学",
+    "sports_science": "体育学",
+    "traditional_chinese_medicine": "中医中药",
+    "virology": "病毒学",
+    "world_history": "世界历史",
+    "world_religions": "世界宗教",
+}
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--base_yaml_path", required=True)
+    parser.add_argument("--save_prefix_path", default="cmmlu")
+    parser.add_argument("--cot_prompt_path", default=None)
+    parser.add_argument("--task_prefix", default="")
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_args()
+
+    # get filename of base_yaml so we can `"include": ` it in our other YAMLs.
+    base_yaml_name = os.path.split(args.base_yaml_path)[-1]
+    with open(args.base_yaml_path, encoding="utf-8") as f:
+        base_yaml = yaml.full_load(f)
+
+    if args.cot_prompt_path is not None:
+        import json
+
+        with open(args.cot_prompt_path, encoding="utf-8") as f:
+            cot_file = json.load(f)
+
+    for subject_eng, subject_zh in tqdm(SUBJECTS.items()):
+        if args.cot_prompt_path is not None:
+            description = cot_file[subject_eng]
+        else:
+            description = (
+                f"以下是关于{subject_zh}的单项选择题，请直接给出正确答案的选项。\n\n"
+            )
+
+        yaml_dict = {
+            "include": base_yaml_name,
+            "task": f"cmmlu_{args.task_prefix}_{subject_eng}"
+            if args.task_prefix != ""
+            else f"cmmlu_{subject_eng}",
+            "dataset_name": subject_eng,
+            "description": description,
+        }
+
+        file_save_path = args.save_prefix_path + f"_{subject_eng}.yaml"
+        eval_logger.info(f"Saving yaml for subset {subject_eng} to {file_save_path}")
+        with open(file_save_path, "w", encoding="utf-8") as yaml_file:
+            yaml.dump(
+                yaml_dict,
+                yaml_file,
+                width=float("inf"),
+                allow_unicode=True,
+                default_style='"',
+            )

LM-Evaluation-Harness-240310/lm_eval/tasks/cmmlu/cmmlu_default_agronomy.yaml

+"dataset_name": "agronomy"
+"description": "以下是关于农学的单项选择题，请直接给出正确答案的选项。\n\n"
+"include": "_default_template_yaml"
+"task": "cmmlu_agronomy"

LM-Evaluation-Harness-240310/lm_eval/tasks/cmmlu/cmmlu_default_anatomy.yaml

+"dataset_name": "anatomy"
+"description": "以下是关于解剖学的单项选择题，请直接给出正确答案的选项。\n\n"
+"include": "_default_template_yaml"
+"task": "cmmlu_anatomy"

LM-Evaluation-Harness-240310/lm_eval/tasks/cmmlu/cmmlu_default_ancient_chinese.yaml

+"dataset_name": "ancient_chinese"
+"description": "以下是关于古汉语的单项选择题，请直接给出正确答案的选项。\n\n"
+"include": "_default_template_yaml"
+"task": "cmmlu_ancient_chinese"