added deepseekv2

LM-Evaluation-Harness-240310/lm_eval/tasks/ceval/_default_ceval_yaml

+group: ceval-valid
+dataset_path: /workspace/ceval
+validation_split: val
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: multiple_choice
+doc_to_text: "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n答案："
+doc_to_choice: ["A", "B", "C", "D"]
+doc_to_target: "{{['A', 'B', 'C', 'D'].index(answer)}}"
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1.0

LM-Evaluation-Harness-240310/lm_eval/tasks/ceval/_generate_configs.py

+"""
+Take in a YAML, and output all other splits with this YAML
+"""
+import argparse
+import os
+
+import yaml
+from tqdm import tqdm
+
+from lm_eval.logger import eval_logger
+
+
+SUBJECTS = {
+    "computer_network": "计算机网络",
+    "operating_system": "操作系统",
+    "computer_architecture": "计算机组成",
+    "college_programming": "大学编程",
+    "college_physics": "大学物理",
+    "college_chemistry": "大学化学",
+    "advanced_mathematics": "高等数学",
+    "probability_and_statistics": "概率统计",
+    "discrete_mathematics": "离散数学",
+    "electrical_engineer": "注册电气工程师",
+    "metrology_engineer": "注册计量师",
+    "high_school_mathematics": "高中数学",
+    "high_school_physics": "高中物理",
+    "high_school_chemistry": "高中化学",
+    "high_school_biology": "高中生物",
+    "middle_school_mathematics": "初中数学",
+    "middle_school_biology": "初中生物",
+    "middle_school_physics": "初中物理",
+    "middle_school_chemistry": "初中化学",
+    "veterinary_medicine": "兽医学",
+    "college_economics": "大学经济学",
+    "business_administration": "工商管理",
+    "marxism": "马克思主义基本原理",
+    "mao_zedong_thought": "毛泽东思想和中国特色社会主义理论体系概论",
+    "education_science": "教育学",
+    "teacher_qualification": "教师资格",
+    "high_school_politics": "高中政治",
+    "high_school_geography": "高中地理",
+    "middle_school_politics": "初中政治",
+    "middle_school_geography": "初中地理",
+    "modern_chinese_history": "近代史纲要",
+    "ideological_and_moral_cultivation": "思想道德修养与法律基础",
+    "logic": "逻辑学",
+    "law": "法学",
+    "chinese_language_and_literature": "中国语言文学",
+    "art_studies": "艺术学",
+    "professional_tour_guide": "导游资格",
+    "legal_professional": "法律职业资格",
+    "high_school_chinese": "高中语文",
+    "high_school_history": "高中历史",
+    "middle_school_history": "初中历史",
+    "civil_servant": "公务员",
+    "sports_science": "体育学",
+    "plant_protection": "植物保护",
+    "basic_medicine": "基础医学",
+    "clinical_medicine": "临床医学",
+    "urban_and_rural_planner": "注册城乡规划师",
+    "accountant": "注册会计师",
+    "fire_engineer": "注册消防工程师",
+    "environmental_impact_assessment_engineer": "环境影响评价工程师",
+    "tax_accountant": "税务师",
+    "physician": "医师资格",
+}
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--base_yaml_path", required=True)
+    parser.add_argument("--save_prefix_path", default="ceval-valid")
+    parser.add_argument("--cot_prompt_path", default=None)
+    parser.add_argument("--task_prefix", default="")
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_args()
+
+    # get filename of base_yaml so we can `"include": ` it in our other YAMLs.
+    base_yaml_name = os.path.split(args.base_yaml_path)[-1]
+    with open(args.base_yaml_path, encoding="utf-8") as f:
+        base_yaml = yaml.full_load(f)
+
+    if args.cot_prompt_path is not None:
+        import json
+
+        with open(args.cot_prompt_path, encoding="utf-8") as f:
+            cot_file = json.load(f)
+
+    for subject_eng, subject_zh in tqdm(SUBJECTS.items()):
+        if args.cot_prompt_path is not None:
+            description = cot_file[subject_eng]
+        else:
+            description = (
+                f"以下是中国关于{subject_zh}的单项选择题，请选出其中的正确答案。\n\n"
+            )
+
+        yaml_dict = {
+            "include": base_yaml_name,
+            "task": f"ceval-valid_{args.task_prefix}_{subject_eng}"
+            if args.task_prefix != ""
+            else f"ceval-valid_{subject_eng}",
+            "dataset_name": subject_eng,
+            "description": description,
+        }
+
+        file_save_path = args.save_prefix_path + f"_{subject_eng}.yaml"
+        eval_logger.info(f"Saving yaml for subset {subject_eng} to {file_save_path}")
+        with open(file_save_path, "w", encoding="utf-8") as yaml_file:
+            yaml.dump(
+                yaml_dict,
+                yaml_file,
+                width=float("inf"),
+                allow_unicode=True,
+                default_style='"',
+            )

LM-Evaluation-Harness-240310/lm_eval/tasks/ceval/ceval-valid_accountant.yaml

+"dataset_name": "accountant"
+"description": "以下是中国关于注册会计师的单项选择题，请选出其中的正确答案。\n\n"
+"include": "_default_ceval_yaml"
+"task": "ceval-valid_accountant"

LM-Evaluation-Harness-240310/lm_eval/tasks/ceval/ceval-valid_advanced_mathematics.yaml

+"dataset_name": "advanced_mathematics"
+"description": "以下是中国关于高等数学的单项选择题，请选出其中的正确答案。\n\n"
+"include": "_default_ceval_yaml"
+"task": "ceval-valid_advanced_mathematics"

LM-Evaluation-Harness-240310/lm_eval/tasks/ceval/ceval-valid_art_studies.yaml

+"dataset_name": "art_studies"
+"description": "以下是中国关于艺术学的单项选择题，请选出其中的正确答案。\n\n"
+"include": "_default_ceval_yaml"
+"task": "ceval-valid_art_studies"

LM-Evaluation-Harness-240310/lm_eval/tasks/ceval/ceval-valid_basic_medicine.yaml

+"dataset_name": "basic_medicine"
+"description": "以下是中国关于基础医学的单项选择题，请选出其中的正确答案。\n\n"
+"include": "_default_ceval_yaml"
+"task": "ceval-valid_basic_medicine"

LM-Evaluation-Harness-240310/lm_eval/tasks/ceval/ceval-valid_business_administration.yaml

+"dataset_name": "business_administration"
+"description": "以下是中国关于工商管理的单项选择题，请选出其中的正确答案。\n\n"
+"include": "_default_ceval_yaml"
+"task": "ceval-valid_business_administration"

LM-Evaluation-Harness-240310/lm_eval/tasks/ceval/ceval-valid_chinese_language_and_literature.yaml

+"dataset_name": "chinese_language_and_literature"
+"description": "以下是中国关于中国语言文学的单项选择题，请选出其中的正确答案。\n\n"
+"include": "_default_ceval_yaml"
+"task": "ceval-valid_chinese_language_and_literature"

LM-Evaluation-Harness-240310/lm_eval/tasks/ceval/ceval-valid_civil_servant.yaml

+"dataset_name": "civil_servant"
+"description": "以下是中国关于公务员的单项选择题，请选出其中的正确答案。\n\n"
+"include": "_default_ceval_yaml"
+"task": "ceval-valid_civil_servant"

LM-Evaluation-Harness-240310/lm_eval/tasks/ceval/ceval-valid_clinical_medicine.yaml

+"dataset_name": "clinical_medicine"
+"description": "以下是中国关于临床医学的单项选择题，请选出其中的正确答案。\n\n"
+"include": "_default_ceval_yaml"
+"task": "ceval-valid_clinical_medicine"

LM-Evaluation-Harness-240310/lm_eval/tasks/ceval/ceval-valid_college_chemistry.yaml

+"dataset_name": "college_chemistry"
+"description": "以下是中国关于大学化学的单项选择题，请选出其中的正确答案。\n\n"
+"include": "_default_ceval_yaml"
+"task": "ceval-valid_college_chemistry"

LM-Evaluation-Harness-240310/lm_eval/tasks/ceval/ceval-valid_college_economics.yaml

+"dataset_name": "college_economics"
+"description": "以下是中国关于大学经济学的单项选择题，请选出其中的正确答案。\n\n"
+"include": "_default_ceval_yaml"
+"task": "ceval-valid_college_economics"

LM-Evaluation-Harness-240310/lm_eval/tasks/ceval/ceval-valid_college_physics.yaml

+"dataset_name": "college_physics"
+"description": "以下是中国关于大学物理的单项选择题，请选出其中的正确答案。\n\n"
+"include": "_default_ceval_yaml"
+"task": "ceval-valid_college_physics"

LM-Evaluation-Harness-240310/lm_eval/tasks/ceval/ceval-valid_college_programming.yaml

+"dataset_name": "college_programming"
+"description": "以下是中国关于大学编程的单项选择题，请选出其中的正确答案。\n\n"
+"include": "_default_ceval_yaml"
+"task": "ceval-valid_college_programming"

LM-Evaluation-Harness-240310/lm_eval/tasks/ceval/ceval-valid_computer_architecture.yaml

+"dataset_name": "computer_architecture"
+"description": "以下是中国关于计算机组成的单项选择题，请选出其中的正确答案。\n\n"
+"include": "_default_ceval_yaml"
+"task": "ceval-valid_computer_architecture"

LM-Evaluation-Harness-240310/lm_eval/tasks/ceval/ceval-valid_computer_network.yaml

+"dataset_name": "computer_network"
+"description": "以下是中国关于计算机网络的单项选择题，请选出其中的正确答案。\n\n"
+"include": "_default_ceval_yaml"
+"task": "ceval-valid_computer_network"

LM-Evaluation-Harness-240310/lm_eval/tasks/ceval/ceval-valid_discrete_mathematics.yaml

+"dataset_name": "discrete_mathematics"
+"description": "以下是中国关于离散数学的单项选择题，请选出其中的正确答案。\n\n"
+"include": "_default_ceval_yaml"
+"task": "ceval-valid_discrete_mathematics"

LM-Evaluation-Harness-240310/lm_eval/tasks/ceval/ceval-valid_education_science.yaml

+"dataset_name": "education_science"
+"description": "以下是中国关于教育学的单项选择题，请选出其中的正确答案。\n\n"
+"include": "_default_ceval_yaml"
+"task": "ceval-valid_education_science"

LM-Evaluation-Harness-240310/lm_eval/tasks/ceval/ceval-valid_electrical_engineer.yaml

+"dataset_name": "electrical_engineer"
+"description": "以下是中国关于注册电气工程师的单项选择题，请选出其中的正确答案。\n\n"
+"include": "_default_ceval_yaml"
+"task": "ceval-valid_electrical_engineer"

LM-Evaluation-Harness-240310/lm_eval/tasks/ceval/ceval-valid_environmental_impact_assessment_engineer.yaml

+"dataset_name": "environmental_impact_assessment_engineer"
+"description": "以下是中国关于环境影响评价工程师的单项选择题，请选出其中的正确答案。\n\n"
+"include": "_default_ceval_yaml"
+"task": "ceval-valid_environmental_impact_assessment_engineer"