[Format] Add config lints (#892)

configs/eval_internlm2_chat_keyset.py

@@ -17,8 +17,8 @@ with read_base():
 
 work_dir = './outputs/internlm2-chat-keyset/'
 
-_origin_datasets = sum([v for k, v in locals().items() if k.endswith("_datasets")], [])
-_origin_models = sum([v for k, v in locals().items() if k.endswith("_model")], [])
+_origin_datasets = sum([v for k, v in locals().items() if k.endswith('_datasets')], [])
+_origin_models = sum([v for k, v in locals().items() if k.endswith('_model')], [])
 
 _vanilla_datasets = [deepcopy(d) for d in _origin_datasets]
 _vanilla_models = []

configs/eval_internlm2_keyset.py

@@ -16,5 +16,5 @@ with read_base():
 
 work_dir = './outputs/internlm2-keyset/'
 
-datasets = sum([v for k, v in locals().items() if k.endswith("_datasets")], [])
-models = sum([v for k, v in locals().items() if k.endswith("_model")], [])
+datasets = sum([v for k, v in locals().items() if k.endswith('_datasets')], [])
+models = sum([v for k, v in locals().items() if k.endswith('_model')], [])

configs/eval_internlm_chat_lmdeploy_apiserver.py

@@ -49,4 +49,3 @@ internlm_chat_7b = dict(
 )
 
 models = [internlm_chat_20b]
-

configs/eval_internlm_chat_lmdeploy_tis.py

@@ -29,7 +29,7 @@ models = [
     dict(
         type=LmdeployTisModel,
         abbr='internlm-chat-20b-lmdeploy-tis',
-        path="internlm/internlm-chat-20b",
+        path='internlm/internlm-chat-20b',
         tis_addr='0.0.0.0:33337',
         max_out_len=100,
         max_seq_len=2048,

configs/eval_internlm_chat_turbomind_tis.py

@@ -29,7 +29,7 @@ models = [
     dict(
         type=TurboMindTisModel,
         abbr='internlm-chat-20b-turbomind',
-        path="internlm",
+        path='internlm',
         tis_addr='0.0.0.0:33337',
         max_out_len=100,
         max_seq_len=2048,

configs/eval_internlm_flames_chat.py

@@ -31,7 +31,7 @@ models = [
     dict(
         type=HuggingFaceCausalLM,
         abbr='internlm2-chat-7b-hf',
-        path="internlm/internlm2-chat-7b",
+        path='internlm/internlm2-chat-7b',
         tokenizer_path='internlm/internlm2-chat-7b',
         model_kwargs=dict(
             trust_remote_code=True,
@@ -49,7 +49,7 @@ models = [
         meta_template=_meta_template,
         run_cfg=dict(num_gpus=1, num_procs=1),
         end_str='<|im_end|>',
-        generation_kwargs = {"eos_token_id": [2, 92542], "do_sample": True},
+        generation_kwargs = {'eos_token_id': [2, 92542], 'do_sample': True},
         batch_padding=True,
     )
 ]
@@ -91,7 +91,7 @@ judge_models = [
             use_fast=False,
             trust_remote_code=True,
         ),
-        generation_kwargs = {"do_sample": True},
+        generation_kwargs = {'do_sample': True},
         max_out_len=512,
         max_seq_len=4096,
         batch_size=8,

configs/eval_internlm_lmdeploy_apiserver.py

@@ -35,4 +35,3 @@ internlm_chat_7b = dict(
 )
 
 models = [internlm_chat_20b]
-

configs/eval_internlm_turbomind.py

@@ -19,7 +19,7 @@ datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), [])
 internlm_7b = dict(
         type=TurboMindModel,
         abbr='internlm-7b-turbomind',
-        path="internlm/internlm-7b",
+        path='internlm/internlm-7b',
         engine_config=dict(session_len=2048,
                            max_batch_size=32,
                            rope_scaling_factor=1.0),
@@ -38,7 +38,7 @@ internlm_7b = dict(
 internlm_20b = dict(
         type=TurboMindModel,
         abbr='internlm-20b-turbomind',
-        path="internlm/internlm-20b",
+        path='internlm/internlm-20b',
         engine_config=dict(session_len=2048,
                            max_batch_size=8,
                            rope_scaling_factor=1.0),

configs/eval_internlm_turbomind_tis.py

@@ -18,7 +18,7 @@ models = [
     dict(
         type=TurboMindTisModel,
         abbr='internlm-chat-20b-turbomind',
-        path="internlm",
+        path='internlm',
         tis_addr='0.0.0.0:33337',
         max_out_len=100,
         max_seq_len=2048,

configs/eval_llama2_7b_lveval.py

@@ -6,11 +6,11 @@ with read_base():
     from .summarizers.lveval import summarizer
 
 models[0][
-    "path"
-] = "/path/to/your/huggingface_models/Llama-2-7b-chat-hf"
+    'path'
+] = '/path/to/your/huggingface_models/Llama-2-7b-chat-hf'
 models[0][
-    "tokenizer_path"
-] = "/path/to/your/huggingface_models/Llama-2-7b-chat-hf"
-models[0]["max_seq_len"] = 4096
-models[0]["generation_kwargs"] = dict(do_sample=False)
-models[0]["mode"] = "mid"  # truncate in the middle
+    'tokenizer_path'
+] = '/path/to/your/huggingface_models/Llama-2-7b-chat-hf'
+models[0]['max_seq_len'] = 4096
+models[0]['generation_kwargs'] = dict(do_sample=False)
+models[0]['mode'] = 'mid'  # truncate in the middle

configs/eval_llama3_instruct.py

@@ -10,7 +10,7 @@ with read_base():
 
 work_dir = 'outputs/debug/llama3-instruct'
 
-models = sum([v for k, v in locals().items() if k.endswith("_model")], [])
+models = sum([v for k, v in locals().items() if k.endswith('_model')], [])
 
 # dataset               version    metric                        mode    llama-3-8b-instruct-hf
 # --------------------  ---------  ----------------------------  ------  ------------------------

configs/eval_math_llm_judge.py

@@ -112,7 +112,7 @@ for d in eng_datasets:
                 ]),
             ),
         ),
-        pred_role="BOT",
+        pred_role='BOT',
     )
 
 infer = dict(

configs/eval_multi_prompt_demo.py

@@ -18,7 +18,7 @@ models=[
     dict(
         type=HuggingFaceCausalLM,
         abbr='internlm-chat-7b-hf',
-        path="internlm/internlm-chat-7b",
+        path='internlm/internlm-chat-7b',
         tokenizer_path='internlm/internlm-chat-7b',
         tokenizer_kwargs=dict(
             padding_side='left',

configs/eval_qwen_7b_chat_lawbench.py

@@ -8,4 +8,4 @@ with read_base():
 
 datasets = lawbench_zero_shot_datasets + lawbench_one_shot_datasets
 for d in datasets:
-    d["infer_cfg"]["inferencer"]["save_every"] = 1
+    d['infer_cfg']['inferencer']['save_every'] = 1

configs/eval_subjective_arena_hard.py

@@ -25,20 +25,20 @@ api_meta_template = dict(
 
 _meta_template = dict(
     round=[
-        dict(role="HUMAN", begin="<|begin_of_text|>user<|end_header_id|>\n\n", end="<|eot_id|>"),
-        dict(role="BOT", begin="<|begin_of_text|>assistant<|end_header_id|>\n\n", end="<|eot_id|>", generate=True),
+        dict(role='HUMAN', begin='<|begin_of_text|>user<|end_header_id|>\n\n', end='<|eot_id|>'),
+        dict(role='BOT', begin='<|begin_of_text|>assistant<|end_header_id|>\n\n', end='<|eot_id|>', generate=True),
     ],
 )
 
 models = [
     dict(
         type=HuggingFaceCausalLM,
-        abbr="llama-3-8b-instruct-hf",
-        path="meta-llama/Meta-Llama-3-8B-Instruct",
-        model_kwargs=dict(device_map="auto"),
+        abbr='llama-3-8b-instruct-hf',
+        path='meta-llama/Meta-Llama-3-8B-Instruct',
+        model_kwargs=dict(device_map='auto'),
         tokenizer_kwargs=dict(
-            padding_side="left",
-            truncation_side="left",
+            padding_side='left',
+            truncation_side='left',
             use_fast=False,
         ),
         meta_template=_meta_template,
@@ -46,7 +46,7 @@ models = [
         max_seq_len=2048,
         batch_size=8,
         run_cfg=dict(num_gpus=1, num_procs=1),
-        generation_kwargs={"eos_token_id": [128001, 128009]},
+        generation_kwargs={'eos_token_id': [128001, 128009]},
         batch_padding=True,
     )
 ]

configs/eval_subjective_functional_multiround.py

@@ -25,8 +25,8 @@ api_meta_template = dict(
 
 _meta_template = dict(
     round=[
-        dict(role="HUMAN", begin='<|im_start|>user\n', end='<|im_end|>\n'),
-        dict(role="BOT", begin="<|im_start|>assistant\n", end='<|im_end|>\n', generate=True),
+        dict(role='HUMAN', begin='<|im_start|>user\n', end='<|im_end|>\n'),
+        dict(role='BOT', begin='<|im_start|>assistant\n', end='<|im_end|>\n', generate=True),
     ],
     eos_token_id=151645,
 )
@@ -35,7 +35,7 @@ models = [
     dict(
         type=HuggingFaceCausalLM,
         abbr='qwen1.5-7b-chat-hf',
-        path="Qwen/Qwen1.5-7B-Chat",
+        path='Qwen/Qwen1.5-7B-Chat',
         model_kwargs=dict(
             device_map='auto',
             trust_remote_code=True