[Format] Add config lints (#892)

configs/datasets/needlebench/atc/atc_choice_20.py

@@ -9,7 +9,7 @@ from opencompass.utils.text_postprocessors import first_option_postprocess
 
 few_shot_prompts = {
     'single_choice_prompts': {
-        "single_choice_cn": [
+        'single_choice_cn': [
             dict(role='HUMAN', prompt='问题: 毕东作为刘红的爸爸，对刘红的成长有重要影响。 \n在上面提供的打乱的家族关系文本中，"刘红"的能够向上追溯到的最年长的亲人是谁？\nA. 毕东\nB. 刘红\nC. 李清亮\nD. 王展\n'),
             dict(role='BOT', prompt='回答: A'),
             dict(role='HUMAN', prompt='问题: 梅建平作为黄玉兰的姥姥，对黄玉兰的成长有重要影响。梅建平的妈妈是朱丽丽。蓝鑫把黄玉兰视为其母亲。焦慧不仅是朱丽丽的妈妈，还是朱丽丽的榜样。 \n在上面提供的打乱的家族关系文本中，"蓝鑫"的能够向上追溯到的最年长的亲人是谁？\nA. 梅建平\nB. 朱丽丽\nC. 蓝鑫\nD. 焦慧\n'),
@@ -20,7 +20,7 @@ few_shot_prompts = {
             dict(role='BOT', prompt='回答: D'),
             dict(role='HUMAN', prompt='问题: {question}'),
         ],
-        "single_choice_en": [
+        'single_choice_en': [
             dict(role='HUMAN', prompt="Question: Jasmine Lane plays the role of James Hill's father in James Hill's life. \nGiven the scrambled family relationships described above, who is the eldest relative that 'James Hill' can trace back to in the context?\nA. Jasmine Lane\nB. James Hill\nC. Christopher Flores\nD. Paula Reeves\n"),
             dict(role='BOT', prompt='Answer: A'),
             dict(role='HUMAN', prompt="Question: For Michael Goodwin, Annette Valdez is not just a dad, but also a friend.For Annette Valdez, Mary Sanders is not just a father, but also a friend. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Michael Goodwin' can trace back to in the context?\nA. Michael Goodwin\nB. Annette Valdez\nC. Mary Sanders\nD. James Mitchell\n"),
@@ -31,7 +31,7 @@ few_shot_prompts = {
             dict(role='BOT', prompt='Answer: B'),
             dict(role='HUMAN', prompt='Question: {question}'),
         ],
-        "single_choice_cn_reasoning": [
+        'single_choice_cn_reasoning': [
             dict(role='HUMAN', prompt='问题: 毕东作为刘红的爸爸，对刘红的成长有重要影响。 \n在上面提供的打乱的家族关系文本中，"刘红"的能够向上追溯到的最年长的亲人是谁？\nA. 毕东\nB. 刘红\nC. 李清亮\nD. 王展\n'),
             dict(role='BOT', prompt='回答: 根据提供的信息，我们可以构建以下家族关系链：\n\n- 刘红的爸爸是毕东。\n\n因此，"刘红"能够向上追溯到的最年长的亲人是毕东。所以答案是：A'),
 
@@ -45,7 +45,7 @@ few_shot_prompts = {
             dict(role='BOT', prompt='回答: 根据提供的信息，我们可以构建以下家族关系链：\n\n- 郑玉珍的外婆是谢平。\n- 谢平的奶奶是魏丽丽。\n- 魏丽丽的外公是魏阳。 \n- 魏阳的姥姥是郭兵。\n- 郭兵的爷爷是李建国。 \n- 李建国的外婆是丁颖。 \n- 丁颖的外公是武波。 \n- 武波的父亲是刘瑜。\n- 刘瑜的爷爷是许玲。\n- 许玲的父亲是余桂芳。 \n\n因此，"郑玉珍"能够向上追溯到的最年长的亲人是余桂芳。所以答案是：D'),
             dict(role='HUMAN', prompt='问题: {question}'),
         ],
-        "single_choice_en_reasoning": [
+        'single_choice_en_reasoning': [
             dict(role='HUMAN', prompt="Question: Sharon House, as Jessica Stewart's father, has a significant impact on Jessica Stewart's upbringing. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Jessica Stewart' can trace back to in the context?\nA. Jack Burch\nB. Jessica Stewart\nC. Sharon House\nD. Carolyn Jackson\n"),
             dict(role='BOT', prompt="Answer: Based on the provided information, we can construct the following family relationship chain：\n\n- Sharon House, as Jessica Stewart's father, has a significant impact on Jessica Stewart's upbringing.\n\nTherefore, the eldest relative that 'Jessica Stewart' can trace back to in the context is Sharon House. The answer is: C"),
             dict(role='HUMAN', prompt="Question: For Robert Hill, Mikayla Scott is not just a paternal grandfather, but also a friend.Jacob Oconnor's paternal grandmother is Robert Hill. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Jacob Oconnor' can trace back to in the context?\nA. Laura Holland\nB. Robert Hill\nC. Jacob Oconnor\nD. Mikayla Scott\n"),
@@ -92,8 +92,8 @@ for _name in list(single_choice_prompts.keys()):
 needlebench_datasets = []
 
 
-needlebench_atc_reader_cfg = dict(input_columns=["question"],
-                                  output_column="answer")
+needlebench_atc_reader_cfg = dict(input_columns=['question'],
+                                  output_column='answer')
 
 for _name in list(single_choice_prompts.keys()):
 
@@ -114,7 +114,7 @@ for _name in list(single_choice_prompts.keys()):
     for num_needles in needle_num_list:
         abbr = (f'NeedleBenchATCDataset-'
                 f'{num_needles}Needle-{"EN" if "en" in _name else "ZH"}')
-        language = "English" if "en" in _name else "Chinese"
+        language = 'English' if 'en' in _name else 'Chinese'
         if 'reasoning' in _name:
             abbr += '-Reasoning'
         dataset_dict = {
@@ -130,4 +130,3 @@ for _name in list(single_choice_prompts.keys()):
             'eval_cfg': needlebench_atc_eval_cfg
         }
         needlebench_datasets.append(dataset_dict)
-

configs/datasets/needlebench/atc/atc_choice_50.py

@@ -24,7 +24,7 @@ for _name in list(single_choice_prompts.keys()):
     for num_needles in needle_num_list:
         abbr = (f'NeedleBenchATCDataset-'
                 f'{num_needles}Needle-{"EN" if "en" in _name else "ZH"}')
-        language = "English" if "en" in _name else "Chinese"
+        language = 'English' if 'en' in _name else 'Chinese'
         if 'reasoning' in _name:
             abbr += '-Reasoning'
         dataset_dict = {
@@ -40,4 +40,3 @@ for _name in list(single_choice_prompts.keys()):
             'eval_cfg': needlebench_atc_eval_cfg
         }
         needlebench_datasets.append(dataset_dict)
-

configs/datasets/needlebench/atc/atc_choice_50_en_reasoning.py

@@ -9,7 +9,7 @@ from opencompass.utils.text_postprocessors import first_option_postprocess
 
 few_shot_prompts = {
     'single_choice_prompts': {
-        "single_choice_en_reasoning": [
+        'single_choice_en_reasoning': [
             dict(role='HUMAN', prompt="Question: Sharon House, as Jessica Stewart's father, has a significant impact on Jessica Stewart's upbringing. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Jessica Stewart' can trace back to in the context?\nA. Jack Burch\nB. Jessica Stewart\nC. Sharon House\nD. Carolyn Jackson\n"),
             dict(role='BOT', prompt="Answer: Based on the provided information, we can construct the following family relationship chain：\n\n- Sharon House, as Jessica Stewart's father, has a significant impact on Jessica Stewart's upbringing.\n\nTherefore, the eldest relative that 'Jessica Stewart' can trace back to in the context is Sharon House. The answer is: C"),
             dict(role='HUMAN', prompt="Question: For Robert Hill, Mikayla Scott is not just a paternal grandfather, but also a friend.Jacob Oconnor's paternal grandmother is Robert Hill. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Jacob Oconnor' can trace back to in the context?\nA. Laura Holland\nB. Robert Hill\nC. Jacob Oconnor\nD. Mikayla Scott\n"),
@@ -56,8 +56,8 @@ for _name in list(single_choice_prompts.keys()):
 needlebench_datasets = []
 
 
-needlebench_atc_reader_cfg = dict(input_columns=["question"],
-                                  output_column="answer")
+needlebench_atc_reader_cfg = dict(input_columns=['question'],
+                                  output_column='answer')
 
 for _name in list(single_choice_prompts.keys()):
 
@@ -78,7 +78,7 @@ for _name in list(single_choice_prompts.keys()):
     for num_needles in needle_num_list:
         abbr = (f'NeedleBenchATCDataset-'
                 f'{num_needles}Needle-{"EN" if "en" in _name else "ZH"}')
-        language = "English" if "en" in _name else "Chinese"
+        language = 'English' if 'en' in _name else 'Chinese'
         if 'reasoning' in _name:
             abbr += '-Reasoning'
         dataset_dict = {

configs/datasets/needlebench/atc/atc_choice_80.py

@@ -24,7 +24,7 @@ for _name in list(single_choice_prompts.keys()):
     for num_needles in needle_num_list:
         abbr = (f'NeedleBenchATCDataset-'
                 f'{num_needles}Needle-{"EN" if "en" in _name else "ZH"}')
-        language = "English" if "en" in _name else "Chinese"
+        language = 'English' if 'en' in _name else 'Chinese'
         if 'reasoning' in _name:
             abbr += '-Reasoning'
         dataset_dict = {
@@ -40,4 +40,3 @@ for _name in list(single_choice_prompts.keys()):
             'eval_cfg': needlebench_atc_eval_cfg
         }
         needlebench_datasets.append(dataset_dict)
-

configs/datasets/needlebench/atc/atc_choice_80_en_reasoning.py

@@ -9,7 +9,7 @@ from opencompass.utils.text_postprocessors import first_option_postprocess
 
 few_shot_prompts = {
     'single_choice_prompts': {
-        "single_choice_en_reasoning": [
+        'single_choice_en_reasoning': [
             dict(role='HUMAN', prompt="Question: Sharon House, as Jessica Stewart's father, has a significant impact on Jessica Stewart's upbringing. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Jessica Stewart' can trace back to in the context?\nA. Jack Burch\nB. Jessica Stewart\nC. Sharon House\nD. Carolyn Jackson\n"),
             dict(role='BOT', prompt="Answer: Based on the provided information, we can construct the following family relationship chain：\n\n- Sharon House, as Jessica Stewart's father, has a significant impact on Jessica Stewart's upbringing.\n\nTherefore, the eldest relative that 'Jessica Stewart' can trace back to in the context is Sharon House. The answer is: C"),
             dict(role='HUMAN', prompt="Question: For Robert Hill, Mikayla Scott is not just a paternal grandfather, but also a friend.Jacob Oconnor's paternal grandmother is Robert Hill. \nGiven the scrambled family relationships described above, who is the eldest relative that 'Jacob Oconnor' can trace back to in the context?\nA. Laura Holland\nB. Robert Hill\nC. Jacob Oconnor\nD. Mikayla Scott\n"),
@@ -56,8 +56,8 @@ for _name in list(single_choice_prompts.keys()):
 needlebench_datasets = []
 
 
-needlebench_atc_reader_cfg = dict(input_columns=["question"],
-                                  output_column="answer")
+needlebench_atc_reader_cfg = dict(input_columns=['question'],
+                                  output_column='answer')
 
 for _name in list(single_choice_prompts.keys()):
 
@@ -78,7 +78,7 @@ for _name in list(single_choice_prompts.keys()):
     for num_needles in needle_num_list:
         abbr = (f'NeedleBenchATCDataset-'
                 f'{num_needles}Needle-{"EN" if "en" in _name else "ZH"}')
-        language = "English" if "en" in _name else "Chinese"
+        language = 'English' if 'en' in _name else 'Chinese'
         if 'reasoning' in _name:
             abbr += '-Reasoning'
         dataset_dict = {

configs/datasets/needlebench/needlebench_1000k/needlebench_multi_reasoning_1000k.py

@@ -16,7 +16,7 @@ def generate_linear_space(start, end, num):
     if num == 1:
         return [start]
     elif num < 1:
-        raise ValueError("num must be at least 1.")
+        raise ValueError('num must be at least 1.')
     step = (end - start) / (num - 1)
     return [start + step * i for i in range(num)]
 

configs/datasets/needlebench/needlebench_1000k/needlebench_multi_retrieval_1000k.py

@@ -16,7 +16,7 @@ def generate_linear_space(start, end, num):
     if num == 1:
         return [start]
     elif num < 1:
-        raise ValueError("num must be at least 1.")
+        raise ValueError('num must be at least 1.')
     step = (end - start) / (num - 1)
     return [start + step * i for i in range(num)]
 
@@ -54,7 +54,7 @@ needlebench_eval_cfg = dict(
 
 context_lengths = list([20000, 160000, 300000, 440000, 580000, 720000, 860000, 1000000])
 document_depth_percent_intervals = 20
-document_depth_percent_interval_type = "linear"
+document_depth_percent_interval_type = 'linear'
 
 base_path = './data/needlebench'
 file_list = ['PaulGrahamEssays.jsonl']

configs/datasets/needlebench/needlebench_1000k/needlebench_single_1000k.py

@@ -16,7 +16,7 @@ def generate_linear_space(start, end, num):
     if num == 1:
         return [start]
     elif num < 1:
-        raise ValueError("num must be at least 1.")
+        raise ValueError('num must be at least 1.')
     step = (end - start) / (num - 1)
     return [start + step * i for i in range(num)]
 

configs/datasets/needlebench/needlebench_128k/needlebench_multi_reasoning_128k.py

@@ -16,7 +16,7 @@ def generate_linear_space(start, end, num):
     if num == 1:
         return [start]
     elif num < 1:
-        raise ValueError("num must be at least 1.")
+        raise ValueError('num must be at least 1.')
     step = (end - start) / (num - 1)
     return [start + step * i for i in range(num)]
 
@@ -55,7 +55,7 @@ needlebench_eval_cfg = dict(
 context_lengths = list([16000, 32000, 48000, 64000, 80000, 96000, 112000, 128000])
 depths_list = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100]
 document_depth_percent_intervals = 20
-document_depth_percent_interval_type = "linear"
+document_depth_percent_interval_type = 'linear'
 
 # ----------English Version----------
 base_path = './data/needlebench'

configs/datasets/needlebench/needlebench_128k/needlebench_multi_retrieval_128k.py

@@ -16,7 +16,7 @@ def generate_linear_space(start, end, num):
     if num == 1:
         return [start]
     elif num < 1:
-        raise ValueError("num must be at least 1.")
+        raise ValueError('num must be at least 1.')
     step = (end - start) / (num - 1)
     return [start + step * i for i in range(num)]
 
@@ -54,7 +54,7 @@ needlebench_eval_cfg = dict(
 
 context_lengths = list([16000, 32000, 48000, 64000, 80000, 96000, 112000, 128000])
 document_depth_percent_intervals = 20
-document_depth_percent_interval_type = "linear"
+document_depth_percent_interval_type = 'linear'
 
 base_path = './data/needlebench'
 file_list = ['PaulGrahamEssays.jsonl']

configs/datasets/needlebench/needlebench_128k/needlebench_single_128k.py

@@ -16,7 +16,7 @@ def generate_linear_space(start, end, num):
     if num == 1:
         return [start]
     elif num < 1:
-        raise ValueError("num must be at least 1.")
+        raise ValueError('num must be at least 1.')
     step = (end - start) / (num - 1)
     return [start + step * i for i in range(num)]
 
@@ -55,7 +55,7 @@ needlebench_eval_cfg = dict(
 context_lengths = list([16000, 32000, 48000, 64000, 80000, 96000, 112000, 128000])
 depths_list = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100]
 document_depth_percent_intervals = 20
-document_depth_percent_interval_type = "linear"
+document_depth_percent_interval_type = 'linear'
 
 base_path = './data/needlebench'
 file_list = ['PaulGrahamEssays.jsonl']

configs/datasets/needlebench/needlebench_200k/needlebench_multi_reasoning_200k.py

@@ -16,7 +16,7 @@ def generate_linear_space(start, end, num):
     if num == 1:
         return [start]
     elif num < 1:
-        raise ValueError("num must be at least 1.")
+        raise ValueError('num must be at least 1.')
     step = (end - start) / (num - 1)
     return [start + step * i for i in range(num)]
 

configs/datasets/needlebench/needlebench_200k/needlebench_multi_retrieval_200k.py

@@ -16,7 +16,7 @@ def generate_linear_space(start, end, num):
     if num == 1:
         return [start]
     elif num < 1:
-        raise ValueError("num must be at least 1.")
+        raise ValueError('num must be at least 1.')
     step = (end - start) / (num - 1)
     return [start + step * i for i in range(num)]
 
@@ -55,7 +55,7 @@ needlebench_eval_cfg = dict(
 # context_lengths = list([16000, 32000, 48000, 64000, 80000, 96000, 112000, 128000, 144000, 160000, 176000, 192000, 200000])
 context_lengths = list([16000, 48000, 80000, 112000, 128000, 144000, 176000, 200000])
 document_depth_percent_intervals = 20
-document_depth_percent_interval_type = "linear"
+document_depth_percent_interval_type = 'linear'
 
 base_path = './data/needlebench'
 file_list = ['PaulGrahamEssays.jsonl']

configs/datasets/needlebench/needlebench_200k/needlebench_single_200k.py

@@ -16,7 +16,7 @@ def generate_linear_space(start, end, num):
     if num == 1:
         return [start]
     elif num < 1:
-        raise ValueError("num must be at least 1.")
+        raise ValueError('num must be at least 1.')
     step = (end - start) / (num - 1)
     return [start + step * i for i in range(num)]
 

configs/datasets/needlebench/needlebench_256k/needlebench_multi_reasoning_256k.py

@@ -16,7 +16,7 @@ def generate_linear_space(start, end, num):
     if num == 1:
         return [start]
     elif num < 1:
-        raise ValueError("num must be at least 1.")
+        raise ValueError('num must be at least 1.')
     step = (end - start) / (num - 1)
     return [start + step * i for i in range(num)]
 

configs/datasets/needlebench/needlebench_256k/needlebench_multi_retrieval_256k.py

@@ -16,7 +16,7 @@ def generate_linear_space(start, end, num):
     if num == 1:
         return [start]
     elif num < 1:
-        raise ValueError("num must be at least 1.")
+        raise ValueError('num must be at least 1.')
     step = (end - start) / (num - 1)
     return [start + step * i for i in range(num)]
 
@@ -55,7 +55,7 @@ needlebench_eval_cfg = dict(
 # context_lengths = list([16000, 32000, 48000, 64000, 80000, 96000, 112000, 128000, 144000, 160000, 176000, 192000, 200000])
 context_lengths = [32000, 128000, 256000]
 document_depth_percent_intervals = 20
-document_depth_percent_interval_type = "linear"
+document_depth_percent_interval_type = 'linear'
 
 base_path = './data/needlebench'
 file_list = ['PaulGrahamEssays.jsonl']

configs/datasets/needlebench/needlebench_256k/needlebench_single_256k.py

@@ -16,7 +16,7 @@ def generate_linear_space(start, end, num):
     if num == 1:
         return [start]
     elif num < 1:
-        raise ValueError("num must be at least 1.")
+        raise ValueError('num must be at least 1.')
     step = (end - start) / (num - 1)
     return [start + step * i for i in range(num)]
 

configs/datasets/needlebench/needlebench_32k/needlebench_multi_reasoning_32k.py

@@ -16,7 +16,7 @@ def generate_linear_space(start, end, num):
     if num == 1:
         return [start]
     elif num < 1:
-        raise ValueError("num must be at least 1.")
+        raise ValueError('num must be at least 1.')
     step = (end - start) / (num - 1)
     return [start + step * i for i in range(num)]
 
@@ -55,7 +55,7 @@ needlebench_eval_cfg = dict(
 context_lengths = list([9000, 13000, 17000, 21000, 25000, 29000, 31000, 32000])
 depths_list = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100]
 document_depth_percent_intervals = 20
-document_depth_percent_interval_type = "linear"
+document_depth_percent_interval_type = 'linear'
 
 # ----------English Version----------
 base_path = './data/needlebench'

configs/datasets/needlebench/needlebench_32k/needlebench_multi_retrieval_32k.py

@@ -16,7 +16,7 @@ def generate_linear_space(start, end, num):
     if num == 1:
         return [start]
     elif num < 1:
-        raise ValueError("num must be at least 1.")
+        raise ValueError('num must be at least 1.')
     step = (end - start) / (num - 1)
     return [start + step * i for i in range(num)]
 
@@ -54,7 +54,7 @@ needlebench_eval_cfg = dict(
 
 context_lengths = list([9000, 13000, 17000, 21000, 25000, 29000, 31000, 32000])
 document_depth_percent_intervals = 20
-document_depth_percent_interval_type = "linear"
+document_depth_percent_interval_type = 'linear'
 
 base_path = './data/needlebench'
 file_list = ['PaulGrahamEssays.jsonl']

configs/datasets/needlebench/needlebench_32k/needlebench_single_32k.py

@@ -16,7 +16,7 @@ def generate_linear_space(start, end, num):
     if num == 1:
         return [start]
     elif num < 1:
-        raise ValueError("num must be at least 1.")
+        raise ValueError('num must be at least 1.')
     step = (end - start) / (num - 1)
     return [start + step * i for i in range(num)]
 
@@ -55,7 +55,7 @@ needlebench_eval_cfg = dict(
 context_lengths = list([9000, 13000, 17000, 21000, 25000, 29000, 31000, 32000])
 depths_list = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100]
 document_depth_percent_intervals = 20
-document_depth_percent_interval_type = "linear"
+document_depth_percent_interval_type = 'linear'
 
 base_path = './data/needlebench'
 file_list = ['PaulGrahamEssays.jsonl']