Skip to content

GitLab

Unverified Commit b50d1632 authored by Mo Li's avatar Mo Li Committed by GitHub
Browse files

[Fix] Refactor Needlebench Configs for CLI Testing Support (#1020) 

* add needlebench datasets suffix

* fix import

* update run.py args for summarizer key and dataset suffix

* update utils/run.py
parent 2d4e5597
Hide whitespace changes
Inline Side-by-side
Showing with 164 additions and 175 deletions
configs/datasets/needlebench/needlebench.py deleted 100644 → 0
View file @ 2d4e5597
from mmengine.config import read_base
with read_base():
from .needlebench_4k.needlebench import needlebench_datasets as needlebench_datasets_4k
from .needlebench_8k.needlebench import needlebench_datasets as needlebench_datasets_8k
from .needlebench_32k.needlebench import needlebench_datasets as needlebench_datasets_32k
from .needlebench_128k.needlebench import needlebench_datasets as needlebench_datasets_128k
from .needlebench_200k.needlebench import needlebench_datasets as needlebench_datasets_200k
from .needlebench_1000k.needlebench import needlebench_datasets as needlebench_datasets_1000k
needlebench_datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), [])
configs/datasets/needlebench/needlebench_1000k/needlebench.py deleted 100644 → 0
View file @ 2d4e5597
from mmengine.config import read_base
with read_base():
from .needlebench_multi_reasoning import needlebench_datasets_2needle_en as needlebench_multi_2needle_en_datasets
from .needlebench_multi_reasoning import needlebench_datasets_3needle_en as needlebench_multi_3needle_en_datasets
from .needlebench_multi_reasoning import needlebench_datasets_4needle_en as needlebench_multi_4needle_en_datasets
from .needlebench_multi_reasoning import needlebench_datasets_5needle_en as needlebench_multi_5needle_en_datasets
from .needlebench_multi_reasoning import needlebench_datasets_2needle_zh as needlebench_multi_2needle_zh_datasets
from .needlebench_multi_reasoning import needlebench_datasets_3needle_zh as needlebench_multi_3needle_zh_datasets
from .needlebench_multi_reasoning import needlebench_datasets_4needle_zh as needlebench_multi_4needle_zh_datasets
from .needlebench_multi_reasoning import needlebench_datasets_5needle_zh as needlebench_multi_5needle_zh_datasets
from .needlebench_single import needlebench_datasets_en as needlebench_origin_en_datasets
from .needlebench_single import needlebench_datasets_zh as needlebench_origin_zh_datasets
from .needlebench_multi_retrieval import needlebench_datasets_en as needlebench_parallel_en_datasets
from .needlebench_multi_retrieval import needlebench_datasets_zh as needlebench_parallel_zh_datasets
needlebench_datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), [])
configs/datasets/needlebench/needlebench_1000k/needlebench_1000k.py 0 → 100644
View file @ b50d1632
from mmengine.config import read_base
with read_base():
from .needlebench_multi_reasoning_1000k import needlebench_2needle_en_datasets as needlebench_multi_2needle_en_datasets
from .needlebench_multi_reasoning_1000k import needlebench_3needle_en_datasets as needlebench_multi_3needle_en_datasets
from .needlebench_multi_reasoning_1000k import needlebench_4needle_en_datasets as needlebench_multi_4needle_en_datasets
from .needlebench_multi_reasoning_1000k import needlebench_5needle_en_datasets as needlebench_multi_5needle_en_datasets
from .needlebench_multi_reasoning_1000k import needlebench_2needle_zh_datasets as needlebench_multi_2needle_zh_datasets
from .needlebench_multi_reasoning_1000k import needlebench_3needle_zh_datasets as needlebench_multi_3needle_zh_datasets
from .needlebench_multi_reasoning_1000k import needlebench_4needle_zh_datasets as needlebench_multi_4needle_zh_datasets
from .needlebench_multi_reasoning_1000k import needlebench_5needle_zh_datasets as needlebench_multi_5needle_zh_datasets
from .needlebench_single_1000k import needlebench_en_datasets as needlebench_origin_en_datasets
from .needlebench_single_1000k import needlebench_zh_datasets as needlebench_origin_zh_datasets
from .needlebench_multi_retrieval_1000k import needlebench_en_datasets as needlebench_parallel_en_datasets
from .needlebench_multi_retrieval_1000k import needlebench_zh_datasets as needlebench_parallel_zh_datasets
needlebench_datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), [])
configs/datasets/needlebench/needlebench_1000k/needlebench_multi_reasoning.py → configs/datasets/needlebench/needlebench_1000k/needlebench_multi_reasoning_1000k.py
View file @ b50d1632
...@@ -62,7 +62,7 @@ file_list = ['PaulGrahamEssays.jsonl'] ...@@ -62,7 +62,7 @@ file_list = ['PaulGrahamEssays.jsonl']
needle_file_name = 'multi_needle_reasoning_en.json' needle_file_name = 'multi_needle_reasoning_en.json'
diff = 10 diff = 10
num_needles = 2 num_needles = 2
needlebench_datasets_2needle_en = [] needlebench_2needle_en_datasets = []
language = 'English' language = 'English'
for original_context_length in context_lengths: for original_context_length in context_lengths:
...@@ -87,10 +87,10 @@ for original_context_length in context_lengths: ...@@ -87,10 +87,10 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_2needle_en.append(dataset_dict) needlebench_2needle_en_datasets.append(dataset_dict)
num_needles = 3 num_needles = 3
needlebench_datasets_3needle_en = [] needlebench_3needle_en_datasets = []
for original_context_length in context_lengths: for original_context_length in context_lengths:
for depth_percent in depths_list: for depth_percent in depths_list:
...@@ -114,10 +114,10 @@ for original_context_length in context_lengths: ...@@ -114,10 +114,10 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_3needle_en.append(dataset_dict) needlebench_3needle_en_datasets.append(dataset_dict)
num_needles = 4 num_needles = 4
needlebench_datasets_4needle_en = [] needlebench_4needle_en_datasets = []
for original_context_length in context_lengths: for original_context_length in context_lengths:
for depth_percent in depths_list: for depth_percent in depths_list:
...@@ -141,10 +141,10 @@ for original_context_length in context_lengths: ...@@ -141,10 +141,10 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_4needle_en.append(dataset_dict) needlebench_4needle_en_datasets.append(dataset_dict)
num_needles = 5 num_needles = 5
needlebench_datasets_5needle_en = [] needlebench_5needle_en_datasets = []
for original_context_length in context_lengths: for original_context_length in context_lengths:
for depth_percent in depths_list: for depth_percent in depths_list:
...@@ -168,7 +168,7 @@ for original_context_length in context_lengths: ...@@ -168,7 +168,7 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_5needle_en.append(dataset_dict) needlebench_5needle_en_datasets.append(dataset_dict)
# ----------Chinese Version---------- # ----------Chinese Version----------
base_path = './data/needlebench' base_path = './data/needlebench'
...@@ -177,7 +177,7 @@ file_list = ['zh_finance.jsonl'] ...@@ -177,7 +177,7 @@ file_list = ['zh_finance.jsonl']
needle_file_name = 'multi_needle_reasoning_zh.json' needle_file_name = 'multi_needle_reasoning_zh.json'
diff = 10 diff = 10
num_needles = 2 num_needles = 2
needlebench_datasets_2needle_zh = [] needlebench_2needle_zh_datasets = []
language = 'Chinese' language = 'Chinese'
for original_context_length in context_lengths: for original_context_length in context_lengths:
...@@ -202,10 +202,10 @@ for original_context_length in context_lengths: ...@@ -202,10 +202,10 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_2needle_zh.append(dataset_dict) needlebench_2needle_zh_datasets.append(dataset_dict)
num_needles = 3 num_needles = 3
needlebench_datasets_3needle_zh = [] needlebench_3needle_zh_datasets = []
for original_context_length in context_lengths: for original_context_length in context_lengths:
for depth_percent in depths_list: for depth_percent in depths_list:
...@@ -229,10 +229,10 @@ for original_context_length in context_lengths: ...@@ -229,10 +229,10 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_3needle_zh.append(dataset_dict) needlebench_3needle_zh_datasets.append(dataset_dict)
num_needles = 4 num_needles = 4
needlebench_datasets_4needle_zh = [] needlebench_4needle_zh_datasets = []
for original_context_length in context_lengths: for original_context_length in context_lengths:
for depth_percent in depths_list: for depth_percent in depths_list:
...@@ -256,10 +256,10 @@ for original_context_length in context_lengths: ...@@ -256,10 +256,10 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_4needle_zh.append(dataset_dict) needlebench_4needle_zh_datasets.append(dataset_dict)
num_needles = 5 num_needles = 5
needlebench_datasets_5needle_zh = [] needlebench_5needle_zh_datasets = []
for original_context_length in context_lengths: for original_context_length in context_lengths:
for depth_percent in depths_list: for depth_percent in depths_list:
...@@ -283,4 +283,4 @@ for original_context_length in context_lengths: ...@@ -283,4 +283,4 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_5needle_zh.append(dataset_dict) needlebench_5needle_zh_datasets.append(dataset_dict)
configs/datasets/needlebench/needlebench_1000k/needlebench_multi_retrieval.py → configs/datasets/needlebench/needlebench_1000k/needlebench_multi_retrieval_1000k.py
View file @ b50d1632
...@@ -58,7 +58,7 @@ document_depth_percent_interval_type = "linear" ...@@ -58,7 +58,7 @@ document_depth_percent_interval_type = "linear"
base_path = './data/needlebench' base_path = './data/needlebench'
file_list = ['PaulGrahamEssays.jsonl'] file_list = ['PaulGrahamEssays.jsonl']
needlebench_datasets_en = [] needlebench_en_datasets = []
needle_file_name = 'needles.jsonl' needle_file_name = 'needles.jsonl'
depths = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100] depths = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100]
...@@ -81,10 +81,10 @@ for original_context_length in context_lengths: ...@@ -81,10 +81,10 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_en.append(dataset_dict) needlebench_en_datasets.append(dataset_dict)
file_list = ['zh_finance.jsonl'] file_list = ['zh_finance.jsonl']
needlebench_datasets_zh = [] needlebench_zh_datasets = []
for original_context_length in context_lengths: for original_context_length in context_lengths:
dataset_dict = { dataset_dict = {
...@@ -105,4 +105,4 @@ for original_context_length in context_lengths: ...@@ -105,4 +105,4 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_zh.append(dataset_dict) needlebench_zh_datasets.append(dataset_dict)
configs/datasets/needlebench/needlebench_1000k/needlebench_single.py → configs/datasets/needlebench/needlebench_1000k/needlebench_single_1000k.py
View file @ b50d1632
...@@ -57,7 +57,7 @@ depths_list = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100] ...@@ -57,7 +57,7 @@ depths_list = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100]
base_path = './data/needlebench' base_path = './data/needlebench'
file_list = ['PaulGrahamEssays.jsonl'] file_list = ['PaulGrahamEssays.jsonl']
needlebench_datasets_en = [] needlebench_en_datasets = []
needle_file_name = 'needles.jsonl' needle_file_name = 'needles.jsonl'
for original_context_length in context_lengths: for original_context_length in context_lengths:
...@@ -80,10 +80,10 @@ for original_context_length in context_lengths: ...@@ -80,10 +80,10 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_en.append(dataset_dict) needlebench_en_datasets.append(dataset_dict)
file_list = ['zh_finance.jsonl'] file_list = ['zh_finance.jsonl']
needlebench_datasets_zh = [] needlebench_zh_datasets = []
needle_file_name = 'needles.jsonl' needle_file_name = 'needles.jsonl'
for original_context_length in context_lengths: for original_context_length in context_lengths:
...@@ -106,4 +106,4 @@ for original_context_length in context_lengths: ...@@ -106,4 +106,4 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_zh.append(dataset_dict) needlebench_zh_datasets.append(dataset_dict)
configs/datasets/needlebench/needlebench_128k/needlebench.py deleted 100644 → 0
View file @ 2d4e5597
from mmengine.config import read_base
with read_base():
from .needlebench_multi_reasoning import needlebench_datasets_2needle_en as needlebench_multi_2needle_en_datasets
from .needlebench_multi_reasoning import needlebench_datasets_3needle_en as needlebench_multi_3needle_en_datasets
from .needlebench_multi_reasoning import needlebench_datasets_4needle_en as needlebench_multi_4needle_en_datasets
from .needlebench_multi_reasoning import needlebench_datasets_5needle_en as needlebench_multi_5needle_en_datasets
from .needlebench_multi_reasoning import needlebench_datasets_2needle_zh as needlebench_multi_2needle_zh_datasets
from .needlebench_multi_reasoning import needlebench_datasets_3needle_zh as needlebench_multi_3needle_zh_datasets
from .needlebench_multi_reasoning import needlebench_datasets_4needle_zh as needlebench_multi_4needle_zh_datasets
from .needlebench_multi_reasoning import needlebench_datasets_5needle_zh as needlebench_multi_5needle_zh_datasets
from .needlebench_single import needlebench_datasets_en as needlebench_origin_en_datasets
from .needlebench_single import needlebench_datasets_zh as needlebench_origin_zh_datasets
from .needlebench_multi_retrieval import needlebench_datasets_en as needlebench_parallel_en_datasets
from .needlebench_multi_retrieval import needlebench_datasets_zh as needlebench_parallel_zh_datasets
needlebench_datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), [])
configs/datasets/needlebench/needlebench_128k/needlebench_128k.py 0 → 100644
View file @ b50d1632
from mmengine.config import read_base
with read_base():
from .needlebench_multi_reasoning_128k import needlebench_2needle_en_datasets as needlebench_multi_2needle_en_datasets
from .needlebench_multi_reasoning_128k import needlebench_3needle_en_datasets as needlebench_multi_3needle_en_datasets
from .needlebench_multi_reasoning_128k import needlebench_4needle_en_datasets as needlebench_multi_4needle_en_datasets
from .needlebench_multi_reasoning_128k import needlebench_5needle_en_datasets as needlebench_multi_5needle_en_datasets
from .needlebench_multi_reasoning_128k import needlebench_2needle_zh_datasets as needlebench_multi_2needle_zh_datasets
from .needlebench_multi_reasoning_128k import needlebench_3needle_zh_datasets as needlebench_multi_3needle_zh_datasets
from .needlebench_multi_reasoning_128k import needlebench_4needle_zh_datasets as needlebench_multi_4needle_zh_datasets
from .needlebench_multi_reasoning_128k import needlebench_5needle_zh_datasets as needlebench_multi_5needle_zh_datasets
from .needlebench_single_128k import needlebench_en_datasets as needlebench_origin_en_datasets
from .needlebench_single_128k import needlebench_zh_datasets as needlebench_origin_zh_datasets
from .needlebench_multi_retrieval_128k import needlebench_en_datasets as needlebench_parallel_en_datasets
from .needlebench_multi_retrieval_128k import needlebench_zh_datasets as needlebench_parallel_zh_datasets
needlebench_datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), [])
configs/datasets/needlebench/needlebench_128k/needlebench_multi_reasoning.py → configs/datasets/needlebench/needlebench_128k/needlebench_multi_reasoning_128k.py
View file @ b50d1632
...@@ -64,7 +64,7 @@ file_list = ['PaulGrahamEssays.jsonl'] ...@@ -64,7 +64,7 @@ file_list = ['PaulGrahamEssays.jsonl']
needle_file_name = 'multi_needle_reasoning_en.json' needle_file_name = 'multi_needle_reasoning_en.json'
diff = 10 diff = 10
num_needles = 2 num_needles = 2
needlebench_datasets_2needle_en = [] needlebench_2needle_en_datasets = []
language = 'English' language = 'English'
for original_context_length in context_lengths: for original_context_length in context_lengths:
...@@ -89,10 +89,10 @@ for original_context_length in context_lengths: ...@@ -89,10 +89,10 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_2needle_en.append(dataset_dict) needlebench_2needle_en_datasets.append(dataset_dict)
num_needles = 3 num_needles = 3
needlebench_datasets_3needle_en = [] needlebench_3needle_en_datasets = []
for original_context_length in context_lengths: for original_context_length in context_lengths:
for depth_percent in depths_list: for depth_percent in depths_list:
...@@ -116,10 +116,10 @@ for original_context_length in context_lengths: ...@@ -116,10 +116,10 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_3needle_en.append(dataset_dict) needlebench_3needle_en_datasets.append(dataset_dict)
num_needles = 4 num_needles = 4
needlebench_datasets_4needle_en = [] needlebench_4needle_en_datasets = []
for original_context_length in context_lengths: for original_context_length in context_lengths:
for depth_percent in depths_list: for depth_percent in depths_list:
...@@ -143,10 +143,10 @@ for original_context_length in context_lengths: ...@@ -143,10 +143,10 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_4needle_en.append(dataset_dict) needlebench_4needle_en_datasets.append(dataset_dict)
num_needles = 5 num_needles = 5
needlebench_datasets_5needle_en = [] needlebench_5needle_en_datasets = []
for original_context_length in context_lengths: for original_context_length in context_lengths:
for depth_percent in depths_list: for depth_percent in depths_list:
...@@ -170,7 +170,7 @@ for original_context_length in context_lengths: ...@@ -170,7 +170,7 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_5needle_en.append(dataset_dict) needlebench_5needle_en_datasets.append(dataset_dict)
# ----------Chinese Version---------- # ----------Chinese Version----------
base_path = './data/needlebench' base_path = './data/needlebench'
...@@ -179,7 +179,7 @@ file_list = ['zh_finance.jsonl'] ...@@ -179,7 +179,7 @@ file_list = ['zh_finance.jsonl']
needle_file_name = 'multi_needle_reasoning_zh.json' needle_file_name = 'multi_needle_reasoning_zh.json'
diff = 10 diff = 10
num_needles = 2 num_needles = 2
needlebench_datasets_2needle_zh = [] needlebench_2needle_zh_datasets = []
language = 'Chinese' language = 'Chinese'
for original_context_length in context_lengths: for original_context_length in context_lengths:
...@@ -204,10 +204,10 @@ for original_context_length in context_lengths: ...@@ -204,10 +204,10 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_2needle_zh.append(dataset_dict) needlebench_2needle_zh_datasets.append(dataset_dict)
num_needles = 3 num_needles = 3
needlebench_datasets_3needle_zh = [] needlebench_3needle_zh_datasets = []
for original_context_length in context_lengths: for original_context_length in context_lengths:
for depth_percent in depths_list: for depth_percent in depths_list:
...@@ -231,10 +231,10 @@ for original_context_length in context_lengths: ...@@ -231,10 +231,10 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_3needle_zh.append(dataset_dict) needlebench_3needle_zh_datasets.append(dataset_dict)
num_needles = 4 num_needles = 4
needlebench_datasets_4needle_zh = [] needlebench_4needle_zh_datasets = []
for original_context_length in context_lengths: for original_context_length in context_lengths:
for depth_percent in depths_list: for depth_percent in depths_list:
...@@ -258,10 +258,10 @@ for original_context_length in context_lengths: ...@@ -258,10 +258,10 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_4needle_zh.append(dataset_dict) needlebench_4needle_zh_datasets.append(dataset_dict)
num_needles = 5 num_needles = 5
needlebench_datasets_5needle_zh = [] needlebench_5needle_zh_datasets = []
for original_context_length in context_lengths: for original_context_length in context_lengths:
for depth_percent in depths_list: for depth_percent in depths_list:
...@@ -285,4 +285,4 @@ for original_context_length in context_lengths: ...@@ -285,4 +285,4 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_5needle_zh.append(dataset_dict) needlebench_5needle_zh_datasets.append(dataset_dict)
configs/datasets/needlebench/needlebench_128k/needlebench_multi_retrieval.py → configs/datasets/needlebench/needlebench_128k/needlebench_multi_retrieval_128k.py
View file @ b50d1632
...@@ -58,7 +58,7 @@ document_depth_percent_interval_type = "linear" ...@@ -58,7 +58,7 @@ document_depth_percent_interval_type = "linear"
base_path = './data/needlebench' base_path = './data/needlebench'
file_list = ['PaulGrahamEssays.jsonl'] file_list = ['PaulGrahamEssays.jsonl']
needlebench_datasets_en = [] needlebench_en_datasets = []
needle_file_name = 'needles.jsonl' needle_file_name = 'needles.jsonl'
depths = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100] depths = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100]
...@@ -81,10 +81,10 @@ for original_context_length in context_lengths: ...@@ -81,10 +81,10 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_en.append(dataset_dict) needlebench_en_datasets.append(dataset_dict)
file_list = ['zh_finance.jsonl'] file_list = ['zh_finance.jsonl']
needlebench_datasets_zh = [] needlebench_zh_datasets = []
for original_context_length in context_lengths: for original_context_length in context_lengths:
dataset_dict = { dataset_dict = {
...@@ -105,4 +105,4 @@ for original_context_length in context_lengths: ...@@ -105,4 +105,4 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_zh.append(dataset_dict) needlebench_zh_datasets.append(dataset_dict)
configs/datasets/needlebench/needlebench_128k/needlebench_single.py → configs/datasets/needlebench/needlebench_128k/needlebench_single_128k.py
View file @ b50d1632
...@@ -59,7 +59,7 @@ document_depth_percent_interval_type = "linear" ...@@ -59,7 +59,7 @@ document_depth_percent_interval_type = "linear"
base_path = './data/needlebench' base_path = './data/needlebench'
file_list = ['PaulGrahamEssays.jsonl'] file_list = ['PaulGrahamEssays.jsonl']
needlebench_datasets_en = [] needlebench_en_datasets = []
needle_file_name = 'needles.jsonl' needle_file_name = 'needles.jsonl'
for original_context_length in context_lengths: for original_context_length in context_lengths:
...@@ -82,10 +82,10 @@ for original_context_length in context_lengths: ...@@ -82,10 +82,10 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_en.append(dataset_dict) needlebench_en_datasets.append(dataset_dict)
file_list = ['zh_finance.jsonl'] file_list = ['zh_finance.jsonl']
needlebench_datasets_zh = [] needlebench_zh_datasets = []
needle_file_name = 'needles.jsonl' needle_file_name = 'needles.jsonl'
for original_context_length in context_lengths: for original_context_length in context_lengths:
...@@ -108,4 +108,4 @@ for original_context_length in context_lengths: ...@@ -108,4 +108,4 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_zh.append(dataset_dict) needlebench_zh_datasets.append(dataset_dict)
configs/datasets/needlebench/needlebench_200k/needlebench.py deleted 100644 → 0
View file @ 2d4e5597
from mmengine.config import read_base
with read_base():
from .needlebench_multi_reasoning import needlebench_datasets_2needle_en as needlebench_multi_2needle_en_datasets
from .needlebench_multi_reasoning import needlebench_datasets_3needle_en as needlebench_multi_3needle_en_datasets
from .needlebench_multi_reasoning import needlebench_datasets_4needle_en as needlebench_multi_4needle_en_datasets
from .needlebench_multi_reasoning import needlebench_datasets_5needle_en as needlebench_multi_5needle_en_datasets
from .needlebench_multi_reasoning import needlebench_datasets_2needle_zh as needlebench_multi_2needle_zh_datasets
from .needlebench_multi_reasoning import needlebench_datasets_3needle_zh as needlebench_multi_3needle_zh_datasets
from .needlebench_multi_reasoning import needlebench_datasets_4needle_zh as needlebench_multi_4needle_zh_datasets
from .needlebench_multi_reasoning import needlebench_datasets_5needle_zh as needlebench_multi_5needle_zh_datasets
from .needlebench_single import needlebench_datasets_en as needlebench_origin_en_datasets
from .needlebench_single import needlebench_datasets_zh as needlebench_origin_zh_datasets
from .needlebench_multi_retrieval import needlebench_datasets_en as needlebench_parallel_en_datasets
from .needlebench_multi_retrieval import needlebench_datasets_zh as needlebench_parallel_zh_datasets
needlebench_datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), [])
configs/datasets/needlebench/needlebench_200k/needlebench_200k.py 0 → 100644
View file @ b50d1632
from mmengine.config import read_base
with read_base():
from .needlebench_multi_reasoning_200k import needlebench_2needle_en_datasets as needlebench_multi_2needle_en_datasets
from .needlebench_multi_reasoning_200k import needlebench_3needle_en_datasets as needlebench_multi_3needle_en_datasets
from .needlebench_multi_reasoning_200k import needlebench_4needle_en_datasets as needlebench_multi_4needle_en_datasets
from .needlebench_multi_reasoning_200k import needlebench_5needle_en_datasets as needlebench_multi_5needle_en_datasets
from .needlebench_multi_reasoning_200k import needlebench_2needle_zh_datasets as needlebench_multi_2needle_zh_datasets
from .needlebench_multi_reasoning_200k import needlebench_3needle_zh_datasets as needlebench_multi_3needle_zh_datasets
from .needlebench_multi_reasoning_200k import needlebench_4needle_zh_datasets as needlebench_multi_4needle_zh_datasets
from .needlebench_multi_reasoning_200k import needlebench_5needle_zh_datasets as needlebench_multi_5needle_zh_datasets
from .needlebench_single_200k import needlebench_en_datasets as needlebench_origin_en_datasets
from .needlebench_single_200k import needlebench_zh_datasets as needlebench_origin_zh_datasets
from .needlebench_multi_retrieval_200k import needlebench_en_datasets as needlebench_parallel_en_datasets
from .needlebench_multi_retrieval_200k import needlebench_zh_datasets as needlebench_parallel_zh_datasets
needlebench_datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), [])
configs/datasets/needlebench/needlebench_200k/needlebench_multi_reasoning.py → configs/datasets/needlebench/needlebench_200k/needlebench_multi_reasoning_200k.py
View file @ b50d1632
...@@ -63,7 +63,7 @@ file_list = ['PaulGrahamEssays.jsonl'] ...@@ -63,7 +63,7 @@ file_list = ['PaulGrahamEssays.jsonl']
needle_file_name = 'multi_needle_reasoning_en.json' needle_file_name = 'multi_needle_reasoning_en.json'
diff = 10 diff = 10
num_needles = 2 num_needles = 2
needlebench_datasets_2needle_en = [] needlebench_2needle_en_datasets = []
language = 'English' language = 'English'
for original_context_length in context_lengths: for original_context_length in context_lengths:
...@@ -88,10 +88,10 @@ for original_context_length in context_lengths: ...@@ -88,10 +88,10 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_2needle_en.append(dataset_dict) needlebench_2needle_en_datasets.append(dataset_dict)
num_needles = 3 num_needles = 3
needlebench_datasets_3needle_en = [] needlebench_3needle_en_datasets = []
for original_context_length in context_lengths: for original_context_length in context_lengths:
for depth_percent in depths_list: for depth_percent in depths_list:
...@@ -115,10 +115,10 @@ for original_context_length in context_lengths: ...@@ -115,10 +115,10 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_3needle_en.append(dataset_dict) needlebench_3needle_en_datasets.append(dataset_dict)
num_needles = 4 num_needles = 4
needlebench_datasets_4needle_en = [] needlebench_4needle_en_datasets = []
for original_context_length in context_lengths: for original_context_length in context_lengths:
for depth_percent in depths_list: for depth_percent in depths_list:
...@@ -142,10 +142,10 @@ for original_context_length in context_lengths: ...@@ -142,10 +142,10 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_4needle_en.append(dataset_dict) needlebench_4needle_en_datasets.append(dataset_dict)
num_needles = 5 num_needles = 5
needlebench_datasets_5needle_en = [] needlebench_5needle_en_datasets = []
for original_context_length in context_lengths: for original_context_length in context_lengths:
for depth_percent in depths_list: for depth_percent in depths_list:
...@@ -169,7 +169,7 @@ for original_context_length in context_lengths: ...@@ -169,7 +169,7 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_5needle_en.append(dataset_dict) needlebench_5needle_en_datasets.append(dataset_dict)
# ----------Chinese Version---------- # ----------Chinese Version----------
base_path = './data/needlebench' base_path = './data/needlebench'
...@@ -178,7 +178,7 @@ file_list = ['zh_finance.jsonl'] ...@@ -178,7 +178,7 @@ file_list = ['zh_finance.jsonl']
needle_file_name = 'multi_needle_reasoning_zh.json' needle_file_name = 'multi_needle_reasoning_zh.json'
diff = 10 diff = 10
num_needles = 2 num_needles = 2
needlebench_datasets_2needle_zh = [] needlebench_2needle_zh_datasets = []
language = 'Chinese' language = 'Chinese'
for original_context_length in context_lengths: for original_context_length in context_lengths:
...@@ -203,10 +203,10 @@ for original_context_length in context_lengths: ...@@ -203,10 +203,10 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_2needle_zh.append(dataset_dict) needlebench_2needle_zh_datasets.append(dataset_dict)
num_needles = 3 num_needles = 3
needlebench_datasets_3needle_zh = [] needlebench_3needle_zh_datasets = []
for original_context_length in context_lengths: for original_context_length in context_lengths:
for depth_percent in depths_list: for depth_percent in depths_list:
...@@ -230,10 +230,10 @@ for original_context_length in context_lengths: ...@@ -230,10 +230,10 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_3needle_zh.append(dataset_dict) needlebench_3needle_zh_datasets.append(dataset_dict)
num_needles = 4 num_needles = 4
needlebench_datasets_4needle_zh = [] needlebench_4needle_zh_datasets = []
for original_context_length in context_lengths: for original_context_length in context_lengths:
for depth_percent in depths_list: for depth_percent in depths_list:
...@@ -257,10 +257,10 @@ for original_context_length in context_lengths: ...@@ -257,10 +257,10 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_4needle_zh.append(dataset_dict) needlebench_4needle_zh_datasets.append(dataset_dict)
num_needles = 5 num_needles = 5
needlebench_datasets_5needle_zh = [] needlebench_5needle_zh_datasets = []
for original_context_length in context_lengths: for original_context_length in context_lengths:
for depth_percent in depths_list: for depth_percent in depths_list:
...@@ -284,4 +284,4 @@ for original_context_length in context_lengths: ...@@ -284,4 +284,4 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_5needle_zh.append(dataset_dict) needlebench_5needle_zh_datasets.append(dataset_dict)
configs/datasets/needlebench/needlebench_200k/needlebench_multi_retrieval.py → configs/datasets/needlebench/needlebench_200k/needlebench_multi_retrieval_200k.py
View file @ b50d1632
...@@ -59,7 +59,7 @@ document_depth_percent_interval_type = "linear" ...@@ -59,7 +59,7 @@ document_depth_percent_interval_type = "linear"
base_path = './data/needlebench' base_path = './data/needlebench'
file_list = ['PaulGrahamEssays.jsonl'] file_list = ['PaulGrahamEssays.jsonl']
needlebench_datasets_en = [] needlebench_en_datasets = []
needle_file_name = 'needles.jsonl' needle_file_name = 'needles.jsonl'
depths = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100] depths = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100]
...@@ -82,10 +82,10 @@ for original_context_length in context_lengths: ...@@ -82,10 +82,10 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_en.append(dataset_dict) needlebench_en_datasets.append(dataset_dict)
file_list = ['zh_finance.jsonl'] file_list = ['zh_finance.jsonl']
needlebench_datasets_zh = [] needlebench_zh_datasets = []
for original_context_length in context_lengths: for original_context_length in context_lengths:
dataset_dict = { dataset_dict = {
...@@ -106,4 +106,4 @@ for original_context_length in context_lengths: ...@@ -106,4 +106,4 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_zh.append(dataset_dict) needlebench_zh_datasets.append(dataset_dict)
configs/datasets/needlebench/needlebench_200k/needlebench_single.py → configs/datasets/needlebench/needlebench_200k/needlebench_single_200k.py
View file @ b50d1632
...@@ -58,7 +58,7 @@ depths_list = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100] ...@@ -58,7 +58,7 @@ depths_list = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100]
base_path = './data/needlebench' base_path = './data/needlebench'
file_list = ['PaulGrahamEssays.jsonl'] file_list = ['PaulGrahamEssays.jsonl']
needlebench_datasets_en = [] needlebench_en_datasets = []
needle_file_name = 'needles.jsonl' needle_file_name = 'needles.jsonl'
for original_context_length in context_lengths: for original_context_length in context_lengths:
...@@ -81,10 +81,10 @@ for original_context_length in context_lengths: ...@@ -81,10 +81,10 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_en.append(dataset_dict) needlebench_en_datasets.append(dataset_dict)
file_list = ['zh_finance.jsonl'] file_list = ['zh_finance.jsonl']
needlebench_datasets_zh = [] needlebench_zh_datasets = []
needle_file_name = 'needles.jsonl' needle_file_name = 'needles.jsonl'
for original_context_length in context_lengths: for original_context_length in context_lengths:
...@@ -107,4 +107,4 @@ for original_context_length in context_lengths: ...@@ -107,4 +107,4 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_zh.append(dataset_dict) needlebench_zh_datasets.append(dataset_dict)
configs/datasets/needlebench/needlebench_32k/needlebench.py deleted 100644 → 0
View file @ 2d4e5597
from mmengine.config import read_base
with read_base():
from .needlebench_multi_reasoning import needlebench_datasets_2needle_en as needlebench_multi_2needle_en_datasets
from .needlebench_multi_reasoning import needlebench_datasets_3needle_en as needlebench_multi_3needle_en_datasets
from .needlebench_multi_reasoning import needlebench_datasets_4needle_en as needlebench_multi_4needle_en_datasets
from .needlebench_multi_reasoning import needlebench_datasets_5needle_en as needlebench_multi_5needle_en_datasets
from .needlebench_multi_reasoning import needlebench_datasets_2needle_zh as needlebench_multi_2needle_zh_datasets
from .needlebench_multi_reasoning import needlebench_datasets_3needle_zh as needlebench_multi_3needle_zh_datasets
from .needlebench_multi_reasoning import needlebench_datasets_4needle_zh as needlebench_multi_4needle_zh_datasets
from .needlebench_multi_reasoning import needlebench_datasets_5needle_zh as needlebench_multi_5needle_zh_datasets
from .needlebench_single import needlebench_datasets_en as needlebench_origin_en_datasets
from .needlebench_single import needlebench_datasets_zh as needlebench_origin_zh_datasets
from .needlebench_multi_retrieval import needlebench_datasets_en as needlebench_parallel_en_datasets
from .needlebench_multi_retrieval import needlebench_datasets_zh as needlebench_parallel_zh_datasets
needlebench_datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), [])
configs/datasets/needlebench/needlebench_32k/needlebench_32k.py 0 → 100644
View file @ b50d1632
from mmengine.config import read_base
with read_base():
from .needlebench_multi_reasoning_32k import needlebench_2needle_en_datasets as needlebench_multi_2needle_en_datasets
from .needlebench_multi_reasoning_32k import needlebench_3needle_en_datasets as needlebench_multi_3needle_en_datasets
from .needlebench_multi_reasoning_32k import needlebench_4needle_en_datasets as needlebench_multi_4needle_en_datasets
from .needlebench_multi_reasoning_32k import needlebench_5needle_en_datasets as needlebench_multi_5needle_en_datasets
from .needlebench_multi_reasoning_32k import needlebench_2needle_zh_datasets as needlebench_multi_2needle_zh_datasets
from .needlebench_multi_reasoning_32k import needlebench_3needle_zh_datasets as needlebench_multi_3needle_zh_datasets
from .needlebench_multi_reasoning_32k import needlebench_4needle_zh_datasets as needlebench_multi_4needle_zh_datasets
from .needlebench_multi_reasoning_32k import needlebench_5needle_zh_datasets as needlebench_multi_5needle_zh_datasets
from .needlebench_single_32k import needlebench_en_datasets as needlebench_origin_en_datasets
from .needlebench_single_32k import needlebench_zh_datasets as needlebench_origin_zh_datasets
from .needlebench_multi_retrieval_32k import needlebench_en_datasets as needlebench_parallel_en_datasets
from .needlebench_multi_retrieval_32k import needlebench_zh_datasets as needlebench_parallel_zh_datasets
needlebench_datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), [])
configs/datasets/needlebench/needlebench_32k/needlebench_multi_reasoning.py → configs/datasets/needlebench/needlebench_32k/needlebench_multi_reasoning_32k.py
View file @ b50d1632
...@@ -64,7 +64,7 @@ file_list = ['PaulGrahamEssays.jsonl'] ...@@ -64,7 +64,7 @@ file_list = ['PaulGrahamEssays.jsonl']
needle_file_name = 'multi_needle_reasoning_en.json' needle_file_name = 'multi_needle_reasoning_en.json'
diff = 10 diff = 10
num_needles = 2 num_needles = 2
needlebench_datasets_2needle_en = [] needlebench_2needle_en_datasets = []
language = 'English' language = 'English'
for original_context_length in context_lengths: for original_context_length in context_lengths:
...@@ -89,10 +89,10 @@ for original_context_length in context_lengths: ...@@ -89,10 +89,10 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_2needle_en.append(dataset_dict) needlebench_2needle_en_datasets.append(dataset_dict)
num_needles = 3 num_needles = 3
needlebench_datasets_3needle_en = [] needlebench_3needle_en_datasets = []
for original_context_length in context_lengths: for original_context_length in context_lengths:
for depth_percent in depths_list: for depth_percent in depths_list:
...@@ -116,10 +116,10 @@ for original_context_length in context_lengths: ...@@ -116,10 +116,10 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_3needle_en.append(dataset_dict) needlebench_3needle_en_datasets.append(dataset_dict)
num_needles = 4 num_needles = 4
needlebench_datasets_4needle_en = [] needlebench_4needle_en_datasets = []
for original_context_length in context_lengths: for original_context_length in context_lengths:
for depth_percent in depths_list: for depth_percent in depths_list:
...@@ -143,10 +143,10 @@ for original_context_length in context_lengths: ...@@ -143,10 +143,10 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_4needle_en.append(dataset_dict) needlebench_4needle_en_datasets.append(dataset_dict)
num_needles = 5 num_needles = 5
needlebench_datasets_5needle_en = [] needlebench_5needle_en_datasets = []
for original_context_length in context_lengths: for original_context_length in context_lengths:
for depth_percent in depths_list: for depth_percent in depths_list:
...@@ -170,7 +170,7 @@ for original_context_length in context_lengths: ...@@ -170,7 +170,7 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_5needle_en.append(dataset_dict) needlebench_5needle_en_datasets.append(dataset_dict)
# ----------Chinese Version---------- # ----------Chinese Version----------
base_path = './data/needlebench' base_path = './data/needlebench'
...@@ -179,7 +179,7 @@ file_list = ['zh_finance.jsonl'] ...@@ -179,7 +179,7 @@ file_list = ['zh_finance.jsonl']
needle_file_name = 'multi_needle_reasoning_zh.json' needle_file_name = 'multi_needle_reasoning_zh.json'
diff = 10 diff = 10
num_needles = 2 num_needles = 2
needlebench_datasets_2needle_zh = [] needlebench_2needle_zh_datasets = []
language = 'Chinese' language = 'Chinese'
for original_context_length in context_lengths: for original_context_length in context_lengths:
...@@ -204,10 +204,10 @@ for original_context_length in context_lengths: ...@@ -204,10 +204,10 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_2needle_zh.append(dataset_dict) needlebench_2needle_zh_datasets.append(dataset_dict)
num_needles = 3 num_needles = 3
needlebench_datasets_3needle_zh = [] needlebench_3needle_zh_datasets = []
for original_context_length in context_lengths: for original_context_length in context_lengths:
for depth_percent in depths_list: for depth_percent in depths_list:
...@@ -231,10 +231,10 @@ for original_context_length in context_lengths: ...@@ -231,10 +231,10 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_3needle_zh.append(dataset_dict) needlebench_3needle_zh_datasets.append(dataset_dict)
num_needles = 4 num_needles = 4
needlebench_datasets_4needle_zh = [] needlebench_4needle_zh_datasets = []
for original_context_length in context_lengths: for original_context_length in context_lengths:
for depth_percent in depths_list: for depth_percent in depths_list:
...@@ -258,10 +258,10 @@ for original_context_length in context_lengths: ...@@ -258,10 +258,10 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_4needle_zh.append(dataset_dict) needlebench_4needle_zh_datasets.append(dataset_dict)
num_needles = 5 num_needles = 5
needlebench_datasets_5needle_zh = [] needlebench_5needle_zh_datasets = []
for original_context_length in context_lengths: for original_context_length in context_lengths:
for depth_percent in depths_list: for depth_percent in depths_list:
...@@ -285,4 +285,4 @@ for original_context_length in context_lengths: ...@@ -285,4 +285,4 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_5needle_zh.append(dataset_dict) needlebench_5needle_zh_datasets.append(dataset_dict)
configs/datasets/needlebench/needlebench_32k/needlebench_multi_retrieval.py → configs/datasets/needlebench/needlebench_32k/needlebench_multi_retrieval_32k.py
View file @ b50d1632
...@@ -58,7 +58,7 @@ document_depth_percent_interval_type = "linear" ...@@ -58,7 +58,7 @@ document_depth_percent_interval_type = "linear"
base_path = './data/needlebench' base_path = './data/needlebench'
file_list = ['PaulGrahamEssays.jsonl'] file_list = ['PaulGrahamEssays.jsonl']
needlebench_datasets_en = [] needlebench_en_datasets = []
needle_file_name = 'needles.jsonl' needle_file_name = 'needles.jsonl'
depths = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100] depths = [0, 10, 21, 31, 42, 52, 63, 73, 84, 94, 100]
...@@ -81,10 +81,10 @@ for original_context_length in context_lengths: ...@@ -81,10 +81,10 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_en.append(dataset_dict) needlebench_en_datasets.append(dataset_dict)
file_list = ['zh_finance.jsonl'] file_list = ['zh_finance.jsonl']
needlebench_datasets_zh = [] needlebench_zh_datasets = []
for original_context_length in context_lengths: for original_context_length in context_lengths:
dataset_dict = { dataset_dict = {
...@@ -105,4 +105,4 @@ for original_context_length in context_lengths: ...@@ -105,4 +105,4 @@ for original_context_length in context_lengths:
'infer_cfg': needlebench_infer_cfg, 'infer_cfg': needlebench_infer_cfg,
'eval_cfg': needlebench_eval_cfg 'eval_cfg': needlebench_eval_cfg
} }
needlebench_datasets_zh.append(dataset_dict) needlebench_zh_datasets.append(dataset_dict)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment