[Feature] Use dataset in local path (#570)

* update commonsenseqa

* update drop

* update flores_first100

* update gsm8k

* update humaneval

* update lambda

* update obqa

* update piqa

* update race

* update siqa

* update story_cloze

* update strategyqa

* update tydiqa

* update winogrande

* update doc

* update hellaswag

* fix obqa

* update collections

* update .zip name

configs/datasets/hellaswag/hellaswag_ppl_47bff9.py

@@ -27,8 +27,9 @@ hellaswag_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
 
 hellaswag_datasets = [
     dict(
+        abbr='hellaswag',
         type=hellaswagDataset,
-        path='hellaswag',
+        path='./data/hellaswag/hellaswag.jsonl',
         reader_cfg=hellaswag_reader_cfg,
         infer_cfg=hellaswag_infer_cfg,
         eval_cfg=hellaswag_eval_cfg)

configs/datasets/hellaswag/hellaswag_ppl_9dbb12.py

@@ -6,9 +6,8 @@ from opencompass.datasets import hellaswagDataset
 
 hellaswag_reader_cfg = dict(
     input_columns=['ctx', 'A', 'B', 'C', 'D'],
-    output_column='label',
-    train_split='validation',
-    test_split='validation')
+    output_column='label'
+)
 
 hellaswag_infer_cfg = dict(
     prompt_template=dict(
@@ -26,8 +25,9 @@ hellaswag_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
 
 hellaswag_datasets = [
     dict(
+        abbr='hellaswag',
         type=hellaswagDataset,
-        path='hellaswag',
+        path='./data/hellaswag/hellaswag.jsonl',
         reader_cfg=hellaswag_reader_cfg,
         infer_cfg=hellaswag_infer_cfg,
         eval_cfg=hellaswag_eval_cfg)

configs/datasets/humaneval/humaneval_gen_6f294d.py

 from opencompass.openicl.icl_prompt_template import PromptTemplate
 from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
-from opencompass.datasets import HFDataset, HumanEvaluator, humaneval_postprocess
+from opencompass.datasets import HumanevalDataset, HumanEvaluator, humaneval_postprocess
 
 humaneval_reader_cfg = dict(
     input_columns=['prompt'], output_column='task_id', train_split='test')
@@ -32,8 +32,9 @@ humaneval_eval_cfg = dict(
 
 humaneval_datasets = [
     dict(
-        type=HFDataset,
-        path='openai_humaneval',
+        abbr='openai_humaneval',
+        type=HumanevalDataset,
+        path='./data/humaneval/human-eval-v2-20210705.jsonl',
         reader_cfg=humaneval_reader_cfg,
         infer_cfg=humaneval_infer_cfg,
         eval_cfg=humaneval_eval_cfg)

configs/datasets/humaneval/humaneval_gen_8e312c.py

 from opencompass.openicl.icl_prompt_template import PromptTemplate
 from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
-from opencompass.datasets import HFDataset, HumanEvaluator, humaneval_postprocess
+from opencompass.datasets import HumanevalDataset, HumanEvaluator, humaneval_postprocess
 
 humaneval_reader_cfg = dict(
     input_columns=['prompt'], output_column='task_id', train_split='test')
@@ -27,8 +27,9 @@ humaneval_eval_cfg = dict(
 
 humaneval_datasets = [
     dict(
-        type=HFDataset,
-        path='openai_humaneval',
+        abbr='openai_humaneval',
+        type=HumanevalDataset,
+        path='./data/humaneval/human-eval-v2-20210705.jsonl',
         reader_cfg=humaneval_reader_cfg,
         infer_cfg=humaneval_infer_cfg,
         eval_cfg=humaneval_eval_cfg)

configs/datasets/humaneval/humaneval_gen_a82cae.py

 from opencompass.openicl.icl_prompt_template import PromptTemplate
 from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
-from opencompass.datasets import HFDataset, HumanEvaluator, humaneval_postprocess
+from opencompass.datasets import HumanevalDataset, HumanEvaluator, humaneval_postprocess
 
 humaneval_reader_cfg = dict(
     input_columns=['prompt'], output_column='task_id', train_split='test')
@@ -27,8 +27,9 @@ humaneval_eval_cfg = dict(
 
 humaneval_datasets = [
     dict(
-        type=HFDataset,
-        path='openai_humaneval',
+        abbr='openai_humaneval',
+        type=HumanevalDataset,
+        path='./data/humaneval/human-eval-v2-20210705.jsonl',
         reader_cfg=humaneval_reader_cfg,
         infer_cfg=humaneval_infer_cfg,
         eval_cfg=humaneval_eval_cfg)

configs/datasets/humaneval/humaneval_gen_fd5822.py

 from opencompass.openicl.icl_prompt_template import PromptTemplate
 from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
-from opencompass.datasets import HFDataset, HumanEvaluator, humaneval_postprocess
+from opencompass.datasets import HumanevalDataset, HumanEvaluator, humaneval_postprocess
 
 humaneval_reader_cfg = dict(
     input_columns=['prompt'], output_column='task_id', train_split='test')
@@ -22,8 +22,9 @@ humaneval_eval_cfg = dict(
 
 humaneval_datasets = [
     dict(
-        type=HFDataset,
-        path='openai_humaneval',
+        abbr='openai_humaneval',
+        type=HumanevalDataset,
+        path='./data/humaneval/human-eval-v2-20210705.jsonl',
         reader_cfg=humaneval_reader_cfg,
         infer_cfg=humaneval_infer_cfg,
         eval_cfg=humaneval_eval_cfg)

configs/datasets/humaneval/humaneval_gen_ff7054.py

 from opencompass.openicl.icl_prompt_template import PromptTemplate
 from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import GenInferencer
-from opencompass.datasets import HFDataset, HumanEvaluator, humaneval_postprocess
+from opencompass.datasets import HumanevalDataset, HumanEvaluator, humaneval_postprocess
 
 humaneval_reader_cfg = dict(
     input_columns=['prompt'], output_column='task_id', train_split='test')
@@ -32,8 +32,9 @@ humaneval_eval_cfg = dict(
 
 humaneval_datasets = [
     dict(
-        type=HFDataset,
-        path='openai_humaneval',
+        abbr='openai_humaneval',
+        type=HumanevalDataset,
+        path='./data/humaneval/human-eval-v2-20210705.jsonl',
         reader_cfg=humaneval_reader_cfg,
         infer_cfg=humaneval_infer_cfg,
         eval_cfg=humaneval_eval_cfg)

configs/datasets/lambada/lambada_gen_217e11.py

@@ -26,7 +26,7 @@ lambada_datasets = [
     dict(
         abbr='lambada',
         type=lambadaDataset,
-        path='craffel/openai_lambada',
+        path='./data/lambada/test.jsonl',
         reader_cfg=lambada_reader_cfg,
         infer_cfg=lambada_infer_cfg,
         eval_cfg=lambada_eval_cfg)

configs/datasets/lambada/lambada_gen_8b48a5.py

@@ -22,7 +22,7 @@ lambada_datasets = [
     dict(
         abbr='lambada',
         type=lambadaDataset,
-        path='craffel/openai_lambada',
+        path='./data/lambada/test.jsonl',
         reader_cfg=lambada_reader_cfg,
         infer_cfg=lambada_infer_cfg,
         eval_cfg=lambada_eval_cfg)

configs/datasets/obqa/obqa_gen_9069e4.py

@@ -32,15 +32,12 @@ obqa_datasets = [
     dict(
         abbr="openbookqa",
         type=OBQADataset,
-        path="openbookqa",
-        split="test",
+        path='./data/openbookqa/Main/test.jsonl',
     ),
     dict(
         abbr="openbookqa_fact",
         type=OBQADataset,
-        path="openbookqa",
-        name="additional",
-        split="test",
+        path='./data/openbookqa/Additional/test_complete.jsonl',
     ),
 ]
 

configs/datasets/obqa/obqa_ppl_1defe8.py

@@ -24,15 +24,12 @@ obqa_datasets = [
     dict(
         abbr="openbookqa",
         type=OBQADataset,
-        path="openbookqa",
-        split="test",
+        path='./data/openbookqa/Main/test.jsonl',
     ),
     dict(
         abbr="openbookqa_fact",
         type=OBQADataset,
-        path="openbookqa",
-        name="additional",
-        split="test",
+        path='./data/openbookqa/Additional/test_complete.jsonl',
     ),
 ]
 for _i in range(2):

configs/datasets/obqa/obqa_ppl_6aac9e.py

@@ -33,9 +33,7 @@ obqa_datasets = [
     dict(
         abbr='openbookqa_fact',
         type=OBQADataset_V2,
-        path='openbookqa',
-        name='additional',
-        split='test',
+        path='./data/openbookqa/Additional/test_complete.jsonl',
         reader_cfg=obqa_reader_cfg,
         infer_cfg=obqa_infer_cfg,
         eval_cfg=obqa_eval_cfg,

configs/datasets/obqa/obqa_ppl_c7c154.py

@@ -37,16 +37,14 @@ _template = [
 
 obqa_datasets = [
     dict(
+        abbr="openbookqa",
         type=OBQADataset,
-        path='openbookqa',
-        split='test',
+        path='./data/openbookqa/Main/test.jsonl',
     ),
     dict(
         abbr='openbookqa_fact',
         type=OBQADataset,
-        path='openbookqa',
-        name='additional',
-        split='test',
+        path='./data/openbookqa/Additional/test_complete.jsonl',
     ),
 ]
 for _i in range(2):

configs/datasets/piqa/piqa_gen_1194eb.py

@@ -34,7 +34,7 @@ piqa_datasets = [
     dict(
         abbr="piqa",
         type=piqaDataset_V2,
-        path="piqa",
+        path='./data/piqa',
         reader_cfg=piqa_reader_cfg,
         infer_cfg=piqa_infer_cfg,
         eval_cfg=piqa_eval_cfg)

configs/datasets/piqa/piqa_ppl_0cfff2.py

@@ -30,7 +30,7 @@ piqa_datasets = [
     dict(
         abbr='piqa',
         type=piqaDataset_V3,
-        path='piqa',
+        path='./data/piqa',
         reader_cfg=piqa_reader_cfg,
         infer_cfg=piqa_infer_cfg,
         eval_cfg=piqa_eval_cfg)

configs/datasets/piqa/piqa_ppl_1cf9f0.py

@@ -2,7 +2,7 @@ from opencompass.openicl.icl_prompt_template import PromptTemplate
 from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import PPLInferencer
 from opencompass.openicl.icl_evaluator import AccEvaluator
-from opencompass.datasets import HFDataset
+from opencompass.datasets import piqaDataset
 
 piqa_reader_cfg = dict(
     input_columns=['goal', 'sol1', 'sol2'],
@@ -23,8 +23,9 @@ piqa_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
 
 piqa_datasets = [
     dict(
-        type=HFDataset,
-        path='piqa',
+        abbr='piqa',
+        type=piqaDataset,
+        path='./data/piqa',
         reader_cfg=piqa_reader_cfg,
         infer_cfg=piqa_infer_cfg,
         eval_cfg=piqa_eval_cfg)

configs/datasets/piqa/piqa_ppl_3431ea.py

@@ -2,7 +2,7 @@ from opencompass.openicl.icl_prompt_template import PromptTemplate
 from opencompass.openicl.icl_retriever import ZeroRetriever
 from opencompass.openicl.icl_inferencer import PPLInferencer
 from opencompass.openicl.icl_evaluator import AccEvaluator
-from opencompass.datasets import HFDataset
+from opencompass.datasets import piqaDataset
 
 piqa_reader_cfg = dict(
     input_columns=['goal', 'sol1', 'sol2'],
@@ -33,8 +33,9 @@ piqa_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
 
 piqa_datasets = [
     dict(
-        type=HFDataset,
-        path='piqa',
+        abbr='piqa',
+        type=piqaDataset,
+        path='./data/piqa',
         reader_cfg=piqa_reader_cfg,
         infer_cfg=piqa_infer_cfg,
         eval_cfg=piqa_eval_cfg)

configs/datasets/race/race_gen_69ee4f.py

@@ -7,7 +7,10 @@ from opencompass.utils.text_postprocessors import first_option_postprocess
 
 race_reader_cfg = dict(
     input_columns=['article', 'question', 'A', 'B', 'C', 'D'],
-    output_column='answer')
+    output_column='answer',
+    train_split="validation",
+    test_split="test"
+)
 
 race_infer_cfg = dict(
     prompt_template=dict(
@@ -29,17 +32,17 @@ race_eval_cfg = dict(
 
 race_datasets = [
     dict(
-        type=RaceDataset,
         abbr='race-middle',
-        path='race',
+        type=RaceDataset,
+        path='./data/race',
         name='middle',
         reader_cfg=race_reader_cfg,
         infer_cfg=race_infer_cfg,
         eval_cfg=race_eval_cfg),
     dict(
-        type=RaceDataset,
         abbr='race-high',
-        path='race',
+        type=RaceDataset,
+        path='./data/race',
         name='high',
         reader_cfg=race_reader_cfg,
         infer_cfg=race_infer_cfg,

configs/datasets/race/race_gen_9302a5.py

@@ -7,7 +7,10 @@ from opencompass.utils.text_postprocessors import first_capital_postprocess
 
 race_reader_cfg = dict(
     input_columns=['article', 'question', 'A', 'B', 'C', 'D'],
-    output_column='answer')
+    output_column='answer',
+    train_split="validation",
+    test_split="test"
+)
 
 race_infer_cfg = dict(
     prompt_template=dict(
@@ -23,17 +26,17 @@ race_eval_cfg = dict(
 
 race_datasets = [
     dict(
-        type=RaceDataset,
         abbr='race-middle',
-        path='race',
+        type=RaceDataset,
+        path='./data/race',
         name='middle',
         reader_cfg=race_reader_cfg,
         infer_cfg=race_infer_cfg,
         eval_cfg=race_eval_cfg),
     dict(
-        type=RaceDataset,
         abbr='race-high',
-        path='race',
+        type=RaceDataset,
+        path='./data/race',
         name='high',
         reader_cfg=race_reader_cfg,
         infer_cfg=race_infer_cfg,

configs/datasets/race/race_ppl_5831a0.py

@@ -6,7 +6,10 @@ from opencompass.datasets import RaceDataset
 
 race_reader_cfg = dict(
     input_columns=['article', 'question', 'A', 'B', 'C', 'D'],
-    output_column='answer')
+    output_column='answer',
+    train_split="validation",
+    test_split="test"
+)
 
 race_infer_cfg = dict(
     prompt_template=dict(
@@ -27,17 +30,17 @@ race_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
 
 race_datasets = [
     dict(
-        type=RaceDataset,
         abbr='race-middle',
-        path='race',
+        type=RaceDataset,
+        path='./data/race',
         name='middle',
         reader_cfg=race_reader_cfg,
         infer_cfg=race_infer_cfg,
         eval_cfg=race_eval_cfg),
     dict(
-        type=RaceDataset,
         abbr='race-high',
-        path='race',
+        type=RaceDataset,
+        path='./data/race',
         name='high',
         reader_cfg=race_reader_cfg,
         infer_cfg=race_infer_cfg,