Skip to content

GitLab

Unverified Commit 689ffe5b authored by Fengzhe Zhou's avatar Fengzhe Zhou Committed by GitHub
Browse files

[Feature] Use dataset in local path (#570) 

* update commonsenseqa

* update drop

* update flores_first100

* update gsm8k

* update humaneval

* update lambda

* update obqa

* update piqa

* update race

* update siqa

* update story_cloze

* update strategyqa

* update tydiqa

* update winogrande

* update doc

* update hellaswag

* fix obqa

* update collections

* update .zip name
parent d6aaac22
Hide whitespace changes
Inline Side-by-side
Showing with 498 additions and 224 deletions
opencompass/datasets/commonsenseqa.py
View file @ 689ffe5b
from datasets import load_dataset import json
import os
from datasets import Dataset, DatasetDict
from opencompass.registry import LOAD_DATASET from opencompass.registry import LOAD_DATASET
...@@ -9,14 +12,33 @@ from .base import BaseDataset ...@@ -9,14 +12,33 @@ from .base import BaseDataset
class commonsenseqaDataset(BaseDataset): class commonsenseqaDataset(BaseDataset):
@staticmethod @staticmethod
def load(**kwargs): def load(path):
dataset = load_dataset(**kwargs) dataset = {}
for split, stub in [
def pre_process(example): ['train', 'train_rand_split.jsonl'],
for i in range(5): ['validation', 'dev_rand_split.jsonl'],
example[chr(ord('A') + i)] = example['choices']['text'][i] ]:
return example data_path = os.path.join(path, stub)
dataset_list = []
with open(data_path, 'r', encoding='utf-8') as f:
for line in f:
line = json.loads(line)
dataset_list.append({
'question':
line['question']['stem'],
'A':
line['question']['choices'][0]['text'],
'B':
line['question']['choices'][1]['text'],
'C':
line['question']['choices'][2]['text'],
'D':
line['question']['choices'][3]['text'],
'E':
line['question']['choices'][4]['text'],
'answerKey':
line['answerKey'],
})
dataset[split] = Dataset.from_list(dataset_list)
dataset = dataset.map(pre_process).remove_columns( return DatasetDict(dataset)
['question_concept', 'id', 'choices'])
return dataset
opencompass/datasets/drop.py
View file @ 689ffe5b
from datasets import DatasetDict, load_dataset import json
from datasets import Dataset, DatasetDict
from opencompass.registry import LOAD_DATASET from opencompass.registry import LOAD_DATASET
...@@ -9,21 +11,37 @@ from .base import BaseDataset ...@@ -9,21 +11,37 @@ from .base import BaseDataset
class dropDataset(BaseDataset): class dropDataset(BaseDataset):
@staticmethod @staticmethod
def load(**kwargs): def get_answers(validated_answers):
dataset = load_dataset(**kwargs, split='validation') answers = []
for answer_item in validated_answers:
def pre_process(example): if answer_item['number']:
example['answers'] = example['answers_spans']['spans'] answers.append(answer_item['number'])
example['prompt'] = example.pop('passage') elif any(answer_item['date'][i] for i in ['day', 'month', 'year']):
return example d = [answer_item['date'][i] for i in ['day', 'month', 'year']]
answers.append(' '.join(d).strip())
def only_number(example): else:
for i in example['answers_spans']['types']: for span in answer_item['spans']:
if i == 'number': answers.append(span)
return True answers = list(set(answers))
return False return answers
dataset = dataset.filter(only_number) @staticmethod
dataset = dataset.map(pre_process).remove_columns( def load(path, only_number=True):
['section_id', 'query_id']) with open(path, 'r', encoding='utf-8') as f:
return DatasetDict({'validation': dataset}) lines = json.load(f)
dataset_list = []
for line in lines.values():
for qa_pair in line['qa_pairs']:
validated_answers = qa_pair['validated_answers']
if only_number and not any(i['number']
for i in validated_answers):
continue
item = {
'prompt': line['passage'],
'question': qa_pair['question'],
'answers': dropDataset.get_answers(validated_answers),
}
dataset_list.append(item)
dataset_list = Dataset.from_list(dataset_list)
return DatasetDict({'validation': dataset_list})
opencompass/datasets/flores.py
View file @ 689ffe5b
import os
import re import re
from datasets import DatasetDict, load_dataset from datasets import Dataset, DatasetDict
from opencompass.registry import LOAD_DATASET, TEXT_POSTPROCESSORS from opencompass.registry import LOAD_DATASET, TEXT_POSTPROCESSORS
...@@ -11,15 +12,30 @@ from .base import BaseDataset ...@@ -11,15 +12,30 @@ from .base import BaseDataset
class FloresFirst100Dataset(BaseDataset): class FloresFirst100Dataset(BaseDataset):
@staticmethod @staticmethod
def load(name): def load_single(src_path, tgt_path, src_lang, tgt_lang):
return DatasetDict({
'dev': with open(src_path, 'r', encoding='utf-8') as f:
load_dataset(path='facebook/flores', name=name, split='dev'), src_lines = f.readlines()
'devtest': with open(tgt_path, 'r', encoding='utf-8') as f:
load_dataset(path='facebook/flores', tgt_lines = f.readlines()
name=name, assert len(src_lines) == len(tgt_lines)
split='devtest[:100]') dataset_list = [{
}) f'sentence_{src_lang}': src_lines[i].strip(),
f'sentence_{tgt_lang}': tgt_lines[i].strip(),
} for i in range(len(src_lines))]
return Dataset.from_list(dataset_list)
@staticmethod
def load(path, name):
src_lang, tgt_lang = name.split('-')
dev_dataset = FloresFirst100Dataset.load_single(
os.path.join(path, 'dev', f'{src_lang}.dev'),
os.path.join(path, 'dev', f'{tgt_lang}.dev'), src_lang, tgt_lang)
devtest_dataset = FloresFirst100Dataset.load_single(
os.path.join(path, 'devtest', f'{src_lang}.devtest'),
os.path.join(path, 'devtest', f'{tgt_lang}.devtest'), src_lang,
tgt_lang)
return DatasetDict({'dev': dev_dataset, 'devtest': devtest_dataset})
@TEXT_POSTPROCESSORS.register_module('flores') @TEXT_POSTPROCESSORS.register_module('flores')
......
opencompass/datasets/gsm8k.py
View file @ 689ffe5b
import json
import os
from datasets import Dataset, DatasetDict
from opencompass.openicl import BaseEvaluator from opencompass.openicl import BaseEvaluator
from opencompass.registry import TEXT_POSTPROCESSORS from opencompass.registry import LOAD_DATASET, TEXT_POSTPROCESSORS
from .base import BaseDataset
@LOAD_DATASET.register_module()
class GSM8KDataset(BaseDataset):
@staticmethod
def load(path):
datasets = {}
for split in ['train', 'test']:
split_path = os.path.join(path, split + '.jsonl')
dataset = []
with open(split_path, 'r', encoding='utf-8') as f:
for line in f:
line = json.loads(line.strip())
line['answer']
dataset.append(line)
datasets[split] = Dataset.from_list(dataset)
return DatasetDict(datasets)
@TEXT_POSTPROCESSORS.register_module('gsm8k_dataset') @TEXT_POSTPROCESSORS.register_module('gsm8k_dataset')
......
opencompass/datasets/hellaswag.py
View file @ 689ffe5b
import json import json
from datasets import Dataset, load_dataset from datasets import Dataset
from opencompass.registry import LOAD_DATASET from opencompass.registry import LOAD_DATASET
...@@ -11,15 +11,20 @@ from .base import BaseDataset ...@@ -11,15 +11,20 @@ from .base import BaseDataset
class hellaswagDataset(BaseDataset): class hellaswagDataset(BaseDataset):
@staticmethod @staticmethod
def load(**kwargs): def load(path):
dataset = load_dataset(**kwargs) dataset = []
with open(path, 'r', encoding='utf-8') as f:
def preprocess(example): for line in f:
for i in range(4): data = json.loads(line)
example[chr(ord('A') + i)] = example['endings'][i] dataset.append({
return example 'ctx': data['query'].split(': ', 2)[-1],
'A': data['choices'][0],
dataset = dataset.map(preprocess).remove_columns(['endings']) 'B': data['choices'][1],
'C': data['choices'][2],
'D': data['choices'][3],
'label': data['gold'],
})
dataset = Dataset.from_list(dataset)
return dataset return dataset
...@@ -27,19 +32,20 @@ class hellaswagDataset(BaseDataset): ...@@ -27,19 +32,20 @@ class hellaswagDataset(BaseDataset):
class hellaswagDataset_V2(BaseDataset): class hellaswagDataset_V2(BaseDataset):
@staticmethod @staticmethod
def load(**kwargs): def load(path):
dataset = load_dataset(**kwargs) dataset = []
with open(path, 'r', encoding='utf-8') as f:
def preprocess(example): for line in f:
for i in range(4): data = json.loads(line)
example[chr(ord('A') + i)] = example['endings'][i] dataset.append({
if example['label']: 'ctx': data['query'].split(': ', 1)[-1],
example['label'] = 'ABCD'[int(example['label'])] 'A': data['choices'][0],
else: 'B': data['choices'][1],
example['label'] = 'NULL' 'C': data['choices'][2],
return example 'D': data['choices'][3],
'label': 'ABCD'[data['gold']],
dataset = dataset.map(preprocess).remove_columns(['endings']) })
dataset = Dataset.from_list(dataset)
return dataset return dataset
......
opencompass/datasets/humaneval.py
View file @ 689ffe5b
import json
import os.path as osp import os.path as osp
import re import re
import tempfile import tempfile
from typing import List from typing import List
from datasets import Dataset
from opencompass.openicl.icl_evaluator import BaseEvaluator from opencompass.openicl.icl_evaluator import BaseEvaluator
from opencompass.registry import LOAD_DATASET
from .base import BaseDataset
@LOAD_DATASET.register_module()
class HumanevalDataset(BaseDataset):
@staticmethod
def load(path):
dataset = []
with open(path, 'r', encoding='utf-8') as f:
for line in f:
dataset.append(json.loads(line.strip()))
return Dataset.from_list(dataset)
class HumanEvaluator(BaseEvaluator): class HumanEvaluator(BaseEvaluator):
......
opencompass/datasets/lambada.py
View file @ 689ffe5b
import json
import re import re
import string import string
from datasets import DatasetDict, load_dataset from datasets import Dataset, DatasetDict
from opencompass.openicl.icl_evaluator import BaseEvaluator from opencompass.openicl.icl_evaluator import BaseEvaluator
from opencompass.registry import ICL_EVALUATORS, LOAD_DATASET from opencompass.registry import ICL_EVALUATORS, LOAD_DATASET
...@@ -14,16 +15,12 @@ from .base import BaseDataset ...@@ -14,16 +15,12 @@ from .base import BaseDataset
class lambadaDataset(BaseDataset): class lambadaDataset(BaseDataset):
@staticmethod @staticmethod
def load(**kwargs): def load(path):
dataset = load_dataset(**kwargs, split='test') dataset = []
with open(path, 'r', encoding='utf-8') as f:
def preprocess(example): for line in f:
prompt, target = example['text'].strip().rsplit(' ', 1) dataset.append(json.loads(line))
example['prompt'] = prompt dataset = Dataset.from_list(dataset)
example['label'] = target
return example
dataset = dataset.map(preprocess)
return DatasetDict({'test': dataset}) return DatasetDict({'test': dataset})
......
opencompass/datasets/obqa.py
View file @ 689ffe5b
from datasets import load_dataset import json
from datasets import Dataset
from opencompass.registry import LOAD_DATASET from opencompass.registry import LOAD_DATASET
...@@ -9,33 +11,46 @@ from .base import BaseDataset ...@@ -9,33 +11,46 @@ from .base import BaseDataset
class OBQADataset(BaseDataset): class OBQADataset(BaseDataset):
@staticmethod @staticmethod
def load(**kwargs): def load(path):
dataset = load_dataset(**kwargs) dataset_list = []
with open(path, 'r') as f:
def pre_process(example): for line in f:
for i in range(4): line = json.loads(line)
example[chr(ord('A') + i)] = example['choices']['text'][i] item = {
return example 'A': line['question']['choices'][0]['text'],
'B': line['question']['choices'][1]['text'],
dataset = dataset.map(pre_process).remove_columns(['id', 'choices']) 'C': line['question']['choices'][2]['text'],
return dataset 'D': line['question']['choices'][3]['text'],
'question_stem': line['question']['stem'],
'answerKey': line['answerKey'],
}
if 'fact1' in line:
item['fact1'] = line['fact1']
dataset_list.append(item)
return Dataset.from_list(dataset_list)
@LOAD_DATASET.register_module() @LOAD_DATASET.register_module()
class OBQADataset_V2(BaseDataset): class OBQADataset_V2(BaseDataset):
@staticmethod @staticmethod
def load(**kwargs): def load(path):
dataset = load_dataset(**kwargs) dataset_list = []
with open(path, 'r') as f:
def pre_process(example): for line in f:
example['A'] = example['choices']['text'][0] line = json.loads(line)
example['B'] = example['choices']['text'][1] question = line['question']['stem']
example['C'] = example['choices']['text'][2] if not question.endswith('?'):
example['D'] = example['choices']['text'][3] question += ' what?'
if not example['question_stem'].endswith('?'): item = {
example['question_stem'] += ' what?' 'A': line['question']['choices'][0]['text'],
return example 'B': line['question']['choices'][1]['text'],
'C': line['question']['choices'][2]['text'],
dataset = dataset.map(pre_process).remove_columns(['id', 'choices']) 'D': line['question']['choices'][3]['text'],
return dataset 'question_stem': question,
'answerKey': line['answerKey'],
}
if 'fact1' in line:
item['fact1'] = line['fact1']
dataset_list.append(item)
return Dataset.from_list(dataset_list)
opencompass/datasets/piqa.py
View file @ 689ffe5b
from datasets import load_dataset import json
import os
from datasets import Dataset, DatasetDict
from opencompass.registry import LOAD_DATASET from opencompass.registry import LOAD_DATASET
...@@ -6,45 +9,100 @@ from .base import BaseDataset ...@@ -6,45 +9,100 @@ from .base import BaseDataset
@LOAD_DATASET.register_module() @LOAD_DATASET.register_module()
class piqaDataset_V2(BaseDataset): class piqaDataset(BaseDataset):
@staticmethod
def load_single(path, data_filename, label_filename):
data_path = os.path.join(path, data_filename)
label_path = os.path.join(path, label_filename)
dataset = []
with open(data_path, 'r', encoding='utf-8') as f:
data_lines = f.readlines()
with open(label_path, 'r', encoding='utf-8') as f:
label_lines = f.readlines()
assert len(data_lines) == len(label_lines)
for data, label in zip(data_lines, label_lines):
i = json.loads(data.strip())
i['label'] = int(label.strip())
dataset.append(i)
return Dataset.from_list(dataset)
@staticmethod @staticmethod
def load(**kwargs): def load(path):
dataset = load_dataset(**kwargs) train_dataset = piqaDataset.load_single(path, 'train.jsonl',
'train-labels.lst')
val_dataset = piqaDataset.load_single(path, 'dev.jsonl',
'dev-labels.lst')
return DatasetDict({'train': train_dataset, 'validation': val_dataset})
@LOAD_DATASET.register_module()
class piqaDataset_V2(BaseDataset):
def preprocess(example): @staticmethod
assert isinstance(example['label'], int) def load_single(path, data_filename, label_filename):
if example['label'] < 0: data_path = os.path.join(path, data_filename)
example['answer'] = 'NULL' label_path = os.path.join(path, label_filename)
dataset = []
with open(data_path, 'r', encoding='utf-8') as f:
data_lines = f.readlines()
with open(label_path, 'r', encoding='utf-8') as f:
label_lines = f.readlines()
assert len(data_lines) == len(label_lines)
for data, label in zip(data_lines, label_lines):
i = json.loads(data.strip())
label = int(label.strip())
if label < 0:
i['answer'] = 'NULL'
else: else:
example['answer'] = 'AB'[example['label']] i['answer'] = 'AB'[label]
example.pop('label') dataset.append(i)
return example
dataset = dataset.map(preprocess) return Dataset.from_list(dataset)
return dataset
@staticmethod
def load(path):
train_dataset = piqaDataset_V2.load_single(path, 'train.jsonl',
'train-labels.lst')
val_dataset = piqaDataset_V2.load_single(path, 'dev.jsonl',
'dev-labels.lst')
return DatasetDict({'train': train_dataset, 'validation': val_dataset})
@LOAD_DATASET.register_module() @LOAD_DATASET.register_module()
class piqaDataset_V3(BaseDataset): class piqaDataset_V3(BaseDataset):
@staticmethod @staticmethod
def load(**kwargs): def load_single(path, data_filename, label_filename):
dataset = load_dataset(**kwargs) data_path = os.path.join(path, data_filename)
label_path = os.path.join(path, label_filename)
def preprocess(example): dataset = []
example['goal'] = example['goal'][0].upper() + example['goal'][1:] with open(data_path, 'r', encoding='utf-8') as f:
if example['goal'].endswith('?') or example['goal'].endswith('.'): data_lines = f.readlines()
example['sol1'] = example['sol1'][0].upper( with open(label_path, 'r', encoding='utf-8') as f:
) + example['sol1'][1:] label_lines = f.readlines()
example['sol2'] = example['sol2'][0].upper( assert len(data_lines) == len(label_lines)
) + example['sol2'][1:] for data, label in zip(data_lines, label_lines):
i = json.loads(data.strip())
i['label'] = int(label.strip())
# some preprocessing
i['goal'] = i['goal'][0].upper() + i['goal'][1:]
if i['goal'].endswith('?') or i['goal'].endswith('.'):
i['sol1'] = i['sol1'][0].upper() + i['sol1'][1:]
i['sol2'] = i['sol2'][0].upper() + i['sol2'][1:]
else: else:
example['sol1'] = example['sol1'][0].lower( i['sol1'] = i['sol1'][0].lower() + i['sol1'][1:]
) + example['sol1'][1:] i['sol2'] = i['sol2'][0].lower() + i['sol2'][1:]
example['sol2'] = example['sol2'][0].lower(
) + example['sol2'][1:] dataset.append(i)
return example
return Dataset.from_list(dataset)
dataset = dataset.map(preprocess)
return dataset @staticmethod
def load(path):
train_dataset = piqaDataset_V3.load_single(path, 'train.jsonl',
'train-labels.lst')
val_dataset = piqaDataset_V3.load_single(path, 'dev.jsonl',
'dev-labels.lst')
return DatasetDict({'train': train_dataset, 'validation': val_dataset})
opencompass/datasets/race.py
View file @ 689ffe5b
from datasets import load_dataset import json
import os
from datasets import Dataset, DatasetDict
from opencompass.registry import LOAD_DATASET from opencompass.registry import LOAD_DATASET
...@@ -10,12 +13,21 @@ class RaceDataset(BaseDataset): ...@@ -10,12 +13,21 @@ class RaceDataset(BaseDataset):
@staticmethod @staticmethod
def load(path: str, name: str): def load(path: str, name: str):
dataset = load_dataset(path, name) dataset = {}
for split in ['validation', 'test']:
def preprocess(x): jsonl_path = os.path.join(path, split, f'{name}.jsonl')
for ans, option in zip(['A', 'B', 'C', 'D'], x['options']): dataset_list = []
x[ans] = option with open(jsonl_path, 'r', encoding='utf-8') as f:
del x['options'] for line in f:
return x line = json.loads(line)
dataset_list.append({
return dataset.map(preprocess) 'article': line['article'],
'question': line['question'],
'A': line['options'][0],
'B': line['options'][1],
'C': line['options'][2],
'D': line['options'][3],
'answer': line['answer'],
})
dataset[split] = Dataset.from_list(dataset_list)
return DatasetDict(dataset)
opencompass/datasets/siqa.py
View file @ 689ffe5b
from datasets import load_dataset import json
import os
from datasets import Dataset, DatasetDict
from opencompass.registry import LOAD_DATASET from opencompass.registry import LOAD_DATASET
...@@ -6,24 +9,72 @@ from .base import BaseDataset ...@@ -6,24 +9,72 @@ from .base import BaseDataset
@LOAD_DATASET.register_module() @LOAD_DATASET.register_module()
class siqaDataset_V2(BaseDataset): class siqaDataset(BaseDataset):
"""Disconnect from HuggingFace version of HFDataset."""
@staticmethod @staticmethod
def load(**kwargs): def load_single(path, data_filename, label_filename):
dataset = load_dataset(**kwargs) data_path = os.path.join(path, data_filename)
label_path = os.path.join(path, label_filename)
dataset = []
with open(data_path, 'r', encoding='utf-8') as f:
data_lines = f.readlines()
with open(label_path, 'r', encoding='utf-8') as f:
label_lines = f.readlines()
assert len(data_lines) == len(label_lines)
for data, label in zip(data_lines, label_lines):
i = json.loads(data.strip())
i['label'] = int(label.strip())
dataset.append(i)
def preprocess(example): return Dataset.from_list(dataset)
example['all_labels'] = {
@staticmethod
def load(path):
train_dataset = siqaDataset.load_single(path, 'train.jsonl',
'train-labels.lst')
val_dataset = siqaDataset.load_single(path, 'dev.jsonl',
'dev-labels.lst')
return DatasetDict({'train': train_dataset, 'validation': val_dataset})
@LOAD_DATASET.register_module()
class siqaDataset_V2(BaseDataset):
"""Disconnect from HuggingFace version of siqaDataset_V2."""
@staticmethod
def load_single(path, data_filename, label_filename):
data_path = os.path.join(path, data_filename)
label_path = os.path.join(path, label_filename)
dataset = []
with open(data_path, 'r', encoding='utf-8') as f:
data_lines = f.readlines()
with open(label_path, 'r', encoding='utf-8') as f:
label_lines = f.readlines()
assert len(data_lines) == len(label_lines)
for data, label in zip(data_lines, label_lines):
i = json.loads(data.strip())
label = int(label.strip())
# some preprocessing
i['all_labels'] = {
'candidates': [ 'candidates': [
f'A. {example["answerA"]}', [f'A. {i["answerA"]}', 'A', i['answerA']],
f'B. {example["answerB"]}', [f'B. {i["answerB"]}', 'B', i['answerB']],
f'C. {example["answerC"]}', [f'C. {i["answerC"]}', 'C', i['answerC']],
], ],
'label': 'label':
int(example['label']) - 1 label - 1
} }
example['label'] = ' ABC'[int(example['label'])] i['label'] = ' ABC'[label]
return example
dataset = dataset.map(preprocess) dataset.append(i)
return dataset
return Dataset.from_list(dataset)
@staticmethod
def load(path):
train_dataset = siqaDataset_V2.load_single(path, 'train.jsonl',
'train-labels.lst')
val_dataset = siqaDataset_V2.load_single(path, 'dev.jsonl',
'dev-labels.lst')
return DatasetDict({'train': train_dataset, 'validation': val_dataset})
opencompass/datasets/storycloze.py
View file @ 689ffe5b
from datasets import DatasetDict, load_dataset import json
import os
from datasets import Dataset, DatasetDict
from opencompass.registry import LOAD_DATASET from opencompass.registry import LOAD_DATASET
...@@ -9,38 +12,39 @@ from .base import BaseDataset ...@@ -9,38 +12,39 @@ from .base import BaseDataset
class storyclozeDataset(BaseDataset): class storyclozeDataset(BaseDataset):
@staticmethod @staticmethod
def load(**kwargs): def load(path, lang):
# special process dataset_list = []
dataset = load_dataset(**kwargs, split='train+eval') for split in ['train', 'eval']:
split_path = os.path.join(path, f'{lang}_{split}.jsonl')
def preprocess(example): with open(split_path, 'r', encoding='utf-8') as f:
example['context'] = ' '.join([ for line in f:
example['input_sentence_1'], example['input_sentence_2'], line = json.loads(line)
example['input_sentence_3'], example['input_sentence_4'] line['context'] = ' '.join([
]) line['input_sentence_1'], line['input_sentence_2'],
return example line['input_sentence_3'], line['input_sentence_4']
])
dataset = dataset.map(preprocess) dataset_list.append(line)
dataset_list = Dataset.from_list(dataset_list)
return DatasetDict({'test': dataset}) return DatasetDict({'test': dataset_list})
@LOAD_DATASET.register_module() @LOAD_DATASET.register_module()
class storyclozeDataset_V2(BaseDataset): class storyclozeDataset_V2(BaseDataset):
@staticmethod @staticmethod
def load(**kwargs): def load(path, lang):
# special process dataset_list = []
dataset = load_dataset(**kwargs, split='train+eval') for split in ['train', 'eval']:
split_path = os.path.join(path, f'{lang}_{split}.jsonl')
def preprocess(example): with open(split_path, 'r', encoding='utf-8') as f:
example['context'] = ' '.join([ for line in f:
example['input_sentence_1'], example['input_sentence_2'], line = json.loads(line)
example['input_sentence_3'], example['input_sentence_4'] line['context'] = ' '.join([
]) line['input_sentence_1'], line['input_sentence_2'],
example['answer_right_ending'] = ' AB'[ line['input_sentence_3'], line['input_sentence_4']
example['answer_right_ending']] ])
return example line['answer_right_ending'] = ' AB'[
line['answer_right_ending']]
dataset = dataset.map(preprocess) dataset_list.append(line)
return dataset dataset_list = Dataset.from_list(dataset_list)
return dataset_list
opencompass/datasets/strategyqa.py
View file @ 689ffe5b
import json
import re import re
from opencompass.registry import TEXT_POSTPROCESSORS from datasets import Dataset
from opencompass.registry import LOAD_DATASET, TEXT_POSTPROCESSORS
from .base import BaseDataset
@TEXT_POSTPROCESSORS.register_module('strategyqa') @TEXT_POSTPROCESSORS.register_module('strategyqa')
...@@ -16,3 +21,13 @@ def strategyqa_pred_postprocess(text: str) -> str: ...@@ -16,3 +21,13 @@ def strategyqa_pred_postprocess(text: str) -> str:
@TEXT_POSTPROCESSORS.register_module('strategyqa_dataset') @TEXT_POSTPROCESSORS.register_module('strategyqa_dataset')
def strategyqa_dataset_postprocess(text: str) -> str: def strategyqa_dataset_postprocess(text: str) -> str:
return 'yes' if str(text) == 'True' else 'no' return 'yes' if str(text) == 'True' else 'no'
@LOAD_DATASET.register_module()
class StrategyQADataset(BaseDataset):
@staticmethod
def load(path):
with open(path, 'r', encoding='utf-8') as f:
dataset = json.load(f)
return Dataset.from_list(dataset)
opencompass/datasets/tydiqa.py
View file @ 689ffe5b
import json
import os
import re import re
from collections import Counter from collections import Counter
from datasets import load_dataset from datasets import Dataset
from opencompass.openicl.icl_evaluator import BaseEvaluator from opencompass.openicl.icl_evaluator import BaseEvaluator
from opencompass.utils.text_postprocessors import general_postprocess from opencompass.utils.text_postprocessors import general_postprocess
...@@ -12,15 +14,16 @@ from .base import BaseDataset ...@@ -12,15 +14,16 @@ from .base import BaseDataset
class TydiQADataset(BaseDataset): class TydiQADataset(BaseDataset):
@staticmethod @staticmethod
def load(**kwargs): def load(path, lang):
dataset = load_dataset(**kwargs) path = os.path.join(path, 'dev', f'{lang}-dev.jsonl')
dataset_list = []
def pre_process(example): with open(path, 'r', encoding='utf-8') as f:
example['answer'] = example['answers']['text'] for line in f:
return example line = json.loads(line)
answer = list(set([i['text'] for i in line['answers']]))
dataset = dataset.map(pre_process).remove_columns(['id', 'answers']) line['answer'] = answer
return dataset dataset_list.append(line)
return Dataset.from_list(dataset_list)
class TydiQAEvaluator(BaseEvaluator): class TydiQAEvaluator(BaseEvaluator):
......
opencompass/datasets/winogrande.py
View file @ 689ffe5b
from datasets import load_dataset import json
import os
from datasets import Dataset
from opencompass.registry import LOAD_DATASET from opencompass.registry import LOAD_DATASET
...@@ -7,38 +10,49 @@ from .base import BaseDataset ...@@ -7,38 +10,49 @@ from .base import BaseDataset
@LOAD_DATASET.register_module() @LOAD_DATASET.register_module()
class winograndeDataset(BaseDataset): class winograndeDataset(BaseDataset):
"""Disconnect from Huggingface, winograndeDataset."""
@staticmethod @staticmethod
def load(**kwargs): def load(path):
path = os.path.join(path, 'dev.jsonl')
dataset = load_dataset(**kwargs) dataset_list = []
with open(path, 'r', encoding='utf-8') as f:
def preprocess(example): for line in f:
prompt = example.pop('sentence') line = json.loads(line)
example['opt1'] = prompt.replace('_', example.pop('option1')) prompt = line['sentence']
example['opt2'] = prompt.replace('_', example.pop('option2')) dataset_list.append({
return example 'opt1':
prompt.replace('_', line['option1']),
return dataset.map(preprocess) 'opt2':
prompt.replace('_', line['option2']),
'answer':
line['answer']
})
dataset_list = Dataset.from_list(dataset_list)
return dataset_list
@LOAD_DATASET.register_module() @LOAD_DATASET.register_module()
class winograndeDataset_V2(BaseDataset): class winograndeDataset_V2(BaseDataset):
"""Disconnect from Huggingface, winograndeDataset_V2."""
@staticmethod @staticmethod
def load(**kwargs): def load(path):
path = os.path.join(path, 'dev.jsonl')
dataset = load_dataset(**kwargs) dataset_list = []
with open(path, 'r', encoding='utf-8') as f:
def preprocess(example): for line in f:
prompt = example.pop('sentence') line = json.loads(line)
example['opt1'] = prompt.replace('_', example.pop('option1')) prompt = line['sentence']
example['opt2'] = prompt.replace('_', example.pop('option2')) answer = line['answer']
answer = example.pop('answer') answer = ' AB'[int(answer)] if answer != '' else 'NULL'
if answer == '': dataset_list.append({
example['label'] = 'NULL' 'opt1':
else: prompt.replace('_', line['option1']),
example['label'] = ' AB'[int(answer)] 'opt2':
return example prompt.replace('_', line['option2']),
'answer':
return dataset.map(preprocess) answer
})
dataset_list = Dataset.from_list(dataset_list)
return dataset_list
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment