Skip to content

GitLab

Unverified Commit aa2dd2b5 authored by Fengzhe Zhou's avatar Fengzhe Zhou Committed by GitHub
Browse files

[Format] Add config lints (#892)

parent 3dbba119
Hide whitespace changes
Inline Side-by-side
Showing with 194 additions and 194 deletions
configs/datasets/OpenFinData/OpenFinData_gen_46dedb.py
View file @ aa2dd2b5
......@@ -18,80 +18,80 @@ OpenFinData_KW_eval_cfg = dict(evaluator=dict(type=OpenFinDataKWEvaluator))
for _name in OpenFinData_all_list:
if _name in OpenFinData_3choices_list:
OpenFinData_infer_cfg = dict(
ice_template=dict(type=PromptTemplate, template=dict(begin="</E>", round=[
dict(role="HUMAN", prompt=f"{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\n答案: "),
dict(role="BOT", prompt="{answer}")]),
ice_token="</E>"), retriever=dict(type=ZeroRetriever), inferencer=dict(type=GenInferencer))
ice_template=dict(type=PromptTemplate, template=dict(begin='</E>', round=[
dict(role='HUMAN', prompt=f'{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\n答案: '),
dict(role='BOT', prompt='{answer}')]),
ice_token='</E>'), retriever=dict(type=ZeroRetriever), inferencer=dict(type=GenInferencer))
OpenFinData_datasets.append(
dict(
type=OpenFinDataDataset,
path="./data/openfindata_release",
path='./data/openfindata_release',
name=_name,
abbr="OpenFinData-" + _name,
abbr='OpenFinData-' + _name,
reader_cfg=dict(
input_columns=["question", "A", "B", "C"],
output_column="answer"),
input_columns=['question', 'A', 'B', 'C'],
output_column='answer'),
infer_cfg=OpenFinData_infer_cfg,
eval_cfg=OpenFinData_eval_cfg,
))
if _name in OpenFinData_4choices_list:
OpenFinData_infer_cfg = dict(
ice_template=dict(type=PromptTemplate, template=dict(begin="</E>", round=[
dict(role="HUMAN", prompt=f"{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n答案: "),
dict(role="BOT", prompt="{answer}")]),
ice_token="</E>"), retriever=dict(type=ZeroRetriever), inferencer=dict(type=GenInferencer))
ice_template=dict(type=PromptTemplate, template=dict(begin='</E>', round=[
dict(role='HUMAN', prompt=f'{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n答案: '),
dict(role='BOT', prompt='{answer}')]),
ice_token='</E>'), retriever=dict(type=ZeroRetriever), inferencer=dict(type=GenInferencer))
OpenFinData_datasets.append(
dict(
type=OpenFinDataDataset,
path="./data/openfindata_release",
path='./data/openfindata_release',
name=_name,
abbr="OpenFinData-" + _name,
abbr='OpenFinData-' + _name,
reader_cfg=dict(
input_columns=["question", "A", "B", "C", "D"],
output_column="answer"),
input_columns=['question', 'A', 'B', 'C', 'D'],
output_column='answer'),
infer_cfg=OpenFinData_infer_cfg,
eval_cfg=OpenFinData_eval_cfg,
))
if _name in OpenFinData_5choices_list:
OpenFinData_infer_cfg = dict(
ice_template=dict(type=PromptTemplate, template=dict(begin="</E>", round=[
dict(role="HUMAN", prompt=f"{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\nE. {{E}}\n答案: "),
dict(role="BOT", prompt="{answer}")]),
ice_token="</E>"), retriever=dict(type=ZeroRetriever), inferencer=dict(type=GenInferencer))
ice_template=dict(type=PromptTemplate, template=dict(begin='</E>', round=[
dict(role='HUMAN', prompt=f'{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\nE. {{E}}\n答案: '),
dict(role='BOT', prompt='{answer}')]),
ice_token='</E>'), retriever=dict(type=ZeroRetriever), inferencer=dict(type=GenInferencer))
OpenFinData_datasets.append(
dict(
type=OpenFinDataDataset,
path="./data/openfindata_release",
path='./data/openfindata_release',
name=_name,
abbr="OpenFinData-" + _name,
abbr='OpenFinData-' + _name,
reader_cfg=dict(
input_columns=["question", "A", "B", "C", "D", "E"],
output_column="answer"),
input_columns=['question', 'A', 'B', 'C', 'D', 'E'],
output_column='answer'),
infer_cfg=OpenFinData_infer_cfg,
eval_cfg=OpenFinData_eval_cfg,
))
if _name in OpenFinData_keyword_list:
OpenFinData_infer_cfg = dict(
ice_template=dict(type=PromptTemplate, template=dict(begin="</E>", round=[
dict(role="HUMAN", prompt=f"{{question}}\n答案: "),
dict(role="BOT", prompt="{answer}")]),
ice_token="</E>"), retriever=dict(type=ZeroRetriever), inferencer=dict(type=GenInferencer))
ice_template=dict(type=PromptTemplate, template=dict(begin='</E>', round=[
dict(role='HUMAN', prompt=f'{{question}}\n答案: '),
dict(role='BOT', prompt='{answer}')]),
ice_token='</E>'), retriever=dict(type=ZeroRetriever), inferencer=dict(type=GenInferencer))
OpenFinData_datasets.append(
dict(
type=OpenFinDataDataset,
path="./data/openfindata_release",
path='./data/openfindata_release',
name=_name,
abbr="OpenFinData-" + _name,
abbr='OpenFinData-' + _name,
reader_cfg=dict(
input_columns=["question"],
output_column="answer"),
input_columns=['question'],
output_column='answer'),
infer_cfg=OpenFinData_infer_cfg,
eval_cfg=OpenFinData_KW_eval_cfg,
))
......
configs/datasets/PJExam/PJExam_gen_8cd97c.py
View file @ aa2dd2b5
......@@ -8,45 +8,45 @@ for _name in [
'gk-2022-v1', 'gk-2022-v1-math', 'gk-2023-v1', 'gk-2023-v1-math',
'gk-2023-v2', 'gk-2023-v2-math', 'zk-2022-v1'
]:
_hint = "请你做一道</major>选择题\n请你一步一步思考并将思考过程写在【解析】和<eoe>之间。你将从A，B，C，D中选出正确的答案，并写在【答案】和<eoa>之间。\n例如：【答案】A<eoa>\n完整的题目回答的格式如下：\n【解析】...<eoe>\n【答案】...<eoa>\n请你严格按照上述格式作答。\n题目如下：\n"
_hint = '请你做一道</major>选择题\n请你一步一步思考并将思考过程写在【解析】和<eoe>之间。你将从A，B，C，D中选出正确的答案，并写在【答案】和<eoa>之间。\n例如：【答案】A<eoa>\n完整的题目回答的格式如下：\n【解析】...<eoe>\n【答案】...<eoa>\n请你严格按照上述格式作答。\n题目如下：\n'
_reader_cfg = {
"input_columns": ['question'],
"output_column": 'std_ans',
'input_columns': ['question'],
'output_column': 'std_ans',
},
_infer_cfg = {
"ice_template": {
"type": PromptTemplate,
"template": {
"round": [{
"role": "HUMAN",
"prompt": _hint + "{question}",
'ice_template': {
'type': PromptTemplate,
'template': {
'round': [{
'role': 'HUMAN',
'prompt': _hint + '{question}',
}]
},
"ice_token": "</E>"
'ice_token': '</E>'
},
"retriever": {
"type": ZeroRetriever
'retriever': {
'type': ZeroRetriever
},
"inferencer": {
"type": GenInferencer,
"max_out_len": 1024,
'inferencer': {
'type': GenInferencer,
'max_out_len': 1024,
}
}
_eval_cfg = {
"evaluator": {
"type": PJExamEvaluator
'evaluator': {
'type': PJExamEvaluator
},
"pred_role": "BOT",
"ds_column": "eval_infos"
'pred_role': 'BOT',
'ds_column': 'eval_infos'
}
_dataset = {
"type": PJExamDataset,
"abbr": "PJExamDataset-" + _name,
"path": './data/PJExam',
"name": _name,
"reader_cfg": _reader_cfg,
"infer_cfg": _infer_cfg,
"eval_cfg": _eval_cfg,
'type': PJExamDataset,
'abbr': 'PJExamDataset-' + _name,
'path': './data/PJExam',
'name': _name,
'reader_cfg': _reader_cfg,
'infer_cfg': _infer_cfg,
'eval_cfg': _eval_cfg,
}
PJExam_datasets.append(_dataset)
......
configs/datasets/QuALITY/QuALITY.md
View file @ aa2dd2b5
......@@ -53,4 +53,4 @@ QuALITY ed2404 all_acc gen 54.65 60
pages = "5336--5358",
abstract = "To enable building and testing models on long-document comprehension, we introduce QuALITY, a multiple-choice QA dataset with context passages in English that have an average length of about 5,000 tokens, much longer than typical current models can process. Unlike in prior work with passages, our questions are written and validated by contributors who have read the entire passage, rather than relying on summaries or excerpts. In addition, only half of the questions are answerable by annotators working under tight time constraints, indicating that skimming and simple search are not enough to consistently perform well. Our baseline models perform poorly on this task (55.4{\%}) and significantly lag behind human performance (93.5{\%}).",
}
```
\ No newline at end of file
```
configs/datasets/QuALITY/QuALITY_gen.py
View file @ aa2dd2b5
from mmengine.config import read_base
with read_base():
from .QuALITY_gen_c407cb import QuALITY_datasets # noqa: F401, F403
\ No newline at end of file
from .QuALITY_gen_c407cb import QuALITY_datasets # noqa: F401, F403
configs/datasets/QuALITY/QuALITY_gen_c407cb.py
View file @ aa2dd2b5
......@@ -14,9 +14,9 @@ QuALITY_infer_cfg = dict(
type=PromptTemplate,
template=dict(round=[
dict(
role="HUMAN",
role='HUMAN',
prompt=
"Read the article, and answer the question.\n\nArticle:\n{article}\n\nQ: {question}\n\nA. {A}\nB. {B}\nC. {C}\nD. {D}"
'Read the article, and answer the question.\n\nArticle:\n{article}\n\nQ: {question}\n\nA. {A}\nB. {B}\nC. {C}\nD. {D}'
),
])),
retriever=dict(type=ZeroRetriever),
......
configs/datasets/SVAMP/svamp_gen_fb25e4.py
View file @ aa2dd2b5
......@@ -9,13 +9,13 @@ svamp_infer_cfg = dict(
template=dict(
round=[
dict(role='HUMAN', prompt="Question: There are 87 oranges and 290 bananas in Philip's collection. If the bananas are organized into 2 groups and oranges are organized into 93 groups How big is each group of bananas?\nLet's think step by step\nAnswer:"),
dict(role='BOT', prompt="To find the size of each group of bananas, we divide the total number of bananas (290) by the number of groups (2): 290 / 2 = 145. Therefore, each group of bananas contains 145 bananas. The answer is 145.\n"),
dict(role='BOT', prompt='To find the size of each group of bananas, we divide the total number of bananas (290) by the number of groups (2): 290 / 2 = 145. Therefore, each group of bananas contains 145 bananas. The answer is 145.\n'),
dict(role='HUMAN', prompt="Question: Marco and his dad went strawberry picking. Marco's dad's strawberries weighed 11 pounds. If together their strawberries weighed 30 pounds. How much did Marco's strawberries weigh?\nLet's think step by step\nAnswer:"),
dict(role='BOT', prompt="To find Marco's strawberries' weight, we subtract his dad's strawberries' weight (11 pounds) from the total weight of their strawberries (30 pounds): 30 - 11 = 19. Therefore, Marco's strawberries weighed 19 pounds. The answer is 19.\n"),
dict(role='HUMAN', prompt="Question: Edward spent $ 6 to buy 2 books each book costing him the same amount of money. Now he has $ 12. How much did each book cost?\nLet's think step by step\nAnswer:"),
dict(role='BOT', prompt="To find the cost of each book, we subtract the initial amount of money Edward had ($6) from the current amount of money he has ($12) and divide it by the number of books (2): (12 - 6) / 2 = 6 / 2 = 3 Therefore, each book cost $3. The answer is 3.\n"),
dict(role='BOT', prompt='To find the cost of each book, we subtract the initial amount of money Edward had ($6) from the current amount of money he has ($12) and divide it by the number of books (2): (12 - 6) / 2 = 6 / 2 = 3 Therefore, each book cost $3. The answer is 3.\n'),
dict(role='HUMAN', prompt="Question: Frank was reading through his favorite book. The book had 3 chapters, each with the same number of pages. It has a total of 594 pages. It took Frank 607 days to finish the book. How many pages are in each chapter?\nLet's think step by step\nAnswer:"),
dict(role='BOT', prompt="To find the number of pages in each chapter, we divide the total number of pages in the book (594) by the number of chapters (3): 594 / 3 = 198. Therefore, each chapter has 198 pages. The answer is 198.\n"),
dict(role='BOT', prompt='To find the number of pages in each chapter, we divide the total number of pages in the book (594) by the number of chapters (3): 594 / 3 = 198. Therefore, each chapter has 198 pages. The answer is 198.\n'),
dict(role='HUMAN', prompt="Question: {question}\nLet's think step by step\nAnswer:"),
],
)),
......
configs/datasets/SuperGLUE_AX_b/SuperGLUE_AX_b_gen_4dfefa.py
View file @ aa2dd2b5
......@@ -6,8 +6,8 @@ from opencompass.datasets import AXDataset_V2
from opencompass.utils.text_postprocessors import first_option_postprocess
AX_b_reader_cfg = dict(
input_columns=["sentence1", "sentence2"],
output_column="label",
input_columns=['sentence1', 'sentence2'],
output_column='label',
)
AX_b_infer_cfg = dict(
......@@ -15,9 +15,9 @@ AX_b_infer_cfg = dict(
type=PromptTemplate,
template=dict(round=[
dict(
role="HUMAN",
role='HUMAN',
prompt=
"{sentence1}\n{sentence2}\nIs the sentence below entailed by the sentence above?\nA. Yes\nB. No\nAnswer:"
'{sentence1}\n{sentence2}\nIs the sentence below entailed by the sentence above?\nA. Yes\nB. No\nAnswer:'
),
]),
),
......@@ -27,15 +27,15 @@ AX_b_infer_cfg = dict(
AX_b_eval_cfg = dict(
evaluator=dict(type=AccEvaluator),
pred_role="BOT",
pred_role='BOT',
pred_postprocessor=dict(type=first_option_postprocess, options='AB'),
)
AX_b_datasets = [
dict(
abbr="AX_b",
abbr='AX_b',
type=AXDataset_V2,
path="./data/SuperGLUE/AX-b/AX-b.jsonl",
path='./data/SuperGLUE/AX-b/AX-b.jsonl',
reader_cfg=AX_b_reader_cfg,
infer_cfg=AX_b_infer_cfg,
eval_cfg=AX_b_eval_cfg,
......
configs/datasets/SuperGLUE_AX_b/SuperGLUE_AX_b_ppl_6db806.py
View file @ aa2dd2b5
......@@ -5,31 +5,31 @@ from opencompass.openicl.icl_evaluator import AccEvaluator
from opencompass.datasets import HFDataset
AX_b_reader_cfg = dict(
input_columns=["sentence1", "sentence2"],
output_column="label",
test_split="train")
input_columns=['sentence1', 'sentence2'],
output_column='label',
test_split='train')
AX_b_infer_cfg = dict(
prompt_template=dict(
type=PromptTemplate,
template={
"entailment":
'entailment':
dict(round=[
dict(
role="HUMAN",
role='HUMAN',
prompt=
"{sentence1}\n{sentence2}\nIs the sentence below entailed by the sentence above?"
'{sentence1}\n{sentence2}\nIs the sentence below entailed by the sentence above?'
),
dict(role="BOT", prompt="Yes"),
dict(role='BOT', prompt='Yes'),
]),
"not_entailment":
'not_entailment':
dict(round=[
dict(
role="HUMAN",
role='HUMAN',
prompt=
"{sentence1}\n{sentence2}\nIs the sentence below entailed by the sentence above?"
'{sentence1}\n{sentence2}\nIs the sentence below entailed by the sentence above?'
),
dict(role="BOT", prompt="No"),
dict(role='BOT', prompt='No'),
])
},
),
......@@ -42,10 +42,10 @@ AX_b_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
AX_b_datasets = [
dict(
type=HFDataset,
abbr="AX_b",
path="json",
data_files="./data/SuperGLUE/AX-b/AX-b.jsonl",
split="train",
abbr='AX_b',
path='json',
data_files='./data/SuperGLUE/AX-b/AX-b.jsonl',
split='train',
reader_cfg=AX_b_reader_cfg,
infer_cfg=AX_b_infer_cfg,
eval_cfg=AX_b_eval_cfg,
......
configs/datasets/SuperGLUE_AX_g/SuperGLUE_AX_g_gen_68aac7.py
View file @ aa2dd2b5
......@@ -6,8 +6,8 @@ from opencompass.datasets import AXDataset_V2
from opencompass.utils.text_postprocessors import first_option_postprocess
AX_g_reader_cfg = dict(
input_columns=["hypothesis", "premise"],
output_column="label",
input_columns=['hypothesis', 'premise'],
output_column='label',
)
AX_g_infer_cfg = dict(
......@@ -15,9 +15,9 @@ AX_g_infer_cfg = dict(
type=PromptTemplate,
template=dict(round=[
dict(
role="HUMAN",
role='HUMAN',
prompt=
"{premise}\n{hypothesis}\nIs the sentence below entailed by the sentence above?\nA. Yes\nB. No\nAnswer:"
'{premise}\n{hypothesis}\nIs the sentence below entailed by the sentence above?\nA. Yes\nB. No\nAnswer:'
),
]),
),
......@@ -27,15 +27,15 @@ AX_g_infer_cfg = dict(
AX_g_eval_cfg = dict(
evaluator=dict(type=AccEvaluator),
pred_role="BOT",
pred_role='BOT',
pred_postprocessor=dict(type=first_option_postprocess, options='AB'),
)
AX_g_datasets = [
dict(
abbr="AX_g",
abbr='AX_g',
type=AXDataset_V2,
path="./data/SuperGLUE/AX-g/AX-g.jsonl",
path='./data/SuperGLUE/AX-g/AX-g.jsonl',
reader_cfg=AX_g_reader_cfg,
infer_cfg=AX_g_infer_cfg,
eval_cfg=AX_g_eval_cfg,
......
configs/datasets/SuperGLUE_AX_g/SuperGLUE_AX_g_ppl_66caf3.py
View file @ aa2dd2b5
......@@ -5,31 +5,31 @@ from opencompass.openicl.icl_evaluator import AccEvaluator
from opencompass.datasets import HFDataset
AX_g_reader_cfg = dict(
input_columns=["hypothesis", "premise"],
output_column="label",
test_split="train")
input_columns=['hypothesis', 'premise'],
output_column='label',
test_split='train')
AX_g_infer_cfg = dict(
prompt_template=dict(
type=PromptTemplate,
template={
"entailment":
'entailment':
dict(round=[
dict(
role="HUMAN",
role='HUMAN',
prompt=
"{premise}\n{hypothesis}\nIs the sentence below entailed by the sentence above?"
'{premise}\n{hypothesis}\nIs the sentence below entailed by the sentence above?'
),
dict(role="BOT", prompt="Yes"),
dict(role='BOT', prompt='Yes'),
]),
"not_entailment":
'not_entailment':
dict(round=[
dict(
role="HUMAN",
role='HUMAN',
prompt=
"{premise}\n{hypothesis}\nIs the sentence below entailed by the sentence above?"
'{premise}\n{hypothesis}\nIs the sentence below entailed by the sentence above?'
),
dict(role="BOT", prompt="No"),
dict(role='BOT', prompt='No'),
])
},
),
......@@ -42,10 +42,10 @@ AX_g_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
AX_g_datasets = [
dict(
type=HFDataset,
abbr="AX_g",
path="json",
data_files="./data/SuperGLUE/AX-g/AX-g.jsonl",
split="train",
abbr='AX_g',
path='json',
data_files='./data/SuperGLUE/AX-g/AX-g.jsonl',
split='train',
reader_cfg=AX_g_reader_cfg,
infer_cfg=AX_g_infer_cfg,
eval_cfg=AX_g_eval_cfg,
......
configs/datasets/SuperGLUE_BoolQ/SuperGLUE_BoolQ_gen_883d50.py
View file @ aa2dd2b5
......@@ -6,8 +6,8 @@ from opencompass.datasets import BoolQDataset_V2
from opencompass.utils.text_postprocessors import first_capital_postprocess
BoolQ_reader_cfg = dict(
input_columns=["question", "passage"],
output_column="label",
input_columns=['question', 'passage'],
output_column='label',
)
BoolQ_infer_cfg = dict(
......@@ -15,8 +15,8 @@ BoolQ_infer_cfg = dict(
type=PromptTemplate,
template=dict(round=[
dict(
role="HUMAN",
prompt="{passage}\nQuestion: {question}\nA. Yes\nB. No\nAnswer:"),
role='HUMAN',
prompt='{passage}\nQuestion: {question}\nA. Yes\nB. No\nAnswer:'),
]),
),
retriever=dict(type=ZeroRetriever),
......@@ -25,15 +25,15 @@ BoolQ_infer_cfg = dict(
BoolQ_eval_cfg = dict(
evaluator=dict(type=AccEvaluator),
pred_role="BOT",
pred_role='BOT',
pred_postprocessor=dict(type=first_capital_postprocess),
)
BoolQ_datasets = [
dict(
abbr="BoolQ",
abbr='BoolQ',
type=BoolQDataset_V2,
path="./data/SuperGLUE/BoolQ/val.jsonl",
path='./data/SuperGLUE/BoolQ/val.jsonl',
reader_cfg=BoolQ_reader_cfg,
infer_cfg=BoolQ_infer_cfg,
eval_cfg=BoolQ_eval_cfg,
......
configs/datasets/SuperGLUE_BoolQ/SuperGLUE_BoolQ_ppl_314797.py
View file @ aa2dd2b5
......@@ -5,9 +5,9 @@ from opencompass.openicl.icl_evaluator import AccEvaluator
from opencompass.datasets import BoolQDataset_V3
BoolQ_reader_cfg = dict(
input_columns=["question", "passage"],
output_column="label",
test_split="train")
input_columns=['question', 'passage'],
output_column='label',
test_split='train')
BoolQ_infer_cfg = dict(
prompt_template=dict(
......@@ -15,13 +15,13 @@ BoolQ_infer_cfg = dict(
template={
'false':
dict(round=[
dict(role="HUMAN", prompt="Passage: {passage}\nQuestion: {question}?"),
dict(role="BOT", prompt="Answer: No"),
dict(role='HUMAN', prompt='Passage: {passage}\nQuestion: {question}?'),
dict(role='BOT', prompt='Answer: No'),
]),
'true':
dict(round=[
dict(role="HUMAN", prompt="Passage: {passage}\nQuestion: {question}?"),
dict(role="BOT", prompt="Answer: Yes"),
dict(role='HUMAN', prompt='Passage: {passage}\nQuestion: {question}?'),
dict(role='BOT', prompt='Answer: Yes'),
]),
},
),
......@@ -33,9 +33,9 @@ BoolQ_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
BoolQ_datasets = [
dict(
abbr="BoolQ",
abbr='BoolQ',
type=BoolQDataset_V3,
path="./data/SuperGLUE/BoolQ/val.jsonl",
path='./data/SuperGLUE/BoolQ/val.jsonl',
reader_cfg=BoolQ_reader_cfg,
infer_cfg=BoolQ_infer_cfg,
eval_cfg=BoolQ_eval_cfg,
......
configs/datasets/SuperGLUE_BoolQ/SuperGLUE_BoolQ_ppl_314b96.py
View file @ aa2dd2b5
......@@ -5,9 +5,9 @@ from opencompass.openicl.icl_evaluator import AccEvaluator
from opencompass.datasets import BoolQDataset
BoolQ_reader_cfg = dict(
input_columns=["question", "passage"],
output_column="answer",
test_split="train")
input_columns=['question', 'passage'],
output_column='answer',
test_split='train')
BoolQ_infer_cfg = dict(
prompt_template=dict(
......@@ -15,13 +15,13 @@ BoolQ_infer_cfg = dict(
template={
0:
dict(round=[
dict(role="HUMAN", prompt="{passage}\nQuestion: {question}?"),
dict(role="BOT", prompt="No"),
dict(role='HUMAN', prompt='{passage}\nQuestion: {question}?'),
dict(role='BOT', prompt='No'),
]),
1:
dict(round=[
dict(role="HUMAN", prompt="{passage}\nQuestion: {question}?"),
dict(role="BOT", prompt="Yes"),
dict(role='HUMAN', prompt='{passage}\nQuestion: {question}?'),
dict(role='BOT', prompt='Yes'),
]),
},
),
......@@ -34,10 +34,10 @@ BoolQ_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
BoolQ_datasets = [
dict(
type=BoolQDataset,
abbr="BoolQ",
path="json",
data_files="./data/SuperGLUE/BoolQ/val.jsonl",
split="train",
abbr='BoolQ',
path='json',
data_files='./data/SuperGLUE/BoolQ/val.jsonl',
split='train',
reader_cfg=BoolQ_reader_cfg,
infer_cfg=BoolQ_infer_cfg,
eval_cfg=BoolQ_eval_cfg,
......
configs/datasets/SuperGLUE_BoolQ/SuperGLUE_BoolQ_ppl_4da4db.py
View file @ aa2dd2b5
......@@ -5,9 +5,9 @@ from opencompass.openicl.icl_evaluator import AccEvaluator
from opencompass.datasets import BoolQDataset
BoolQ_reader_cfg = dict(
input_columns=["question", "passage"],
output_column="answer",
test_split="train")
input_columns=['question', 'passage'],
output_column='answer',
test_split='train')
BoolQ_infer_cfg = dict(
prompt_template=dict(
......@@ -15,13 +15,13 @@ BoolQ_infer_cfg = dict(
template={
0:
dict(round=[
dict(role="HUMAN", prompt="{passage}\nQuestion: {question}"),
dict(role="BOT", prompt="No."),
dict(role='HUMAN', prompt='{passage}\nQuestion: {question}'),
dict(role='BOT', prompt='No.'),
]),
1:
dict(round=[
dict(role="HUMAN", prompt="{passage}\nQuestion: {question}"),
dict(role="BOT", prompt="Yes."),
dict(role='HUMAN', prompt='{passage}\nQuestion: {question}'),
dict(role='BOT', prompt='Yes.'),
]),
},
),
......@@ -34,10 +34,10 @@ BoolQ_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
BoolQ_datasets = [
dict(
type=BoolQDataset,
abbr="BoolQ",
path="json",
data_files="./data/SuperGLUE/BoolQ/val.jsonl",
split="train",
abbr='BoolQ',
path='json',
data_files='./data/SuperGLUE/BoolQ/val.jsonl',
split='train',
reader_cfg=BoolQ_reader_cfg,
infer_cfg=BoolQ_infer_cfg,
eval_cfg=BoolQ_eval_cfg,
......
configs/datasets/SuperGLUE_BoolQ/SuperGLUE_BoolQ_ppl_9619db.py
View file @ aa2dd2b5
......@@ -13,8 +13,8 @@ BoolQ_infer_cfg = dict(
prompt_template=dict(
type=PromptTemplate,
template={
0: "Passage:{passage}。\nQuestion:{question}。\nAnswer: No.",
1: "Passage:{passage}。\nQuestion:{question}。\nAnswer: Yes.",
0: 'Passage:{passage}。\nQuestion:{question}。\nAnswer: No.',
1: 'Passage:{passage}。\nQuestion:{question}。\nAnswer: Yes.',
}),
retriever=dict(type=ZeroRetriever),
inferencer=dict(type=PPLInferencer))
......
configs/datasets/SuperGLUE_CB/SuperGLUE_CB_gen_854c6c.py
View file @ aa2dd2b5
......@@ -6,8 +6,8 @@ from opencompass.datasets import CBDataset_V2
from opencompass.utils.text_postprocessors import first_option_postprocess
CB_reader_cfg = dict(
input_columns=["premise", "hypothesis"],
output_column="label",
input_columns=['premise', 'hypothesis'],
output_column='label',
)
CB_infer_cfg = dict(
......@@ -16,9 +16,9 @@ CB_infer_cfg = dict(
template=dict(
round=[
dict(
role="HUMAN",
role='HUMAN',
prompt=
"{premise}\n{hypothesis}\nWhat is the relation between the two sentences?\nA. Contradiction\nB. Entailment\nC. Neutral\nAnswer:"
'{premise}\n{hypothesis}\nWhat is the relation between the two sentences?\nA. Contradiction\nB. Entailment\nC. Neutral\nAnswer:'
),
], ),
),
......@@ -28,15 +28,15 @@ CB_infer_cfg = dict(
CB_eval_cfg = dict(
evaluator=dict(type=AccEvaluator),
pred_role="BOT",
pred_role='BOT',
pred_postprocessor=dict(type=first_option_postprocess, options='ABC'),
)
CB_datasets = [
dict(
abbr="CB",
abbr='CB',
type=CBDataset_V2,
path="./data/SuperGLUE/CB/val.jsonl",
path='./data/SuperGLUE/CB/val.jsonl',
reader_cfg=CB_reader_cfg,
infer_cfg=CB_infer_cfg,
eval_cfg=CB_eval_cfg,
......
configs/datasets/SuperGLUE_CB/SuperGLUE_CB_ppl_0143fe.py
View file @ aa2dd2b5
......@@ -5,40 +5,40 @@ from opencompass.openicl.icl_evaluator import AccEvaluator
from opencompass.datasets import HFDataset
CB_reader_cfg = dict(
input_columns=["premise", "hypothesis"],
output_column="label",
input_columns=['premise', 'hypothesis'],
output_column='label',
)
CB_infer_cfg = dict(
prompt_template=dict(
type=PromptTemplate,
template={
"contradiction":
'contradiction':
dict(round=[
dict(
role="HUMAN",
role='HUMAN',
prompt=
"{premise}\n{hypothesis}\nWhat is the relation between the two sentences?"
'{premise}\n{hypothesis}\nWhat is the relation between the two sentences?'
),
dict(role="BOT", prompt="Contradiction"),
dict(role='BOT', prompt='Contradiction'),
]),
"entailment":
'entailment':
dict(round=[
dict(
role="HUMAN",
role='HUMAN',
prompt=
"{premise}\n{hypothesis}\nWhat is the relation between the two sentences?"
'{premise}\n{hypothesis}\nWhat is the relation between the two sentences?'
),
dict(role="BOT", prompt="Entailment"),
dict(role='BOT', prompt='Entailment'),
]),
"neutral":
'neutral':
dict(round=[
dict(
role="HUMAN",
role='HUMAN',
prompt=
"{premise}\n{hypothesis}\nWhat is the relation between the two sentences?"
'{premise}\n{hypothesis}\nWhat is the relation between the two sentences?'
),
dict(role="BOT", prompt="Neutral"),
dict(role='BOT', prompt='Neutral'),
]),
},
),
......@@ -51,10 +51,10 @@ CB_eval_cfg = dict(evaluator=dict(type=AccEvaluator), )
CB_datasets = [
dict(
type=HFDataset,
abbr="CB",
path="json",
split="train",
data_files="./data/SuperGLUE/CB/val.jsonl",
abbr='CB',
path='json',
split='train',
data_files='./data/SuperGLUE/CB/val.jsonl',
reader_cfg=CB_reader_cfg,
infer_cfg=CB_infer_cfg,
eval_cfg=CB_eval_cfg,
......
configs/datasets/SuperGLUE_COPA/SuperGLUE_COPA_gen_91ca53.py
View file @ aa2dd2b5
......@@ -6,8 +6,8 @@ from opencompass.datasets import COPADataset_V2
from opencompass.utils.text_postprocessors import first_option_postprocess
COPA_reader_cfg = dict(
input_columns=["question", "premise", "choice1", "choice2"],
output_column="label",
input_columns=['question', 'premise', 'choice1', 'choice2'],
output_column='label',
)
COPA_infer_cfg = dict(
......@@ -16,9 +16,9 @@ COPA_infer_cfg = dict(
template=dict(
round=[
dict(
role="HUMAN",
role='HUMAN',
prompt=
"{premise}\nQuestion: Which may be the {question}?\nA. {choice1}\nB. {choice2}\nAnswer:"
'{premise}\nQuestion: Which may be the {question}?\nA. {choice1}\nB. {choice2}\nAnswer:'
),
], ),
),
......@@ -28,15 +28,15 @@ COPA_infer_cfg = dict(
COPA_eval_cfg = dict(
evaluator=dict(type=AccEvaluator),
pred_role="BOT",
pred_role='BOT',
pred_postprocessor=dict(type=first_option_postprocess, options='AB'),
)
COPA_datasets = [
dict(
abbr="COPA",
abbr='COPA',
type=COPADataset_V2,
path="./data/SuperGLUE/COPA/val.jsonl",
path='./data/SuperGLUE/COPA/val.jsonl',
reader_cfg=COPA_reader_cfg,
infer_cfg=COPA_infer_cfg,
eval_cfg=COPA_eval_cfg,
......
configs/datasets/SuperGLUE_COPA/SuperGLUE_COPA_ppl_54058d.py
View file @ aa2dd2b5
......@@ -13,8 +13,8 @@ COPA_infer_cfg = dict(
prompt_template=dict(
type=PromptTemplate,
template={
0: "Premise:{premise}。\nQuestion:{question}。\nAnswer: {choice1}.",
1: "Passage:{premise}。\nQuestion:{question}。\nAnswer: {choice2}.",
0: 'Premise:{premise}。\nQuestion:{question}。\nAnswer: {choice1}.',
1: 'Passage:{premise}。\nQuestion:{question}。\nAnswer: {choice2}.',
}),
retriever=dict(type=ZeroRetriever),
inferencer=dict(type=PPLInferencer))
......
configs/datasets/SuperGLUE_COPA/SuperGLUE_COPA_ppl_5c24f1.py
View file @ aa2dd2b5
......@@ -5,9 +5,9 @@ from opencompass.openicl.icl_evaluator import AccEvaluator
from opencompass.datasets import HFDataset
COPA_reader_cfg = dict(
input_columns=["question", "premise", "choice1", "choice2"],
output_column="label",
test_split="train")
input_columns=['question', 'premise', 'choice1', 'choice2'],
output_column='label',
test_split='train')
COPA_infer_cfg = dict(
prompt_template=dict(
......@@ -15,13 +15,13 @@ COPA_infer_cfg = dict(
template={
0:
dict(round=[
dict(role="HUMAN", prompt="{premise}\nQuestion: {question}\nAnswer:"),
dict(role="BOT", prompt="{choice1}"),
dict(role='HUMAN', prompt='{premise}\nQuestion: {question}\nAnswer:'),
dict(role='BOT', prompt='{choice1}'),
]),
1:
dict(round=[
dict(role="HUMAN", prompt="{premise}\nQuestion: {question}\nAnswer:"),
dict(role="BOT", prompt="{choice2}"),
dict(role='HUMAN', prompt='{premise}\nQuestion: {question}\nAnswer:'),
dict(role='BOT', prompt='{choice2}'),
]),
},
),
......@@ -34,10 +34,10 @@ COPA_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
COPA_datasets = [
dict(
type=HFDataset,
abbr="COPA",
path="json",
data_files="./data/SuperGLUE/COPA/val.jsonl",
split="train",
abbr='COPA',
path='json',
data_files='./data/SuperGLUE/COPA/val.jsonl',
split='train',
reader_cfg=COPA_reader_cfg,
infer_cfg=COPA_infer_cfg,
eval_cfg=COPA_eval_cfg,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment