Skip to content

GitLab

Unverified Commit aa2dd2b5 authored by Fengzhe Zhou's avatar Fengzhe Zhou Committed by GitHub
Browse files

[Format] Add config lints (#892)

parent 3dbba119
Show whitespace changes
Inline Side-by-side
Showing with 194 additions and 194 deletions
configs/datasets/OpenFinData/OpenFinData_gen_46dedb.py
View file @ aa2dd2b5
......@@ -18,80 +18,80 @@ OpenFinData_KW_eval_cfg = dict(evaluator=dict(type=OpenFinDataKWEvaluator))
for _name in OpenFinData_all_list:
if _name in OpenFinData_3choices_list:
OpenFinData_infer_cfg = dict(
ice_template=dict(type=PromptTemplate, template=dict(begin="</E>", round=[
dict(role="HUMAN", prompt=f"{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\n答案: "),
dict(role="BOT", prompt="{answer}")]),
ice_token="</E>"), retriever=dict(type=ZeroRetriever), inferencer=dict(type=GenInferencer))
ice_template=dict(type=PromptTemplate, template=dict(begin='</E>', round=[
dict(role='HUMAN', prompt=f'{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\n答案: '),
dict(role='BOT', prompt='{answer}')]),
ice_token='</E>'), retriever=dict(type=ZeroRetriever), inferencer=dict(type=GenInferencer))
OpenFinData_datasets.append(
dict(
type=OpenFinDataDataset,
path="./data/openfindata_release",
path='./data/openfindata_release',
name=_name,
abbr="OpenFinData-" + _name,
abbr='OpenFinData-' + _name,
reader_cfg=dict(
input_columns=["question", "A", "B", "C"],
output_column="answer"),
input_columns=['question', 'A', 'B', 'C'],
output_column='answer'),
infer_cfg=OpenFinData_infer_cfg,
eval_cfg=OpenFinData_eval_cfg,
))
if _name in OpenFinData_4choices_list:
OpenFinData_infer_cfg = dict(
ice_template=dict(type=PromptTemplate, template=dict(begin="</E>", round=[
dict(role="HUMAN", prompt=f"{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n答案: "),
dict(role="BOT", prompt="{answer}")]),
ice_token="</E>"), retriever=dict(type=ZeroRetriever), inferencer=dict(type=GenInferencer))
ice_template=dict(type=PromptTemplate, template=dict(begin='</E>', round=[
dict(role='HUMAN', prompt=f'{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n答案: '),
dict(role='BOT', prompt='{answer}')]),
ice_token='</E>'), retriever=dict(type=ZeroRetriever), inferencer=dict(type=GenInferencer))
OpenFinData_datasets.append(
dict(
type=OpenFinDataDataset,
path="./data/openfindata_release",
path='./data/openfindata_release',
name=_name,
abbr="OpenFinData-" + _name,
abbr='OpenFinData-' + _name,
reader_cfg=dict(
input_columns=["question", "A", "B", "C", "D"],
output_column="answer"),
input_columns=['question', 'A', 'B', 'C', 'D'],
output_column='answer'),
infer_cfg=OpenFinData_infer_cfg,
eval_cfg=OpenFinData_eval_cfg,
))
if _name in OpenFinData_5choices_list:
OpenFinData_infer_cfg = dict(
ice_template=dict(type=PromptTemplate, template=dict(begin="</E>", round=[
dict(role="HUMAN", prompt=f"{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\nE. {{E}}\n答案: "),
dict(role="BOT", prompt="{answer}")]),
ice_token="</E>"), retriever=dict(type=ZeroRetriever), inferencer=dict(type=GenInferencer))
ice_template=dict(type=PromptTemplate, template=dict(begin='</E>', round=[
dict(role='HUMAN', prompt=f'{{question}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\nE. {{E}}\n答案: '),
dict(role='BOT', prompt='{answer}')]),
ice_token='</E>'), retriever=dict(type=ZeroRetriever), inferencer=dict(type=GenInferencer))
OpenFinData_datasets.append(
dict(
type=OpenFinDataDataset,
path="./data/openfindata_release",
path='./data/openfindata_release',
name=_name,
abbr="OpenFinData-" + _name,
abbr='OpenFinData-' + _name,
reader_cfg=dict(
input_columns=["question", "A", "B", "C", "D", "E"],
output_column="answer"),
input_columns=['question', 'A', 'B', 'C', 'D', 'E'],
output_column='answer'),
infer_cfg=OpenFinData_infer_cfg,
eval_cfg=OpenFinData_eval_cfg,
))
if _name in OpenFinData_keyword_list:
OpenFinData_infer_cfg = dict(
ice_template=dict(type=PromptTemplate, template=dict(begin="</E>", round=[
dict(role="HUMAN", prompt=f"{{question}}\n答案: "),
dict(role="BOT", prompt="{answer}")]),
ice_token="</E>"), retriever=dict(type=ZeroRetriever), inferencer=dict(type=GenInferencer))
ice_template=dict(type=PromptTemplate, template=dict(begin='</E>', round=[
dict(role='HUMAN', prompt=f'{{question}}\n答案: '),
dict(role='BOT', prompt='{answer}')]),
ice_token='</E>'), retriever=dict(type=ZeroRetriever), inferencer=dict(type=GenInferencer))
OpenFinData_datasets.append(
dict(
type=OpenFinDataDataset,
path="./data/openfindata_release",
path='./data/openfindata_release',
name=_name,
abbr="OpenFinData-" + _name,
abbr='OpenFinData-' + _name,
reader_cfg=dict(
input_columns=["question"],
output_column="answer"),
input_columns=['question'],
output_column='answer'),
infer_cfg=OpenFinData_infer_cfg,
eval_cfg=OpenFinData_KW_eval_cfg,
))
......
configs/datasets/PJExam/PJExam_gen_8cd97c.py
View file @ aa2dd2b5
......@@ -8,45 +8,45 @@ for _name in [
'gk-2022-v1', 'gk-2022-v1-math', 'gk-2023-v1', 'gk-2023-v1-math',
'gk-2023-v2', 'gk-2023-v2-math', 'zk-2022-v1'
]:
_hint = "请你做一道</major>选择题\n请你一步一步思考并将思考过程写在【解析】和<eoe>之间。你将从A，B，C，D中选出正确的答案，并写在【答案】和<eoa>之间。\n例如：【答案】A<eoa>\n完整的题目回答的格式如下：\n【解析】...<eoe>\n【答案】...<eoa>\n请你严格按照上述格式作答。\n题目如下：\n"
_hint = '请你做一道</major>选择题\n请你一步一步思考并将思考过程写在【解析】和<eoe>之间。你将从A，B，C，D中选出正确的答案，并写在【答案】和<eoa>之间。\n例如：【答案】A<eoa>\n完整的题目回答的格式如下：\n【解析】...<eoe>\n【答案】...<eoa>\n请你严格按照上述格式作答。\n题目如下：\n'
_reader_cfg = {
"input_columns": ['question'],
"output_column": 'std_ans',
'input_columns': ['question'],
'output_column': 'std_ans',
},
_infer_cfg = {
"ice_template": {
"type": PromptTemplate,
"template": {
"round": [{
"role": "HUMAN",
"prompt": _hint + "{question}",
'ice_template': {
'type': PromptTemplate,
'template': {
'round': [{
'role': 'HUMAN',
'prompt': _hint + '{question}',
}]
},
"ice_token": "</E>"
'ice_token': '</E>'
},
"retriever": {
"type": ZeroRetriever
'retriever': {
'type': ZeroRetriever
},
"inferencer": {
"type": GenInferencer,
"max_out_len": 1024,
'inferencer': {
'type': GenInferencer,
'max_out_len': 1024,
}
}
_eval_cfg = {
"evaluator": {
"type": PJExamEvaluator
'evaluator': {
'type': PJExamEvaluator
},
"pred_role": "BOT",
"ds_column": "eval_infos"
'pred_role': 'BOT',
'ds_column': 'eval_infos'
}
_dataset = {
"type": PJExamDataset,
"abbr": "PJExamDataset-" + _name,
"path": './data/PJExam',
"name": _name,
"reader_cfg": _reader_cfg,
"infer_cfg": _infer_cfg,
"eval_cfg": _eval_cfg,
'type': PJExamDataset,
'abbr': 'PJExamDataset-' + _name,
'path': './data/PJExam',
'name': _name,
'reader_cfg': _reader_cfg,
'infer_cfg': _infer_cfg,
'eval_cfg': _eval_cfg,
}
PJExam_datasets.append(_dataset)
......
configs/datasets/QuALITY/QuALITY_gen_c407cb.py
View file @ aa2dd2b5
......@@ -14,9 +14,9 @@ QuALITY_infer_cfg = dict(
type=PromptTemplate,
template=dict(round=[
dict(
role="HUMAN",
role='HUMAN',
prompt=
"Read the article, and answer the question.\n\nArticle:\n{article}\n\nQ: {question}\n\nA. {A}\nB. {B}\nC. {C}\nD. {D}"
'Read the article, and answer the question.\n\nArticle:\n{article}\n\nQ: {question}\n\nA. {A}\nB. {B}\nC. {C}\nD. {D}'
),
])),
retriever=dict(type=ZeroRetriever),
......
configs/datasets/SVAMP/svamp_gen_fb25e4.py
View file @ aa2dd2b5
......@@ -9,13 +9,13 @@ svamp_infer_cfg = dict(
template=dict(
round=[
dict(role='HUMAN', prompt="Question: There are 87 oranges and 290 bananas in Philip's collection. If the bananas are organized into 2 groups and oranges are organized into 93 groups How big is each group of bananas?\nLet's think step by step\nAnswer:"),
dict(role='BOT', prompt="To find the size of each group of bananas, we divide the total number of bananas (290) by the number of groups (2): 290 / 2 = 145. Therefore, each group of bananas contains 145 bananas. The answer is 145.\n"),
dict(role='BOT', prompt='To find the size of each group of bananas, we divide the total number of bananas (290) by the number of groups (2): 290 / 2 = 145. Therefore, each group of bananas contains 145 bananas. The answer is 145.\n'),
dict(role='HUMAN', prompt="Question: Marco and his dad went strawberry picking. Marco's dad's strawberries weighed 11 pounds. If together their strawberries weighed 30 pounds. How much did Marco's strawberries weigh?\nLet's think step by step\nAnswer:"),
dict(role='BOT', prompt="To find Marco's strawberries' weight, we subtract his dad's strawberries' weight (11 pounds) from the total weight of their strawberries (30 pounds): 30 - 11 = 19. Therefore, Marco's strawberries weighed 19 pounds. The answer is 19.\n"),
dict(role='HUMAN', prompt="Question: Edward spent $ 6 to buy 2 books each book costing him the same amount of money. Now he has $ 12. How much did each book cost?\nLet's think step by step\nAnswer:"),
dict(role='BOT', prompt="To find the cost of each book, we subtract the initial amount of money Edward had ($6) from the current amount of money he has ($12) and divide it by the number of books (2): (12 - 6) / 2 = 6 / 2 = 3 Therefore, each book cost $3. The answer is 3.\n"),
dict(role='BOT', prompt='To find the cost of each book, we subtract the initial amount of money Edward had ($6) from the current amount of money he has ($12) and divide it by the number of books (2): (12 - 6) / 2 = 6 / 2 = 3 Therefore, each book cost $3. The answer is 3.\n'),
dict(role='HUMAN', prompt="Question: Frank was reading through his favorite book. The book had 3 chapters, each with the same number of pages. It has a total of 594 pages. It took Frank 607 days to finish the book. How many pages are in each chapter?\nLet's think step by step\nAnswer:"),
dict(role='BOT', prompt="To find the number of pages in each chapter, we divide the total number of pages in the book (594) by the number of chapters (3): 594 / 3 = 198. Therefore, each chapter has 198 pages. The answer is 198.\n"),
dict(role='BOT', prompt='To find the number of pages in each chapter, we divide the total number of pages in the book (594) by the number of chapters (3): 594 / 3 = 198. Therefore, each chapter has 198 pages. The answer is 198.\n'),
dict(role='HUMAN', prompt="Question: {question}\nLet's think step by step\nAnswer:"),
],
)),
......
configs/datasets/SuperGLUE_AX_b/SuperGLUE_AX_b_gen_4dfefa.py
View file @ aa2dd2b5
......@@ -6,8 +6,8 @@ from opencompass.datasets import AXDataset_V2
from opencompass.utils.text_postprocessors import first_option_postprocess
AX_b_reader_cfg = dict(
input_columns=["sentence1", "sentence2"],
output_column="label",
input_columns=['sentence1', 'sentence2'],
output_column='label',
)
AX_b_infer_cfg = dict(
......@@ -15,9 +15,9 @@ AX_b_infer_cfg = dict(
type=PromptTemplate,
template=dict(round=[
dict(
role="HUMAN",
role='HUMAN',
prompt=
"{sentence1}\n{sentence2}\nIs the sentence below entailed by the sentence above?\nA. Yes\nB. No\nAnswer:"
'{sentence1}\n{sentence2}\nIs the sentence below entailed by the sentence above?\nA. Yes\nB. No\nAnswer:'
),
]),
),
......@@ -27,15 +27,15 @@ AX_b_infer_cfg = dict(
AX_b_eval_cfg = dict(
evaluator=dict(type=AccEvaluator),
pred_role="BOT",
pred_role='BOT',
pred_postprocessor=dict(type=first_option_postprocess, options='AB'),
)
AX_b_datasets = [
dict(
abbr="AX_b",
abbr='AX_b',
type=AXDataset_V2,
path="./data/SuperGLUE/AX-b/AX-b.jsonl",
path='./data/SuperGLUE/AX-b/AX-b.jsonl',
reader_cfg=AX_b_reader_cfg,
infer_cfg=AX_b_infer_cfg,
eval_cfg=AX_b_eval_cfg,
......
configs/datasets/SuperGLUE_AX_b/SuperGLUE_AX_b_ppl_6db806.py
View file @ aa2dd2b5
......@@ -5,31 +5,31 @@ from opencompass.openicl.icl_evaluator import AccEvaluator
from opencompass.datasets import HFDataset
AX_b_reader_cfg = dict(
input_columns=["sentence1", "sentence2"],
output_column="label",
test_split="train")
input_columns=['sentence1', 'sentence2'],
output_column='label',
test_split='train')
AX_b_infer_cfg = dict(
prompt_template=dict(
type=PromptTemplate,
template={
"entailment":
'entailment':
dict(round=[
dict(
role="HUMAN",
role='HUMAN',
prompt=
"{sentence1}\n{sentence2}\nIs the sentence below entailed by the sentence above?"
'{sentence1}\n{sentence2}\nIs the sentence below entailed by the sentence above?'
),
dict(role="BOT", prompt="Yes"),
dict(role='BOT', prompt='Yes'),
]),
"not_entailment":
'not_entailment':
dict(round=[
dict(
role="HUMAN",
role='HUMAN',
prompt=
"{sentence1}\n{sentence2}\nIs the sentence below entailed by the sentence above?"
'{sentence1}\n{sentence2}\nIs the sentence below entailed by the sentence above?'
),
dict(role="BOT", prompt="No"),
dict(role='BOT', prompt='No'),
])
},
),
......@@ -42,10 +42,10 @@ AX_b_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
AX_b_datasets = [
dict(
type=HFDataset,
abbr="AX_b",
path="json",
data_files="./data/SuperGLUE/AX-b/AX-b.jsonl",
split="train",
abbr='AX_b',
path='json',
data_files='./data/SuperGLUE/AX-b/AX-b.jsonl',
split='train',
reader_cfg=AX_b_reader_cfg,
infer_cfg=AX_b_infer_cfg,
eval_cfg=AX_b_eval_cfg,
......
configs/datasets/SuperGLUE_AX_g/SuperGLUE_AX_g_gen_68aac7.py
View file @ aa2dd2b5
......@@ -6,8 +6,8 @@ from opencompass.datasets import AXDataset_V2
from opencompass.utils.text_postprocessors import first_option_postprocess
AX_g_reader_cfg = dict(
input_columns=["hypothesis", "premise"],
output_column="label",
input_columns=['hypothesis', 'premise'],
output_column='label',
)
AX_g_infer_cfg = dict(
......@@ -15,9 +15,9 @@ AX_g_infer_cfg = dict(
type=PromptTemplate,
template=dict(round=[
dict(
role="HUMAN",
role='HUMAN',
prompt=
"{premise}\n{hypothesis}\nIs the sentence below entailed by the sentence above?\nA. Yes\nB. No\nAnswer:"
'{premise}\n{hypothesis}\nIs the sentence below entailed by the sentence above?\nA. Yes\nB. No\nAnswer:'
),
]),
),
......@@ -27,15 +27,15 @@ AX_g_infer_cfg = dict(
AX_g_eval_cfg = dict(
evaluator=dict(type=AccEvaluator),
pred_role="BOT",
pred_role='BOT',
pred_postprocessor=dict(type=first_option_postprocess, options='AB'),
)
AX_g_datasets = [
dict(
abbr="AX_g",
abbr='AX_g',
type=AXDataset_V2,
path="./data/SuperGLUE/AX-g/AX-g.jsonl",
path='./data/SuperGLUE/AX-g/AX-g.jsonl',
reader_cfg=AX_g_reader_cfg,
infer_cfg=AX_g_infer_cfg,
eval_cfg=AX_g_eval_cfg,
......
configs/datasets/SuperGLUE_AX_g/SuperGLUE_AX_g_ppl_66caf3.py
View file @ aa2dd2b5
......@@ -5,31 +5,31 @@ from opencompass.openicl.icl_evaluator import AccEvaluator
from opencompass.datasets import HFDataset
AX_g_reader_cfg = dict(
input_columns=["hypothesis", "premise"],
output_column="label",
test_split="train")
input_columns=['hypothesis', 'premise'],
output_column='label',
test_split='train')
AX_g_infer_cfg = dict(
prompt_template=dict(
type=PromptTemplate,
template={
"entailment":
'entailment':
dict(round=[
dict(
role="HUMAN",
role='HUMAN',
prompt=
"{premise}\n{hypothesis}\nIs the sentence below entailed by the sentence above?"
'{premise}\n{hypothesis}\nIs the sentence below entailed by the sentence above?'
),
dict(role="BOT", prompt="Yes"),
dict(role='BOT', prompt='Yes'),
]),
"not_entailment":
'not_entailment':
dict(round=[
dict(
role="HUMAN",
role='HUMAN',
prompt=
"{premise}\n{hypothesis}\nIs the sentence below entailed by the sentence above?"
'{premise}\n{hypothesis}\nIs the sentence below entailed by the sentence above?'
),
dict(role="BOT", prompt="No"),
dict(role='BOT', prompt='No'),
])
},
),
......@@ -42,10 +42,10 @@ AX_g_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
AX_g_datasets = [
dict(
type=HFDataset,
abbr="AX_g",
path="json",
data_files="./data/SuperGLUE/AX-g/AX-g.jsonl",
split="train",
abbr='AX_g',
path='json',
data_files='./data/SuperGLUE/AX-g/AX-g.jsonl',
split='train',
reader_cfg=AX_g_reader_cfg,
infer_cfg=AX_g_infer_cfg,
eval_cfg=AX_g_eval_cfg,
......
configs/datasets/SuperGLUE_BoolQ/SuperGLUE_BoolQ_gen_883d50.py
View file @ aa2dd2b5
......@@ -6,8 +6,8 @@ from opencompass.datasets import BoolQDataset_V2
from opencompass.utils.text_postprocessors import first_capital_postprocess
BoolQ_reader_cfg = dict(
input_columns=["question", "passage"],
output_column="label",
input_columns=['question', 'passage'],
output_column='label',
)
BoolQ_infer_cfg = dict(
......@@ -15,8 +15,8 @@ BoolQ_infer_cfg = dict(
type=PromptTemplate,
template=dict(round=[
dict(
role="HUMAN",
prompt="{passage}\nQuestion: {question}\nA. Yes\nB. No\nAnswer:"),
role='HUMAN',
prompt='{passage}\nQuestion: {question}\nA. Yes\nB. No\nAnswer:'),
]),
),
retriever=dict(type=ZeroRetriever),
......@@ -25,15 +25,15 @@ BoolQ_infer_cfg = dict(
BoolQ_eval_cfg = dict(
evaluator=dict(type=AccEvaluator),
pred_role="BOT",
pred_role='BOT',
pred_postprocessor=dict(type=first_capital_postprocess),
)
BoolQ_datasets = [
dict(
abbr="BoolQ",
abbr='BoolQ',
type=BoolQDataset_V2,
path="./data/SuperGLUE/BoolQ/val.jsonl",
path='./data/SuperGLUE/BoolQ/val.jsonl',
reader_cfg=BoolQ_reader_cfg,
infer_cfg=BoolQ_infer_cfg,
eval_cfg=BoolQ_eval_cfg,
......
configs/datasets/SuperGLUE_BoolQ/SuperGLUE_BoolQ_ppl_314797.py
View file @ aa2dd2b5
......@@ -5,9 +5,9 @@ from opencompass.openicl.icl_evaluator import AccEvaluator
from opencompass.datasets import BoolQDataset_V3
BoolQ_reader_cfg = dict(
input_columns=["question", "passage"],
output_column="label",
test_split="train")
input_columns=['question', 'passage'],
output_column='label',
test_split='train')
BoolQ_infer_cfg = dict(
prompt_template=dict(
......@@ -15,13 +15,13 @@ BoolQ_infer_cfg = dict(
template={
'false':
dict(round=[
dict(role="HUMAN", prompt="Passage: {passage}\nQuestion: {question}?"),
dict(role="BOT", prompt="Answer: No"),
dict(role='HUMAN', prompt='Passage: {passage}\nQuestion: {question}?'),
dict(role='BOT', prompt='Answer: No'),
]),
'true':
dict(round=[
dict(role="HUMAN", prompt="Passage: {passage}\nQuestion: {question}?"),
dict(role="BOT", prompt="Answer: Yes"),
dict(role='HUMAN', prompt='Passage: {passage}\nQuestion: {question}?'),
dict(role='BOT', prompt='Answer: Yes'),
]),
},
),
......@@ -33,9 +33,9 @@ BoolQ_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
BoolQ_datasets = [
dict(
abbr="BoolQ",
abbr='BoolQ',
type=BoolQDataset_V3,
path="./data/SuperGLUE/BoolQ/val.jsonl",
path='./data/SuperGLUE/BoolQ/val.jsonl',
reader_cfg=BoolQ_reader_cfg,
infer_cfg=BoolQ_infer_cfg,
eval_cfg=BoolQ_eval_cfg,
......
configs/datasets/SuperGLUE_BoolQ/SuperGLUE_BoolQ_ppl_314b96.py
View file @ aa2dd2b5
......@@ -5,9 +5,9 @@ from opencompass.openicl.icl_evaluator import AccEvaluator
from opencompass.datasets import BoolQDataset
BoolQ_reader_cfg = dict(
input_columns=["question", "passage"],
output_column="answer",
test_split="train")
input_columns=['question', 'passage'],
output_column='answer',
test_split='train')
BoolQ_infer_cfg = dict(
prompt_template=dict(
......@@ -15,13 +15,13 @@ BoolQ_infer_cfg = dict(
template={
0:
dict(round=[
dict(role="HUMAN", prompt="{passage}\nQuestion: {question}?"),
dict(role="BOT", prompt="No"),
dict(role='HUMAN', prompt='{passage}\nQuestion: {question}?'),
dict(role='BOT', prompt='No'),
]),
1:
dict(round=[
dict(role="HUMAN", prompt="{passage}\nQuestion: {question}?"),
dict(role="BOT", prompt="Yes"),
dict(role='HUMAN', prompt='{passage}\nQuestion: {question}?'),
dict(role='BOT', prompt='Yes'),
]),
},
),
......@@ -34,10 +34,10 @@ BoolQ_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
BoolQ_datasets = [
dict(
type=BoolQDataset,
abbr="BoolQ",
path="json",
data_files="./data/SuperGLUE/BoolQ/val.jsonl",
split="train",
abbr='BoolQ',
path='json',
data_files='./data/SuperGLUE/BoolQ/val.jsonl',
split='train',
reader_cfg=BoolQ_reader_cfg,
infer_cfg=BoolQ_infer_cfg,
eval_cfg=BoolQ_eval_cfg,
......
configs/datasets/SuperGLUE_BoolQ/SuperGLUE_BoolQ_ppl_4da4db.py
View file @ aa2dd2b5
......@@ -5,9 +5,9 @@ from opencompass.openicl.icl_evaluator import AccEvaluator
from opencompass.datasets import BoolQDataset
BoolQ_reader_cfg = dict(
input_columns=["question", "passage"],
output_column="answer",
test_split="train")
input_columns=['question', 'passage'],
output_column='answer',
test_split='train')
BoolQ_infer_cfg = dict(
prompt_template=dict(
......@@ -15,13 +15,13 @@ BoolQ_infer_cfg = dict(
template={
0:
dict(round=[
dict(role="HUMAN", prompt="{passage}\nQuestion: {question}"),
dict(role="BOT", prompt="No."),
dict(role='HUMAN', prompt='{passage}\nQuestion: {question}'),
dict(role='BOT', prompt='No.'),
]),
1:
dict(round=[
dict(role="HUMAN", prompt="{passage}\nQuestion: {question}"),
dict(role="BOT", prompt="Yes."),
dict(role='HUMAN', prompt='{passage}\nQuestion: {question}'),
dict(role='BOT', prompt='Yes.'),
]),
},
),
......@@ -34,10 +34,10 @@ BoolQ_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
BoolQ_datasets = [
dict(
type=BoolQDataset,
abbr="BoolQ",
path="json",
data_files="./data/SuperGLUE/BoolQ/val.jsonl",
split="train",
abbr='BoolQ',
path='json',
data_files='./data/SuperGLUE/BoolQ/val.jsonl',
split='train',
reader_cfg=BoolQ_reader_cfg,
infer_cfg=BoolQ_infer_cfg,
eval_cfg=BoolQ_eval_cfg,
......
configs/datasets/SuperGLUE_BoolQ/SuperGLUE_BoolQ_ppl_9619db.py
View file @ aa2dd2b5
......@@ -13,8 +13,8 @@ BoolQ_infer_cfg = dict(
prompt_template=dict(
type=PromptTemplate,
template={
0: "Passage:{passage}。\nQuestion:{question}。\nAnswer: No.",
1: "Passage:{passage}。\nQuestion:{question}。\nAnswer: Yes.",
0: 'Passage:{passage}。\nQuestion:{question}。\nAnswer: No.',
1: 'Passage:{passage}。\nQuestion:{question}。\nAnswer: Yes.',
}),
retriever=dict(type=ZeroRetriever),
inferencer=dict(type=PPLInferencer))
......
configs/datasets/SuperGLUE_CB/SuperGLUE_CB_gen_854c6c.py
View file @ aa2dd2b5
......@@ -6,8 +6,8 @@ from opencompass.datasets import CBDataset_V2
from opencompass.utils.text_postprocessors import first_option_postprocess
CB_reader_cfg = dict(
input_columns=["premise", "hypothesis"],
output_column="label",
input_columns=['premise', 'hypothesis'],
output_column='label',
)
CB_infer_cfg = dict(
......@@ -16,9 +16,9 @@ CB_infer_cfg = dict(
template=dict(
round=[
dict(
role="HUMAN",
role='HUMAN',
prompt=
"{premise}\n{hypothesis}\nWhat is the relation between the two sentences?\nA. Contradiction\nB. Entailment\nC. Neutral\nAnswer:"
'{premise}\n{hypothesis}\nWhat is the relation between the two sentences?\nA. Contradiction\nB. Entailment\nC. Neutral\nAnswer:'
),
], ),
),
......@@ -28,15 +28,15 @@ CB_infer_cfg = dict(
CB_eval_cfg = dict(
evaluator=dict(type=AccEvaluator),
pred_role="BOT",
pred_role='BOT',
pred_postprocessor=dict(type=first_option_postprocess, options='ABC'),
)
CB_datasets = [
dict(
abbr="CB",
abbr='CB',
type=CBDataset_V2,
path="./data/SuperGLUE/CB/val.jsonl",
path='./data/SuperGLUE/CB/val.jsonl',
reader_cfg=CB_reader_cfg,
infer_cfg=CB_infer_cfg,
eval_cfg=CB_eval_cfg,
......
configs/datasets/SuperGLUE_CB/SuperGLUE_CB_ppl_0143fe.py
View file @ aa2dd2b5
......@@ -5,40 +5,40 @@ from opencompass.openicl.icl_evaluator import AccEvaluator
from opencompass.datasets import HFDataset
CB_reader_cfg = dict(
input_columns=["premise", "hypothesis"],
output_column="label",
input_columns=['premise', 'hypothesis'],
output_column='label',
)
CB_infer_cfg = dict(
prompt_template=dict(
type=PromptTemplate,
template={
"contradiction":
'contradiction':
dict(round=[
dict(
role="HUMAN",
role='HUMAN',
prompt=
"{premise}\n{hypothesis}\nWhat is the relation between the two sentences?"
'{premise}\n{hypothesis}\nWhat is the relation between the two sentences?'
),
dict(role="BOT", prompt="Contradiction"),
dict(role='BOT', prompt='Contradiction'),
]),
"entailment":
'entailment':
dict(round=[
dict(
role="HUMAN",
role='HUMAN',
prompt=
"{premise}\n{hypothesis}\nWhat is the relation between the two sentences?"
'{premise}\n{hypothesis}\nWhat is the relation between the two sentences?'
),
dict(role="BOT", prompt="Entailment"),
dict(role='BOT', prompt='Entailment'),
]),
"neutral":
'neutral':
dict(round=[
dict(
role="HUMAN",
role='HUMAN',
prompt=
"{premise}\n{hypothesis}\nWhat is the relation between the two sentences?"
'{premise}\n{hypothesis}\nWhat is the relation between the two sentences?'
),
dict(role="BOT", prompt="Neutral"),
dict(role='BOT', prompt='Neutral'),
]),
},
),
......@@ -51,10 +51,10 @@ CB_eval_cfg = dict(evaluator=dict(type=AccEvaluator), )
CB_datasets = [
dict(
type=HFDataset,
abbr="CB",
path="json",
split="train",
data_files="./data/SuperGLUE/CB/val.jsonl",
abbr='CB',
path='json',
split='train',
data_files='./data/SuperGLUE/CB/val.jsonl',
reader_cfg=CB_reader_cfg,
infer_cfg=CB_infer_cfg,
eval_cfg=CB_eval_cfg,
......
configs/datasets/SuperGLUE_COPA/SuperGLUE_COPA_gen_91ca53.py
View file @ aa2dd2b5
......@@ -6,8 +6,8 @@ from opencompass.datasets import COPADataset_V2
from opencompass.utils.text_postprocessors import first_option_postprocess
COPA_reader_cfg = dict(
input_columns=["question", "premise", "choice1", "choice2"],
output_column="label",
input_columns=['question', 'premise', 'choice1', 'choice2'],
output_column='label',
)
COPA_infer_cfg = dict(
......@@ -16,9 +16,9 @@ COPA_infer_cfg = dict(
template=dict(
round=[
dict(
role="HUMAN",
role='HUMAN',
prompt=
"{premise}\nQuestion: Which may be the {question}?\nA. {choice1}\nB. {choice2}\nAnswer:"
'{premise}\nQuestion: Which may be the {question}?\nA. {choice1}\nB. {choice2}\nAnswer:'
),
], ),
),
......@@ -28,15 +28,15 @@ COPA_infer_cfg = dict(
COPA_eval_cfg = dict(
evaluator=dict(type=AccEvaluator),
pred_role="BOT",
pred_role='BOT',
pred_postprocessor=dict(type=first_option_postprocess, options='AB'),
)
COPA_datasets = [
dict(
abbr="COPA",
abbr='COPA',
type=COPADataset_V2,
path="./data/SuperGLUE/COPA/val.jsonl",
path='./data/SuperGLUE/COPA/val.jsonl',
reader_cfg=COPA_reader_cfg,
infer_cfg=COPA_infer_cfg,
eval_cfg=COPA_eval_cfg,
......
configs/datasets/SuperGLUE_COPA/SuperGLUE_COPA_ppl_54058d.py
View file @ aa2dd2b5
......@@ -13,8 +13,8 @@ COPA_infer_cfg = dict(
prompt_template=dict(
type=PromptTemplate,
template={
0: "Premise:{premise}。\nQuestion:{question}。\nAnswer: {choice1}.",
1: "Passage:{premise}。\nQuestion:{question}。\nAnswer: {choice2}.",
0: 'Premise:{premise}。\nQuestion:{question}。\nAnswer: {choice1}.',
1: 'Passage:{premise}。\nQuestion:{question}。\nAnswer: {choice2}.',
}),
retriever=dict(type=ZeroRetriever),
inferencer=dict(type=PPLInferencer))
......
configs/datasets/SuperGLUE_COPA/SuperGLUE_COPA_ppl_5c24f1.py
View file @ aa2dd2b5
......@@ -5,9 +5,9 @@ from opencompass.openicl.icl_evaluator import AccEvaluator
from opencompass.datasets import HFDataset
COPA_reader_cfg = dict(
input_columns=["question", "premise", "choice1", "choice2"],
output_column="label",
test_split="train")
input_columns=['question', 'premise', 'choice1', 'choice2'],
output_column='label',
test_split='train')
COPA_infer_cfg = dict(
prompt_template=dict(
......@@ -15,13 +15,13 @@ COPA_infer_cfg = dict(
template={
0:
dict(round=[
dict(role="HUMAN", prompt="{premise}\nQuestion: {question}\nAnswer:"),
dict(role="BOT", prompt="{choice1}"),
dict(role='HUMAN', prompt='{premise}\nQuestion: {question}\nAnswer:'),
dict(role='BOT', prompt='{choice1}'),
]),
1:
dict(round=[
dict(role="HUMAN", prompt="{premise}\nQuestion: {question}\nAnswer:"),
dict(role="BOT", prompt="{choice2}"),
dict(role='HUMAN', prompt='{premise}\nQuestion: {question}\nAnswer:'),
dict(role='BOT', prompt='{choice2}'),
]),
},
),
......@@ -34,10 +34,10 @@ COPA_eval_cfg = dict(evaluator=dict(type=AccEvaluator))
COPA_datasets = [
dict(
type=HFDataset,
abbr="COPA",
path="json",
data_files="./data/SuperGLUE/COPA/val.jsonl",
split="train",
abbr='COPA',
path='json',
data_files='./data/SuperGLUE/COPA/val.jsonl',
split='train',
reader_cfg=COPA_reader_cfg,
infer_cfg=COPA_infer_cfg,
eval_cfg=COPA_eval_cfg,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment