Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
OpenDAS
opencompass
Commits
35aace77
Unverified
Commit
35aace77
authored
Jan 26, 2024
by
Xiaoming Shi
Committed by
GitHub
Jan 26, 2024
Browse files
[Fix] Update MedBench (#845)
parent
8ed022b4
Changes
2
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
161 additions
and
255 deletions
+161
-255
configs/datasets/MedBench/medbench_gen_0b4fff.py
configs/datasets/MedBench/medbench_gen_0b4fff.py
+3
-27
opencompass/datasets/medbench/medbench.py
opencompass/datasets/medbench/medbench.py
+158
-228
No files found.
configs/datasets/MedBench/medbench_gen_0b4fff.py
View file @
35aace77
...
@@ -2,13 +2,13 @@ from opencompass.openicl.icl_prompt_template import PromptTemplate
...
@@ -2,13 +2,13 @@ from opencompass.openicl.icl_prompt_template import PromptTemplate
from
opencompass.openicl.icl_retriever
import
ZeroRetriever
from
opencompass.openicl.icl_retriever
import
ZeroRetriever
from
opencompass.openicl.icl_inferencer
import
GenInferencer
from
opencompass.openicl.icl_inferencer
import
GenInferencer
from
opencompass.openicl.icl_evaluator
import
AccEvaluator
from
opencompass.openicl.icl_evaluator
import
AccEvaluator
from
opencompass.datasets
import
MedBenchDataset
,
MedBenchEvaluator
,
MedBenchEvaluator_Cloze
,
MedBenchEvaluator_IE
,
MedBenchEvaluator_mcq
,
MedBenchEvaluator_CMeEE
,
MedBenchEvaluator_CMeIE
,
MedBenchEvaluator_CHIP_CDEE
,
MedBenchEvaluator_CHIP_CDN
,
MedBenchEvaluator_CHIP_CTC
,
MedBenchEvaluator_NLG
,
MedBenchEvaluator_TF
,
MedBenchEvaluator_DBMHG
,
MedBenchEvaluator_SMDoc
,
MedBenchEvaluator_IMCS_V2_MRG
from
opencompass.datasets
import
MedBenchDataset
,
MedBenchEvaluator
,
MedBenchEvaluator_Cloze
,
MedBenchEvaluator_CMeEE
,
MedBenchEvaluator_CMeIE
,
MedBenchEvaluator_CHIP_CDEE
,
MedBenchEvaluator_CHIP_CDN
,
MedBenchEvaluator_CHIP_CTC
,
MedBenchEvaluator_NLG
,
MedBenchEvaluator_TF
,
MedBenchEvaluator_DBMHG
,
MedBenchEvaluator_SMDoc
,
MedBenchEvaluator_IMCS_V2_MRG
from
opencompass.utils.text_postprocessors
import
first_capital_postprocess
from
opencompass.utils.text_postprocessors
import
first_capital_postprocess
medbench_reader_cfg
=
dict
(
medbench_reader_cfg
=
dict
(
input_columns
=
[
'problem_input'
],
output_column
=
'label'
)
input_columns
=
[
'problem_input'
],
output_column
=
'label'
)
medbench_multiple_choices_sets
=
[
'Med-Exam'
,
'DDx-basic'
,
'DDx-advanced'
,
'Safety
Bench
'
]
# 选择题,用acc判断
medbench_multiple_choices_sets
=
[
'Med-Exam'
,
'DDx-basic'
,
'DDx-advanced'
,
'
Med
Safety'
]
# 选择题,用acc判断
medbench_qa_sets
=
[
'MedHC'
,
'MedMC'
,
'MedDG'
,
'MedSpeQA'
,
'MedTreat'
,
'CMB-Clin'
]
# 开放式QA,有标答
medbench_qa_sets
=
[
'MedHC'
,
'MedMC'
,
'MedDG'
,
'MedSpeQA'
,
'MedTreat'
,
'CMB-Clin'
]
# 开放式QA,有标答
...
@@ -20,31 +20,7 @@ medbench_ie_sets = ['DBMHG', 'CMeEE', 'CMeIE', 'CHIP-CDEE', 'CHIP-CDN', 'CHIP-CT
...
@@ -20,31 +20,7 @@ medbench_ie_sets = ['DBMHG', 'CMeEE', 'CMeIE', 'CHIP-CDEE', 'CHIP-CDN', 'CHIP-CT
medbench_datasets
=
[]
medbench_datasets
=
[]
for
name
in
medbench_single_choice_sets
:
for
name
in
medbench_single_choice_sets
+
medbench_multiple_choices_sets
:
medbench_infer_cfg
=
dict
(
prompt_template
=
dict
(
type
=
PromptTemplate
,
template
=
dict
(
round
=
[
dict
(
role
=
"HUMAN"
,
prompt
=
'{problem_input}'
)])),
retriever
=
dict
(
type
=
ZeroRetriever
),
# retriver 不起作用,以输入参数为准 (zero-shot / few-shot)
inferencer
=
dict
(
type
=
GenInferencer
))
medbench_eval_cfg
=
dict
(
evaluator
=
dict
(
type
=
MedBenchEvaluator_TF
),
pred_role
=
"BOT"
)
medbench_datasets
.
append
(
dict
(
type
=
MedBenchDataset
,
path
=
'./data/MedBench/'
+
name
,
name
=
name
,
abbr
=
'medbench-'
+
name
,
setting_name
=
'zero-shot'
,
reader_cfg
=
medbench_reader_cfg
,
infer_cfg
=
medbench_infer_cfg
.
copy
(),
eval_cfg
=
medbench_eval_cfg
.
copy
()))
for
name
in
medbench_multiple_choices_sets
:
medbench_infer_cfg
=
dict
(
medbench_infer_cfg
=
dict
(
prompt_template
=
dict
(
prompt_template
=
dict
(
type
=
PromptTemplate
,
type
=
PromptTemplate
,
...
...
opencompass/datasets/medbench/medbench.py
View file @
35aace77
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment