Initial commit

opencompass/configs/datasets/humaneval_cn/humaneval_cn_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .humaneval_cn_gen_6313aa import humaneval_cn_datasets  # noqa: F401, F403

opencompass/configs/datasets/humaneval_cn/humaneval_cn_gen_6313aa.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import HumanevalDataset, HumanEvalEvaluator, humaneval_postprocess_v2
+
+humaneval_reader_cfg = dict(
+    input_columns=['prompt'], output_column='task_id', train_split='test')
+
+# TODO: allow empty output-column
+humaneval_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt='完成以下Python代码任务:\n{prompt}'),
+        ])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=512))
+
+humaneval_eval_cfg = dict(
+    evaluator=dict(type=HumanEvalEvaluator),
+    pred_role='BOT',
+    k=[1, 10, 100],  # the parameter only for humaneval
+    pred_postprocessor=dict(type=humaneval_postprocess_v2),
+)
+
+humaneval_cn_datasets = [
+    dict(
+        abbr='openai_humaneval_cn',
+        type=HumanevalDataset,
+        path='opencompass/humaneval_cn',
+        reader_cfg=humaneval_reader_cfg,
+        infer_cfg=humaneval_infer_cfg,
+        eval_cfg=humaneval_eval_cfg)
+
+]

opencompass/configs/datasets/humaneval_cn/humaneval_cn_passk_gen_6313aa.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import HumanevalDataset, HumanEvalEvaluator, humaneval_postprocess_v2
+
+humaneval_reader_cfg = dict(
+    input_columns=['prompt'], output_column='task_id', train_split='test')
+
+# TODO: allow empty output-column
+humaneval_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt='完成以下Python代码任务:\n{prompt}'),
+        ])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=512))
+
+humaneval_eval_cfg = dict(
+    evaluator=dict(type=HumanEvalEvaluator),
+    pred_role='BOT',
+    k=[1, 10, 100],  # the parameter only for humaneval
+    pred_postprocessor=dict(type=humaneval_postprocess_v2),
+)
+
+humaneval_cn_datasets = [
+    dict(
+        abbr='openai_humaneval_cn_passk',
+        type=HumanevalDataset,
+        path='opencompass/humaneval_cn',
+        reader_cfg=humaneval_reader_cfg,
+        infer_cfg=humaneval_infer_cfg,
+        eval_cfg=humaneval_eval_cfg)
+
+]

opencompass/configs/datasets/humaneval_cn/humaneval_cn_repeat10_gen_6313aa.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import HumanevalDataset, HumanEvalEvaluator, humaneval_postprocess_v2
+
+humaneval_reader_cfg = dict(
+    input_columns=['prompt'], output_column='task_id', train_split='test')
+
+# TODO: allow empty output-column
+humaneval_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(round=[
+            dict(
+                role='HUMAN',
+                prompt='完成以下Python代码任务:\n{prompt}'),
+        ])),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=512))
+
+humaneval_eval_cfg = dict(
+    evaluator=dict(type=HumanEvalEvaluator),
+    pred_role='BOT',
+    k=[1, 10, 100],  # the parameter only for humaneval
+    pred_postprocessor=dict(type=humaneval_postprocess_v2),
+)
+
+humaneval_cn_datasets = [
+    dict(
+        abbr='openai_humaneval_cn_repeat10',
+        type=HumanevalDataset,
+        path='opencompass/humaneval_cn',
+        num_repeats=10,
+        reader_cfg=humaneval_reader_cfg,
+        infer_cfg=humaneval_infer_cfg,
+        eval_cfg=humaneval_eval_cfg)
+
+]

opencompass/configs/datasets/humaneval_multi/humaneval_multi_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .humaneval_multi_gen_82cf85 import humaneval_multi_datasets  # noqa: F401, F403

opencompass/configs/datasets/humaneval_multi/humaneval_multi_gen_82cf85.py

+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.datasets import HumanevalMultiDataset, HumanevalMultiEvaluator
+
+humaneval_multi_reader_cfg = dict(input_columns=['prompt'], output_column='tests')
+
+humaneval_multi_infer_cfg = dict(
+    prompt_template=dict(type=PromptTemplate, template='{prompt}'),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer, max_out_len=1024),
+)
+
+humaneval_multi_eval_cfg = {
+    lang: dict(
+        evaluator=dict(
+            type=HumanevalMultiEvaluator,
+            language=lang,
+            ip_address='localhost',  # replace to your code_eval_server ip_address, port
+            port=5000,
+        ),  # refer to https://opencompass.readthedocs.io/en/latest/advanced_guides/code_eval_service.html to launch a server
+        pred_role='BOT',
+    ) for lang in ['cpp', 'cs', 'd', 'go', 'java', 'jl', 'js', 'lua', 'php', 'pl', 'py', 'r', 'rb', 'rkt', 'rs', 'scala', 'sh', 'swift', 'ts']
+}
+
+'''there are four versions of humaneval-{LANG}-{version}.jsonl:
+['keep', 'transform', 'reworded', 'remove']
+SRCDATA-LANG-keep is the same as SRCDATA-LANG, but the text of the prompt is totally unchanged. If the original prompt had Python doctests, they remain as Python instead of being translated to LANG. If the original prompt had Python-specific terminology, e.g., 'list', it remains 'list', instead of being translated, e.g., to 'vector' for C++.
+SRCDATA-LANG-transform transforms the doctests to LANG but leaves the natural language text of the prompt unchanged.
+SRCDATA-LANG-reworded transforms both the doctests and the natural language text of the prompt to LANG.
+SRCDATA-LANG-remove removes the doctests from the prompt.
+'''
+
+humaneval_multi_datasets = [
+    dict(
+        type=HumanevalMultiDataset,
+        abbr=f'humaneval_multiple-{lang}',
+        language=lang,
+        version='reworded',  # choose from ['keep', 'transform', 'reworded', 'remove']
+        num_repeats=1,
+        path='./data/multi-data/humaneval_multipl-e/',
+        reader_cfg=humaneval_multi_reader_cfg,
+        infer_cfg=humaneval_multi_infer_cfg,
+        eval_cfg=humaneval_multi_eval_cfg[lang],
+    ) for lang in ['cpp', 'cs', 'd', 'go', 'java', 'jl', 'js', 'lua', 'php', 'pl', 'py', 'r', 'rb', 'rkt', 'rs', 'scala', 'sh', 'swift', 'ts']
+]

opencompass/configs/datasets/humaneval_plus/humaneval_plus_gen.py

+from mmengine.config import read_base
+
+with read_base():
+    from .humaneval_plus_gen_8e312c import humaneval_plus_datasets  # noqa: F401, F403