mbpp.yaml

task: mbpp
dataset_path: google-research-datasets/mbpp
dataset_name: full
unsafe_code: true
output_type: generate_until
test_split: test
repeats: 20
doc_to_text: "{{text|trim}}\n{{code}}.split(':')[0]:\n"
doc_to_target: "{% if is_fewshot is defined %}{{code}}\n{% else %}{{test_list[0]}}\n{{test_list[1]}}\n{{test_list[2]}}{% endif %}"
target_delimiter: ""
gen_prefix: "Here is the completed function:\n\n```python\n"
metric_list:
  - metric: !function utils.pass_at_k
    aggregation: mean
    higher_is_better: true
    k: [ 10 ]
generation_kwargs:
  until: [
    "\nclass",
    "\nassert",
    '\n"""',
    "\nprint",
    "\nif",
    "\n```",
    "\n#",
    "\n<|/",
    "<|eot_id|>",
  ]
  do_sample: false
fewshot_config:
  sampler: first_n
  samples: !function utils.list_fewshot_samples
metadata:
  version: 2.0