niah_single_1.yaml

tag:
  - longcxt
task: niah_single_1
dataset_path: ""
dataset_name: ""
output_type: generate_until
test_split: test
download_dataset: !function utils.niah_single_1
doc_to_text: "{{input}}"
doc_to_target: "{{outputs[0]}}"
gen_prefix: "{{gen_prefix}}"
process_results: !function utils.process_results
metric_list:
  - metric: "4096"
    aggregation: !function utils.aggregate_metrics
    higher_is_better: true
  - metric: "8192"
    aggregation: !function utils.aggregate_metrics
    higher_is_better: true
  - metric: "16384"
    aggregation: !function utils.aggregate_metrics
    higher_is_better: true
#  - metric: "32768"
#    aggregation: !function utils.aggregate_metrics
#    higher_is_better: true
#  - metric: "65536"
#    aggregation: !function utils.aggregate_metrics
#    higher_is_better: true
#  - metric: "131072"
#    aggregation: !function utils.aggregate_metrics
#    higher_is_better: true
generation_kwargs:
  do_sample: false
  temperature: 0.0
  max_gen_toks: 64
  until: []
repeats: 1
metadata:
  version: 1.0