niah_1.yaml

tag:
  - ruler
task: niah_1
dataset_path: ""
dataset_name: ""
output_type: generate_until
test_split: test
download_dataset: !function utils.niah_single_1
doc_to_text: "{{input}}"
doc_to_target: "{{outputs[0]}}"
process_results: !function utils.process_results

metric_list:
  - metric: "4096"
    aggregation: !function utils.aggregate_metrics
    higher_is_better: true
  - metric: "8192"
    aggregation: !function utils.aggregate_metrics
    higher_is_better: true
  - metric: "16384"
    aggregation: !function utils.aggregate_metrics
    higher_is_better: true
#  - metric: "32768"
#    aggregation: !function utils.aggregate_metrics
#    higher_is_better: true
#  - metric: "65536"
#    aggregation: !function utils.aggregate_metrics
#    higher_is_better: true
#  - metric: "131072"
#    aggregation: !function utils.aggregate_metrics
#    higher_is_better: true
generation_kwargs:
  do_sample: true
  temperature: 1.0
  max_gen_toks: 128
  until: []
repeats: 1
metadata:
  version: 1.0