Skip to content

GitLab

Switch branch/tag
Find file Normal viewHistoryPermalink
sniah_1.yaml 1 KB
Newer Older
Baber's avatar
add niah  
Baber committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39 
tag:
  - ruler
task: niah_1
dataset_path: ""
dataset_name: ""
output_type: generate_until
test_split: test
download_dataset: !function utils.niah_single_2
doc_to_text: "{{input}}"
doc_to_target: "{{outputs[0]}}" #" {{answer.split('### ')[-1].rstrip()}}"
process_results: !function utils.process_results

metric_list:
  - metric: "4096"
    aggregation: !function utils.aggregate_metrics
    higher_is_better: true
  - metric: "8192"
    aggregation: !function utils.aggregate_metrics
    higher_is_better: true
  - metric: "16384"
    aggregation: !function utils.aggregate_metrics
    higher_is_better: true
  - metric: "32768"
    aggregation: !function utils.aggregate_metrics
    higher_is_better: true
  - metric: "65536"
    aggregation: !function utils.aggregate_metrics
    higher_is_better: true
  - metric: "131072"
    aggregation: !function utils.aggregate_metrics
    higher_is_better: true
generation_kwargs:
  do_sample: true
  temperature: 1.0
  max_gen_toks: 128
  until: []
repeats: 1
metadata:
  version: 3.0