"megatron/legacy/model/language_model.py" did not exist on "2eaf6c790d71928924fc2d679a2f9193e9d3be30"
lambada_openai.yaml 491 Bytes
Newer Older
haileyschoelkopf's avatar
haileyschoelkopf committed
1
2
group:
  - lambada
3
task: lambada_openai
4
5
6
7
8
9
10
11
12
13
14
dataset_path: EleutherAI/lambada_openai
dataset_name: default
output_type: loglikelihood
test_split: test
doc_to_text: "{{text.split(' ')[:-1]|join(' ')}}"
doc_to_target: "{{' '+text.split(' ')[-1]}}"
should_decontaminate: true
doc_to_decontamination_query: "{{text}}"
metric_list:
  - metric: perplexity
    aggregation: perplexity
haileyschoelkopf's avatar
haileyschoelkopf committed
15
    higher_is_better: false
haileyschoelkopf's avatar
haileyschoelkopf committed
16
  - metric: acc
17
18
    aggregation: mean
    higher_is_better: true
lintangsutawika's avatar
lintangsutawika committed
19
metadata:
20
  version: 1.0