Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
gaoqiong
lm-evaluation-harness
Commits
8c997e53
Commit
8c997e53
authored
May 03, 2022
by
jon-tow
Browse files
Revert `tests/testdata` changes and address flake8 issues
parent
d95a4333
Changes
627
Hide whitespace changes
Inline
Side-by-side
Showing
20 changed files
with
20 additions
and
20 deletions
+20
-20
tests/testdata/triviaqa-v0-res.json
tests/testdata/triviaqa-v0-res.json
+1
-1
tests/testdata/truthfulqa_gen-v0-greedy_until
tests/testdata/truthfulqa_gen-v0-greedy_until
+1
-1
tests/testdata/truthfulqa_gen-v0-res.json
tests/testdata/truthfulqa_gen-v0-res.json
+1
-1
tests/testdata/truthfulqa_gen-v1-greedy_until
tests/testdata/truthfulqa_gen-v1-greedy_until
+1
-1
tests/testdata/truthfulqa_gen-v1-res.json
tests/testdata/truthfulqa_gen-v1-res.json
+1
-1
tests/testdata/truthfulqa_mc-v0-loglikelihood
tests/testdata/truthfulqa_mc-v0-loglikelihood
+1
-1
tests/testdata/truthfulqa_mc-v0-res.json
tests/testdata/truthfulqa_mc-v0-res.json
+1
-1
tests/testdata/truthfulqa_mc-v1-loglikelihood
tests/testdata/truthfulqa_mc-v1-loglikelihood
+1
-1
tests/testdata/truthfulqa_mc-v1-res.json
tests/testdata/truthfulqa_mc-v1-res.json
+1
-1
tests/testdata/webqs-v0-loglikelihood
tests/testdata/webqs-v0-loglikelihood
+1
-1
tests/testdata/webqs-v0-res.json
tests/testdata/webqs-v0-res.json
+1
-1
tests/testdata/wic-v0-loglikelihood
tests/testdata/wic-v0-loglikelihood
+1
-1
tests/testdata/wic-v0-res.json
tests/testdata/wic-v0-res.json
+1
-1
tests/testdata/wikitext-v0-loglikelihood_rolling
tests/testdata/wikitext-v0-loglikelihood_rolling
+1
-1
tests/testdata/wikitext-v0-res.json
tests/testdata/wikitext-v0-res.json
+1
-1
tests/testdata/wikitext-v1-loglikelihood_rolling
tests/testdata/wikitext-v1-loglikelihood_rolling
+1
-1
tests/testdata/wikitext-v1-res.json
tests/testdata/wikitext-v1-res.json
+1
-1
tests/testdata/winogrande-v0-loglikelihood
tests/testdata/winogrande-v0-loglikelihood
+1
-1
tests/testdata/winogrande-v0-res.json
tests/testdata/winogrande-v0-res.json
+1
-1
tests/testdata/wmt14-en-fr-v0-greedy_until
tests/testdata/wmt14-en-fr-v0-greedy_until
+1
-1
No files found.
tests/testdata/triviaqa-v0-res.json
View file @
8c997e53
{
"results"
:
{
"triviaqa"
:
{
"acc"
:
0.0
,
"acc_stderr"
:
0.0
}},
"versions"
:
{
"triviaqa"
:
0
}}
{
"results"
:
{
"triviaqa"
:
{
"acc"
:
0.0
,
"acc_stderr"
:
0.0
}},
"versions"
:
{
"triviaqa"
:
0
}}
\ No newline at end of file
tests/testdata/truthfulqa_gen-v0-greedy_until
View file @
8c997e53
0d7c56e1aa71ffd8f94bde28f6e8dfdd35f7aaadffa0620bd2a27704253d6c14
0d7c56e1aa71ffd8f94bde28f6e8dfdd35f7aaadffa0620bd2a27704253d6c14
\ No newline at end of file
tests/testdata/truthfulqa_gen-v0-res.json
View file @
8c997e53
{
"results"
:
{
"truthfulqa_gen"
:
{
"bleu_acc"
:
0.0
,
"bleu_acc_stderr"
:
0.0
,
"bleu_diff"
:
0.0
,
"bleu_diff_stderr"
:
0.0
,
"bleu_max"
:
0.0
,
"bleu_max_stderr"
:
0.0
,
"bleurt_acc"
:
0.8372093023255814
,
"bleurt_acc_stderr"
:
0.012923696051772253
,
"bleurt_diff"
:
0.13967358205134603
,
"bleurt_diff_stderr"
:
0.00532907098769571
,
"bleurt_max"
:
-1.4402793981454072
,
"bleurt_max_stderr"
:
0.0021884846359458963
,
"rouge1_acc"
:
0.0
,
"rouge1_acc_stderr"
:
0.0
,
"rouge1_diff"
:
0.0
,
"rouge1_diff_stderr"
:
0.0
,
"rouge1_max"
:
0.0
,
"rouge1_max_stderr"
:
0.0
,
"rouge2_acc"
:
0.0
,
"rouge2_acc_stderr"
:
0.0
,
"rouge2_diff"
:
0.0
,
"rouge2_diff_stderr"
:
0.0
,
"rouge2_max"
:
0.0
,
"rouge2_max_stderr"
:
0.0
,
"rougeL_acc"
:
0.0
,
"rougeL_acc_stderr"
:
0.0
,
"rougeL_diff"
:
0.0
,
"rougeL_diff_stderr"
:
0.0
,
"rougeL_max"
:
0.0
,
"rougeL_max_stderr"
:
0.0
}},
"versions"
:
{
"truthfulqa_gen"
:
0
}}
{
"results"
:
{
"truthfulqa_gen"
:
{
"bleu_acc"
:
0.0
,
"bleu_acc_stderr"
:
0.0
,
"bleu_diff"
:
0.0
,
"bleu_diff_stderr"
:
0.0
,
"bleu_max"
:
0.0
,
"bleu_max_stderr"
:
0.0
,
"bleurt_acc"
:
0.8372093023255814
,
"bleurt_acc_stderr"
:
0.012923696051772253
,
"bleurt_diff"
:
0.13967358205134603
,
"bleurt_diff_stderr"
:
0.00532907098769571
,
"bleurt_max"
:
-1.4402793981454072
,
"bleurt_max_stderr"
:
0.0021884846359458963
,
"rouge1_acc"
:
0.0
,
"rouge1_acc_stderr"
:
0.0
,
"rouge1_diff"
:
0.0
,
"rouge1_diff_stderr"
:
0.0
,
"rouge1_max"
:
0.0
,
"rouge1_max_stderr"
:
0.0
,
"rouge2_acc"
:
0.0
,
"rouge2_acc_stderr"
:
0.0
,
"rouge2_diff"
:
0.0
,
"rouge2_diff_stderr"
:
0.0
,
"rouge2_max"
:
0.0
,
"rouge2_max_stderr"
:
0.0
,
"rougeL_acc"
:
0.0
,
"rougeL_acc_stderr"
:
0.0
,
"rougeL_diff"
:
0.0
,
"rougeL_diff_stderr"
:
0.0
,
"rougeL_max"
:
0.0
,
"rougeL_max_stderr"
:
0.0
}},
"versions"
:
{
"truthfulqa_gen"
:
0
}}
\ No newline at end of file
tests/testdata/truthfulqa_gen-v1-greedy_until
View file @
8c997e53
1a280973bbac2b7ac29dd64dddac474fb4749585f7de893483b4034814466c67
1a280973bbac2b7ac29dd64dddac474fb4749585f7de893483b4034814466c67
\ No newline at end of file
tests/testdata/truthfulqa_gen-v1-res.json
View file @
8c997e53
{
"results"
:
{
"truthfulqa_gen"
:
{
"bleu_acc"
:
0.0
,
"bleu_acc_stderr"
:
0.0
,
"bleu_diff"
:
0.0
,
"bleu_diff_stderr"
:
0.0
,
"bleu_max"
:
0.0
,
"bleu_max_stderr"
:
0.0
,
"bleurt_acc"
:
0.835985312117503
,
"bleurt_acc_stderr"
:
0.012962704327492454
,
"bleurt_diff"
:
0.14077322143090107
,
"bleurt_diff_stderr"
:
0.005459888909582694
,
"bleurt_max"
:
-1.4399358725752065
,
"bleurt_max_stderr"
:
0.0022126992369197133
,
"rouge1_acc"
:
0.0
,
"rouge1_acc_stderr"
:
0.0
,
"rouge1_diff"
:
0.0
,
"rouge1_diff_stderr"
:
0.0
,
"rouge1_max"
:
0.0
,
"rouge1_max_stderr"
:
0.0
,
"rouge2_acc"
:
0.0
,
"rouge2_acc_stderr"
:
0.0
,
"rouge2_diff"
:
0.0
,
"rouge2_diff_stderr"
:
0.0
,
"rouge2_max"
:
0.0
,
"rouge2_max_stderr"
:
0.0
,
"rougeL_acc"
:
0.0
,
"rougeL_acc_stderr"
:
0.0
,
"rougeL_diff"
:
0.0
,
"rougeL_diff_stderr"
:
0.0
,
"rougeL_max"
:
0.0
,
"rougeL_max_stderr"
:
0.0
}},
"versions"
:
{
"truthfulqa_gen"
:
1
}}
{
"results"
:
{
"truthfulqa_gen"
:
{
"bleu_acc"
:
0.0
,
"bleu_acc_stderr"
:
0.0
,
"bleu_diff"
:
0.0
,
"bleu_diff_stderr"
:
0.0
,
"bleu_max"
:
0.0
,
"bleu_max_stderr"
:
0.0
,
"bleurt_acc"
:
0.835985312117503
,
"bleurt_acc_stderr"
:
0.012962704327492454
,
"bleurt_diff"
:
0.14077322143090107
,
"bleurt_diff_stderr"
:
0.005459888909582694
,
"bleurt_max"
:
-1.4399358725752065
,
"bleurt_max_stderr"
:
0.0022126992369197133
,
"rouge1_acc"
:
0.0
,
"rouge1_acc_stderr"
:
0.0
,
"rouge1_diff"
:
0.0
,
"rouge1_diff_stderr"
:
0.0
,
"rouge1_max"
:
0.0
,
"rouge1_max_stderr"
:
0.0
,
"rouge2_acc"
:
0.0
,
"rouge2_acc_stderr"
:
0.0
,
"rouge2_diff"
:
0.0
,
"rouge2_diff_stderr"
:
0.0
,
"rouge2_max"
:
0.0
,
"rouge2_max_stderr"
:
0.0
,
"rougeL_acc"
:
0.0
,
"rougeL_acc_stderr"
:
0.0
,
"rougeL_diff"
:
0.0
,
"rougeL_diff_stderr"
:
0.0
,
"rougeL_max"
:
0.0
,
"rougeL_max_stderr"
:
0.0
}},
"versions"
:
{
"truthfulqa_gen"
:
1
}}
\ No newline at end of file
tests/testdata/truthfulqa_mc-v0-loglikelihood
View file @
8c997e53
226a6783976177dc9ceda5688623ff37023242eff30ddf270b886bf7b9b32228
226a6783976177dc9ceda5688623ff37023242eff30ddf270b886bf7b9b32228
\ No newline at end of file
tests/testdata/truthfulqa_mc-v0-res.json
View file @
8c997e53
{
"results"
:
{
"truthfulqa_mc"
:
{
"mc1"
:
0.2141982864137087
,
"mc1_stderr"
:
0.01436214815569045
,
"mc2"
:
0.465436996173817
,
"mc2_stderr"
:
0.0048422530880316405
}},
"versions"
:
{
"truthfulqa_mc"
:
0
}}
{
"results"
:
{
"truthfulqa_mc"
:
{
"mc1"
:
0.2141982864137087
,
"mc1_stderr"
:
0.01436214815569045
,
"mc2"
:
0.465436996173817
,
"mc2_stderr"
:
0.0048422530880316405
}},
"versions"
:
{
"truthfulqa_mc"
:
0
}}
\ No newline at end of file
tests/testdata/truthfulqa_mc-v1-loglikelihood
View file @
8c997e53
1e07020e9cf41d46ed65312eb39d2b8e6599673d4f0d6b67c0d0eba0efb493bb
1e07020e9cf41d46ed65312eb39d2b8e6599673d4f0d6b67c0d0eba0efb493bb
\ No newline at end of file
tests/testdata/truthfulqa_mc-v1-res.json
View file @
8c997e53
{
"results"
:
{
"truthfulqa_mc"
:
{
"mc1"
:
0.23255813953488372
,
"mc1_stderr"
:
0.01478915753108052
,
"mc2"
:
0.4462325560722362
,
"mc2_stderr"
:
0.004986523944692003
}},
"versions"
:
{
"truthfulqa_mc"
:
1
}}
{
"results"
:
{
"truthfulqa_mc"
:
{
"mc1"
:
0.23255813953488372
,
"mc1_stderr"
:
0.01478915753108052
,
"mc2"
:
0.4462325560722362
,
"mc2_stderr"
:
0.004986523944692003
}},
"versions"
:
{
"truthfulqa_mc"
:
1
}}
\ No newline at end of file
tests/testdata/webqs-v0-loglikelihood
View file @
8c997e53
96b218173468cc94552a0b946193bda89faba51f1bfc3e7945531f9dff8d6fe9
96b218173468cc94552a0b946193bda89faba51f1bfc3e7945531f9dff8d6fe9
\ No newline at end of file
tests/testdata/webqs-v0-res.json
View file @
8c997e53
{
"results"
:
{
"webqs"
:
{
"acc"
:
0.0
,
"acc_stderr"
:
0.0
}},
"versions"
:
{
"webqs"
:
0
}}
{
"results"
:
{
"webqs"
:
{
"acc"
:
0.0
,
"acc_stderr"
:
0.0
}},
"versions"
:
{
"webqs"
:
0
}}
\ No newline at end of file
tests/testdata/wic-v0-loglikelihood
View file @
8c997e53
403a08da05e4c44d7e3dd3358382a7ba489c41d223e24cd1a9ed82ef1a2d004b
403a08da05e4c44d7e3dd3358382a7ba489c41d223e24cd1a9ed82ef1a2d004b
\ No newline at end of file
tests/testdata/wic-v0-res.json
View file @
8c997e53
{
"results"
:
{
"wic"
:
{
"acc"
:
0.49216300940438873
,
"acc_stderr"
:
0.01980828765781383
}},
"versions"
:
{
"wic"
:
0
}}
{
"results"
:
{
"wic"
:
{
"acc"
:
0.49216300940438873
,
"acc_stderr"
:
0.01980828765781383
}},
"versions"
:
{
"wic"
:
0
}}
\ No newline at end of file
tests/testdata/wikitext-v0-loglikelihood_rolling
View file @
8c997e53
b6f83e6cf7535ee41b0057c3e2ec2cf7f2fa5a9119b305c479a83091d1142b2c
b6f83e6cf7535ee41b0057c3e2ec2cf7f2fa5a9119b305c479a83091d1142b2c
\ No newline at end of file
tests/testdata/wikitext-v0-res.json
View file @
8c997e53
{
"results"
:
{
"wikitext"
:
{
"bits_per_byte"
:
2.219817611605802e-05
,
"byte_perplexity"
:
1.0000221984224973
,
"word_perplexity"
:
1.000118710696617
}},
"versions"
:
{
"wikitext"
:
0
}}
{
"results"
:
{
"wikitext"
:
{
"bits_per_byte"
:
2.219817611605802e-05
,
"byte_perplexity"
:
1.0000221984224973
,
"word_perplexity"
:
1.000118710696617
}},
"versions"
:
{
"wikitext"
:
0
}}
\ No newline at end of file
tests/testdata/wikitext-v1-loglikelihood_rolling
View file @
8c997e53
b6f83e6cf7535ee41b0057c3e2ec2cf7f2fa5a9119b305c479a83091d1142b2c
b6f83e6cf7535ee41b0057c3e2ec2cf7f2fa5a9119b305c479a83091d1142b2c
\ No newline at end of file
tests/testdata/wikitext-v1-res.json
View file @
8c997e53
{
"results"
:
{
"wikitext"
:
{
"bits_per_byte"
:
3.202519859941674e-05
,
"byte_perplexity"
:
1.0000221984224973
,
"word_perplexity"
:
1.000118710696617
}},
"versions"
:
{
"wikitext"
:
1
}}
{
"results"
:
{
"wikitext"
:
{
"bits_per_byte"
:
3.202519859941674e-05
,
"byte_perplexity"
:
1.0000221984224973
,
"word_perplexity"
:
1.000118710696617
}},
"versions"
:
{
"wikitext"
:
1
}}
\ No newline at end of file
tests/testdata/winogrande-v0-loglikelihood
View file @
8c997e53
90a3eff49de9173964d46f5ed57bcf9a78a72dd1bfe0e5323b25cebb40b49ea9
90a3eff49de9173964d46f5ed57bcf9a78a72dd1bfe0e5323b25cebb40b49ea9
\ No newline at end of file
tests/testdata/winogrande-v0-res.json
View file @
8c997e53
{
"results"
:
{
"winogrande"
:
{
"acc"
:
0.516179952644041
,
"acc_stderr"
:
0.014045126130978606
}},
"versions"
:
{
"winogrande"
:
0
}}
{
"results"
:
{
"winogrande"
:
{
"acc"
:
0.516179952644041
,
"acc_stderr"
:
0.014045126130978606
}},
"versions"
:
{
"winogrande"
:
0
}}
\ No newline at end of file
tests/testdata/wmt14-en-fr-v0-greedy_until
View file @
8c997e53
368ae7eec0f902b5123f2d5197caa5109a23942011c53fe68d9eaeee20180e46
368ae7eec0f902b5123f2d5197caa5109a23942011c53fe68d9eaeee20180e46
\ No newline at end of file
Prev
1
…
24
25
26
27
28
29
30
31
32
Next
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment