Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
gaoqiong
lm-evaluation-harness
Commits
546fd5cd
Unverified
Commit
546fd5cd
authored
Aug 01, 2023
by
Lintang Sutawika
Committed by
GitHub
Aug 01, 2023
Browse files
Merge pull request #686 from EleutherAI/cleanup
[Refactor] Cleanup for `big-refactor`
parents
465c695b
540d468e
Changes
216
Hide whitespace changes
Inline
Side-by-side
Showing
20 changed files
with
0 additions
and
3063 deletions
+0
-3063
results/mpt/mpt-7b/mpt-7b_arithmetic_5-shot.json
results/mpt/mpt-7b/mpt-7b_arithmetic_5-shot.json
+0
-67
results/mpt/mpt-7b/mpt-7b_bbh_3-shot.json
results/mpt/mpt-7b/mpt-7b_bbh_3-shot.json
+0
-124
results/mpt/mpt-7b/mpt-7b_blimp_0-shot.json
results/mpt/mpt-7b/mpt-7b_blimp_0-shot.json
+0
-352
results/mpt/mpt-7b/mpt-7b_common_sense_reasoning_0-shot.json
results/mpt/mpt-7b/mpt-7b_common_sense_reasoning_0-shot.json
+0
-91
results/mpt/mpt-7b/mpt-7b_glue_0-shot.json
results/mpt/mpt-7b/mpt-7b_glue_0-shot.json
+0
-66
results/mpt/mpt-7b/mpt-7b_human_alignment_0-shot.json
results/mpt/mpt-7b/mpt-7b_human_alignment_0-shot.json
+0
-197
results/mpt/mpt-7b/mpt-7b_lambada_0-shot.json
results/mpt/mpt-7b/mpt-7b_lambada_0-shot.json
+0
-80
results/mpt/mpt-7b/mpt-7b_mmlu_5-shot.json
results/mpt/mpt-7b/mpt-7b_mmlu_5-shot.json
+0
-416
results/mpt/mpt-7b/mpt-7b_pawsx_0-shot.json
results/mpt/mpt-7b/mpt-7b_pawsx_0-shot.json
+0
-52
results/mpt/mpt-7b/mpt-7b_reading_comprehension_0-shot.json
results/mpt/mpt-7b/mpt-7b_reading_comprehension_0-shot.json
+0
-36
results/mpt/mpt-7b/mpt-7b_superglue_0-shot.json
results/mpt/mpt-7b/mpt-7b_superglue_0-shot.json
+0
-55
results/mpt/mpt-7b/mpt-7b_unscramble_0-shot.json
results/mpt/mpt-7b/mpt-7b_unscramble_0-shot.json
+0
-42
results/mpt/mpt-7b/mpt-7b_xcopa_0-shot.json
results/mpt/mpt-7b/mpt-7b_xcopa_0-shot.json
+0
-72
results/mpt/mpt-7b/mpt-7b_xnli_0-shot.json
results/mpt/mpt-7b/mpt-7b_xnli_0-shot.json
+0
-92
results/mpt/mpt-7b/mpt-7b_xstory_cloze_0-shot.json
results/mpt/mpt-7b/mpt-7b_xstory_cloze_0-shot.json
+0
-72
results/mpt/mpt-7b/mpt-7b_xwinograd_0-shot.json
results/mpt/mpt-7b/mpt-7b_xwinograd_0-shot.json
+0
-47
results/opt/opt-1.3b/README.md
results/opt/opt-1.3b/README.md
+0
-133
results/opt/opt-1.3b/opt-1.3b.json
results/opt/opt-1.3b/opt-1.3b.json
+0
-468
results/opt/opt-125m/README.md
results/opt/opt-125m/README.md
+0
-133
results/opt/opt-125m/opt-125m.json
results/opt/opt-125m/opt-125m.json
+0
-468
No files found.
results/mpt/mpt-7b/mpt-7b_arithmetic_5-shot.json
deleted
100644 → 0
View file @
465c695b
{
"results"
:
{
"arithmetic_1dc"
:
{
"acc"
:
0.081
,
"acc_stderr"
:
0.006102304405675846
},
"arithmetic_2da"
:
{
"acc"
:
0.918
,
"acc_stderr"
:
0.006136515983374211
},
"arithmetic_2dm"
:
{
"acc"
:
0.256
,
"acc_stderr"
:
0.009761129023832868
},
"arithmetic_2ds"
:
{
"acc"
:
0.7875
,
"acc_stderr"
:
0.009149520854445372
},
"arithmetic_3da"
:
{
"acc"
:
0.2915
,
"acc_stderr"
:
0.010164424861564223
},
"arithmetic_3ds"
:
{
"acc"
:
0.428
,
"acc_stderr"
:
0.011066581884995264
},
"arithmetic_4da"
:
{
"acc"
:
0.026
,
"acc_stderr"
:
0.0035592603398856676
},
"arithmetic_4ds"
:
{
"acc"
:
0.026
,
"acc_stderr"
:
0.003559260339885677
},
"arithmetic_5da"
:
{
"acc"
:
0.0045
,
"acc_stderr"
:
0.0014969954902233202
},
"arithmetic_5ds"
:
{
"acc"
:
0.002
,
"acc_stderr"
:
0.0009992493430694993
}
},
"versions"
:
{
"arithmetic_1dc"
:
0
,
"arithmetic_2da"
:
0
,
"arithmetic_2dm"
:
0
,
"arithmetic_2ds"
:
0
,
"arithmetic_3da"
:
0
,
"arithmetic_3ds"
:
0
,
"arithmetic_4da"
:
0
,
"arithmetic_4ds"
:
0
,
"arithmetic_5da"
:
0
,
"arithmetic_5ds"
:
0
},
"config"
:
{
"model"
:
"hf-causal-experimental"
,
"model_args"
:
"pretrained=mosaicml/mpt-7b,trust_remote_code=True"
,
"num_fewshot"
:
5
,
"batch_size"
:
"auto"
,
"device"
:
"cuda"
,
"no_cache"
:
true
,
"limit"
:
null
,
"bootstrap_iters"
:
100000
,
"description_dict"
:
{}
}
}
results/mpt/mpt-7b/mpt-7b_bbh_3-shot.json
deleted
100644 → 0
View file @
465c695b
{
"results"
:
{
"bigbench_causal_judgement"
:
{
"multiple_choice_grade"
:
0.5631578947368421
,
"multiple_choice_grade_stderr"
:
0.036078330444807245
},
"bigbench_date_understanding"
:
{
"multiple_choice_grade"
:
0.5826558265582655
,
"multiple_choice_grade_stderr"
:
0.025705692903559226
},
"bigbench_disambiguation_qa"
:
{
"multiple_choice_grade"
:
0.3643410852713178
,
"multiple_choice_grade_stderr"
:
0.03001923241206336
},
"bigbench_dyck_languages"
:
{
"multiple_choice_grade"
:
0.123
,
"multiple_choice_grade_stderr"
:
0.010391293421849874
},
"bigbench_formal_fallacies_syllogisms_negation"
:
{
"multiple_choice_grade"
:
0.49915492957746477
,
"multiple_choice_grade_stderr"
:
0.0041960485493055645
},
"bigbench_geometric_shapes"
:
{
"multiple_choice_grade"
:
0.20334261838440112
,
"multiple_choice_grade_stderr"
:
0.021272007856536258
,
"exact_str_match"
:
0.12256267409470752
,
"exact_str_match_stderr"
:
0.017331879192703025
},
"bigbench_hyperbaton"
:
{
"multiple_choice_grade"
:
0.4936
,
"multiple_choice_grade_stderr"
:
0.002235907150490653
},
"bigbench_logical_deduction_five_objects"
:
{
"multiple_choice_grade"
:
0.24
,
"multiple_choice_grade_stderr"
:
0.01911886665375975
},
"bigbench_logical_deduction_seven_objects"
:
{
"multiple_choice_grade"
:
0.1657142857142857
,
"multiple_choice_grade_stderr"
:
0.014063673984033173
},
"bigbench_logical_deduction_three_objects"
:
{
"multiple_choice_grade"
:
0.38666666666666666
,
"multiple_choice_grade_stderr"
:
0.028163138908196852
},
"bigbench_movie_recommendation"
:
{
"multiple_choice_grade"
:
0.438
,
"multiple_choice_grade_stderr"
:
0.022210326363977417
},
"bigbench_navigate"
:
{
"multiple_choice_grade"
:
0.486
,
"multiple_choice_grade_stderr"
:
0.01581309754773099
},
"bigbench_reasoning_about_colored_objects"
:
{
"multiple_choice_grade"
:
0.2985
,
"multiple_choice_grade_stderr"
:
0.010234805842091589
},
"bigbench_ruin_names"
:
{
"multiple_choice_grade"
:
0.296875
,
"multiple_choice_grade_stderr"
:
0.021609729061250887
},
"bigbench_salient_translation_error_detection"
:
{
"multiple_choice_grade"
:
0.17935871743486975
,
"multiple_choice_grade_stderr"
:
0.012150393578288319
},
"bigbench_snarks"
:
{
"multiple_choice_grade"
:
0.5303867403314917
,
"multiple_choice_grade_stderr"
:
0.03719891321680327
},
"bigbench_sports_understanding"
:
{
"multiple_choice_grade"
:
0.4949290060851927
,
"multiple_choice_grade_stderr"
:
0.015930505328489487
},
"bigbench_temporal_sequences"
:
{
"multiple_choice_grade"
:
0.296
,
"multiple_choice_grade_stderr"
:
0.014442734941575016
},
"bigbench_tracking_shuffled_objects_five_objects"
:
{
"multiple_choice_grade"
:
0.1944
,
"multiple_choice_grade_stderr"
:
0.011197643581460408
},
"bigbench_tracking_shuffled_objects_seven_objects"
:
{
"multiple_choice_grade"
:
0.13428571428571429
,
"multiple_choice_grade_stderr"
:
0.008152809490408933
},
"bigbench_tracking_shuffled_objects_three_objects"
:
{
"multiple_choice_grade"
:
0.38666666666666666
,
"multiple_choice_grade_stderr"
:
0.028163138908196852
}
},
"versions"
:
{
"bigbench_causal_judgement"
:
0
,
"bigbench_date_understanding"
:
0
,
"bigbench_disambiguation_qa"
:
0
,
"bigbench_dyck_languages"
:
0
,
"bigbench_formal_fallacies_syllogisms_negation"
:
0
,
"bigbench_geometric_shapes"
:
0
,
"bigbench_hyperbaton"
:
0
,
"bigbench_logical_deduction_five_objects"
:
0
,
"bigbench_logical_deduction_seven_objects"
:
0
,
"bigbench_logical_deduction_three_objects"
:
0
,
"bigbench_movie_recommendation"
:
0
,
"bigbench_navigate"
:
0
,
"bigbench_reasoning_about_colored_objects"
:
0
,
"bigbench_ruin_names"
:
0
,
"bigbench_salient_translation_error_detection"
:
0
,
"bigbench_snarks"
:
0
,
"bigbench_sports_understanding"
:
0
,
"bigbench_temporal_sequences"
:
0
,
"bigbench_tracking_shuffled_objects_five_objects"
:
0
,
"bigbench_tracking_shuffled_objects_seven_objects"
:
0
,
"bigbench_tracking_shuffled_objects_three_objects"
:
0
},
"config"
:
{
"model"
:
"hf-causal-experimental"
,
"model_args"
:
"pretrained=mosaicml/mpt-7b,trust_remote_code=True"
,
"num_fewshot"
:
3
,
"batch_size"
:
"auto"
,
"device"
:
"cuda"
,
"no_cache"
:
true
,
"limit"
:
null
,
"bootstrap_iters"
:
100000
,
"description_dict"
:
{}
}
}
results/mpt/mpt-7b/mpt-7b_blimp_0-shot.json
deleted
100644 → 0
View file @
465c695b
{
"results"
:
{
"blimp_adjunct_island"
:
{
"acc"
:
0.878
,
"acc_stderr"
:
0.010354864712936724
},
"blimp_anaphor_gender_agreement"
:
{
"acc"
:
0.995
,
"acc_stderr"
:
0.002231586874844882
},
"blimp_anaphor_number_agreement"
:
{
"acc"
:
0.995
,
"acc_stderr"
:
0.0022315868748448817
},
"blimp_animate_subject_passive"
:
{
"acc"
:
0.775
,
"acc_stderr"
:
0.013211720158614753
},
"blimp_animate_subject_trans"
:
{
"acc"
:
0.884
,
"acc_stderr"
:
0.010131468138756997
},
"blimp_causative"
:
{
"acc"
:
0.747
,
"acc_stderr"
:
0.01375427861358708
},
"blimp_complex_NP_island"
:
{
"acc"
:
0.528
,
"acc_stderr"
:
0.01579447578951147
},
"blimp_coordinate_structure_constraint_complex_left_branch"
:
{
"acc"
:
0.779
,
"acc_stderr"
:
0.013127502859696239
},
"blimp_coordinate_structure_constraint_object_extraction"
:
{
"acc"
:
0.848
,
"acc_stderr"
:
0.011358918303475293
},
"blimp_determiner_noun_agreement_1"
:
{
"acc"
:
0.992
,
"acc_stderr"
:
0.002818500300504507
},
"blimp_determiner_noun_agreement_2"
:
{
"acc"
:
0.974
,
"acc_stderr"
:
0.0050348137353182195
},
"blimp_determiner_noun_agreement_irregular_1"
:
{
"acc"
:
0.936
,
"acc_stderr"
:
0.007743640226919308
},
"blimp_determiner_noun_agreement_irregular_2"
:
{
"acc"
:
0.928
,
"acc_stderr"
:
0.008178195576218681
},
"blimp_determiner_noun_agreement_with_adj_2"
:
{
"acc"
:
0.935
,
"acc_stderr"
:
0.007799733061832037
},
"blimp_determiner_noun_agreement_with_adj_irregular_1"
:
{
"acc"
:
0.883
,
"acc_stderr"
:
0.010169287802713327
},
"blimp_determiner_noun_agreement_with_adj_irregular_2"
:
{
"acc"
:
0.919
,
"acc_stderr"
:
0.008632121032139973
},
"blimp_determiner_noun_agreement_with_adjective_1"
:
{
"acc"
:
0.972
,
"acc_stderr"
:
0.005219506034410047
},
"blimp_distractor_agreement_relational_noun"
:
{
"acc"
:
0.889
,
"acc_stderr"
:
0.009938701010583726
},
"blimp_distractor_agreement_relative_clause"
:
{
"acc"
:
0.743
,
"acc_stderr"
:
0.013825416526895042
},
"blimp_drop_argument"
:
{
"acc"
:
0.787
,
"acc_stderr"
:
0.012953717566737234
},
"blimp_ellipsis_n_bar_1"
:
{
"acc"
:
0.79
,
"acc_stderr"
:
0.012886662332274522
},
"blimp_ellipsis_n_bar_2"
:
{
"acc"
:
0.928
,
"acc_stderr"
:
0.008178195576218681
},
"blimp_existential_there_object_raising"
:
{
"acc"
:
0.834
,
"acc_stderr"
:
0.01177211037081219
},
"blimp_existential_there_quantifiers_1"
:
{
"acc"
:
0.988
,
"acc_stderr"
:
0.003444977194099841
},
"blimp_existential_there_quantifiers_2"
:
{
"acc"
:
0.27
,
"acc_stderr"
:
0.014046255632633908
},
"blimp_existential_there_subject_raising"
:
{
"acc"
:
0.888
,
"acc_stderr"
:
0.00997775303139724
},
"blimp_expletive_it_object_raising"
:
{
"acc"
:
0.8
,
"acc_stderr"
:
0.012655439943366646
},
"blimp_inchoative"
:
{
"acc"
:
0.673
,
"acc_stderr"
:
0.01484221315341125
},
"blimp_intransitive"
:
{
"acc"
:
0.832
,
"acc_stderr"
:
0.01182860583145426
},
"blimp_irregular_past_participle_adjectives"
:
{
"acc"
:
0.972
,
"acc_stderr"
:
0.005219506034410053
},
"blimp_irregular_past_participle_verbs"
:
{
"acc"
:
0.885
,
"acc_stderr"
:
0.010093407594904614
},
"blimp_irregular_plural_subject_verb_agreement_1"
:
{
"acc"
:
0.92
,
"acc_stderr"
:
0.008583336977753651
},
"blimp_irregular_plural_subject_verb_agreement_2"
:
{
"acc"
:
0.908
,
"acc_stderr"
:
0.009144376393151112
},
"blimp_left_branch_island_echo_question"
:
{
"acc"
:
0.432
,
"acc_stderr"
:
0.01567232023733621
},
"blimp_left_branch_island_simple_question"
:
{
"acc"
:
0.897
,
"acc_stderr"
:
0.009616833339695798
},
"blimp_matrix_question_npi_licensor_present"
:
{
"acc"
:
0.705
,
"acc_stderr"
:
0.014428554438445514
},
"blimp_npi_present_1"
:
{
"acc"
:
0.579
,
"acc_stderr"
:
0.015620595475301318
},
"blimp_npi_present_2"
:
{
"acc"
:
0.688
,
"acc_stderr"
:
0.01465847437050901
},
"blimp_only_npi_licensor_present"
:
{
"acc"
:
0.916
,
"acc_stderr"
:
0.008776162089491096
},
"blimp_only_npi_scope"
:
{
"acc"
:
0.732
,
"acc_stderr"
:
0.014013292702729491
},
"blimp_passive_1"
:
{
"acc"
:
0.887
,
"acc_stderr"
:
0.010016552866696863
},
"blimp_passive_2"
:
{
"acc"
:
0.895
,
"acc_stderr"
:
0.009698921026024947
},
"blimp_principle_A_c_command"
:
{
"acc"
:
0.75
,
"acc_stderr"
:
0.013699915608779773
},
"blimp_principle_A_case_1"
:
{
"acc"
:
1.0
,
"acc_stderr"
:
0.0
},
"blimp_principle_A_case_2"
:
{
"acc"
:
0.94
,
"acc_stderr"
:
0.007513751157474928
},
"blimp_principle_A_domain_1"
:
{
"acc"
:
0.997
,
"acc_stderr"
:
0.0017303161543469417
},
"blimp_principle_A_domain_2"
:
{
"acc"
:
0.828
,
"acc_stderr"
:
0.011939788882495321
},
"blimp_principle_A_domain_3"
:
{
"acc"
:
0.761
,
"acc_stderr"
:
0.013493000446937591
},
"blimp_principle_A_reconstruction"
:
{
"acc"
:
0.41
,
"acc_stderr"
:
0.015560917136921646
},
"blimp_regular_plural_subject_verb_agreement_1"
:
{
"acc"
:
0.971
,
"acc_stderr"
:
0.005309160685756986
},
"blimp_regular_plural_subject_verb_agreement_2"
:
{
"acc"
:
0.907
,
"acc_stderr"
:
0.009188875634996659
},
"blimp_sentential_negation_npi_licensor_present"
:
{
"acc"
:
0.989
,
"acc_stderr"
:
0.0032999833166078166
},
"blimp_sentential_negation_npi_scope"
:
{
"acc"
:
0.733
,
"acc_stderr"
:
0.013996674851796264
},
"blimp_sentential_subject_island"
:
{
"acc"
:
0.399
,
"acc_stderr"
:
0.015493193313162906
},
"blimp_superlative_quantifiers_1"
:
{
"acc"
:
0.822
,
"acc_stderr"
:
0.012102167676183596
},
"blimp_superlative_quantifiers_2"
:
{
"acc"
:
0.897
,
"acc_stderr"
:
0.009616833339695796
},
"blimp_tough_vs_raising_1"
:
{
"acc"
:
0.69
,
"acc_stderr"
:
0.014632638658632898
},
"blimp_tough_vs_raising_2"
:
{
"acc"
:
0.829
,
"acc_stderr"
:
0.011912216456264606
},
"blimp_transitive"
:
{
"acc"
:
0.872
,
"acc_stderr"
:
0.010570133761108658
},
"blimp_wh_island"
:
{
"acc"
:
0.813
,
"acc_stderr"
:
0.012336254828074126
},
"blimp_wh_questions_object_gap"
:
{
"acc"
:
0.763
,
"acc_stderr"
:
0.01345407046257794
},
"blimp_wh_questions_subject_gap"
:
{
"acc"
:
0.89
,
"acc_stderr"
:
0.009899393819724435
},
"blimp_wh_questions_subject_gap_long_distance"
:
{
"acc"
:
0.893
,
"acc_stderr"
:
0.009779910359847169
},
"blimp_wh_vs_that_no_gap"
:
{
"acc"
:
0.946
,
"acc_stderr"
:
0.007150883521295428
},
"blimp_wh_vs_that_no_gap_long_distance"
:
{
"acc"
:
0.951
,
"acc_stderr"
:
0.006829761756140921
},
"blimp_wh_vs_that_with_gap"
:
{
"acc"
:
0.321
,
"acc_stderr"
:
0.01477082181793466
},
"blimp_wh_vs_that_with_gap_long_distance"
:
{
"acc"
:
0.292
,
"acc_stderr"
:
0.014385511563477347
}
},
"versions"
:
{
"blimp_adjunct_island"
:
0
,
"blimp_anaphor_gender_agreement"
:
0
,
"blimp_anaphor_number_agreement"
:
0
,
"blimp_animate_subject_passive"
:
0
,
"blimp_animate_subject_trans"
:
0
,
"blimp_causative"
:
0
,
"blimp_complex_NP_island"
:
0
,
"blimp_coordinate_structure_constraint_complex_left_branch"
:
0
,
"blimp_coordinate_structure_constraint_object_extraction"
:
0
,
"blimp_determiner_noun_agreement_1"
:
0
,
"blimp_determiner_noun_agreement_2"
:
0
,
"blimp_determiner_noun_agreement_irregular_1"
:
0
,
"blimp_determiner_noun_agreement_irregular_2"
:
0
,
"blimp_determiner_noun_agreement_with_adj_2"
:
0
,
"blimp_determiner_noun_agreement_with_adj_irregular_1"
:
0
,
"blimp_determiner_noun_agreement_with_adj_irregular_2"
:
0
,
"blimp_determiner_noun_agreement_with_adjective_1"
:
0
,
"blimp_distractor_agreement_relational_noun"
:
0
,
"blimp_distractor_agreement_relative_clause"
:
0
,
"blimp_drop_argument"
:
0
,
"blimp_ellipsis_n_bar_1"
:
0
,
"blimp_ellipsis_n_bar_2"
:
0
,
"blimp_existential_there_object_raising"
:
0
,
"blimp_existential_there_quantifiers_1"
:
0
,
"blimp_existential_there_quantifiers_2"
:
0
,
"blimp_existential_there_subject_raising"
:
0
,
"blimp_expletive_it_object_raising"
:
0
,
"blimp_inchoative"
:
0
,
"blimp_intransitive"
:
0
,
"blimp_irregular_past_participle_adjectives"
:
0
,
"blimp_irregular_past_participle_verbs"
:
0
,
"blimp_irregular_plural_subject_verb_agreement_1"
:
0
,
"blimp_irregular_plural_subject_verb_agreement_2"
:
0
,
"blimp_left_branch_island_echo_question"
:
0
,
"blimp_left_branch_island_simple_question"
:
0
,
"blimp_matrix_question_npi_licensor_present"
:
0
,
"blimp_npi_present_1"
:
0
,
"blimp_npi_present_2"
:
0
,
"blimp_only_npi_licensor_present"
:
0
,
"blimp_only_npi_scope"
:
0
,
"blimp_passive_1"
:
0
,
"blimp_passive_2"
:
0
,
"blimp_principle_A_c_command"
:
0
,
"blimp_principle_A_case_1"
:
0
,
"blimp_principle_A_case_2"
:
0
,
"blimp_principle_A_domain_1"
:
0
,
"blimp_principle_A_domain_2"
:
0
,
"blimp_principle_A_domain_3"
:
0
,
"blimp_principle_A_reconstruction"
:
0
,
"blimp_regular_plural_subject_verb_agreement_1"
:
0
,
"blimp_regular_plural_subject_verb_agreement_2"
:
0
,
"blimp_sentential_negation_npi_licensor_present"
:
0
,
"blimp_sentential_negation_npi_scope"
:
0
,
"blimp_sentential_subject_island"
:
0
,
"blimp_superlative_quantifiers_1"
:
0
,
"blimp_superlative_quantifiers_2"
:
0
,
"blimp_tough_vs_raising_1"
:
0
,
"blimp_tough_vs_raising_2"
:
0
,
"blimp_transitive"
:
0
,
"blimp_wh_island"
:
0
,
"blimp_wh_questions_object_gap"
:
0
,
"blimp_wh_questions_subject_gap"
:
0
,
"blimp_wh_questions_subject_gap_long_distance"
:
0
,
"blimp_wh_vs_that_no_gap"
:
0
,
"blimp_wh_vs_that_no_gap_long_distance"
:
0
,
"blimp_wh_vs_that_with_gap"
:
0
,
"blimp_wh_vs_that_with_gap_long_distance"
:
0
},
"config"
:
{
"model"
:
"hf-causal-experimental"
,
"model_args"
:
"pretrained=mosaicml/mpt-7b,trust_remote_code=True"
,
"num_fewshot"
:
0
,
"batch_size"
:
"auto"
,
"device"
:
"cuda"
,
"no_cache"
:
true
,
"limit"
:
null
,
"bootstrap_iters"
:
100000
,
"description_dict"
:
{}
}
}
results/mpt/mpt-7b/mpt-7b_common_sense_reasoning_0-shot.json
deleted
100644 → 0
View file @
465c695b
{
"results"
:
{
"arc_challenge"
:
{
"acc"
:
0.4061433447098976
,
"acc_stderr"
:
0.014351656690097862
,
"acc_norm"
:
0.4180887372013652
,
"acc_norm_stderr"
:
0.01441398839699608
},
"arc_easy"
:
{
"acc"
:
0.7487373737373737
,
"acc_stderr"
:
0.008900141191221648
,
"acc_norm"
:
0.7028619528619529
,
"acc_norm_stderr"
:
0.009377397867796849
},
"boolq"
:
{
"acc"
:
0.7351681957186544
,
"acc_stderr"
:
0.007717399182659714
},
"copa"
:
{
"acc"
:
0.85
,
"acc_stderr"
:
0.03588702812826373
},
"hellaswag"
:
{
"acc"
:
0.5723959370643298
,
"acc_stderr"
:
0.004937199759947679
,
"acc_norm"
:
0.761202947619996
,
"acc_norm_stderr"
:
0.004254771367531344
},
"mc_taco"
:
{
"em"
:
0.13513513513513514
,
"f1"
:
0.45480193909643063
},
"openbookqa"
:
{
"acc"
:
0.32
,
"acc_stderr"
:
0.020882340488761805
,
"acc_norm"
:
0.426
,
"acc_norm_stderr"
:
0.022136577335085637
},
"piqa"
:
{
"acc"
:
0.7916213275299239
,
"acc_stderr"
:
0.009476125383049457
,
"acc_norm"
:
0.8041349292709467
,
"acc_norm_stderr"
:
0.009259518041395765
},
"prost"
:
{
"acc"
:
0.25731212638770284
,
"acc_stderr"
:
0.0031937906462958443
,
"acc_norm"
:
0.30118488471391974
,
"acc_norm_stderr"
:
0.003351744324251047
},
"swag"
:
{
"acc"
:
0.5616814955513346
,
"acc_stderr"
:
0.0035080896485241934
,
"acc_norm"
:
0.7579726082175348
,
"acc_norm_stderr"
:
0.003028236139561354
},
"winogrande"
:
{
"acc"
:
0.6866614048934491
,
"acc_stderr"
:
0.013036512096747986
},
"wsc273"
:
{
"acc"
:
0.8571428571428571
,
"acc_stderr"
:
0.021217447349500165
}
},
"versions"
:
{
"arc_challenge"
:
0
,
"arc_easy"
:
0
,
"boolq"
:
1
,
"copa"
:
0
,
"hellaswag"
:
0
,
"mc_taco"
:
0
,
"openbookqa"
:
0
,
"piqa"
:
0
,
"prost"
:
0
,
"swag"
:
0
,
"winogrande"
:
0
,
"wsc273"
:
0
},
"config"
:
{
"model"
:
"hf-causal-experimental"
,
"model_args"
:
"pretrained=mosaicml/mpt-7b,trust_remote_code=True"
,
"num_fewshot"
:
0
,
"batch_size"
:
"auto"
,
"device"
:
"cuda"
,
"no_cache"
:
true
,
"limit"
:
null
,
"bootstrap_iters"
:
100000
,
"description_dict"
:
{}
}
}
results/mpt/mpt-7b/mpt-7b_glue_0-shot.json
deleted
100644 → 0
View file @
465c695b
{
"results"
:
{
"cola"
:
{
"mcc"
:
-0.044109532013985
,
"mcc_stderr"
:
0.031207420387242075
},
"mnli"
:
{
"acc"
:
0.3782985226693836
,
"acc_stderr"
:
0.0048953668826959605
},
"mnli_mismatched"
:
{
"acc"
:
0.37489829129373475
,
"acc_stderr"
:
0.004882398521925481
},
"mrpc"
:
{
"acc"
:
0.6299019607843137
,
"acc_stderr"
:
0.023933029030729185
,
"f1"
:
0.7560581583198709
,
"f1_stderr"
:
0.019295620717095126
},
"qnli"
:
{
"acc"
:
0.513454146073586
,
"acc_stderr"
:
0.006762960839582666
},
"qqp"
:
{
"acc"
:
0.5035864457086322
,
"acc_stderr"
:
0.002486636660034357
,
"f1"
:
0.5414038936111873
,
"f1_stderr"
:
0.0028706161186533327
},
"rte"
:
{
"acc"
:
0.6389891696750902
,
"acc_stderr"
:
0.028910281676964168
},
"sst"
:
{
"acc"
:
0.768348623853211
,
"acc_stderr"
:
0.014295110726150017
},
"wnli"
:
{
"acc"
:
0.4788732394366197
,
"acc_stderr"
:
0.05970805879899505
}
},
"versions"
:
{
"cola"
:
0
,
"mnli"
:
0
,
"mnli_mismatched"
:
0
,
"mrpc"
:
0
,
"qnli"
:
0
,
"qqp"
:
0
,
"rte"
:
0
,
"sst"
:
0
,
"wnli"
:
1
},
"config"
:
{
"model"
:
"hf-causal-experimental"
,
"model_args"
:
"pretrained=mosaicml/mpt-7b,trust_remote_code=True"
,
"num_fewshot"
:
0
,
"batch_size"
:
"auto"
,
"device"
:
"cuda"
,
"no_cache"
:
true
,
"limit"
:
null
,
"bootstrap_iters"
:
100000
,
"description_dict"
:
{}
}
}
results/mpt/mpt-7b/mpt-7b_human_alignment_0-shot.json
deleted
100644 → 0
View file @
465c695b
{
"results"
:
{
"crows_pairs_english_age"
:
{
"likelihood_difference"
:
4.1510989010989015
,
"likelihood_difference_stderr"
:
0.38322042294913006
,
"pct_stereotype"
:
0.7362637362637363
,
"pct_stereotype_stderr"
:
0.04644942852497395
},
"crows_pairs_english_autre"
:
{
"likelihood_difference"
:
5.056818181818182
,
"likelihood_difference_stderr"
:
1.7702853227564594
,
"pct_stereotype"
:
0.7272727272727273
,
"pct_stereotype_stderr"
:
0.14083575804390605
},
"crows_pairs_english_disability"
:
{
"likelihood_difference"
:
6.019230769230769
,
"likelihood_difference_stderr"
:
0.6330801651940536
,
"pct_stereotype"
:
0.7692307692307693
,
"pct_stereotype_stderr"
:
0.05266563052934291
},
"crows_pairs_english_gender"
:
{
"likelihood_difference"
:
2.682421875
,
"likelihood_difference_stderr"
:
0.17011984636779479
,
"pct_stereotype"
:
0.6375
,
"pct_stereotype_stderr"
:
0.02691527109619775
},
"crows_pairs_english_nationality"
:
{
"likelihood_difference"
:
3.498263888888889
,
"likelihood_difference_stderr"
:
0.21512426671182808
,
"pct_stereotype"
:
0.6157407407407407
,
"pct_stereotype_stderr"
:
0.03317354514310742
},
"crows_pairs_english_physical_appearance"
:
{
"likelihood_difference"
:
3.7378472222222223
,
"likelihood_difference_stderr"
:
0.3384578916047944
,
"pct_stereotype"
:
0.7222222222222222
,
"pct_stereotype_stderr"
:
0.053156331218399945
},
"crows_pairs_english_race_color"
:
{
"likelihood_difference"
:
3.3619586614173227
,
"likelihood_difference_stderr"
:
0.14103384370541044
,
"pct_stereotype"
:
0.5728346456692913
,
"pct_stereotype_stderr"
:
0.021968918082519016
},
"crows_pairs_english_religion"
:
{
"likelihood_difference"
:
3.6644144144144146
,
"likelihood_difference_stderr"
:
0.33863382204528913
,
"pct_stereotype"
:
0.7297297297297297
,
"pct_stereotype_stderr"
:
0.04234321361084538
},
"crows_pairs_english_sexual_orientation"
:
{
"likelihood_difference"
:
4.630376344086022
,
"likelihood_difference_stderr"
:
0.4574502494208299
,
"pct_stereotype"
:
0.8279569892473119
,
"pct_stereotype_stderr"
:
0.03934852812061865
},
"crows_pairs_english_socioeconomic"
:
{
"likelihood_difference"
:
4.065131578947368
,
"likelihood_difference_stderr"
:
0.23523235884669547
,
"pct_stereotype"
:
0.6789473684210526
,
"pct_stereotype_stderr"
:
0.03396059335824887
},
"crows_pairs_french_age"
:
{
"likelihood_difference"
:
3.609722222222222
,
"likelihood_difference_stderr"
:
0.3615396777446631
,
"pct_stereotype"
:
0.4222222222222222
,
"pct_stereotype_stderr"
:
0.05235473399540658
},
"crows_pairs_french_autre"
:
{
"likelihood_difference"
:
2.6923076923076925
,
"likelihood_difference_stderr"
:
0.9229767573858816
,
"pct_stereotype"
:
0.6153846153846154
,
"pct_stereotype_stderr"
:
0.14044168141158106
},
"crows_pairs_french_disability"
:
{
"likelihood_difference"
:
4.958333333333333
,
"likelihood_difference_stderr"
:
0.42689940749326905
,
"pct_stereotype"
:
0.6363636363636364
,
"pct_stereotype_stderr"
:
0.05966637484671758
},
"crows_pairs_french_gender"
:
{
"likelihood_difference"
:
3.213785046728972
,
"likelihood_difference_stderr"
:
0.17588386956758606
,
"pct_stereotype"
:
0.5109034267912772
,
"pct_stereotype_stderr"
:
0.027944203070818633
},
"crows_pairs_french_nationality"
:
{
"likelihood_difference"
:
3.883399209486166
,
"likelihood_difference_stderr"
:
0.21844567559439967
,
"pct_stereotype"
:
0.3438735177865613
,
"pct_stereotype_stderr"
:
0.029922155720849428
},
"crows_pairs_french_physical_appearance"
:
{
"likelihood_difference"
:
3.2274305555555554
,
"likelihood_difference_stderr"
:
0.4328667471914375
,
"pct_stereotype"
:
0.5972222222222222
,
"pct_stereotype_stderr"
:
0.05820650942569533
},
"crows_pairs_french_race_color"
:
{
"likelihood_difference"
:
3.161413043478261
,
"likelihood_difference_stderr"
:
0.16557903974411925
,
"pct_stereotype"
:
0.4369565217391304
,
"pct_stereotype_stderr"
:
0.023151745316873387
},
"crows_pairs_french_religion"
:
{
"likelihood_difference"
:
3.5673913043478263
,
"likelihood_difference_stderr"
:
0.3368331015818195
,
"pct_stereotype"
:
0.6260869565217392
,
"pct_stereotype_stderr"
:
0.045315858286449635
},
"crows_pairs_french_sexual_orientation"
:
{
"likelihood_difference"
:
4.791208791208791
,
"likelihood_difference_stderr"
:
0.4009539855629619
,
"pct_stereotype"
:
0.7802197802197802
,
"pct_stereotype_stderr"
:
0.043649726328985346
},
"crows_pairs_french_socioeconomic"
:
{
"likelihood_difference"
:
3.9939413265306123
,
"likelihood_difference_stderr"
:
0.263093158126228
,
"pct_stereotype"
:
0.6581632653061225
,
"pct_stereotype_stderr"
:
0.033967132039868675
},
"ethics_cm"
:
{
"acc"
:
0.5459459459459459
,
"acc_stderr"
:
0.007988936899457039
},
"ethics_deontology"
:
{
"acc"
:
0.5025027808676307
,
"acc_stderr"
:
0.008339021933755771
,
"em"
:
0.004449388209121246
},
"ethics_justice"
:
{
"acc"
:
0.5196005917159763
,
"acc_stderr"
:
0.009609770755397009
,
"em"
:
0.011834319526627219
},
"ethics_utilitarianism"
:
{
"acc"
:
0.5748752079866889
,
"acc_stderr"
:
0.007130302336230959
},
"ethics_utilitarianism_original"
:
{
"acc"
:
0.9956322795341098
,
"acc_stderr"
:
0.000951129914345755
},
"ethics_virtue"
:
{
"acc"
:
0.8040201005025126
,
"acc_stderr"
:
0.005628417801676332
,
"em"
:
0.12562814070351758
},
"toxigen"
:
{
"acc"
:
0.4319148936170213
,
"acc_stderr"
:
0.016164899004911828
,
"acc_norm"
:
0.4319148936170213
,
"acc_norm_stderr"
:
0.016164899004911828
}
},
"versions"
:
{
"crows_pairs_english_age"
:
0
,
"crows_pairs_english_autre"
:
0
,
"crows_pairs_english_disability"
:
0
,
"crows_pairs_english_gender"
:
0
,
"crows_pairs_english_nationality"
:
0
,
"crows_pairs_english_physical_appearance"
:
0
,
"crows_pairs_english_race_color"
:
0
,
"crows_pairs_english_religion"
:
0
,
"crows_pairs_english_sexual_orientation"
:
0
,
"crows_pairs_english_socioeconomic"
:
0
,
"crows_pairs_french_age"
:
0
,
"crows_pairs_french_autre"
:
0
,
"crows_pairs_french_disability"
:
0
,
"crows_pairs_french_gender"
:
0
,
"crows_pairs_french_nationality"
:
0
,
"crows_pairs_french_physical_appearance"
:
0
,
"crows_pairs_french_race_color"
:
0
,
"crows_pairs_french_religion"
:
0
,
"crows_pairs_french_sexual_orientation"
:
0
,
"crows_pairs_french_socioeconomic"
:
0
,
"ethics_cm"
:
0
,
"ethics_deontology"
:
0
,
"ethics_justice"
:
0
,
"ethics_utilitarianism"
:
0
,
"ethics_utilitarianism_original"
:
0
,
"ethics_virtue"
:
0
,
"toxigen"
:
0
},
"config"
:
{
"model"
:
"hf-causal-experimental"
,
"model_args"
:
"pretrained=mosaicml/mpt-7b,trust_remote_code=True"
,
"num_fewshot"
:
0
,
"batch_size"
:
"auto"
,
"device"
:
"cuda"
,
"no_cache"
:
true
,
"limit"
:
null
,
"bootstrap_iters"
:
100000
,
"description_dict"
:
{}
}
}
results/mpt/mpt-7b/mpt-7b_lambada_0-shot.json
deleted
100644 → 0
View file @
465c695b
{
"results"
:
{
"lambada_openai"
:
{
"ppl"
:
3.8685393479651173
,
"ppl_stderr"
:
0.08079962928213236
,
"acc"
:
0.6834853483407723
,
"acc_stderr"
:
0.006479978824925181
},
"lambada_openai_cloze"
:
{
"ppl"
:
26.557641771948866
,
"ppl_stderr"
:
0.7040059319391023
,
"acc"
:
0.3964680768484378
,
"acc_stderr"
:
0.006815007030417622
},
"lambada_openai_mt_de"
:
{
"ppl"
:
70.11746381055114
,
"ppl_stderr"
:
4.044349022196945
,
"acc"
:
0.33766737822627596
,
"acc_stderr"
:
0.006588623616680426
},
"lambada_openai_mt_en"
:
{
"ppl"
:
3.8685393479651173
,
"ppl_stderr"
:
0.08079962928213236
,
"acc"
:
0.6834853483407723
,
"acc_stderr"
:
0.006479978824925181
},
"lambada_openai_mt_es"
:
{
"ppl"
:
67.22873557270539
,
"ppl_stderr"
:
3.6853291413042055
,
"acc"
:
0.3694934989326606
,
"acc_stderr"
:
0.006724504027913079
},
"lambada_openai_mt_fr"
:
{
"ppl"
:
42.93424455960906
,
"ppl_stderr"
:
2.3689294613751506
,
"acc"
:
0.4302348146710654
,
"acc_stderr"
:
0.006897835015074962
},
"lambada_openai_mt_it"
:
{
"ppl"
:
65.76277239773485
,
"ppl_stderr"
:
3.8678074045404967
,
"acc"
:
0.392004657481079
,
"acc_stderr"
:
0.006801548708056975
},
"lambada_standard"
:
{
"ppl"
:
4.922814417244969
,
"ppl_stderr"
:
0.10844800336124351
,
"acc"
:
0.619056860081506
,
"acc_stderr"
:
0.006765617200231529
},
"lambada_standard_cloze"
:
{
"ppl"
:
109.09886744922697
,
"ppl_stderr"
:
3.0351063735026287
,
"acc"
:
0.1674752571317679
,
"acc_stderr"
:
0.005202187177767864
}
},
"versions"
:
{
"lambada_openai"
:
0
,
"lambada_openai_cloze"
:
0
,
"lambada_openai_mt_de"
:
0
,
"lambada_openai_mt_en"
:
0
,
"lambada_openai_mt_es"
:
0
,
"lambada_openai_mt_fr"
:
0
,
"lambada_openai_mt_it"
:
0
,
"lambada_standard"
:
0
,
"lambada_standard_cloze"
:
0
},
"config"
:
{
"model"
:
"hf-causal-experimental"
,
"model_args"
:
"pretrained=mosaicml/mpt-7b,trust_remote_code=True"
,
"num_fewshot"
:
0
,
"batch_size"
:
"auto"
,
"device"
:
"cuda"
,
"no_cache"
:
true
,
"limit"
:
null
,
"bootstrap_iters"
:
100000
,
"description_dict"
:
{}
}
}
results/mpt/mpt-7b/mpt-7b_mmlu_5-shot.json
deleted
100644 → 0
View file @
465c695b
{
"results"
:
{
"hendrycksTest-abstract_algebra"
:
{
"acc"
:
0.18
,
"acc_stderr"
:
0.03861229196653695
,
"acc_norm"
:
0.21
,
"acc_norm_stderr"
:
0.040936018074033256
},
"hendrycksTest-anatomy"
:
{
"acc"
:
0.3851851851851852
,
"acc_stderr"
:
0.042039210401562783
,
"acc_norm"
:
0.37777777777777777
,
"acc_norm_stderr"
:
0.04188307537595853
},
"hendrycksTest-astronomy"
:
{
"acc"
:
0.39473684210526316
,
"acc_stderr"
:
0.039777499346220734
,
"acc_norm"
:
0.42105263157894735
,
"acc_norm_stderr"
:
0.04017901275981748
},
"hendrycksTest-business_ethics"
:
{
"acc"
:
0.49
,
"acc_stderr"
:
0.05024183937956911
,
"acc_norm"
:
0.48
,
"acc_norm_stderr"
:
0.050211673156867795
},
"hendrycksTest-clinical_knowledge"
:
{
"acc"
:
0.3320754716981132
,
"acc_stderr"
:
0.028985455652334388
,
"acc_norm"
:
0.37735849056603776
,
"acc_norm_stderr"
:
0.029832808114796005
},
"hendrycksTest-college_biology"
:
{
"acc"
:
0.3819444444444444
,
"acc_stderr"
:
0.040629907841466674
,
"acc_norm"
:
0.3541666666666667
,
"acc_norm_stderr"
:
0.039994111357535424
},
"hendrycksTest-college_chemistry"
:
{
"acc"
:
0.39
,
"acc_stderr"
:
0.04902071300001974
,
"acc_norm"
:
0.41
,
"acc_norm_stderr"
:
0.049431107042371025
},
"hendrycksTest-college_computer_science"
:
{
"acc"
:
0.34
,
"acc_stderr"
:
0.04760952285695235
,
"acc_norm"
:
0.32
,
"acc_norm_stderr"
:
0.046882617226215034
},
"hendrycksTest-college_mathematics"
:
{
"acc"
:
0.27
,
"acc_stderr"
:
0.044619604333847394
,
"acc_norm"
:
0.33
,
"acc_norm_stderr"
:
0.047258156262526045
},
"hendrycksTest-college_medicine"
:
{
"acc"
:
0.36416184971098264
,
"acc_stderr"
:
0.03669072477416907
,
"acc_norm"
:
0.3468208092485549
,
"acc_norm_stderr"
:
0.036291466701596636
},
"hendrycksTest-college_physics"
:
{
"acc"
:
0.30392156862745096
,
"acc_stderr"
:
0.045766654032077615
,
"acc_norm"
:
0.3333333333333333
,
"acc_norm_stderr"
:
0.04690650298201943
},
"hendrycksTest-computer_security"
:
{
"acc"
:
0.41
,
"acc_stderr"
:
0.04943110704237102
,
"acc_norm"
:
0.41
,
"acc_norm_stderr"
:
0.049431107042371025
},
"hendrycksTest-conceptual_physics"
:
{
"acc"
:
0.3276595744680851
,
"acc_stderr"
:
0.030683020843231015
,
"acc_norm"
:
0.2553191489361702
,
"acc_norm_stderr"
:
0.02850485647051418
},
"hendrycksTest-econometrics"
:
{
"acc"
:
0.2719298245614035
,
"acc_stderr"
:
0.04185774424022056
,
"acc_norm"
:
0.23684210526315788
,
"acc_norm_stderr"
:
0.039994238792813365
},
"hendrycksTest-electrical_engineering"
:
{
"acc"
:
0.36551724137931035
,
"acc_stderr"
:
0.040131241954243856
,
"acc_norm"
:
0.33793103448275863
,
"acc_norm_stderr"
:
0.03941707632064889
},
"hendrycksTest-elementary_mathematics"
:
{
"acc"
:
0.29894179894179895
,
"acc_stderr"
:
0.023577604791655802
,
"acc_norm"
:
0.28835978835978837
,
"acc_norm_stderr"
:
0.023330654054535892
},
"hendrycksTest-formal_logic"
:
{
"acc"
:
0.30952380952380953
,
"acc_stderr"
:
0.04134913018303316
,
"acc_norm"
:
0.2857142857142857
,
"acc_norm_stderr"
:
0.040406101782088394
},
"hendrycksTest-global_facts"
:
{
"acc"
:
0.35
,
"acc_stderr"
:
0.047937248544110196
,
"acc_norm"
:
0.33
,
"acc_norm_stderr"
:
0.047258156262526045
},
"hendrycksTest-high_school_biology"
:
{
"acc"
:
0.36451612903225805
,
"acc_stderr"
:
0.02737987122994325
,
"acc_norm"
:
0.3903225806451613
,
"acc_norm_stderr"
:
0.027751256636969583
},
"hendrycksTest-high_school_chemistry"
:
{
"acc"
:
0.21182266009852216
,
"acc_stderr"
:
0.02874898368994106
,
"acc_norm"
:
0.21674876847290642
,
"acc_norm_stderr"
:
0.028990331252516235
},
"hendrycksTest-high_school_computer_science"
:
{
"acc"
:
0.43
,
"acc_stderr"
:
0.04975698519562428
,
"acc_norm"
:
0.41
,
"acc_norm_stderr"
:
0.049431107042371025
},
"hendrycksTest-high_school_european_history"
:
{
"acc"
:
0.38181818181818183
,
"acc_stderr"
:
0.03793713171165635
,
"acc_norm"
:
0.37575757575757573
,
"acc_norm_stderr"
:
0.03781887353205983
},
"hendrycksTest-high_school_geography"
:
{
"acc"
:
0.3838383838383838
,
"acc_stderr"
:
0.03464881675016339
,
"acc_norm"
:
0.40404040404040403
,
"acc_norm_stderr"
:
0.03496130972056128
},
"hendrycksTest-high_school_government_and_politics"
:
{
"acc"
:
0.41450777202072536
,
"acc_stderr"
:
0.03555300319557673
,
"acc_norm"
:
0.41450777202072536
,
"acc_norm_stderr"
:
0.03555300319557672
},
"hendrycksTest-high_school_macroeconomics"
:
{
"acc"
:
0.3487179487179487
,
"acc_stderr"
:
0.024162780284017717
,
"acc_norm"
:
0.29743589743589743
,
"acc_norm_stderr"
:
0.02317740813146594
},
"hendrycksTest-high_school_mathematics"
:
{
"acc"
:
0.29259259259259257
,
"acc_stderr"
:
0.027738969632176088
,
"acc_norm"
:
0.3037037037037037
,
"acc_norm_stderr"
:
0.02803792996911499
},
"hendrycksTest-high_school_microeconomics"
:
{
"acc"
:
0.33613445378151263
,
"acc_stderr"
:
0.030684737115135353
,
"acc_norm"
:
0.3697478991596639
,
"acc_norm_stderr"
:
0.03135709599613591
},
"hendrycksTest-high_school_physics"
:
{
"acc"
:
0.2781456953642384
,
"acc_stderr"
:
0.03658603262763743
,
"acc_norm"
:
0.2781456953642384
,
"acc_norm_stderr"
:
0.03658603262763743
},
"hendrycksTest-high_school_psychology"
:
{
"acc"
:
0.46972477064220186
,
"acc_stderr"
:
0.021397988604936965
,
"acc_norm"
:
0.44587155963302755
,
"acc_norm_stderr"
:
0.02131133500970858
},
"hendrycksTest-high_school_statistics"
:
{
"acc"
:
0.3287037037037037
,
"acc_stderr"
:
0.03203614084670058
,
"acc_norm"
:
0.32407407407407407
,
"acc_norm_stderr"
:
0.03191923445686185
},
"hendrycksTest-high_school_us_history"
:
{
"acc"
:
0.3431372549019608
,
"acc_stderr"
:
0.03332139944668085
,
"acc_norm"
:
0.3137254901960784
,
"acc_norm_stderr"
:
0.032566854844603886
},
"hendrycksTest-high_school_world_history"
:
{
"acc"
:
0.29535864978902954
,
"acc_stderr"
:
0.029696338713422876
,
"acc_norm"
:
0.2869198312236287
,
"acc_norm_stderr"
:
0.02944377302259469
},
"hendrycksTest-human_aging"
:
{
"acc"
:
0.336322869955157
,
"acc_stderr"
:
0.031708824268455
,
"acc_norm"
:
0.3273542600896861
,
"acc_norm_stderr"
:
0.031493846709941306
},
"hendrycksTest-human_sexuality"
:
{
"acc"
:
0.2748091603053435
,
"acc_stderr"
:
0.03915345408847836
,
"acc_norm"
:
0.3282442748091603
,
"acc_norm_stderr"
:
0.041184385658062976
},
"hendrycksTest-international_law"
:
{
"acc"
:
0.371900826446281
,
"acc_stderr"
:
0.04412015806624504
,
"acc_norm"
:
0.49586776859504134
,
"acc_norm_stderr"
:
0.045641987674327526
},
"hendrycksTest-jurisprudence"
:
{
"acc"
:
0.3425925925925926
,
"acc_stderr"
:
0.045879047413018105
,
"acc_norm"
:
0.39814814814814814
,
"acc_norm_stderr"
:
0.04732332615978814
},
"hendrycksTest-logical_fallacies"
:
{
"acc"
:
0.3803680981595092
,
"acc_stderr"
:
0.038142698932618374
,
"acc_norm"
:
0.36809815950920244
,
"acc_norm_stderr"
:
0.03789213935838395
},
"hendrycksTest-machine_learning"
:
{
"acc"
:
0.26785714285714285
,
"acc_stderr"
:
0.04203277291467763
,
"acc_norm"
:
0.24107142857142858
,
"acc_norm_stderr"
:
0.04059867246952686
},
"hendrycksTest-management"
:
{
"acc"
:
0.42718446601941745
,
"acc_stderr"
:
0.04897957737781169
,
"acc_norm"
:
0.39805825242718446
,
"acc_norm_stderr"
:
0.0484674825397724
},
"hendrycksTest-marketing"
:
{
"acc"
:
0.5512820512820513
,
"acc_stderr"
:
0.032583346493868806
,
"acc_norm"
:
0.5512820512820513
,
"acc_norm_stderr"
:
0.032583346493868806
},
"hendrycksTest-medical_genetics"
:
{
"acc"
:
0.39
,
"acc_stderr"
:
0.04902071300001975
,
"acc_norm"
:
0.38
,
"acc_norm_stderr"
:
0.04878317312145633
},
"hendrycksTest-miscellaneous"
:
{
"acc"
:
0.5555555555555556
,
"acc_stderr"
:
0.017769250583533246
,
"acc_norm"
:
0.5568326947637292
,
"acc_norm_stderr"
:
0.01776408503534841
},
"hendrycksTest-moral_disputes"
:
{
"acc"
:
0.3208092485549133
,
"acc_stderr"
:
0.025131000233647904
,
"acc_norm"
:
0.30057803468208094
,
"acc_norm_stderr"
:
0.024685316867257796
},
"hendrycksTest-moral_scenarios"
:
{
"acc"
:
0.26033519553072626
,
"acc_stderr"
:
0.014676252009319483
,
"acc_norm"
:
0.27262569832402234
,
"acc_norm_stderr"
:
0.014893391735249614
},
"hendrycksTest-nutrition"
:
{
"acc"
:
0.3431372549019608
,
"acc_stderr"
:
0.027184498909941616
,
"acc_norm"
:
0.4019607843137255
,
"acc_norm_stderr"
:
0.02807415894760066
},
"hendrycksTest-philosophy"
:
{
"acc"
:
0.3762057877813505
,
"acc_stderr"
:
0.027513925683549427
,
"acc_norm"
:
0.36977491961414793
,
"acc_norm_stderr"
:
0.027417996705630998
},
"hendrycksTest-prehistory"
:
{
"acc"
:
0.33641975308641975
,
"acc_stderr"
:
0.02628973494595293
,
"acc_norm"
:
0.3055555555555556
,
"acc_norm_stderr"
:
0.025630824975621344
},
"hendrycksTest-professional_accounting"
:
{
"acc"
:
0.3049645390070922
,
"acc_stderr"
:
0.027464708442022135
,
"acc_norm"
:
0.2907801418439716
,
"acc_norm_stderr"
:
0.027090664368353178
},
"hendrycksTest-professional_law"
:
{
"acc"
:
0.25945241199478486
,
"acc_stderr"
:
0.011195262076350299
,
"acc_norm"
:
0.2842242503259452
,
"acc_norm_stderr"
:
0.011519880596516074
},
"hendrycksTest-professional_medicine"
:
{
"acc"
:
0.29411764705882354
,
"acc_stderr"
:
0.027678468642144703
,
"acc_norm"
:
0.3161764705882353
,
"acc_norm_stderr"
:
0.02824568739146291
},
"hendrycksTest-professional_psychology"
:
{
"acc"
:
0.315359477124183
,
"acc_stderr"
:
0.018798086284886883
,
"acc_norm"
:
0.3022875816993464
,
"acc_norm_stderr"
:
0.01857923271111388
},
"hendrycksTest-public_relations"
:
{
"acc"
:
0.41818181818181815
,
"acc_stderr"
:
0.04724577405731571
,
"acc_norm"
:
0.42727272727272725
,
"acc_norm_stderr"
:
0.04738198703545483
},
"hendrycksTest-security_studies"
:
{
"acc"
:
0.2816326530612245
,
"acc_stderr"
:
0.028795185574291282
,
"acc_norm"
:
0.24081632653061225
,
"acc_norm_stderr"
:
0.027372942201788163
},
"hendrycksTest-sociology"
:
{
"acc"
:
0.34328358208955223
,
"acc_stderr"
:
0.03357379665433431
,
"acc_norm"
:
0.3681592039800995
,
"acc_norm_stderr"
:
0.034104105654953025
},
"hendrycksTest-us_foreign_policy"
:
{
"acc"
:
0.38
,
"acc_stderr"
:
0.04878317312145632
,
"acc_norm"
:
0.39
,
"acc_norm_stderr"
:
0.04902071300001975
},
"hendrycksTest-virology"
:
{
"acc"
:
0.3253012048192771
,
"acc_stderr"
:
0.03647168523683229
,
"acc_norm"
:
0.3253012048192771
,
"acc_norm_stderr"
:
0.03647168523683227
},
"hendrycksTest-world_religions"
:
{
"acc"
:
0.543859649122807
,
"acc_stderr"
:
0.03820042586602966
,
"acc_norm"
:
0.5789473684210527
,
"acc_norm_stderr"
:
0.03786720706234214
}
},
"versions"
:
{
"hendrycksTest-abstract_algebra"
:
0
,
"hendrycksTest-anatomy"
:
0
,
"hendrycksTest-astronomy"
:
0
,
"hendrycksTest-business_ethics"
:
0
,
"hendrycksTest-clinical_knowledge"
:
0
,
"hendrycksTest-college_biology"
:
0
,
"hendrycksTest-college_chemistry"
:
0
,
"hendrycksTest-college_computer_science"
:
0
,
"hendrycksTest-college_mathematics"
:
0
,
"hendrycksTest-college_medicine"
:
0
,
"hendrycksTest-college_physics"
:
0
,
"hendrycksTest-computer_security"
:
0
,
"hendrycksTest-conceptual_physics"
:
0
,
"hendrycksTest-econometrics"
:
0
,
"hendrycksTest-electrical_engineering"
:
0
,
"hendrycksTest-elementary_mathematics"
:
0
,
"hendrycksTest-formal_logic"
:
0
,
"hendrycksTest-global_facts"
:
0
,
"hendrycksTest-high_school_biology"
:
0
,
"hendrycksTest-high_school_chemistry"
:
0
,
"hendrycksTest-high_school_computer_science"
:
0
,
"hendrycksTest-high_school_european_history"
:
0
,
"hendrycksTest-high_school_geography"
:
0
,
"hendrycksTest-high_school_government_and_politics"
:
0
,
"hendrycksTest-high_school_macroeconomics"
:
0
,
"hendrycksTest-high_school_mathematics"
:
0
,
"hendrycksTest-high_school_microeconomics"
:
0
,
"hendrycksTest-high_school_physics"
:
0
,
"hendrycksTest-high_school_psychology"
:
0
,
"hendrycksTest-high_school_statistics"
:
0
,
"hendrycksTest-high_school_us_history"
:
0
,
"hendrycksTest-high_school_world_history"
:
0
,
"hendrycksTest-human_aging"
:
0
,
"hendrycksTest-human_sexuality"
:
0
,
"hendrycksTest-international_law"
:
0
,
"hendrycksTest-jurisprudence"
:
0
,
"hendrycksTest-logical_fallacies"
:
0
,
"hendrycksTest-machine_learning"
:
0
,
"hendrycksTest-management"
:
0
,
"hendrycksTest-marketing"
:
0
,
"hendrycksTest-medical_genetics"
:
0
,
"hendrycksTest-miscellaneous"
:
0
,
"hendrycksTest-moral_disputes"
:
0
,
"hendrycksTest-moral_scenarios"
:
0
,
"hendrycksTest-nutrition"
:
0
,
"hendrycksTest-philosophy"
:
0
,
"hendrycksTest-prehistory"
:
0
,
"hendrycksTest-professional_accounting"
:
0
,
"hendrycksTest-professional_law"
:
0
,
"hendrycksTest-professional_medicine"
:
0
,
"hendrycksTest-professional_psychology"
:
0
,
"hendrycksTest-public_relations"
:
0
,
"hendrycksTest-security_studies"
:
0
,
"hendrycksTest-sociology"
:
0
,
"hendrycksTest-us_foreign_policy"
:
0
,
"hendrycksTest-virology"
:
0
,
"hendrycksTest-world_religions"
:
0
},
"config"
:
{
"model"
:
"hf-causal-experimental"
,
"model_args"
:
"pretrained=mosaicml/mpt-7b,trust_remote_code=True,dtype=bfloat16"
,
"num_fewshot"
:
5
,
"batch_size"
:
"auto"
,
"device"
:
"cuda"
,
"no_cache"
:
true
,
"limit"
:
null
,
"bootstrap_iters"
:
100000
,
"description_dict"
:
{}
}
}
results/mpt/mpt-7b/mpt-7b_pawsx_0-shot.json
deleted
100644 → 0
View file @
465c695b
{
"results"
:
{
"pawsx_de"
:
{
"acc"
:
0.614
,
"acc_stderr"
:
0.010888584877766427
},
"pawsx_en"
:
{
"acc"
:
0.7035
,
"acc_stderr"
:
0.010214991337441791
},
"pawsx_es"
:
{
"acc"
:
0.6495
,
"acc_stderr"
:
0.01067154233969731
},
"pawsx_fr"
:
{
"acc"
:
0.6285
,
"acc_stderr"
:
0.01080751017293364
},
"pawsx_ja"
:
{
"acc"
:
0.493
,
"acc_stderr"
:
0.011182040020027768
},
"pawsx_ko"
:
{
"acc"
:
0.5365
,
"acc_stderr"
:
0.011153298751334327
},
"pawsx_zh"
:
{
"acc"
:
0.5625
,
"acc_stderr"
:
0.011095423796079503
}
},
"versions"
:
{
"pawsx_de"
:
0
,
"pawsx_en"
:
0
,
"pawsx_es"
:
0
,
"pawsx_fr"
:
0
,
"pawsx_ja"
:
0
,
"pawsx_ko"
:
0
,
"pawsx_zh"
:
0
},
"config"
:
{
"model"
:
"hf-causal-experimental"
,
"model_args"
:
"pretrained=mosaicml/mpt-7b,trust_remote_code=True,dtype=bfloat16"
,
"num_fewshot"
:
0
,
"batch_size"
:
"auto"
,
"device"
:
"cuda"
,
"no_cache"
:
true
,
"limit"
:
null
,
"bootstrap_iters"
:
100000
,
"description_dict"
:
{}
}
}
results/mpt/mpt-7b/mpt-7b_reading_comprehension_0-shot.json
deleted
100644 → 0
View file @
465c695b
{
"results"
:
{
"coqa"
:
{
"f1"
:
0.7650867255895625
,
"f1_stderr"
:
0.01481717694356494
,
"em"
:
0.6301666666666667
,
"em_stderr"
:
0.018680205213012713
},
"drop"
:
{
"em"
:
0.03429110738255033
,
"em_stderr"
:
0.0018636035184959787
,
"f1"
:
0.1338569630872482
,
"f1_stderr"
:
0.0025165760673094154
},
"race"
:
{
"acc"
:
0.3866028708133971
,
"acc_stderr"
:
0.01507138477304713
}
},
"versions"
:
{
"coqa"
:
1
,
"drop"
:
1
,
"race"
:
1
},
"config"
:
{
"model"
:
"hf-causal-experimental"
,
"model_args"
:
"pretrained=mosaicml/mpt-7b,trust_remote_code=True"
,
"num_fewshot"
:
0
,
"batch_size"
:
"auto"
,
"device"
:
"cuda"
,
"no_cache"
:
true
,
"limit"
:
null
,
"bootstrap_iters"
:
100000
,
"description_dict"
:
{}
}
}
results/mpt/mpt-7b/mpt-7b_superglue_0-shot.json
deleted
100644 → 0
View file @
465c695b
{
"results"
:
{
"boolq"
:
{
"acc"
:
0.7382262996941896
,
"acc_stderr"
:
0.007688653730439849
},
"cb"
:
{
"acc"
:
0.4107142857142857
,
"acc_stderr"
:
0.0663363415035954
,
"f1"
:
0.2126984126984127
},
"copa"
:
{
"acc"
:
0.84
,
"acc_stderr"
:
0.036845294917747115
},
"multirc"
:
{
"acc"
:
0.008394543546694649
,
"acc_stderr"
:
0.002956987733350658
},
"record"
:
{
"f1"
:
0.900992857142857
,
"f1_stderr"
:
0.0029401923751195343
,
"em"
:
0.893
,
"em_stderr"
:
0.0030912870467751773
},
"wic"
:
{
"acc"
:
0.4843260188087774
,
"acc_stderr"
:
0.01980098495534785
},
"wsc"
:
{
"acc"
:
0.6346153846153846
,
"acc_stderr"
:
0.0474473339327792
}
},
"versions"
:
{
"boolq"
:
1
,
"cb"
:
1
,
"copa"
:
0
,
"multirc"
:
1
,
"record"
:
0
,
"wic"
:
0
,
"wsc"
:
0
},
"config"
:
{
"model"
:
"hf-causal-experimental"
,
"model_args"
:
"pretrained=mosaicml/mpt-7b,trust_remote_code=True"
,
"num_fewshot"
:
0
,
"batch_size"
:
"auto"
,
"device"
:
"cuda"
,
"no_cache"
:
true
,
"limit"
:
null
,
"bootstrap_iters"
:
100000
,
"description_dict"
:
{}
}
}
results/mpt/mpt-7b/mpt-7b_unscramble_0-shot.json
deleted
100644 → 0
View file @
465c695b
{
"results"
:
{
"anagrams1"
:
{
"acc"
:
0.0
,
"acc_stderr"
:
0.0
},
"anagrams2"
:
{
"acc"
:
0.0001
,
"acc_stderr"
:
0.0001000000000000119
},
"cycle_letters"
:
{
"acc"
:
0.0
,
"acc_stderr"
:
0.0
},
"random_insertion"
:
{
"acc"
:
0.0004
,
"acc_stderr"
:
0.00019996999474889126
},
"reversed_words"
:
{
"acc"
:
0.0
,
"acc_stderr"
:
0.0
}
},
"versions"
:
{
"anagrams1"
:
0
,
"anagrams2"
:
0
,
"cycle_letters"
:
0
,
"random_insertion"
:
0
,
"reversed_words"
:
0
},
"config"
:
{
"model"
:
"hf-causal-experimental"
,
"model_args"
:
"pretrained=mosaicml/mpt-7b,trust_remote_code=True"
,
"num_fewshot"
:
0
,
"batch_size"
:
"auto"
,
"device"
:
"cuda"
,
"no_cache"
:
true
,
"limit"
:
null
,
"bootstrap_iters"
:
100000
,
"description_dict"
:
{}
}
}
results/mpt/mpt-7b/mpt-7b_xcopa_0-shot.json
deleted
100644 → 0
View file @
465c695b
{
"results"
:
{
"xcopa_et"
:
{
"acc"
:
0.474
,
"acc_stderr"
:
0.02235279165091416
},
"xcopa_ht"
:
{
"acc"
:
0.498
,
"acc_stderr"
:
0.02238289498648353
},
"xcopa_id"
:
{
"acc"
:
0.568
,
"acc_stderr"
:
0.02217510926561316
},
"xcopa_it"
:
{
"acc"
:
0.594
,
"acc_stderr"
:
0.02198396209008634
},
"xcopa_qu"
:
{
"acc"
:
0.484
,
"acc_stderr"
:
0.0223716109825804
},
"xcopa_sw"
:
{
"acc"
:
0.516
,
"acc_stderr"
:
0.0223716109825804
},
"xcopa_ta"
:
{
"acc"
:
0.54
,
"acc_stderr"
:
0.022311333245289663
},
"xcopa_th"
:
{
"acc"
:
0.542
,
"acc_stderr"
:
0.02230396677426995
},
"xcopa_tr"
:
{
"acc"
:
0.516
,
"acc_stderr"
:
0.022371610982580396
},
"xcopa_vi"
:
{
"acc"
:
0.536
,
"acc_stderr"
:
0.022324981738385246
},
"xcopa_zh"
:
{
"acc"
:
0.632
,
"acc_stderr"
:
0.02158898256835354
}
},
"versions"
:
{
"xcopa_et"
:
0
,
"xcopa_ht"
:
0
,
"xcopa_id"
:
0
,
"xcopa_it"
:
0
,
"xcopa_qu"
:
0
,
"xcopa_sw"
:
0
,
"xcopa_ta"
:
0
,
"xcopa_th"
:
0
,
"xcopa_tr"
:
0
,
"xcopa_vi"
:
0
,
"xcopa_zh"
:
0
},
"config"
:
{
"model"
:
"hf-causal-experimental"
,
"model_args"
:
"pretrained=mosaicml/mpt-7b,trust_remote_code=True,dtype=bfloat16"
,
"num_fewshot"
:
0
,
"batch_size"
:
"auto"
,
"device"
:
"cuda"
,
"no_cache"
:
true
,
"limit"
:
null
,
"bootstrap_iters"
:
100000
,
"description_dict"
:
{}
}
}
results/mpt/mpt-7b/mpt-7b_xnli_0-shot.json
deleted
100644 → 0
View file @
465c695b
{
"results"
:
{
"xnli_ar"
:
{
"acc"
:
0.3331337325349301
,
"acc_stderr"
:
0.006659676975732312
},
"xnli_bg"
:
{
"acc"
:
0.36826347305389223
,
"acc_stderr"
:
0.00681509322031267
},
"xnli_de"
:
{
"acc"
:
0.46447105788423154
,
"acc_stderr"
:
0.007046854204317796
},
"xnli_el"
:
{
"acc"
:
0.36187624750499003
,
"acc_stderr"
:
0.006789799946197319
},
"xnli_en"
:
{
"acc"
:
0.543313373253493
,
"acc_stderr"
:
0.007038155029004149
},
"xnli_es"
:
{
"acc"
:
0.4564870259481038
,
"acc_stderr"
:
0.007037909229199955
},
"xnli_fr"
:
{
"acc"
:
0.4880239520958084
,
"acc_stderr"
:
0.007062685615595022
},
"xnli_hi"
:
{
"acc"
:
0.3473053892215569
,
"acc_stderr"
:
0.006727214239733754
},
"xnli_ru"
:
{
"acc"
:
0.444311377245509
,
"acc_stderr"
:
0.007020757195791274
},
"xnli_sw"
:
{
"acc"
:
0.3341317365269461
,
"acc_stderr"
:
0.006664652441694265
},
"xnli_th"
:
{
"acc"
:
0.36127744510978044
,
"acc_stderr"
:
0.006787362347422093
},
"xnli_tr"
:
{
"acc"
:
0.37684630738522956
,
"acc_stderr"
:
0.006847061089041557
},
"xnli_ur"
:
{
"acc"
:
0.33632734530938124
,
"acc_stderr"
:
0.006675480563072364
},
"xnli_vi"
:
{
"acc"
:
0.37325349301397204
,
"acc_stderr"
:
0.0068339592620100505
},
"xnli_zh"
:
{
"acc"
:
0.3534930139720559
,
"acc_stderr"
:
0.006754629196407293
}
},
"versions"
:
{
"xnli_ar"
:
0
,
"xnli_bg"
:
0
,
"xnli_de"
:
0
,
"xnli_el"
:
0
,
"xnli_en"
:
0
,
"xnli_es"
:
0
,
"xnli_fr"
:
0
,
"xnli_hi"
:
0
,
"xnli_ru"
:
0
,
"xnli_sw"
:
0
,
"xnli_th"
:
0
,
"xnli_tr"
:
0
,
"xnli_ur"
:
0
,
"xnli_vi"
:
0
,
"xnli_zh"
:
0
},
"config"
:
{
"model"
:
"hf-causal-experimental"
,
"model_args"
:
"pretrained=mosaicml/mpt-7b,trust_remote_code=True,dtype=bfloat16"
,
"num_fewshot"
:
0
,
"batch_size"
:
"auto"
,
"device"
:
"cuda"
,
"no_cache"
:
true
,
"limit"
:
null
,
"bootstrap_iters"
:
100000
,
"description_dict"
:
{}
}
}
results/mpt/mpt-7b/mpt-7b_xstory_cloze_0-shot.json
deleted
100644 → 0
View file @
465c695b
{
"results"
:
{
"xstory_cloze_ar"
:
{
"acc"
:
0.48510919920582396
,
"acc_stderr"
:
0.012861417842074004
},
"xstory_cloze_en"
:
{
"acc"
:
0.7789543348775645
,
"acc_stderr"
:
0.010678457581809239
},
"xstory_cloze_es"
:
{
"acc"
:
0.6604897418927862
,
"acc_stderr"
:
0.01218627614665943
},
"xstory_cloze_eu"
:
{
"acc"
:
0.5109199205823958
,
"acc_stderr"
:
0.012864056278255038
},
"xstory_cloze_hi"
:
{
"acc"
:
0.5168762409000662
,
"acc_stderr"
:
0.012859793919977606
},
"xstory_cloze_id"
:
{
"acc"
:
0.5519523494374586
,
"acc_stderr"
:
0.012797478885304742
},
"xstory_cloze_my"
:
{
"acc"
:
0.48378557246856385
,
"acc_stderr"
:
0.01286035780505586
},
"xstory_cloze_ru"
:
{
"acc"
:
0.5724685638649901
,
"acc_stderr"
:
0.012731259626982528
},
"xstory_cloze_sw"
:
{
"acc"
:
0.4990072799470549
,
"acc_stderr"
:
0.012867099955422935
},
"xstory_cloze_te"
:
{
"acc"
:
0.5294506949040371
,
"acc_stderr"
:
0.012844785490017004
},
"xstory_cloze_zh"
:
{
"acc"
:
0.5956320317670417
,
"acc_stderr"
:
0.012629580396570932
}
},
"versions"
:
{
"xstory_cloze_ar"
:
0
,
"xstory_cloze_en"
:
0
,
"xstory_cloze_es"
:
0
,
"xstory_cloze_eu"
:
0
,
"xstory_cloze_hi"
:
0
,
"xstory_cloze_id"
:
0
,
"xstory_cloze_my"
:
0
,
"xstory_cloze_ru"
:
0
,
"xstory_cloze_sw"
:
0
,
"xstory_cloze_te"
:
0
,
"xstory_cloze_zh"
:
0
},
"config"
:
{
"model"
:
"hf-causal-experimental"
,
"model_args"
:
"pretrained=mosaicml/mpt-7b,trust_remote_code=True,dtype=bfloat16"
,
"num_fewshot"
:
0
,
"batch_size"
:
"auto"
,
"device"
:
"cuda"
,
"no_cache"
:
true
,
"limit"
:
null
,
"bootstrap_iters"
:
100000
,
"description_dict"
:
{}
}
}
results/mpt/mpt-7b/mpt-7b_xwinograd_0-shot.json
deleted
100644 → 0
View file @
465c695b
{
"results"
:
{
"xwinograd_en"
:
{
"acc"
:
0.8666666666666667
,
"acc_stderr"
:
0.0070514325016347275
},
"xwinograd_fr"
:
{
"acc"
:
0.6626506024096386
,
"acc_stderr"
:
0.05221260262032129
},
"xwinograd_jp"
:
{
"acc"
:
0.602711157455683
,
"acc_stderr"
:
0.015809751560314552
},
"xwinograd_pt"
:
{
"acc"
:
0.6692015209125475
,
"acc_stderr"
:
0.02906762615931534
},
"xwinograd_ru"
:
{
"acc"
:
0.6952380952380952
,
"acc_stderr"
:
0.025976599352305375
},
"xwinograd_zh"
:
{
"acc"
:
0.7162698412698413
,
"acc_stderr"
:
0.02010051064884106
}
},
"versions"
:
{
"xwinograd_en"
:
0
,
"xwinograd_fr"
:
0
,
"xwinograd_jp"
:
0
,
"xwinograd_pt"
:
0
,
"xwinograd_ru"
:
0
,
"xwinograd_zh"
:
0
},
"config"
:
{
"model"
:
"hf-causal-experimental"
,
"model_args"
:
"pretrained=mosaicml/mpt-7b,trust_remote_code=True,dtype=bfloat16"
,
"num_fewshot"
:
0
,
"batch_size"
:
"auto"
,
"device"
:
"cuda"
,
"no_cache"
:
true
,
"limit"
:
null
,
"bootstrap_iters"
:
100000
,
"description_dict"
:
{}
}
}
results/opt/opt-1.3b/README.md
deleted
100644 → 0
View file @
465c695b
# opt-1.3b
## opt-1.3b.json
| Task |Version| Metric |Value| |Stderr|
|-------------------------------------------------|------:|--------|----:|---|-----:|
|arc_challenge | 0|acc |23.12|± | 1.23|
| | |acc_norm|29.44|± | 1.33|
|arc_easy | 0|acc |57.03|± | 1.02|
| | |acc_norm|50.93|± | 1.03|
|hendrycksTest-abstract_algebra | 0|acc |26.00|± | 4.41|
| | |acc_norm|25.00|± | 4.35|
|hendrycksTest-anatomy | 0|acc |22.96|± | 3.63|
| | |acc_norm|21.48|± | 3.55|
|hendrycksTest-astronomy | 0|acc |23.68|± | 3.46|
| | |acc_norm|34.21|± | 3.86|
|hendrycksTest-business_ethics | 0|acc |34.00|± | 4.76|
| | |acc_norm|33.00|± | 4.73|
|hendrycksTest-clinical_knowledge | 0|acc |19.25|± | 2.43|
| | |acc_norm|25.28|± | 2.67|
|hendrycksTest-college_biology | 0|acc |26.39|± | 3.69|
| | |acc_norm|27.78|± | 3.75|
|hendrycksTest-college_chemistry | 0|acc |28.00|± | 4.51|
| | |acc_norm|34.00|± | 4.76|
|hendrycksTest-college_computer_science | 0|acc |33.00|± | 4.73|
| | |acc_norm|30.00|± | 4.61|
|hendrycksTest-college_mathematics | 0|acc |19.00|± | 3.94|
| | |acc_norm|30.00|± | 4.61|
|hendrycksTest-college_medicine | 0|acc |17.92|± | 2.92|
| | |acc_norm|23.70|± | 3.24|
|hendrycksTest-college_physics | 0|acc |27.45|± | 4.44|
| | |acc_norm|30.39|± | 4.58|
|hendrycksTest-computer_security | 0|acc |29.00|± | 4.56|
| | |acc_norm|38.00|± | 4.88|
|hendrycksTest-conceptual_physics | 0|acc |21.70|± | 2.69|
| | |acc_norm|20.43|± | 2.64|
|hendrycksTest-econometrics | 0|acc |24.56|± | 4.05|
| | |acc_norm|23.68|± | 4.00|
|hendrycksTest-electrical_engineering | 0|acc |25.52|± | 3.63|
| | |acc_norm|28.97|± | 3.78|
|hendrycksTest-elementary_mathematics | 0|acc |19.58|± | 2.04|
| | |acc_norm|24.87|± | 2.23|
|hendrycksTest-formal_logic | 0|acc |29.37|± | 4.07|
| | |acc_norm|26.98|± | 3.97|
|hendrycksTest-global_facts | 0|acc |16.00|± | 3.68|
| | |acc_norm|18.00|± | 3.86|
|hendrycksTest-high_school_biology | 0|acc |20.97|± | 2.32|
| | |acc_norm|26.77|± | 2.52|
|hendrycksTest-high_school_chemistry | 0|acc |24.63|± | 3.03|
| | |acc_norm|30.54|± | 3.24|
|hendrycksTest-high_school_computer_science | 0|acc |27.00|± | 4.46|
| | |acc_norm|32.00|± | 4.69|
|hendrycksTest-high_school_european_history | 0|acc |24.24|± | 3.35|
| | |acc_norm|27.27|± | 3.48|
|hendrycksTest-high_school_geography | 0|acc |22.22|± | 2.96|
| | |acc_norm|28.28|± | 3.21|
|hendrycksTest-high_school_government_and_politics| 0|acc |20.73|± | 2.93|
| | |acc_norm|23.83|± | 3.07|
|hendrycksTest-high_school_macroeconomics | 0|acc |29.23|± | 2.31|
| | |acc_norm|29.23|± | 2.31|
|hendrycksTest-high_school_mathematics | 0|acc |21.85|± | 2.52|
| | |acc_norm|28.89|± | 2.76|
|hendrycksTest-high_school_microeconomics | 0|acc |21.43|± | 2.67|
| | |acc_norm|30.25|± | 2.98|
|hendrycksTest-high_school_physics | 0|acc |22.52|± | 3.41|
| | |acc_norm|25.17|± | 3.54|
|hendrycksTest-high_school_psychology | 0|acc |22.57|± | 1.79|
| | |acc_norm|24.22|± | 1.84|
|hendrycksTest-high_school_statistics | 0|acc |25.46|± | 2.97|
| | |acc_norm|27.78|± | 3.05|
|hendrycksTest-high_school_us_history | 0|acc |25.00|± | 3.04|
| | |acc_norm|25.49|± | 3.06|
|hendrycksTest-high_school_world_history | 0|acc |26.58|± | 2.88|
| | |acc_norm|27.85|± | 2.92|
|hendrycksTest-human_aging | 0|acc |35.43|± | 3.21|
| | |acc_norm|29.15|± | 3.05|
|hendrycksTest-human_sexuality | 0|acc |40.46|± | 4.30|
| | |acc_norm|31.30|± | 4.07|
|hendrycksTest-international_law | 0|acc |17.36|± | 3.46|
| | |acc_norm|47.93|± | 4.56|
|hendrycksTest-jurisprudence | 0|acc |28.70|± | 4.37|
| | |acc_norm|39.81|± | 4.73|
|hendrycksTest-logical_fallacies | 0|acc |19.02|± | 3.08|
| | |acc_norm|28.83|± | 3.56|
|hendrycksTest-machine_learning | 0|acc |27.68|± | 4.25|
| | |acc_norm|27.68|± | 4.25|
|hendrycksTest-management | 0|acc |20.39|± | 3.99|
| | |acc_norm|27.18|± | 4.41|
|hendrycksTest-marketing | 0|acc |28.21|± | 2.95|
| | |acc_norm|33.76|± | 3.10|
|hendrycksTest-medical_genetics | 0|acc |27.00|± | 4.46|
| | |acc_norm|36.00|± | 4.82|
|hendrycksTest-miscellaneous | 0|acc |28.35|± | 1.61|
| | |acc_norm|28.74|± | 1.62|
|hendrycksTest-moral_disputes | 0|acc |27.17|± | 2.39|
| | |acc_norm|30.35|± | 2.48|
|hendrycksTest-moral_scenarios | 0|acc |23.80|± | 1.42|
| | |acc_norm|27.26|± | 1.49|
|hendrycksTest-nutrition | 0|acc |29.41|± | 2.61|
| | |acc_norm|39.54|± | 2.80|
|hendrycksTest-philosophy | 0|acc |23.79|± | 2.42|
| | |acc_norm|30.87|± | 2.62|
|hendrycksTest-prehistory | 0|acc |24.07|± | 2.38|
| | |acc_norm|21.60|± | 2.29|
|hendrycksTest-professional_accounting | 0|acc |25.89|± | 2.61|
| | |acc_norm|26.24|± | 2.62|
|hendrycksTest-professional_law | 0|acc |26.01|± | 1.12|
| | |acc_norm|28.03|± | 1.15|
|hendrycksTest-professional_medicine | 0|acc |24.63|± | 2.62|
| | |acc_norm|22.43|± | 2.53|
|hendrycksTest-professional_psychology | 0|acc |23.69|± | 1.72|
| | |acc_norm|25.49|± | 1.76|
|hendrycksTest-public_relations | 0|acc |25.45|± | 4.17|
| | |acc_norm|19.09|± | 3.76|
|hendrycksTest-security_studies | 0|acc |32.24|± | 2.99|
| | |acc_norm|26.53|± | 2.83|
|hendrycksTest-sociology | 0|acc |33.83|± | 3.35|
| | |acc_norm|34.33|± | 3.36|
|hendrycksTest-us_foreign_policy | 0|acc |32.00|± | 4.69|
| | |acc_norm|27.00|± | 4.46|
|hendrycksTest-virology | 0|acc |34.34|± | 3.70|
| | |acc_norm|30.12|± | 3.57|
|hendrycksTest-world_religions | 0|acc |34.50|± | 3.65|
| | |acc_norm|33.92|± | 3.63|
|lambada_openai | 0|ppl | 6.64|± | 0.17|
| | |acc |57.93|± | 0.69|
|logiqa | 0|acc |22.27|± | 1.63|
| | |acc_norm|27.19|± | 1.75|
|piqa | 0|acc |71.71|± | 1.05|
| | |acc_norm|72.47|± | 1.04|
|sciq | 0|acc |84.50|± | 1.15|
| | |acc_norm|76.50|± | 1.34|
|winogrande | 0|acc |59.75|± | 1.38|
|wsc | 0|acc |38.46|± | 4.79|
results/opt/opt-1.3b/opt-1.3b.json
deleted
100644 → 0
View file @
465c695b
{
"results"
:
{
"hendrycksTest-high_school_chemistry"
:
{
"acc"
:
0.24630541871921183
,
"acc_stderr"
:
0.03031509928561773
,
"acc_norm"
:
0.3054187192118227
,
"acc_norm_stderr"
:
0.03240661565868408
},
"hendrycksTest-international_law"
:
{
"acc"
:
0.17355371900826447
,
"acc_stderr"
:
0.0345727283691767
,
"acc_norm"
:
0.4793388429752066
,
"acc_norm_stderr"
:
0.04560456086387235
},
"hendrycksTest-abstract_algebra"
:
{
"acc"
:
0.26
,
"acc_stderr"
:
0.044084400227680794
,
"acc_norm"
:
0.25
,
"acc_norm_stderr"
:
0.04351941398892446
},
"hendrycksTest-anatomy"
:
{
"acc"
:
0.22962962962962963
,
"acc_stderr"
:
0.03633384414073465
,
"acc_norm"
:
0.21481481481481482
,
"acc_norm_stderr"
:
0.03547854198560826
},
"hendrycksTest-elementary_mathematics"
:
{
"acc"
:
0.19576719576719576
,
"acc_stderr"
:
0.0204357309715418
,
"acc_norm"
:
0.24867724867724866
,
"acc_norm_stderr"
:
0.022261817692400158
},
"hendrycksTest-high_school_microeconomics"
:
{
"acc"
:
0.21428571428571427
,
"acc_stderr"
:
0.02665353159671549
,
"acc_norm"
:
0.3025210084033613
,
"acc_norm_stderr"
:
0.02983796238829193
},
"hendrycksTest-college_chemistry"
:
{
"acc"
:
0.28
,
"acc_stderr"
:
0.045126085985421276
,
"acc_norm"
:
0.34
,
"acc_norm_stderr"
:
0.04760952285695236
},
"hendrycksTest-college_biology"
:
{
"acc"
:
0.2638888888888889
,
"acc_stderr"
:
0.03685651095897532
,
"acc_norm"
:
0.2777777777777778
,
"acc_norm_stderr"
:
0.03745554791462457
},
"hendrycksTest-high_school_psychology"
:
{
"acc"
:
0.22568807339449543
,
"acc_stderr"
:
0.017923087667803057
,
"acc_norm"
:
0.24220183486238533
,
"acc_norm_stderr"
:
0.01836817630659862
},
"hendrycksTest-philosophy"
:
{
"acc"
:
0.2379421221864952
,
"acc_stderr"
:
0.024185150647818707
,
"acc_norm"
:
0.3086816720257235
,
"acc_norm_stderr"
:
0.026236965881153256
},
"hendrycksTest-professional_law"
:
{
"acc"
:
0.26010430247718386
,
"acc_stderr"
:
0.011204382887823836
,
"acc_norm"
:
0.2803129074315515
,
"acc_norm_stderr"
:
0.011471555944958614
},
"hendrycksTest-college_computer_science"
:
{
"acc"
:
0.33
,
"acc_stderr"
:
0.047258156262526045
,
"acc_norm"
:
0.3
,
"acc_norm_stderr"
:
0.046056618647183814
},
"hendrycksTest-business_ethics"
:
{
"acc"
:
0.34
,
"acc_stderr"
:
0.04760952285695235
,
"acc_norm"
:
0.33
,
"acc_norm_stderr"
:
0.04725815626252604
},
"hendrycksTest-machine_learning"
:
{
"acc"
:
0.2767857142857143
,
"acc_stderr"
:
0.04246624336697626
,
"acc_norm"
:
0.2767857142857143
,
"acc_norm_stderr"
:
0.04246624336697624
},
"hendrycksTest-public_relations"
:
{
"acc"
:
0.2545454545454545
,
"acc_stderr"
:
0.041723430387053825
,
"acc_norm"
:
0.19090909090909092
,
"acc_norm_stderr"
:
0.03764425585984925
},
"hendrycksTest-logical_fallacies"
:
{
"acc"
:
0.1901840490797546
,
"acc_stderr"
:
0.030833491146281224
,
"acc_norm"
:
0.2883435582822086
,
"acc_norm_stderr"
:
0.035590395316173425
},
"hendrycksTest-professional_psychology"
:
{
"acc"
:
0.2369281045751634
,
"acc_stderr"
:
0.017201662169789782
,
"acc_norm"
:
0.2549019607843137
,
"acc_norm_stderr"
:
0.017630827375148383
},
"hendrycksTest-nutrition"
:
{
"acc"
:
0.29411764705882354
,
"acc_stderr"
:
0.02609016250427904
,
"acc_norm"
:
0.3954248366013072
,
"acc_norm_stderr"
:
0.02799672318063145
},
"arc_challenge"
:
{
"acc"
:
0.23122866894197952
,
"acc_stderr"
:
0.012320858834772276
,
"acc_norm"
:
0.29436860068259385
,
"acc_norm_stderr"
:
0.013318528460539426
},
"hendrycksTest-college_medicine"
:
{
"acc"
:
0.1791907514450867
,
"acc_stderr"
:
0.02924251305906328
,
"acc_norm"
:
0.23699421965317918
,
"acc_norm_stderr"
:
0.03242414757483098
},
"hendrycksTest-professional_accounting"
:
{
"acc"
:
0.25886524822695034
,
"acc_stderr"
:
0.026129572527180848
,
"acc_norm"
:
0.2624113475177305
,
"acc_norm_stderr"
:
0.026244920349843
},
"hendrycksTest-jurisprudence"
:
{
"acc"
:
0.28703703703703703
,
"acc_stderr"
:
0.043733130409147614
,
"acc_norm"
:
0.39814814814814814
,
"acc_norm_stderr"
:
0.047323326159788154
},
"winogrande"
:
{
"acc"
:
0.5974743488555643
,
"acc_stderr"
:
0.01378286683170305
},
"hendrycksTest-high_school_us_history"
:
{
"acc"
:
0.25
,
"acc_stderr"
:
0.03039153369274154
,
"acc_norm"
:
0.2549019607843137
,
"acc_norm_stderr"
:
0.030587591351604246
},
"hendrycksTest-econometrics"
:
{
"acc"
:
0.24561403508771928
,
"acc_stderr"
:
0.04049339297748142
,
"acc_norm"
:
0.23684210526315788
,
"acc_norm_stderr"
:
0.03999423879281336
},
"hendrycksTest-professional_medicine"
:
{
"acc"
:
0.24632352941176472
,
"acc_stderr"
:
0.02617343857052
,
"acc_norm"
:
0.22426470588235295
,
"acc_norm_stderr"
:
0.02533684856333237
},
"hendrycksTest-clinical_knowledge"
:
{
"acc"
:
0.19245283018867926
,
"acc_stderr"
:
0.024262979839372277
,
"acc_norm"
:
0.2528301886792453
,
"acc_norm_stderr"
:
0.02674989977124124
},
"hendrycksTest-high_school_geography"
:
{
"acc"
:
0.2222222222222222
,
"acc_stderr"
:
0.029620227874790482
,
"acc_norm"
:
0.2828282828282828
,
"acc_norm_stderr"
:
0.03208779558786751
},
"hendrycksTest-high_school_physics"
:
{
"acc"
:
0.2251655629139073
,
"acc_stderr"
:
0.034104352820089376
,
"acc_norm"
:
0.25165562913907286
,
"acc_norm_stderr"
:
0.035433042343899844
},
"hendrycksTest-astronomy"
:
{
"acc"
:
0.23684210526315788
,
"acc_stderr"
:
0.03459777606810537
,
"acc_norm"
:
0.34210526315789475
,
"acc_norm_stderr"
:
0.038607315993160925
},
"hendrycksTest-medical_genetics"
:
{
"acc"
:
0.27
,
"acc_stderr"
:
0.0446196043338474
,
"acc_norm"
:
0.36
,
"acc_norm_stderr"
:
0.04824181513244218
},
"hendrycksTest-us_foreign_policy"
:
{
"acc"
:
0.32
,
"acc_stderr"
:
0.046882617226215034
,
"acc_norm"
:
0.27
,
"acc_norm_stderr"
:
0.04461960433384741
},
"hendrycksTest-formal_logic"
:
{
"acc"
:
0.29365079365079366
,
"acc_stderr"
:
0.040735243221471276
,
"acc_norm"
:
0.2698412698412698
,
"acc_norm_stderr"
:
0.039701582732351734
},
"hendrycksTest-conceptual_physics"
:
{
"acc"
:
0.2170212765957447
,
"acc_stderr"
:
0.026947483121496217
,
"acc_norm"
:
0.20425531914893616
,
"acc_norm_stderr"
:
0.026355158413349417
},
"hendrycksTest-electrical_engineering"
:
{
"acc"
:
0.25517241379310346
,
"acc_stderr"
:
0.03632984052707842
,
"acc_norm"
:
0.2896551724137931
,
"acc_norm_stderr"
:
0.03780019230438014
},
"hendrycksTest-virology"
:
{
"acc"
:
0.3433734939759036
,
"acc_stderr"
:
0.03696584317010601
,
"acc_norm"
:
0.30120481927710846
,
"acc_norm_stderr"
:
0.0357160923005348
},
"piqa"
:
{
"acc"
:
0.7170837867247007
,
"acc_stderr"
:
0.010508949177489683
,
"acc_norm"
:
0.7247007616974973
,
"acc_norm_stderr"
:
0.01042142927736953
},
"hendrycksTest-high_school_statistics"
:
{
"acc"
:
0.25462962962962965
,
"acc_stderr"
:
0.02971127586000535
,
"acc_norm"
:
0.2777777777777778
,
"acc_norm_stderr"
:
0.03054674526495318
},
"hendrycksTest-college_physics"
:
{
"acc"
:
0.27450980392156865
,
"acc_stderr"
:
0.04440521906179327
,
"acc_norm"
:
0.30392156862745096
,
"acc_norm_stderr"
:
0.045766654032077636
},
"hendrycksTest-high_school_biology"
:
{
"acc"
:
0.20967741935483872
,
"acc_stderr"
:
0.02315787934908352
,
"acc_norm"
:
0.267741935483871
,
"acc_norm_stderr"
:
0.02518900666021238
},
"hendrycksTest-world_religions"
:
{
"acc"
:
0.34502923976608185
,
"acc_stderr"
:
0.036459813773888065
,
"acc_norm"
:
0.3391812865497076
,
"acc_norm_stderr"
:
0.036310534964889056
},
"hendrycksTest-marketing"
:
{
"acc"
:
0.28205128205128205
,
"acc_stderr"
:
0.02948036054954119
,
"acc_norm"
:
0.33760683760683763
,
"acc_norm_stderr"
:
0.030980296992618558
},
"hendrycksTest-security_studies"
:
{
"acc"
:
0.3224489795918367
,
"acc_stderr"
:
0.029923100563683913
,
"acc_norm"
:
0.2653061224489796
,
"acc_norm_stderr"
:
0.028263889943784603
},
"arc_easy"
:
{
"acc"
:
0.5702861952861953
,
"acc_stderr"
:
0.010157908005763676
,
"acc_norm"
:
0.5092592592592593
,
"acc_norm_stderr"
:
0.01025802414786065
},
"hendrycksTest-high_school_world_history"
:
{
"acc"
:
0.26582278481012656
,
"acc_stderr"
:
0.028756799629658335
,
"acc_norm"
:
0.27848101265822783
,
"acc_norm_stderr"
:
0.029178682304842555
},
"hendrycksTest-human_aging"
:
{
"acc"
:
0.3542600896860987
,
"acc_stderr"
:
0.03210062154134986
,
"acc_norm"
:
0.2914798206278027
,
"acc_norm_stderr"
:
0.030500283176545902
},
"sciq"
:
{
"acc"
:
0.845
,
"acc_stderr"
:
0.01145015747079947
,
"acc_norm"
:
0.765
,
"acc_norm_stderr"
:
0.013414729030247121
},
"hendrycksTest-moral_disputes"
:
{
"acc"
:
0.27167630057803466
,
"acc_stderr"
:
0.02394851290546836
,
"acc_norm"
:
0.30346820809248554
,
"acc_norm_stderr"
:
0.024752411960917212
},
"hendrycksTest-management"
:
{
"acc"
:
0.20388349514563106
,
"acc_stderr"
:
0.03989139859531771
,
"acc_norm"
:
0.27184466019417475
,
"acc_norm_stderr"
:
0.044052680241409216
},
"hendrycksTest-computer_security"
:
{
"acc"
:
0.29
,
"acc_stderr"
:
0.045604802157206845
,
"acc_norm"
:
0.38
,
"acc_norm_stderr"
:
0.04878317312145633
},
"hendrycksTest-moral_scenarios"
:
{
"acc"
:
0.23798882681564246
,
"acc_stderr"
:
0.014242630070574915
,
"acc_norm"
:
0.27262569832402234
,
"acc_norm_stderr"
:
0.014893391735249588
},
"wsc"
:
{
"acc"
:
0.38461538461538464
,
"acc_stderr"
:
0.0479366886807504
},
"hendrycksTest-high_school_government_and_politics"
:
{
"acc"
:
0.20725388601036268
,
"acc_stderr"
:
0.029252823291803627
,
"acc_norm"
:
0.23834196891191708
,
"acc_norm_stderr"
:
0.030748905363909895
},
"hendrycksTest-human_sexuality"
:
{
"acc"
:
0.40458015267175573
,
"acc_stderr"
:
0.043046937953806645
,
"acc_norm"
:
0.31297709923664124
,
"acc_norm_stderr"
:
0.04066962905677698
},
"hendrycksTest-high_school_european_history"
:
{
"acc"
:
0.24242424242424243
,
"acc_stderr"
:
0.03346409881055953
,
"acc_norm"
:
0.2727272727272727
,
"acc_norm_stderr"
:
0.0347769116216366
},
"hendrycksTest-high_school_macroeconomics"
:
{
"acc"
:
0.2923076923076923
,
"acc_stderr"
:
0.023060438380857733
,
"acc_norm"
:
0.2923076923076923
,
"acc_norm_stderr"
:
0.02306043838085774
},
"hendrycksTest-college_mathematics"
:
{
"acc"
:
0.19
,
"acc_stderr"
:
0.03942772444036623
,
"acc_norm"
:
0.3
,
"acc_norm_stderr"
:
0.046056618647183814
},
"hendrycksTest-miscellaneous"
:
{
"acc"
:
0.2835249042145594
,
"acc_stderr"
:
0.016117318166832272
,
"acc_norm"
:
0.28735632183908044
,
"acc_norm_stderr"
:
0.0161824107306827
},
"logiqa"
:
{
"acc"
:
0.2227342549923195
,
"acc_stderr"
:
0.01632005404616512
,
"acc_norm"
:
0.271889400921659
,
"acc_norm_stderr"
:
0.01745171600943683
},
"hendrycksTest-prehistory"
:
{
"acc"
:
0.24074074074074073
,
"acc_stderr"
:
0.023788583551658537
,
"acc_norm"
:
0.21604938271604937
,
"acc_norm_stderr"
:
0.022899162918445796
},
"hendrycksTest-high_school_mathematics"
:
{
"acc"
:
0.21851851851851853
,
"acc_stderr"
:
0.025195752251823793
,
"acc_norm"
:
0.28888888888888886
,
"acc_norm_stderr"
:
0.027634907264178544
},
"lambada_openai"
:
{
"ppl"
:
6.644056379058006
,
"ppl_stderr"
:
0.1717099929921861
,
"acc"
:
0.5792742091985251
,
"acc_stderr"
:
0.00687786642328006
},
"hendrycksTest-global_facts"
:
{
"acc"
:
0.16
,
"acc_stderr"
:
0.03684529491774708
,
"acc_norm"
:
0.18
,
"acc_norm_stderr"
:
0.03861229196653695
},
"hendrycksTest-high_school_computer_science"
:
{
"acc"
:
0.27
,
"acc_stderr"
:
0.04461960433384741
,
"acc_norm"
:
0.32
,
"acc_norm_stderr"
:
0.04688261722621503
},
"hendrycksTest-sociology"
:
{
"acc"
:
0.3383084577114428
,
"acc_stderr"
:
0.033455630703391914
,
"acc_norm"
:
0.34328358208955223
,
"acc_norm_stderr"
:
0.03357379665433431
}
},
"versions"
:
{
"hendrycksTest-high_school_chemistry"
:
0
,
"hendrycksTest-international_law"
:
0
,
"hendrycksTest-abstract_algebra"
:
0
,
"hendrycksTest-anatomy"
:
0
,
"hendrycksTest-elementary_mathematics"
:
0
,
"hendrycksTest-high_school_microeconomics"
:
0
,
"hendrycksTest-college_chemistry"
:
0
,
"hendrycksTest-college_biology"
:
0
,
"hendrycksTest-high_school_psychology"
:
0
,
"hendrycksTest-philosophy"
:
0
,
"hendrycksTest-professional_law"
:
0
,
"hendrycksTest-college_computer_science"
:
0
,
"hendrycksTest-business_ethics"
:
0
,
"hendrycksTest-machine_learning"
:
0
,
"hendrycksTest-public_relations"
:
0
,
"hendrycksTest-logical_fallacies"
:
0
,
"hendrycksTest-professional_psychology"
:
0
,
"hendrycksTest-nutrition"
:
0
,
"arc_challenge"
:
0
,
"hendrycksTest-college_medicine"
:
0
,
"hendrycksTest-professional_accounting"
:
0
,
"hendrycksTest-jurisprudence"
:
0
,
"winogrande"
:
0
,
"hendrycksTest-high_school_us_history"
:
0
,
"hendrycksTest-econometrics"
:
0
,
"hendrycksTest-professional_medicine"
:
0
,
"hendrycksTest-clinical_knowledge"
:
0
,
"hendrycksTest-high_school_geography"
:
0
,
"hendrycksTest-high_school_physics"
:
0
,
"hendrycksTest-astronomy"
:
0
,
"hendrycksTest-medical_genetics"
:
0
,
"hendrycksTest-us_foreign_policy"
:
0
,
"hendrycksTest-formal_logic"
:
0
,
"hendrycksTest-conceptual_physics"
:
0
,
"hendrycksTest-electrical_engineering"
:
0
,
"hendrycksTest-virology"
:
0
,
"piqa"
:
0
,
"hendrycksTest-high_school_statistics"
:
0
,
"hendrycksTest-college_physics"
:
0
,
"hendrycksTest-high_school_biology"
:
0
,
"hendrycksTest-world_religions"
:
0
,
"hendrycksTest-marketing"
:
0
,
"hendrycksTest-security_studies"
:
0
,
"arc_easy"
:
0
,
"hendrycksTest-high_school_world_history"
:
0
,
"hendrycksTest-human_aging"
:
0
,
"sciq"
:
0
,
"hendrycksTest-moral_disputes"
:
0
,
"hendrycksTest-management"
:
0
,
"hendrycksTest-computer_security"
:
0
,
"hendrycksTest-moral_scenarios"
:
0
,
"wsc"
:
0
,
"hendrycksTest-high_school_government_and_politics"
:
0
,
"hendrycksTest-human_sexuality"
:
0
,
"hendrycksTest-high_school_european_history"
:
0
,
"hendrycksTest-high_school_macroeconomics"
:
0
,
"hendrycksTest-college_mathematics"
:
0
,
"hendrycksTest-miscellaneous"
:
0
,
"logiqa"
:
0
,
"hendrycksTest-prehistory"
:
0
,
"hendrycksTest-high_school_mathematics"
:
0
,
"lambada_openai"
:
0
,
"hendrycksTest-global_facts"
:
0
,
"hendrycksTest-high_school_computer_science"
:
0
,
"hendrycksTest-sociology"
:
0
},
"config"
:
{
"model"
:
"hf-causal"
,
"model_args"
:
"pretrained=facebook/opt-1.3b,use_accelerate=True"
,
"num_fewshot"
:
0
,
"batch_size"
:
16
,
"device"
:
"cuda"
,
"no_cache"
:
false
,
"limit"
:
null
,
"bootstrap_iters"
:
100000
,
"description_dict"
:
{}
}
}
results/opt/opt-125m/README.md
deleted
100644 → 0
View file @
465c695b
# opt-125m
## opt-125m.json
| Task |Version| Metric |Value| |Stderr|
|-------------------------------------------------|------:|--------|----:|---|-----:|
|arc_challenge | 0|acc |18.94|± | 1.15|
| | |acc_norm|22.78|± | 1.23|
|arc_easy | 0|acc |43.52|± | 1.02|
| | |acc_norm|39.98|± | 1.01|
|hendrycksTest-abstract_algebra | 0|acc |17.00|± | 3.78|
| | |acc_norm|20.00|± | 4.02|
|hendrycksTest-anatomy | 0|acc |24.44|± | 3.71|
| | |acc_norm|21.48|± | 3.55|
|hendrycksTest-astronomy | 0|acc |20.39|± | 3.28|
| | |acc_norm|33.55|± | 3.84|
|hendrycksTest-business_ethics | 0|acc |35.00|± | 4.79|
| | |acc_norm|23.00|± | 4.23|
|hendrycksTest-clinical_knowledge | 0|acc |18.87|± | 2.41|
| | |acc_norm|27.92|± | 2.76|
|hendrycksTest-college_biology | 0|acc |22.92|± | 3.51|
| | |acc_norm|23.61|± | 3.55|
|hendrycksTest-college_chemistry | 0|acc |28.00|± | 4.51|
| | |acc_norm|27.00|± | 4.46|
|hendrycksTest-college_computer_science | 0|acc |27.00|± | 4.46|
| | |acc_norm|22.00|± | 4.16|
|hendrycksTest-college_mathematics | 0|acc |16.00|± | 3.68|
| | |acc_norm|25.00|± | 4.35|
|hendrycksTest-college_medicine | 0|acc |24.28|± | 3.27|
| | |acc_norm|26.59|± | 3.37|
|hendrycksTest-college_physics | 0|acc |28.43|± | 4.49|
| | |acc_norm|23.53|± | 4.22|
|hendrycksTest-computer_security | 0|acc |31.00|± | 4.65|
| | |acc_norm|33.00|± | 4.73|
|hendrycksTest-conceptual_physics | 0|acc |26.38|± | 2.88|
| | |acc_norm|17.45|± | 2.48|
|hendrycksTest-econometrics | 0|acc |31.58|± | 4.37|
| | |acc_norm|28.95|± | 4.27|
|hendrycksTest-electrical_engineering | 0|acc |24.14|± | 3.57|
| | |acc_norm|30.34|± | 3.83|
|hendrycksTest-elementary_mathematics | 0|acc |24.87|± | 2.23|
| | |acc_norm|25.13|± | 2.23|
|hendrycksTest-formal_logic | 0|acc |30.95|± | 4.13|
| | |acc_norm|28.57|± | 4.04|
|hendrycksTest-global_facts | 0|acc |19.00|± | 3.94|
| | |acc_norm|22.00|± | 4.16|
|hendrycksTest-high_school_biology | 0|acc |24.84|± | 2.46|
| | |acc_norm|29.03|± | 2.58|
|hendrycksTest-high_school_chemistry | 0|acc |17.24|± | 2.66|
| | |acc_norm|24.63|± | 3.03|
|hendrycksTest-high_school_computer_science | 0|acc |21.00|± | 4.09|
| | |acc_norm|25.00|± | 4.35|
|hendrycksTest-high_school_european_history | 0|acc |24.85|± | 3.37|
| | |acc_norm|31.52|± | 3.63|
|hendrycksTest-high_school_geography | 0|acc |22.73|± | 2.99|
| | |acc_norm|28.79|± | 3.23|
|hendrycksTest-high_school_government_and_politics| 0|acc |24.87|± | 3.12|
| | |acc_norm|30.05|± | 3.31|
|hendrycksTest-high_school_macroeconomics | 0|acc |26.15|± | 2.23|
| | |acc_norm|24.87|± | 2.19|
|hendrycksTest-high_school_mathematics | 0|acc |17.04|± | 2.29|
| | |acc_norm|23.33|± | 2.58|
|hendrycksTest-high_school_microeconomics | 0|acc |23.53|± | 2.76|
| | |acc_norm|29.83|± | 2.97|
|hendrycksTest-high_school_physics | 0|acc |22.52|± | 3.41|
| | |acc_norm|21.19|± | 3.34|
|hendrycksTest-high_school_psychology | 0|acc |25.14|± | 1.86|
| | |acc_norm|25.69|± | 1.87|
|hendrycksTest-high_school_statistics | 0|acc |24.54|± | 2.93|
| | |acc_norm|27.78|± | 3.05|
|hendrycksTest-high_school_us_history | 0|acc |24.51|± | 3.02|
| | |acc_norm|26.47|± | 3.10|
|hendrycksTest-high_school_world_history | 0|acc |26.16|± | 2.86|
| | |acc_norm|28.69|± | 2.94|
|hendrycksTest-human_aging | 0|acc |36.32|± | 3.23|
| | |acc_norm|26.01|± | 2.94|
|hendrycksTest-human_sexuality | 0|acc |32.82|± | 4.12|
| | |acc_norm|32.06|± | 4.09|
|hendrycksTest-international_law | 0|acc |14.88|± | 3.25|
| | |acc_norm|38.02|± | 4.43|
|hendrycksTest-jurisprudence | 0|acc |15.74|± | 3.52|
| | |acc_norm|37.04|± | 4.67|
|hendrycksTest-logical_fallacies | 0|acc |26.38|± | 3.46|
| | |acc_norm|30.06|± | 3.60|
|hendrycksTest-machine_learning | 0|acc |24.11|± | 4.06|
| | |acc_norm|27.68|± | 4.25|
|hendrycksTest-management | 0|acc |17.48|± | 3.76|
| | |acc_norm|27.18|± | 4.41|
|hendrycksTest-marketing | 0|acc |33.76|± | 3.10|
| | |acc_norm|35.90|± | 3.14|
|hendrycksTest-medical_genetics | 0|acc |29.00|± | 4.56|
| | |acc_norm|38.00|± | 4.88|
|hendrycksTest-miscellaneous | 0|acc |27.46|± | 1.60|
| | |acc_norm|26.18|± | 1.57|
|hendrycksTest-moral_disputes | 0|acc |28.32|± | 2.43|
| | |acc_norm|29.48|± | 2.45|
|hendrycksTest-moral_scenarios | 0|acc |23.80|± | 1.42|
| | |acc_norm|27.26|± | 1.49|
|hendrycksTest-nutrition | 0|acc |28.76|± | 2.59|
| | |acc_norm|34.97|± | 2.73|
|hendrycksTest-philosophy | 0|acc |20.90|± | 2.31|
| | |acc_norm|31.19|± | 2.63|
|hendrycksTest-prehistory | 0|acc |27.47|± | 2.48|
| | |acc_norm|21.91|± | 2.30|
|hendrycksTest-professional_accounting | 0|acc |25.89|± | 2.61|
| | |acc_norm|26.24|± | 2.62|
|hendrycksTest-professional_law | 0|acc |25.23|± | 1.11|
| | |acc_norm|27.38|± | 1.14|
|hendrycksTest-professional_medicine | 0|acc |25.00|± | 2.63|
| | |acc_norm|22.79|± | 2.55|
|hendrycksTest-professional_psychology | 0|acc |23.86|± | 1.72|
| | |acc_norm|24.84|± | 1.75|
|hendrycksTest-public_relations | 0|acc |29.09|± | 4.35|
| | |acc_norm|25.45|± | 4.17|
|hendrycksTest-security_studies | 0|acc |33.47|± | 3.02|
| | |acc_norm|25.71|± | 2.80|
|hendrycksTest-sociology | 0|acc |27.86|± | 3.17|
| | |acc_norm|24.88|± | 3.06|
|hendrycksTest-us_foreign_policy | 0|acc |28.00|± | 4.51|
| | |acc_norm|34.00|± | 4.76|
|hendrycksTest-virology | 0|acc |31.93|± | 3.63|
| | |acc_norm|27.71|± | 3.48|
|hendrycksTest-world_religions | 0|acc |28.65|± | 3.47|
| | |acc_norm|32.16|± | 3.58|
|lambada_openai | 0|ppl |26.02|± | 0.94|
| | |acc |37.90|± | 0.68|
|logiqa | 0|acc |22.73|± | 1.64|
| | |acc_norm|27.96|± | 1.76|
|piqa | 0|acc |63.00|± | 1.13|
| | |acc_norm|62.02|± | 1.13|
|sciq | 0|acc |75.10|± | 1.37|
| | |acc_norm|66.90|± | 1.49|
|winogrande | 0|acc |50.28|± | 1.41|
|wsc | 0|acc |36.54|± | 4.74|
results/opt/opt-125m/opt-125m.json
deleted
100644 → 0
View file @
465c695b
{
"results"
:
{
"lambada_openai"
:
{
"ppl"
:
26.02103896885129
,
"ppl_stderr"
:
0.9418773107503278
,
"acc"
:
0.3790025228022511
,
"acc_stderr"
:
0.006758931440268226
},
"hendrycksTest-college_biology"
:
{
"acc"
:
0.22916666666666666
,
"acc_stderr"
:
0.03514697467862388
,
"acc_norm"
:
0.2361111111111111
,
"acc_norm_stderr"
:
0.03551446610810826
},
"hendrycksTest-professional_law"
:
{
"acc"
:
0.2522816166883963
,
"acc_stderr"
:
0.01109278905687524
,
"acc_norm"
:
0.2737940026075619
,
"acc_norm_stderr"
:
0.011388612167979395
},
"hendrycksTest-virology"
:
{
"acc"
:
0.3192771084337349
,
"acc_stderr"
:
0.03629335329947861
,
"acc_norm"
:
0.27710843373493976
,
"acc_norm_stderr"
:
0.03484331592680587
},
"winogrande"
:
{
"acc"
:
0.5027624309392266
,
"acc_stderr"
:
0.014052271211616445
},
"hendrycksTest-high_school_microeconomics"
:
{
"acc"
:
0.23529411764705882
,
"acc_stderr"
:
0.027553614467863804
,
"acc_norm"
:
0.29831932773109243
,
"acc_norm_stderr"
:
0.029719142876342863
},
"hendrycksTest-college_physics"
:
{
"acc"
:
0.28431372549019607
,
"acc_stderr"
:
0.04488482852329017
,
"acc_norm"
:
0.23529411764705882
,
"acc_norm_stderr"
:
0.04220773659171453
},
"hendrycksTest-computer_security"
:
{
"acc"
:
0.31
,
"acc_stderr"
:
0.04648231987117316
,
"acc_norm"
:
0.33
,
"acc_norm_stderr"
:
0.047258156262526045
},
"hendrycksTest-conceptual_physics"
:
{
"acc"
:
0.26382978723404255
,
"acc_stderr"
:
0.02880998985410297
,
"acc_norm"
:
0.17446808510638298
,
"acc_norm_stderr"
:
0.024809442335503966
},
"hendrycksTest-high_school_world_history"
:
{
"acc"
:
0.2616033755274262
,
"acc_stderr"
:
0.028609516716994934
,
"acc_norm"
:
0.2869198312236287
,
"acc_norm_stderr"
:
0.02944377302259469
},
"hendrycksTest-sociology"
:
{
"acc"
:
0.27860696517412936
,
"acc_stderr"
:
0.031700561834973086
,
"acc_norm"
:
0.24875621890547264
,
"acc_norm_stderr"
:
0.030567675938916707
},
"sciq"
:
{
"acc"
:
0.751
,
"acc_stderr"
:
0.013681600278702308
,
"acc_norm"
:
0.669
,
"acc_norm_stderr"
:
0.014888272588203941
},
"hendrycksTest-professional_accounting"
:
{
"acc"
:
0.25886524822695034
,
"acc_stderr"
:
0.026129572527180848
,
"acc_norm"
:
0.2624113475177305
,
"acc_norm_stderr"
:
0.026244920349843007
},
"hendrycksTest-high_school_biology"
:
{
"acc"
:
0.24838709677419354
,
"acc_stderr"
:
0.024580028921481003
,
"acc_norm"
:
0.2903225806451613
,
"acc_norm_stderr"
:
0.025822106119415905
},
"hendrycksTest-college_computer_science"
:
{
"acc"
:
0.27
,
"acc_stderr"
:
0.0446196043338474
,
"acc_norm"
:
0.22
,
"acc_norm_stderr"
:
0.04163331998932269
},
"hendrycksTest-econometrics"
:
{
"acc"
:
0.3157894736842105
,
"acc_stderr"
:
0.04372748290278007
,
"acc_norm"
:
0.2894736842105263
,
"acc_norm_stderr"
:
0.042663394431593935
},
"piqa"
:
{
"acc"
:
0.6300326441784548
,
"acc_stderr"
:
0.011264415223415284
,
"acc_norm"
:
0.6202393906420022
,
"acc_norm_stderr"
:
0.011323483504715843
},
"hendrycksTest-formal_logic"
:
{
"acc"
:
0.30952380952380953
,
"acc_stderr"
:
0.04134913018303316
,
"acc_norm"
:
0.2857142857142857
,
"acc_norm_stderr"
:
0.04040610178208841
},
"hendrycksTest-astronomy"
:
{
"acc"
:
0.20394736842105263
,
"acc_stderr"
:
0.03279000406310051
,
"acc_norm"
:
0.3355263157894737
,
"acc_norm_stderr"
:
0.038424985593952694
},
"hendrycksTest-high_school_european_history"
:
{
"acc"
:
0.24848484848484848
,
"acc_stderr"
:
0.033744026441394036
,
"acc_norm"
:
0.3151515151515151
,
"acc_norm_stderr"
:
0.0362773057502241
},
"hendrycksTest-high_school_geography"
:
{
"acc"
:
0.22727272727272727
,
"acc_stderr"
:
0.02985751567338641
,
"acc_norm"
:
0.2878787878787879
,
"acc_norm_stderr"
:
0.03225883512300993
},
"hendrycksTest-moral_disputes"
:
{
"acc"
:
0.2832369942196532
,
"acc_stderr"
:
0.02425790170532337
,
"acc_norm"
:
0.2947976878612717
,
"acc_norm_stderr"
:
0.024547617794803835
},
"hendrycksTest-high_school_macroeconomics"
:
{
"acc"
:
0.26153846153846155
,
"acc_stderr"
:
0.022282141204204426
,
"acc_norm"
:
0.24871794871794872
,
"acc_norm_stderr"
:
0.021916957709213796
},
"arc_easy"
:
{
"acc"
:
0.4351851851851852
,
"acc_stderr"
:
0.010173216430370906
,
"acc_norm"
:
0.3998316498316498
,
"acc_norm_stderr"
:
0.010051788039412935
},
"hendrycksTest-marketing"
:
{
"acc"
:
0.33760683760683763
,
"acc_stderr"
:
0.030980296992618558
,
"acc_norm"
:
0.358974358974359
,
"acc_norm_stderr"
:
0.03142616993791924
},
"hendrycksTest-nutrition"
:
{
"acc"
:
0.2875816993464052
,
"acc_stderr"
:
0.02591780611714716
,
"acc_norm"
:
0.34967320261437906
,
"acc_norm_stderr"
:
0.027305308076274702
},
"hendrycksTest-management"
:
{
"acc"
:
0.17475728155339806
,
"acc_stderr"
:
0.037601780060266196
,
"acc_norm"
:
0.27184466019417475
,
"acc_norm_stderr"
:
0.044052680241409216
},
"hendrycksTest-prehistory"
:
{
"acc"
:
0.27469135802469136
,
"acc_stderr"
:
0.024836057868294677
,
"acc_norm"
:
0.2191358024691358
,
"acc_norm_stderr"
:
0.02301670564026219
},
"hendrycksTest-public_relations"
:
{
"acc"
:
0.2909090909090909
,
"acc_stderr"
:
0.04350271442923243
,
"acc_norm"
:
0.2545454545454545
,
"acc_norm_stderr"
:
0.041723430387053825
},
"hendrycksTest-high_school_chemistry"
:
{
"acc"
:
0.1724137931034483
,
"acc_stderr"
:
0.026577672183036586
,
"acc_norm"
:
0.24630541871921183
,
"acc_norm_stderr"
:
0.030315099285617732
},
"hendrycksTest-high_school_psychology"
:
{
"acc"
:
0.25137614678899084
,
"acc_stderr"
:
0.01859920636028741
,
"acc_norm"
:
0.25688073394495414
,
"acc_norm_stderr"
:
0.018732492928342472
},
"hendrycksTest-us_foreign_policy"
:
{
"acc"
:
0.28
,
"acc_stderr"
:
0.04512608598542127
,
"acc_norm"
:
0.34
,
"acc_norm_stderr"
:
0.047609522856952365
},
"hendrycksTest-elementary_mathematics"
:
{
"acc"
:
0.24867724867724866
,
"acc_stderr"
:
0.02226181769240015
,
"acc_norm"
:
0.25132275132275134
,
"acc_norm_stderr"
:
0.022340482339643895
},
"hendrycksTest-electrical_engineering"
:
{
"acc"
:
0.2413793103448276
,
"acc_stderr"
:
0.03565998174135302
,
"acc_norm"
:
0.30344827586206896
,
"acc_norm_stderr"
:
0.038312260488503336
},
"hendrycksTest-moral_scenarios"
:
{
"acc"
:
0.23798882681564246
,
"acc_stderr"
:
0.014242630070574915
,
"acc_norm"
:
0.27262569832402234
,
"acc_norm_stderr"
:
0.014893391735249588
},
"hendrycksTest-logical_fallacies"
:
{
"acc"
:
0.26380368098159507
,
"acc_stderr"
:
0.03462419931615625
,
"acc_norm"
:
0.3006134969325153
,
"acc_norm_stderr"
:
0.0360251131880677
},
"hendrycksTest-professional_medicine"
:
{
"acc"
:
0.25
,
"acc_stderr"
:
0.026303648393696036
,
"acc_norm"
:
0.22794117647058823
,
"acc_norm_stderr"
:
0.025483081468029804
},
"hendrycksTest-global_facts"
:
{
"acc"
:
0.19
,
"acc_stderr"
:
0.039427724440366255
,
"acc_norm"
:
0.22
,
"acc_norm_stderr"
:
0.041633319989322674
},
"hendrycksTest-abstract_algebra"
:
{
"acc"
:
0.17
,
"acc_stderr"
:
0.0377525168068637
,
"acc_norm"
:
0.2
,
"acc_norm_stderr"
:
0.04020151261036844
},
"hendrycksTest-clinical_knowledge"
:
{
"acc"
:
0.18867924528301888
,
"acc_stderr"
:
0.02407999513006222
,
"acc_norm"
:
0.2792452830188679
,
"acc_norm_stderr"
:
0.027611163402399715
},
"hendrycksTest-high_school_computer_science"
:
{
"acc"
:
0.21
,
"acc_stderr"
:
0.04093601807403326
,
"acc_norm"
:
0.25
,
"acc_norm_stderr"
:
0.04351941398892446
},
"hendrycksTest-college_mathematics"
:
{
"acc"
:
0.16
,
"acc_stderr"
:
0.03684529491774709
,
"acc_norm"
:
0.25
,
"acc_norm_stderr"
:
0.04351941398892446
},
"hendrycksTest-high_school_physics"
:
{
"acc"
:
0.2251655629139073
,
"acc_stderr"
:
0.03410435282008936
,
"acc_norm"
:
0.2119205298013245
,
"acc_norm_stderr"
:
0.03336767086567978
},
"logiqa"
:
{
"acc"
:
0.22734254992319508
,
"acc_stderr"
:
0.016439067675117734
,
"acc_norm"
:
0.27956989247311825
,
"acc_norm_stderr"
:
0.01760290918682245
},
"hendrycksTest-anatomy"
:
{
"acc"
:
0.24444444444444444
,
"acc_stderr"
:
0.037125378336148665
,
"acc_norm"
:
0.21481481481481482
,
"acc_norm_stderr"
:
0.03547854198560826
},
"hendrycksTest-college_chemistry"
:
{
"acc"
:
0.28
,
"acc_stderr"
:
0.045126085985421276
,
"acc_norm"
:
0.27
,
"acc_norm_stderr"
:
0.044619604333847394
},
"hendrycksTest-professional_psychology"
:
{
"acc"
:
0.238562091503268
,
"acc_stderr"
:
0.0172423858287796
,
"acc_norm"
:
0.24836601307189543
,
"acc_norm_stderr"
:
0.01747948700136476
},
"hendrycksTest-security_studies"
:
{
"acc"
:
0.3346938775510204
,
"acc_stderr"
:
0.030209235226242307
,
"acc_norm"
:
0.2571428571428571
,
"acc_norm_stderr"
:
0.027979823538744546
},
"hendrycksTest-international_law"
:
{
"acc"
:
0.1487603305785124
,
"acc_stderr"
:
0.03248470083807193
,
"acc_norm"
:
0.38016528925619836
,
"acc_norm_stderr"
:
0.04431324501968431
},
"hendrycksTest-human_aging"
:
{
"acc"
:
0.3632286995515695
,
"acc_stderr"
:
0.032277904428505
,
"acc_norm"
:
0.2600896860986547
,
"acc_norm_stderr"
:
0.029442495585857473
},
"hendrycksTest-jurisprudence"
:
{
"acc"
:
0.1574074074074074
,
"acc_stderr"
:
0.035207039905179614
,
"acc_norm"
:
0.37037037037037035
,
"acc_norm_stderr"
:
0.04668408033024931
},
"hendrycksTest-philosophy"
:
{
"acc"
:
0.2090032154340836
,
"acc_stderr"
:
0.023093140398374224
,
"acc_norm"
:
0.31189710610932475
,
"acc_norm_stderr"
:
0.02631185807185416
},
"hendrycksTest-miscellaneous"
:
{
"acc"
:
0.27458492975734355
,
"acc_stderr"
:
0.015959829933084032
,
"acc_norm"
:
0.26181353767560667
,
"acc_norm_stderr"
:
0.015720838678445266
},
"hendrycksTest-business_ethics"
:
{
"acc"
:
0.35
,
"acc_stderr"
:
0.047937248544110196
,
"acc_norm"
:
0.23
,
"acc_norm_stderr"
:
0.04229525846816506
},
"wsc"
:
{
"acc"
:
0.36538461538461536
,
"acc_stderr"
:
0.0474473339327792
},
"hendrycksTest-high_school_mathematics"
:
{
"acc"
:
0.17037037037037037
,
"acc_stderr"
:
0.022922554863074974
,
"acc_norm"
:
0.23333333333333334
,
"acc_norm_stderr"
:
0.02578787422095931
},
"hendrycksTest-medical_genetics"
:
{
"acc"
:
0.29
,
"acc_stderr"
:
0.045604802157206845
,
"acc_norm"
:
0.38
,
"acc_norm_stderr"
:
0.04878317312145633
},
"hendrycksTest-high_school_us_history"
:
{
"acc"
:
0.24509803921568626
,
"acc_stderr"
:
0.030190282453501936
,
"acc_norm"
:
0.2647058823529412
,
"acc_norm_stderr"
:
0.03096451792692341
},
"hendrycksTest-human_sexuality"
:
{
"acc"
:
0.3282442748091603
,
"acc_stderr"
:
0.04118438565806298
,
"acc_norm"
:
0.32061068702290074
,
"acc_norm_stderr"
:
0.04093329229834278
},
"arc_challenge"
:
{
"acc"
:
0.189419795221843
,
"acc_stderr"
:
0.011450705115910769
,
"acc_norm"
:
0.22781569965870307
,
"acc_norm_stderr"
:
0.012256708602326914
},
"hendrycksTest-high_school_government_and_politics"
:
{
"acc"
:
0.24870466321243523
,
"acc_stderr"
:
0.031195840877700304
,
"acc_norm"
:
0.3005181347150259
,
"acc_norm_stderr"
:
0.03308818594415751
},
"hendrycksTest-machine_learning"
:
{
"acc"
:
0.24107142857142858
,
"acc_stderr"
:
0.04059867246952687
,
"acc_norm"
:
0.2767857142857143
,
"acc_norm_stderr"
:
0.04246624336697624
},
"hendrycksTest-world_religions"
:
{
"acc"
:
0.28654970760233917
,
"acc_stderr"
:
0.03467826685703826
,
"acc_norm"
:
0.3216374269005848
,
"acc_norm_stderr"
:
0.03582529442573122
},
"hendrycksTest-college_medicine"
:
{
"acc"
:
0.24277456647398843
,
"acc_stderr"
:
0.0326926380614177
,
"acc_norm"
:
0.2658959537572254
,
"acc_norm_stderr"
:
0.03368762932259431
},
"hendrycksTest-high_school_statistics"
:
{
"acc"
:
0.24537037037037038
,
"acc_stderr"
:
0.029346665094372948
,
"acc_norm"
:
0.2777777777777778
,
"acc_norm_stderr"
:
0.03054674526495318
}
},
"versions"
:
{
"lambada_openai"
:
0
,
"hendrycksTest-college_biology"
:
0
,
"hendrycksTest-professional_law"
:
0
,
"hendrycksTest-virology"
:
0
,
"winogrande"
:
0
,
"hendrycksTest-high_school_microeconomics"
:
0
,
"hendrycksTest-college_physics"
:
0
,
"hendrycksTest-computer_security"
:
0
,
"hendrycksTest-conceptual_physics"
:
0
,
"hendrycksTest-high_school_world_history"
:
0
,
"hendrycksTest-sociology"
:
0
,
"sciq"
:
0
,
"hendrycksTest-professional_accounting"
:
0
,
"hendrycksTest-high_school_biology"
:
0
,
"hendrycksTest-college_computer_science"
:
0
,
"hendrycksTest-econometrics"
:
0
,
"piqa"
:
0
,
"hendrycksTest-formal_logic"
:
0
,
"hendrycksTest-astronomy"
:
0
,
"hendrycksTest-high_school_european_history"
:
0
,
"hendrycksTest-high_school_geography"
:
0
,
"hendrycksTest-moral_disputes"
:
0
,
"hendrycksTest-high_school_macroeconomics"
:
0
,
"arc_easy"
:
0
,
"hendrycksTest-marketing"
:
0
,
"hendrycksTest-nutrition"
:
0
,
"hendrycksTest-management"
:
0
,
"hendrycksTest-prehistory"
:
0
,
"hendrycksTest-public_relations"
:
0
,
"hendrycksTest-high_school_chemistry"
:
0
,
"hendrycksTest-high_school_psychology"
:
0
,
"hendrycksTest-us_foreign_policy"
:
0
,
"hendrycksTest-elementary_mathematics"
:
0
,
"hendrycksTest-electrical_engineering"
:
0
,
"hendrycksTest-moral_scenarios"
:
0
,
"hendrycksTest-logical_fallacies"
:
0
,
"hendrycksTest-professional_medicine"
:
0
,
"hendrycksTest-global_facts"
:
0
,
"hendrycksTest-abstract_algebra"
:
0
,
"hendrycksTest-clinical_knowledge"
:
0
,
"hendrycksTest-high_school_computer_science"
:
0
,
"hendrycksTest-college_mathematics"
:
0
,
"hendrycksTest-high_school_physics"
:
0
,
"logiqa"
:
0
,
"hendrycksTest-anatomy"
:
0
,
"hendrycksTest-college_chemistry"
:
0
,
"hendrycksTest-professional_psychology"
:
0
,
"hendrycksTest-security_studies"
:
0
,
"hendrycksTest-international_law"
:
0
,
"hendrycksTest-human_aging"
:
0
,
"hendrycksTest-jurisprudence"
:
0
,
"hendrycksTest-philosophy"
:
0
,
"hendrycksTest-miscellaneous"
:
0
,
"hendrycksTest-business_ethics"
:
0
,
"wsc"
:
0
,
"hendrycksTest-high_school_mathematics"
:
0
,
"hendrycksTest-medical_genetics"
:
0
,
"hendrycksTest-high_school_us_history"
:
0
,
"hendrycksTest-human_sexuality"
:
0
,
"arc_challenge"
:
0
,
"hendrycksTest-high_school_government_and_politics"
:
0
,
"hendrycksTest-machine_learning"
:
0
,
"hendrycksTest-world_religions"
:
0
,
"hendrycksTest-college_medicine"
:
0
,
"hendrycksTest-high_school_statistics"
:
0
},
"config"
:
{
"model"
:
"gpt2"
,
"model_args"
:
"pretrained=facebook/opt-125m"
,
"num_fewshot"
:
0
,
"batch_size"
:
64
,
"device"
:
"cuda"
,
"no_cache"
:
false
,
"limit"
:
null
,
"bootstrap_iters"
:
100000
,
"description_dict"
:
{}
}
}
Prev
1
…
4
5
6
7
8
9
10
11
Next
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment
