Skip to content

GitLab

"tests/t5/test_modeling_t5.py" did not exist on "552f8d30917cabd738d1c32a9e047f2da3ae1b28"
Unverified Commit 546fd5cd authored by Lintang Sutawika's avatar Lintang Sutawika Committed by GitHub
Browse files

Merge pull request #686 from EleutherAI/cleanup 

[Refactor] Cleanup for `big-refactor`
parents 465c695b 540d468e
Hide whitespace changes
Inline Side-by-side
Showing with 0 additions and 3063 deletions
results/mpt/mpt-7b/mpt-7b_arithmetic_5-shot.json deleted 100644 → 0
View file @ 465c695b
{
"results": {
"arithmetic_1dc": {
"acc": 0.081,
"acc_stderr": 0.006102304405675846
},
"arithmetic_2da": {
"acc": 0.918,
"acc_stderr": 0.006136515983374211
},
"arithmetic_2dm": {
"acc": 0.256,
"acc_stderr": 0.009761129023832868
},
"arithmetic_2ds": {
"acc": 0.7875,
"acc_stderr": 0.009149520854445372
},
"arithmetic_3da": {
"acc": 0.2915,
"acc_stderr": 0.010164424861564223
},
"arithmetic_3ds": {
"acc": 0.428,
"acc_stderr": 0.011066581884995264
},
"arithmetic_4da": {
"acc": 0.026,
"acc_stderr": 0.0035592603398856676
},
"arithmetic_4ds": {
"acc": 0.026,
"acc_stderr": 0.003559260339885677
},
"arithmetic_5da": {
"acc": 0.0045,
"acc_stderr": 0.0014969954902233202
},
"arithmetic_5ds": {
"acc": 0.002,
"acc_stderr": 0.0009992493430694993
}
},
"versions": {
"arithmetic_1dc": 0,
"arithmetic_2da": 0,
"arithmetic_2dm": 0,
"arithmetic_2ds": 0,
"arithmetic_3da": 0,
"arithmetic_3ds": 0,
"arithmetic_4da": 0,
"arithmetic_4ds": 0,
"arithmetic_5da": 0,
"arithmetic_5ds": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=mosaicml/mpt-7b,trust_remote_code=True",
"num_fewshot": 5,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/mpt/mpt-7b/mpt-7b_bbh_3-shot.json deleted 100644 → 0
View file @ 465c695b
{
"results": {
"bigbench_causal_judgement": {
"multiple_choice_grade": 0.5631578947368421,
"multiple_choice_grade_stderr": 0.036078330444807245
},
"bigbench_date_understanding": {
"multiple_choice_grade": 0.5826558265582655,
"multiple_choice_grade_stderr": 0.025705692903559226
},
"bigbench_disambiguation_qa": {
"multiple_choice_grade": 0.3643410852713178,
"multiple_choice_grade_stderr": 0.03001923241206336
},
"bigbench_dyck_languages": {
"multiple_choice_grade": 0.123,
"multiple_choice_grade_stderr": 0.010391293421849874
},
"bigbench_formal_fallacies_syllogisms_negation": {
"multiple_choice_grade": 0.49915492957746477,
"multiple_choice_grade_stderr": 0.0041960485493055645
},
"bigbench_geometric_shapes": {
"multiple_choice_grade": 0.20334261838440112,
"multiple_choice_grade_stderr": 0.021272007856536258,
"exact_str_match": 0.12256267409470752,
"exact_str_match_stderr": 0.017331879192703025
},
"bigbench_hyperbaton": {
"multiple_choice_grade": 0.4936,
"multiple_choice_grade_stderr": 0.002235907150490653
},
"bigbench_logical_deduction_five_objects": {
"multiple_choice_grade": 0.24,
"multiple_choice_grade_stderr": 0.01911886665375975
},
"bigbench_logical_deduction_seven_objects": {
"multiple_choice_grade": 0.1657142857142857,
"multiple_choice_grade_stderr": 0.014063673984033173
},
"bigbench_logical_deduction_three_objects": {
"multiple_choice_grade": 0.38666666666666666,
"multiple_choice_grade_stderr": 0.028163138908196852
},
"bigbench_movie_recommendation": {
"multiple_choice_grade": 0.438,
"multiple_choice_grade_stderr": 0.022210326363977417
},
"bigbench_navigate": {
"multiple_choice_grade": 0.486,
"multiple_choice_grade_stderr": 0.01581309754773099
},
"bigbench_reasoning_about_colored_objects": {
"multiple_choice_grade": 0.2985,
"multiple_choice_grade_stderr": 0.010234805842091589
},
"bigbench_ruin_names": {
"multiple_choice_grade": 0.296875,
"multiple_choice_grade_stderr": 0.021609729061250887
},
"bigbench_salient_translation_error_detection": {
"multiple_choice_grade": 0.17935871743486975,
"multiple_choice_grade_stderr": 0.012150393578288319
},
"bigbench_snarks": {
"multiple_choice_grade": 0.5303867403314917,
"multiple_choice_grade_stderr": 0.03719891321680327
},
"bigbench_sports_understanding": {
"multiple_choice_grade": 0.4949290060851927,
"multiple_choice_grade_stderr": 0.015930505328489487
},
"bigbench_temporal_sequences": {
"multiple_choice_grade": 0.296,
"multiple_choice_grade_stderr": 0.014442734941575016
},
"bigbench_tracking_shuffled_objects_five_objects": {
"multiple_choice_grade": 0.1944,
"multiple_choice_grade_stderr": 0.011197643581460408
},
"bigbench_tracking_shuffled_objects_seven_objects": {
"multiple_choice_grade": 0.13428571428571429,
"multiple_choice_grade_stderr": 0.008152809490408933
},
"bigbench_tracking_shuffled_objects_three_objects": {
"multiple_choice_grade": 0.38666666666666666,
"multiple_choice_grade_stderr": 0.028163138908196852
}
},
"versions": {
"bigbench_causal_judgement": 0,
"bigbench_date_understanding": 0,
"bigbench_disambiguation_qa": 0,
"bigbench_dyck_languages": 0,
"bigbench_formal_fallacies_syllogisms_negation": 0,
"bigbench_geometric_shapes": 0,
"bigbench_hyperbaton": 0,
"bigbench_logical_deduction_five_objects": 0,
"bigbench_logical_deduction_seven_objects": 0,
"bigbench_logical_deduction_three_objects": 0,
"bigbench_movie_recommendation": 0,
"bigbench_navigate": 0,
"bigbench_reasoning_about_colored_objects": 0,
"bigbench_ruin_names": 0,
"bigbench_salient_translation_error_detection": 0,
"bigbench_snarks": 0,
"bigbench_sports_understanding": 0,
"bigbench_temporal_sequences": 0,
"bigbench_tracking_shuffled_objects_five_objects": 0,
"bigbench_tracking_shuffled_objects_seven_objects": 0,
"bigbench_tracking_shuffled_objects_three_objects": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=mosaicml/mpt-7b,trust_remote_code=True",
"num_fewshot": 3,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/mpt/mpt-7b/mpt-7b_blimp_0-shot.json deleted 100644 → 0
View file @ 465c695b
{
"results": {
"blimp_adjunct_island": {
"acc": 0.878,
"acc_stderr": 0.010354864712936724
},
"blimp_anaphor_gender_agreement": {
"acc": 0.995,
"acc_stderr": 0.002231586874844882
},
"blimp_anaphor_number_agreement": {
"acc": 0.995,
"acc_stderr": 0.0022315868748448817
},
"blimp_animate_subject_passive": {
"acc": 0.775,
"acc_stderr": 0.013211720158614753
},
"blimp_animate_subject_trans": {
"acc": 0.884,
"acc_stderr": 0.010131468138756997
},
"blimp_causative": {
"acc": 0.747,
"acc_stderr": 0.01375427861358708
},
"blimp_complex_NP_island": {
"acc": 0.528,
"acc_stderr": 0.01579447578951147
},
"blimp_coordinate_structure_constraint_complex_left_branch": {
"acc": 0.779,
"acc_stderr": 0.013127502859696239
},
"blimp_coordinate_structure_constraint_object_extraction": {
"acc": 0.848,
"acc_stderr": 0.011358918303475293
},
"blimp_determiner_noun_agreement_1": {
"acc": 0.992,
"acc_stderr": 0.002818500300504507
},
"blimp_determiner_noun_agreement_2": {
"acc": 0.974,
"acc_stderr": 0.0050348137353182195
},
"blimp_determiner_noun_agreement_irregular_1": {
"acc": 0.936,
"acc_stderr": 0.007743640226919308
},
"blimp_determiner_noun_agreement_irregular_2": {
"acc": 0.928,
"acc_stderr": 0.008178195576218681
},
"blimp_determiner_noun_agreement_with_adj_2": {
"acc": 0.935,
"acc_stderr": 0.007799733061832037
},
"blimp_determiner_noun_agreement_with_adj_irregular_1": {
"acc": 0.883,
"acc_stderr": 0.010169287802713327
},
"blimp_determiner_noun_agreement_with_adj_irregular_2": {
"acc": 0.919,
"acc_stderr": 0.008632121032139973
},
"blimp_determiner_noun_agreement_with_adjective_1": {
"acc": 0.972,
"acc_stderr": 0.005219506034410047
},
"blimp_distractor_agreement_relational_noun": {
"acc": 0.889,
"acc_stderr": 0.009938701010583726
},
"blimp_distractor_agreement_relative_clause": {
"acc": 0.743,
"acc_stderr": 0.013825416526895042
},
"blimp_drop_argument": {
"acc": 0.787,
"acc_stderr": 0.012953717566737234
},
"blimp_ellipsis_n_bar_1": {
"acc": 0.79,
"acc_stderr": 0.012886662332274522
},
"blimp_ellipsis_n_bar_2": {
"acc": 0.928,
"acc_stderr": 0.008178195576218681
},
"blimp_existential_there_object_raising": {
"acc": 0.834,
"acc_stderr": 0.01177211037081219
},
"blimp_existential_there_quantifiers_1": {
"acc": 0.988,
"acc_stderr": 0.003444977194099841
},
"blimp_existential_there_quantifiers_2": {
"acc": 0.27,
"acc_stderr": 0.014046255632633908
},
"blimp_existential_there_subject_raising": {
"acc": 0.888,
"acc_stderr": 0.00997775303139724
},
"blimp_expletive_it_object_raising": {
"acc": 0.8,
"acc_stderr": 0.012655439943366646
},
"blimp_inchoative": {
"acc": 0.673,
"acc_stderr": 0.01484221315341125
},
"blimp_intransitive": {
"acc": 0.832,
"acc_stderr": 0.01182860583145426
},
"blimp_irregular_past_participle_adjectives": {
"acc": 0.972,
"acc_stderr": 0.005219506034410053
},
"blimp_irregular_past_participle_verbs": {
"acc": 0.885,
"acc_stderr": 0.010093407594904614
},
"blimp_irregular_plural_subject_verb_agreement_1": {
"acc": 0.92,
"acc_stderr": 0.008583336977753651
},
"blimp_irregular_plural_subject_verb_agreement_2": {
"acc": 0.908,
"acc_stderr": 0.009144376393151112
},
"blimp_left_branch_island_echo_question": {
"acc": 0.432,
"acc_stderr": 0.01567232023733621
},
"blimp_left_branch_island_simple_question": {
"acc": 0.897,
"acc_stderr": 0.009616833339695798
},
"blimp_matrix_question_npi_licensor_present": {
"acc": 0.705,
"acc_stderr": 0.014428554438445514
},
"blimp_npi_present_1": {
"acc": 0.579,
"acc_stderr": 0.015620595475301318
},
"blimp_npi_present_2": {
"acc": 0.688,
"acc_stderr": 0.01465847437050901
},
"blimp_only_npi_licensor_present": {
"acc": 0.916,
"acc_stderr": 0.008776162089491096
},
"blimp_only_npi_scope": {
"acc": 0.732,
"acc_stderr": 0.014013292702729491
},
"blimp_passive_1": {
"acc": 0.887,
"acc_stderr": 0.010016552866696863
},
"blimp_passive_2": {
"acc": 0.895,
"acc_stderr": 0.009698921026024947
},
"blimp_principle_A_c_command": {
"acc": 0.75,
"acc_stderr": 0.013699915608779773
},
"blimp_principle_A_case_1": {
"acc": 1.0,
"acc_stderr": 0.0
},
"blimp_principle_A_case_2": {
"acc": 0.94,
"acc_stderr": 0.007513751157474928
},
"blimp_principle_A_domain_1": {
"acc": 0.997,
"acc_stderr": 0.0017303161543469417
},
"blimp_principle_A_domain_2": {
"acc": 0.828,
"acc_stderr": 0.011939788882495321
},
"blimp_principle_A_domain_3": {
"acc": 0.761,
"acc_stderr": 0.013493000446937591
},
"blimp_principle_A_reconstruction": {
"acc": 0.41,
"acc_stderr": 0.015560917136921646
},
"blimp_regular_plural_subject_verb_agreement_1": {
"acc": 0.971,
"acc_stderr": 0.005309160685756986
},
"blimp_regular_plural_subject_verb_agreement_2": {
"acc": 0.907,
"acc_stderr": 0.009188875634996659
},
"blimp_sentential_negation_npi_licensor_present": {
"acc": 0.989,
"acc_stderr": 0.0032999833166078166
},
"blimp_sentential_negation_npi_scope": {
"acc": 0.733,
"acc_stderr": 0.013996674851796264
},
"blimp_sentential_subject_island": {
"acc": 0.399,
"acc_stderr": 0.015493193313162906
},
"blimp_superlative_quantifiers_1": {
"acc": 0.822,
"acc_stderr": 0.012102167676183596
},
"blimp_superlative_quantifiers_2": {
"acc": 0.897,
"acc_stderr": 0.009616833339695796
},
"blimp_tough_vs_raising_1": {
"acc": 0.69,
"acc_stderr": 0.014632638658632898
},
"blimp_tough_vs_raising_2": {
"acc": 0.829,
"acc_stderr": 0.011912216456264606
},
"blimp_transitive": {
"acc": 0.872,
"acc_stderr": 0.010570133761108658
},
"blimp_wh_island": {
"acc": 0.813,
"acc_stderr": 0.012336254828074126
},
"blimp_wh_questions_object_gap": {
"acc": 0.763,
"acc_stderr": 0.01345407046257794
},
"blimp_wh_questions_subject_gap": {
"acc": 0.89,
"acc_stderr": 0.009899393819724435
},
"blimp_wh_questions_subject_gap_long_distance": {
"acc": 0.893,
"acc_stderr": 0.009779910359847169
},
"blimp_wh_vs_that_no_gap": {
"acc": 0.946,
"acc_stderr": 0.007150883521295428
},
"blimp_wh_vs_that_no_gap_long_distance": {
"acc": 0.951,
"acc_stderr": 0.006829761756140921
},
"blimp_wh_vs_that_with_gap": {
"acc": 0.321,
"acc_stderr": 0.01477082181793466
},
"blimp_wh_vs_that_with_gap_long_distance": {
"acc": 0.292,
"acc_stderr": 0.014385511563477347
}
},
"versions": {
"blimp_adjunct_island": 0,
"blimp_anaphor_gender_agreement": 0,
"blimp_anaphor_number_agreement": 0,
"blimp_animate_subject_passive": 0,
"blimp_animate_subject_trans": 0,
"blimp_causative": 0,
"blimp_complex_NP_island": 0,
"blimp_coordinate_structure_constraint_complex_left_branch": 0,
"blimp_coordinate_structure_constraint_object_extraction": 0,
"blimp_determiner_noun_agreement_1": 0,
"blimp_determiner_noun_agreement_2": 0,
"blimp_determiner_noun_agreement_irregular_1": 0,
"blimp_determiner_noun_agreement_irregular_2": 0,
"blimp_determiner_noun_agreement_with_adj_2": 0,
"blimp_determiner_noun_agreement_with_adj_irregular_1": 0,
"blimp_determiner_noun_agreement_with_adj_irregular_2": 0,
"blimp_determiner_noun_agreement_with_adjective_1": 0,
"blimp_distractor_agreement_relational_noun": 0,
"blimp_distractor_agreement_relative_clause": 0,
"blimp_drop_argument": 0,
"blimp_ellipsis_n_bar_1": 0,
"blimp_ellipsis_n_bar_2": 0,
"blimp_existential_there_object_raising": 0,
"blimp_existential_there_quantifiers_1": 0,
"blimp_existential_there_quantifiers_2": 0,
"blimp_existential_there_subject_raising": 0,
"blimp_expletive_it_object_raising": 0,
"blimp_inchoative": 0,
"blimp_intransitive": 0,
"blimp_irregular_past_participle_adjectives": 0,
"blimp_irregular_past_participle_verbs": 0,
"blimp_irregular_plural_subject_verb_agreement_1": 0,
"blimp_irregular_plural_subject_verb_agreement_2": 0,
"blimp_left_branch_island_echo_question": 0,
"blimp_left_branch_island_simple_question": 0,
"blimp_matrix_question_npi_licensor_present": 0,
"blimp_npi_present_1": 0,
"blimp_npi_present_2": 0,
"blimp_only_npi_licensor_present": 0,
"blimp_only_npi_scope": 0,
"blimp_passive_1": 0,
"blimp_passive_2": 0,
"blimp_principle_A_c_command": 0,
"blimp_principle_A_case_1": 0,
"blimp_principle_A_case_2": 0,
"blimp_principle_A_domain_1": 0,
"blimp_principle_A_domain_2": 0,
"blimp_principle_A_domain_3": 0,
"blimp_principle_A_reconstruction": 0,
"blimp_regular_plural_subject_verb_agreement_1": 0,
"blimp_regular_plural_subject_verb_agreement_2": 0,
"blimp_sentential_negation_npi_licensor_present": 0,
"blimp_sentential_negation_npi_scope": 0,
"blimp_sentential_subject_island": 0,
"blimp_superlative_quantifiers_1": 0,
"blimp_superlative_quantifiers_2": 0,
"blimp_tough_vs_raising_1": 0,
"blimp_tough_vs_raising_2": 0,
"blimp_transitive": 0,
"blimp_wh_island": 0,
"blimp_wh_questions_object_gap": 0,
"blimp_wh_questions_subject_gap": 0,
"blimp_wh_questions_subject_gap_long_distance": 0,
"blimp_wh_vs_that_no_gap": 0,
"blimp_wh_vs_that_no_gap_long_distance": 0,
"blimp_wh_vs_that_with_gap": 0,
"blimp_wh_vs_that_with_gap_long_distance": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=mosaicml/mpt-7b,trust_remote_code=True",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/mpt/mpt-7b/mpt-7b_common_sense_reasoning_0-shot.json deleted 100644 → 0
View file @ 465c695b
{
"results": {
"arc_challenge": {
"acc": 0.4061433447098976,
"acc_stderr": 0.014351656690097862,
"acc_norm": 0.4180887372013652,
"acc_norm_stderr": 0.01441398839699608
},
"arc_easy": {
"acc": 0.7487373737373737,
"acc_stderr": 0.008900141191221648,
"acc_norm": 0.7028619528619529,
"acc_norm_stderr": 0.009377397867796849
},
"boolq": {
"acc": 0.7351681957186544,
"acc_stderr": 0.007717399182659714
},
"copa": {
"acc": 0.85,
"acc_stderr": 0.03588702812826373
},
"hellaswag": {
"acc": 0.5723959370643298,
"acc_stderr": 0.004937199759947679,
"acc_norm": 0.761202947619996,
"acc_norm_stderr": 0.004254771367531344
},
"mc_taco": {
"em": 0.13513513513513514,
"f1": 0.45480193909643063
},
"openbookqa": {
"acc": 0.32,
"acc_stderr": 0.020882340488761805,
"acc_norm": 0.426,
"acc_norm_stderr": 0.022136577335085637
},
"piqa": {
"acc": 0.7916213275299239,
"acc_stderr": 0.009476125383049457,
"acc_norm": 0.8041349292709467,
"acc_norm_stderr": 0.009259518041395765
},
"prost": {
"acc": 0.25731212638770284,
"acc_stderr": 0.0031937906462958443,
"acc_norm": 0.30118488471391974,
"acc_norm_stderr": 0.003351744324251047
},
"swag": {
"acc": 0.5616814955513346,
"acc_stderr": 0.0035080896485241934,
"acc_norm": 0.7579726082175348,
"acc_norm_stderr": 0.003028236139561354
},
"winogrande": {
"acc": 0.6866614048934491,
"acc_stderr": 0.013036512096747986
},
"wsc273": {
"acc": 0.8571428571428571,
"acc_stderr": 0.021217447349500165
}
},
"versions": {
"arc_challenge": 0,
"arc_easy": 0,
"boolq": 1,
"copa": 0,
"hellaswag": 0,
"mc_taco": 0,
"openbookqa": 0,
"piqa": 0,
"prost": 0,
"swag": 0,
"winogrande": 0,
"wsc273": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=mosaicml/mpt-7b,trust_remote_code=True",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/mpt/mpt-7b/mpt-7b_glue_0-shot.json deleted 100644 → 0
View file @ 465c695b
{
"results": {
"cola": {
"mcc": -0.044109532013985,
"mcc_stderr": 0.031207420387242075
},
"mnli": {
"acc": 0.3782985226693836,
"acc_stderr": 0.0048953668826959605
},
"mnli_mismatched": {
"acc": 0.37489829129373475,
"acc_stderr": 0.004882398521925481
},
"mrpc": {
"acc": 0.6299019607843137,
"acc_stderr": 0.023933029030729185,
"f1": 0.7560581583198709,
"f1_stderr": 0.019295620717095126
},
"qnli": {
"acc": 0.513454146073586,
"acc_stderr": 0.006762960839582666
},
"qqp": {
"acc": 0.5035864457086322,
"acc_stderr": 0.002486636660034357,
"f1": 0.5414038936111873,
"f1_stderr": 0.0028706161186533327
},
"rte": {
"acc": 0.6389891696750902,
"acc_stderr": 0.028910281676964168
},
"sst": {
"acc": 0.768348623853211,
"acc_stderr": 0.014295110726150017
},
"wnli": {
"acc": 0.4788732394366197,
"acc_stderr": 0.05970805879899505
}
},
"versions": {
"cola": 0,
"mnli": 0,
"mnli_mismatched": 0,
"mrpc": 0,
"qnli": 0,
"qqp": 0,
"rte": 0,
"sst": 0,
"wnli": 1
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=mosaicml/mpt-7b,trust_remote_code=True",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/mpt/mpt-7b/mpt-7b_human_alignment_0-shot.json deleted 100644 → 0
View file @ 465c695b
{
"results": {
"crows_pairs_english_age": {
"likelihood_difference": 4.1510989010989015,
"likelihood_difference_stderr": 0.38322042294913006,
"pct_stereotype": 0.7362637362637363,
"pct_stereotype_stderr": 0.04644942852497395
},
"crows_pairs_english_autre": {
"likelihood_difference": 5.056818181818182,
"likelihood_difference_stderr": 1.7702853227564594,
"pct_stereotype": 0.7272727272727273,
"pct_stereotype_stderr": 0.14083575804390605
},
"crows_pairs_english_disability": {
"likelihood_difference": 6.019230769230769,
"likelihood_difference_stderr": 0.6330801651940536,
"pct_stereotype": 0.7692307692307693,
"pct_stereotype_stderr": 0.05266563052934291
},
"crows_pairs_english_gender": {
"likelihood_difference": 2.682421875,
"likelihood_difference_stderr": 0.17011984636779479,
"pct_stereotype": 0.6375,
"pct_stereotype_stderr": 0.02691527109619775
},
"crows_pairs_english_nationality": {
"likelihood_difference": 3.498263888888889,
"likelihood_difference_stderr": 0.21512426671182808,
"pct_stereotype": 0.6157407407407407,
"pct_stereotype_stderr": 0.03317354514310742
},
"crows_pairs_english_physical_appearance": {
"likelihood_difference": 3.7378472222222223,
"likelihood_difference_stderr": 0.3384578916047944,
"pct_stereotype": 0.7222222222222222,
"pct_stereotype_stderr": 0.053156331218399945
},
"crows_pairs_english_race_color": {
"likelihood_difference": 3.3619586614173227,
"likelihood_difference_stderr": 0.14103384370541044,
"pct_stereotype": 0.5728346456692913,
"pct_stereotype_stderr": 0.021968918082519016
},
"crows_pairs_english_religion": {
"likelihood_difference": 3.6644144144144146,
"likelihood_difference_stderr": 0.33863382204528913,
"pct_stereotype": 0.7297297297297297,
"pct_stereotype_stderr": 0.04234321361084538
},
"crows_pairs_english_sexual_orientation": {
"likelihood_difference": 4.630376344086022,
"likelihood_difference_stderr": 0.4574502494208299,
"pct_stereotype": 0.8279569892473119,
"pct_stereotype_stderr": 0.03934852812061865
},
"crows_pairs_english_socioeconomic": {
"likelihood_difference": 4.065131578947368,
"likelihood_difference_stderr": 0.23523235884669547,
"pct_stereotype": 0.6789473684210526,
"pct_stereotype_stderr": 0.03396059335824887
},
"crows_pairs_french_age": {
"likelihood_difference": 3.609722222222222,
"likelihood_difference_stderr": 0.3615396777446631,
"pct_stereotype": 0.4222222222222222,
"pct_stereotype_stderr": 0.05235473399540658
},
"crows_pairs_french_autre": {
"likelihood_difference": 2.6923076923076925,
"likelihood_difference_stderr": 0.9229767573858816,
"pct_stereotype": 0.6153846153846154,
"pct_stereotype_stderr": 0.14044168141158106
},
"crows_pairs_french_disability": {
"likelihood_difference": 4.958333333333333,
"likelihood_difference_stderr": 0.42689940749326905,
"pct_stereotype": 0.6363636363636364,
"pct_stereotype_stderr": 0.05966637484671758
},
"crows_pairs_french_gender": {
"likelihood_difference": 3.213785046728972,
"likelihood_difference_stderr": 0.17588386956758606,
"pct_stereotype": 0.5109034267912772,
"pct_stereotype_stderr": 0.027944203070818633
},
"crows_pairs_french_nationality": {
"likelihood_difference": 3.883399209486166,
"likelihood_difference_stderr": 0.21844567559439967,
"pct_stereotype": 0.3438735177865613,
"pct_stereotype_stderr": 0.029922155720849428
},
"crows_pairs_french_physical_appearance": {
"likelihood_difference": 3.2274305555555554,
"likelihood_difference_stderr": 0.4328667471914375,
"pct_stereotype": 0.5972222222222222,
"pct_stereotype_stderr": 0.05820650942569533
},
"crows_pairs_french_race_color": {
"likelihood_difference": 3.161413043478261,
"likelihood_difference_stderr": 0.16557903974411925,
"pct_stereotype": 0.4369565217391304,
"pct_stereotype_stderr": 0.023151745316873387
},
"crows_pairs_french_religion": {
"likelihood_difference": 3.5673913043478263,
"likelihood_difference_stderr": 0.3368331015818195,
"pct_stereotype": 0.6260869565217392,
"pct_stereotype_stderr": 0.045315858286449635
},
"crows_pairs_french_sexual_orientation": {
"likelihood_difference": 4.791208791208791,
"likelihood_difference_stderr": 0.4009539855629619,
"pct_stereotype": 0.7802197802197802,
"pct_stereotype_stderr": 0.043649726328985346
},
"crows_pairs_french_socioeconomic": {
"likelihood_difference": 3.9939413265306123,
"likelihood_difference_stderr": 0.263093158126228,
"pct_stereotype": 0.6581632653061225,
"pct_stereotype_stderr": 0.033967132039868675
},
"ethics_cm": {
"acc": 0.5459459459459459,
"acc_stderr": 0.007988936899457039
},
"ethics_deontology": {
"acc": 0.5025027808676307,
"acc_stderr": 0.008339021933755771,
"em": 0.004449388209121246
},
"ethics_justice": {
"acc": 0.5196005917159763,
"acc_stderr": 0.009609770755397009,
"em": 0.011834319526627219
},
"ethics_utilitarianism": {
"acc": 0.5748752079866889,
"acc_stderr": 0.007130302336230959
},
"ethics_utilitarianism_original": {
"acc": 0.9956322795341098,
"acc_stderr": 0.000951129914345755
},
"ethics_virtue": {
"acc": 0.8040201005025126,
"acc_stderr": 0.005628417801676332,
"em": 0.12562814070351758
},
"toxigen": {
"acc": 0.4319148936170213,
"acc_stderr": 0.016164899004911828,
"acc_norm": 0.4319148936170213,
"acc_norm_stderr": 0.016164899004911828
}
},
"versions": {
"crows_pairs_english_age": 0,
"crows_pairs_english_autre": 0,
"crows_pairs_english_disability": 0,
"crows_pairs_english_gender": 0,
"crows_pairs_english_nationality": 0,
"crows_pairs_english_physical_appearance": 0,
"crows_pairs_english_race_color": 0,
"crows_pairs_english_religion": 0,
"crows_pairs_english_sexual_orientation": 0,
"crows_pairs_english_socioeconomic": 0,
"crows_pairs_french_age": 0,
"crows_pairs_french_autre": 0,
"crows_pairs_french_disability": 0,
"crows_pairs_french_gender": 0,
"crows_pairs_french_nationality": 0,
"crows_pairs_french_physical_appearance": 0,
"crows_pairs_french_race_color": 0,
"crows_pairs_french_religion": 0,
"crows_pairs_french_sexual_orientation": 0,
"crows_pairs_french_socioeconomic": 0,
"ethics_cm": 0,
"ethics_deontology": 0,
"ethics_justice": 0,
"ethics_utilitarianism": 0,
"ethics_utilitarianism_original": 0,
"ethics_virtue": 0,
"toxigen": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=mosaicml/mpt-7b,trust_remote_code=True",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/mpt/mpt-7b/mpt-7b_lambada_0-shot.json deleted 100644 → 0
View file @ 465c695b
{
"results": {
"lambada_openai": {
"ppl": 3.8685393479651173,
"ppl_stderr": 0.08079962928213236,
"acc": 0.6834853483407723,
"acc_stderr": 0.006479978824925181
},
"lambada_openai_cloze": {
"ppl": 26.557641771948866,
"ppl_stderr": 0.7040059319391023,
"acc": 0.3964680768484378,
"acc_stderr": 0.006815007030417622
},
"lambada_openai_mt_de": {
"ppl": 70.11746381055114,
"ppl_stderr": 4.044349022196945,
"acc": 0.33766737822627596,
"acc_stderr": 0.006588623616680426
},
"lambada_openai_mt_en": {
"ppl": 3.8685393479651173,
"ppl_stderr": 0.08079962928213236,
"acc": 0.6834853483407723,
"acc_stderr": 0.006479978824925181
},
"lambada_openai_mt_es": {
"ppl": 67.22873557270539,
"ppl_stderr": 3.6853291413042055,
"acc": 0.3694934989326606,
"acc_stderr": 0.006724504027913079
},
"lambada_openai_mt_fr": {
"ppl": 42.93424455960906,
"ppl_stderr": 2.3689294613751506,
"acc": 0.4302348146710654,
"acc_stderr": 0.006897835015074962
},
"lambada_openai_mt_it": {
"ppl": 65.76277239773485,
"ppl_stderr": 3.8678074045404967,
"acc": 0.392004657481079,
"acc_stderr": 0.006801548708056975
},
"lambada_standard": {
"ppl": 4.922814417244969,
"ppl_stderr": 0.10844800336124351,
"acc": 0.619056860081506,
"acc_stderr": 0.006765617200231529
},
"lambada_standard_cloze": {
"ppl": 109.09886744922697,
"ppl_stderr": 3.0351063735026287,
"acc": 0.1674752571317679,
"acc_stderr": 0.005202187177767864
}
},
"versions": {
"lambada_openai": 0,
"lambada_openai_cloze": 0,
"lambada_openai_mt_de": 0,
"lambada_openai_mt_en": 0,
"lambada_openai_mt_es": 0,
"lambada_openai_mt_fr": 0,
"lambada_openai_mt_it": 0,
"lambada_standard": 0,
"lambada_standard_cloze": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=mosaicml/mpt-7b,trust_remote_code=True",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/mpt/mpt-7b/mpt-7b_mmlu_5-shot.json deleted 100644 → 0
View file @ 465c695b
{
"results": {
"hendrycksTest-abstract_algebra": {
"acc": 0.18,
"acc_stderr": 0.03861229196653695,
"acc_norm": 0.21,
"acc_norm_stderr": 0.040936018074033256
},
"hendrycksTest-anatomy": {
"acc": 0.3851851851851852,
"acc_stderr": 0.042039210401562783,
"acc_norm": 0.37777777777777777,
"acc_norm_stderr": 0.04188307537595853
},
"hendrycksTest-astronomy": {
"acc": 0.39473684210526316,
"acc_stderr": 0.039777499346220734,
"acc_norm": 0.42105263157894735,
"acc_norm_stderr": 0.04017901275981748
},
"hendrycksTest-business_ethics": {
"acc": 0.49,
"acc_stderr": 0.05024183937956911,
"acc_norm": 0.48,
"acc_norm_stderr": 0.050211673156867795
},
"hendrycksTest-clinical_knowledge": {
"acc": 0.3320754716981132,
"acc_stderr": 0.028985455652334388,
"acc_norm": 0.37735849056603776,
"acc_norm_stderr": 0.029832808114796005
},
"hendrycksTest-college_biology": {
"acc": 0.3819444444444444,
"acc_stderr": 0.040629907841466674,
"acc_norm": 0.3541666666666667,
"acc_norm_stderr": 0.039994111357535424
},
"hendrycksTest-college_chemistry": {
"acc": 0.39,
"acc_stderr": 0.04902071300001974,
"acc_norm": 0.41,
"acc_norm_stderr": 0.049431107042371025
},
"hendrycksTest-college_computer_science": {
"acc": 0.34,
"acc_stderr": 0.04760952285695235,
"acc_norm": 0.32,
"acc_norm_stderr": 0.046882617226215034
},
"hendrycksTest-college_mathematics": {
"acc": 0.27,
"acc_stderr": 0.044619604333847394,
"acc_norm": 0.33,
"acc_norm_stderr": 0.047258156262526045
},
"hendrycksTest-college_medicine": {
"acc": 0.36416184971098264,
"acc_stderr": 0.03669072477416907,
"acc_norm": 0.3468208092485549,
"acc_norm_stderr": 0.036291466701596636
},
"hendrycksTest-college_physics": {
"acc": 0.30392156862745096,
"acc_stderr": 0.045766654032077615,
"acc_norm": 0.3333333333333333,
"acc_norm_stderr": 0.04690650298201943
},
"hendrycksTest-computer_security": {
"acc": 0.41,
"acc_stderr": 0.04943110704237102,
"acc_norm": 0.41,
"acc_norm_stderr": 0.049431107042371025
},
"hendrycksTest-conceptual_physics": {
"acc": 0.3276595744680851,
"acc_stderr": 0.030683020843231015,
"acc_norm": 0.2553191489361702,
"acc_norm_stderr": 0.02850485647051418
},
"hendrycksTest-econometrics": {
"acc": 0.2719298245614035,
"acc_stderr": 0.04185774424022056,
"acc_norm": 0.23684210526315788,
"acc_norm_stderr": 0.039994238792813365
},
"hendrycksTest-electrical_engineering": {
"acc": 0.36551724137931035,
"acc_stderr": 0.040131241954243856,
"acc_norm": 0.33793103448275863,
"acc_norm_stderr": 0.03941707632064889
},
"hendrycksTest-elementary_mathematics": {
"acc": 0.29894179894179895,
"acc_stderr": 0.023577604791655802,
"acc_norm": 0.28835978835978837,
"acc_norm_stderr": 0.023330654054535892
},
"hendrycksTest-formal_logic": {
"acc": 0.30952380952380953,
"acc_stderr": 0.04134913018303316,
"acc_norm": 0.2857142857142857,
"acc_norm_stderr": 0.040406101782088394
},
"hendrycksTest-global_facts": {
"acc": 0.35,
"acc_stderr": 0.047937248544110196,
"acc_norm": 0.33,
"acc_norm_stderr": 0.047258156262526045
},
"hendrycksTest-high_school_biology": {
"acc": 0.36451612903225805,
"acc_stderr": 0.02737987122994325,
"acc_norm": 0.3903225806451613,
"acc_norm_stderr": 0.027751256636969583
},
"hendrycksTest-high_school_chemistry": {
"acc": 0.21182266009852216,
"acc_stderr": 0.02874898368994106,
"acc_norm": 0.21674876847290642,
"acc_norm_stderr": 0.028990331252516235
},
"hendrycksTest-high_school_computer_science": {
"acc": 0.43,
"acc_stderr": 0.04975698519562428,
"acc_norm": 0.41,
"acc_norm_stderr": 0.049431107042371025
},
"hendrycksTest-high_school_european_history": {
"acc": 0.38181818181818183,
"acc_stderr": 0.03793713171165635,
"acc_norm": 0.37575757575757573,
"acc_norm_stderr": 0.03781887353205983
},
"hendrycksTest-high_school_geography": {
"acc": 0.3838383838383838,
"acc_stderr": 0.03464881675016339,
"acc_norm": 0.40404040404040403,
"acc_norm_stderr": 0.03496130972056128
},
"hendrycksTest-high_school_government_and_politics": {
"acc": 0.41450777202072536,
"acc_stderr": 0.03555300319557673,
"acc_norm": 0.41450777202072536,
"acc_norm_stderr": 0.03555300319557672
},
"hendrycksTest-high_school_macroeconomics": {
"acc": 0.3487179487179487,
"acc_stderr": 0.024162780284017717,
"acc_norm": 0.29743589743589743,
"acc_norm_stderr": 0.02317740813146594
},
"hendrycksTest-high_school_mathematics": {
"acc": 0.29259259259259257,
"acc_stderr": 0.027738969632176088,
"acc_norm": 0.3037037037037037,
"acc_norm_stderr": 0.02803792996911499
},
"hendrycksTest-high_school_microeconomics": {
"acc": 0.33613445378151263,
"acc_stderr": 0.030684737115135353,
"acc_norm": 0.3697478991596639,
"acc_norm_stderr": 0.03135709599613591
},
"hendrycksTest-high_school_physics": {
"acc": 0.2781456953642384,
"acc_stderr": 0.03658603262763743,
"acc_norm": 0.2781456953642384,
"acc_norm_stderr": 0.03658603262763743
},
"hendrycksTest-high_school_psychology": {
"acc": 0.46972477064220186,
"acc_stderr": 0.021397988604936965,
"acc_norm": 0.44587155963302755,
"acc_norm_stderr": 0.02131133500970858
},
"hendrycksTest-high_school_statistics": {
"acc": 0.3287037037037037,
"acc_stderr": 0.03203614084670058,
"acc_norm": 0.32407407407407407,
"acc_norm_stderr": 0.03191923445686185
},
"hendrycksTest-high_school_us_history": {
"acc": 0.3431372549019608,
"acc_stderr": 0.03332139944668085,
"acc_norm": 0.3137254901960784,
"acc_norm_stderr": 0.032566854844603886
},
"hendrycksTest-high_school_world_history": {
"acc": 0.29535864978902954,
"acc_stderr": 0.029696338713422876,
"acc_norm": 0.2869198312236287,
"acc_norm_stderr": 0.02944377302259469
},
"hendrycksTest-human_aging": {
"acc": 0.336322869955157,
"acc_stderr": 0.031708824268455,
"acc_norm": 0.3273542600896861,
"acc_norm_stderr": 0.031493846709941306
},
"hendrycksTest-human_sexuality": {
"acc": 0.2748091603053435,
"acc_stderr": 0.03915345408847836,
"acc_norm": 0.3282442748091603,
"acc_norm_stderr": 0.041184385658062976
},
"hendrycksTest-international_law": {
"acc": 0.371900826446281,
"acc_stderr": 0.04412015806624504,
"acc_norm": 0.49586776859504134,
"acc_norm_stderr": 0.045641987674327526
},
"hendrycksTest-jurisprudence": {
"acc": 0.3425925925925926,
"acc_stderr": 0.045879047413018105,
"acc_norm": 0.39814814814814814,
"acc_norm_stderr": 0.04732332615978814
},
"hendrycksTest-logical_fallacies": {
"acc": 0.3803680981595092,
"acc_stderr": 0.038142698932618374,
"acc_norm": 0.36809815950920244,
"acc_norm_stderr": 0.03789213935838395
},
"hendrycksTest-machine_learning": {
"acc": 0.26785714285714285,
"acc_stderr": 0.04203277291467763,
"acc_norm": 0.24107142857142858,
"acc_norm_stderr": 0.04059867246952686
},
"hendrycksTest-management": {
"acc": 0.42718446601941745,
"acc_stderr": 0.04897957737781169,
"acc_norm": 0.39805825242718446,
"acc_norm_stderr": 0.0484674825397724
},
"hendrycksTest-marketing": {
"acc": 0.5512820512820513,
"acc_stderr": 0.032583346493868806,
"acc_norm": 0.5512820512820513,
"acc_norm_stderr": 0.032583346493868806
},
"hendrycksTest-medical_genetics": {
"acc": 0.39,
"acc_stderr": 0.04902071300001975,
"acc_norm": 0.38,
"acc_norm_stderr": 0.04878317312145633
},
"hendrycksTest-miscellaneous": {
"acc": 0.5555555555555556,
"acc_stderr": 0.017769250583533246,
"acc_norm": 0.5568326947637292,
"acc_norm_stderr": 0.01776408503534841
},
"hendrycksTest-moral_disputes": {
"acc": 0.3208092485549133,
"acc_stderr": 0.025131000233647904,
"acc_norm": 0.30057803468208094,
"acc_norm_stderr": 0.024685316867257796
},
"hendrycksTest-moral_scenarios": {
"acc": 0.26033519553072626,
"acc_stderr": 0.014676252009319483,
"acc_norm": 0.27262569832402234,
"acc_norm_stderr": 0.014893391735249614
},
"hendrycksTest-nutrition": {
"acc": 0.3431372549019608,
"acc_stderr": 0.027184498909941616,
"acc_norm": 0.4019607843137255,
"acc_norm_stderr": 0.02807415894760066
},
"hendrycksTest-philosophy": {
"acc": 0.3762057877813505,
"acc_stderr": 0.027513925683549427,
"acc_norm": 0.36977491961414793,
"acc_norm_stderr": 0.027417996705630998
},
"hendrycksTest-prehistory": {
"acc": 0.33641975308641975,
"acc_stderr": 0.02628973494595293,
"acc_norm": 0.3055555555555556,
"acc_norm_stderr": 0.025630824975621344
},
"hendrycksTest-professional_accounting": {
"acc": 0.3049645390070922,
"acc_stderr": 0.027464708442022135,
"acc_norm": 0.2907801418439716,
"acc_norm_stderr": 0.027090664368353178
},
"hendrycksTest-professional_law": {
"acc": 0.25945241199478486,
"acc_stderr": 0.011195262076350299,
"acc_norm": 0.2842242503259452,
"acc_norm_stderr": 0.011519880596516074
},
"hendrycksTest-professional_medicine": {
"acc": 0.29411764705882354,
"acc_stderr": 0.027678468642144703,
"acc_norm": 0.3161764705882353,
"acc_norm_stderr": 0.02824568739146291
},
"hendrycksTest-professional_psychology": {
"acc": 0.315359477124183,
"acc_stderr": 0.018798086284886883,
"acc_norm": 0.3022875816993464,
"acc_norm_stderr": 0.01857923271111388
},
"hendrycksTest-public_relations": {
"acc": 0.41818181818181815,
"acc_stderr": 0.04724577405731571,
"acc_norm": 0.42727272727272725,
"acc_norm_stderr": 0.04738198703545483
},
"hendrycksTest-security_studies": {
"acc": 0.2816326530612245,
"acc_stderr": 0.028795185574291282,
"acc_norm": 0.24081632653061225,
"acc_norm_stderr": 0.027372942201788163
},
"hendrycksTest-sociology": {
"acc": 0.34328358208955223,
"acc_stderr": 0.03357379665433431,
"acc_norm": 0.3681592039800995,
"acc_norm_stderr": 0.034104105654953025
},
"hendrycksTest-us_foreign_policy": {
"acc": 0.38,
"acc_stderr": 0.04878317312145632,
"acc_norm": 0.39,
"acc_norm_stderr": 0.04902071300001975
},
"hendrycksTest-virology": {
"acc": 0.3253012048192771,
"acc_stderr": 0.03647168523683229,
"acc_norm": 0.3253012048192771,
"acc_norm_stderr": 0.03647168523683227
},
"hendrycksTest-world_religions": {
"acc": 0.543859649122807,
"acc_stderr": 0.03820042586602966,
"acc_norm": 0.5789473684210527,
"acc_norm_stderr": 0.03786720706234214
}
},
"versions": {
"hendrycksTest-abstract_algebra": 0,
"hendrycksTest-anatomy": 0,
"hendrycksTest-astronomy": 0,
"hendrycksTest-business_ethics": 0,
"hendrycksTest-clinical_knowledge": 0,
"hendrycksTest-college_biology": 0,
"hendrycksTest-college_chemistry": 0,
"hendrycksTest-college_computer_science": 0,
"hendrycksTest-college_mathematics": 0,
"hendrycksTest-college_medicine": 0,
"hendrycksTest-college_physics": 0,
"hendrycksTest-computer_security": 0,
"hendrycksTest-conceptual_physics": 0,
"hendrycksTest-econometrics": 0,
"hendrycksTest-electrical_engineering": 0,
"hendrycksTest-elementary_mathematics": 0,
"hendrycksTest-formal_logic": 0,
"hendrycksTest-global_facts": 0,
"hendrycksTest-high_school_biology": 0,
"hendrycksTest-high_school_chemistry": 0,
"hendrycksTest-high_school_computer_science": 0,
"hendrycksTest-high_school_european_history": 0,
"hendrycksTest-high_school_geography": 0,
"hendrycksTest-high_school_government_and_politics": 0,
"hendrycksTest-high_school_macroeconomics": 0,
"hendrycksTest-high_school_mathematics": 0,
"hendrycksTest-high_school_microeconomics": 0,
"hendrycksTest-high_school_physics": 0,
"hendrycksTest-high_school_psychology": 0,
"hendrycksTest-high_school_statistics": 0,
"hendrycksTest-high_school_us_history": 0,
"hendrycksTest-high_school_world_history": 0,
"hendrycksTest-human_aging": 0,
"hendrycksTest-human_sexuality": 0,
"hendrycksTest-international_law": 0,
"hendrycksTest-jurisprudence": 0,
"hendrycksTest-logical_fallacies": 0,
"hendrycksTest-machine_learning": 0,
"hendrycksTest-management": 0,
"hendrycksTest-marketing": 0,
"hendrycksTest-medical_genetics": 0,
"hendrycksTest-miscellaneous": 0,
"hendrycksTest-moral_disputes": 0,
"hendrycksTest-moral_scenarios": 0,
"hendrycksTest-nutrition": 0,
"hendrycksTest-philosophy": 0,
"hendrycksTest-prehistory": 0,
"hendrycksTest-professional_accounting": 0,
"hendrycksTest-professional_law": 0,
"hendrycksTest-professional_medicine": 0,
"hendrycksTest-professional_psychology": 0,
"hendrycksTest-public_relations": 0,
"hendrycksTest-security_studies": 0,
"hendrycksTest-sociology": 0,
"hendrycksTest-us_foreign_policy": 0,
"hendrycksTest-virology": 0,
"hendrycksTest-world_religions": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=mosaicml/mpt-7b,trust_remote_code=True,dtype=bfloat16",
"num_fewshot": 5,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/mpt/mpt-7b/mpt-7b_pawsx_0-shot.json deleted 100644 → 0
View file @ 465c695b
{
"results": {
"pawsx_de": {
"acc": 0.614,
"acc_stderr": 0.010888584877766427
},
"pawsx_en": {
"acc": 0.7035,
"acc_stderr": 0.010214991337441791
},
"pawsx_es": {
"acc": 0.6495,
"acc_stderr": 0.01067154233969731
},
"pawsx_fr": {
"acc": 0.6285,
"acc_stderr": 0.01080751017293364
},
"pawsx_ja": {
"acc": 0.493,
"acc_stderr": 0.011182040020027768
},
"pawsx_ko": {
"acc": 0.5365,
"acc_stderr": 0.011153298751334327
},
"pawsx_zh": {
"acc": 0.5625,
"acc_stderr": 0.011095423796079503
}
},
"versions": {
"pawsx_de": 0,
"pawsx_en": 0,
"pawsx_es": 0,
"pawsx_fr": 0,
"pawsx_ja": 0,
"pawsx_ko": 0,
"pawsx_zh": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=mosaicml/mpt-7b,trust_remote_code=True,dtype=bfloat16",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/mpt/mpt-7b/mpt-7b_reading_comprehension_0-shot.json deleted 100644 → 0
View file @ 465c695b
{
"results": {
"coqa": {
"f1": 0.7650867255895625,
"f1_stderr": 0.01481717694356494,
"em": 0.6301666666666667,
"em_stderr": 0.018680205213012713
},
"drop": {
"em": 0.03429110738255033,
"em_stderr": 0.0018636035184959787,
"f1": 0.1338569630872482,
"f1_stderr": 0.0025165760673094154
},
"race": {
"acc": 0.3866028708133971,
"acc_stderr": 0.01507138477304713
}
},
"versions": {
"coqa": 1,
"drop": 1,
"race": 1
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=mosaicml/mpt-7b,trust_remote_code=True",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/mpt/mpt-7b/mpt-7b_superglue_0-shot.json deleted 100644 → 0
View file @ 465c695b
{
"results": {
"boolq": {
"acc": 0.7382262996941896,
"acc_stderr": 0.007688653730439849
},
"cb": {
"acc": 0.4107142857142857,
"acc_stderr": 0.0663363415035954,
"f1": 0.2126984126984127
},
"copa": {
"acc": 0.84,
"acc_stderr": 0.036845294917747115
},
"multirc": {
"acc": 0.008394543546694649,
"acc_stderr": 0.002956987733350658
},
"record": {
"f1": 0.900992857142857,
"f1_stderr": 0.0029401923751195343,
"em": 0.893,
"em_stderr": 0.0030912870467751773
},
"wic": {
"acc": 0.4843260188087774,
"acc_stderr": 0.01980098495534785
},
"wsc": {
"acc": 0.6346153846153846,
"acc_stderr": 0.0474473339327792
}
},
"versions": {
"boolq": 1,
"cb": 1,
"copa": 0,
"multirc": 1,
"record": 0,
"wic": 0,
"wsc": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=mosaicml/mpt-7b,trust_remote_code=True",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/mpt/mpt-7b/mpt-7b_unscramble_0-shot.json deleted 100644 → 0
View file @ 465c695b
{
"results": {
"anagrams1": {
"acc": 0.0,
"acc_stderr": 0.0
},
"anagrams2": {
"acc": 0.0001,
"acc_stderr": 0.0001000000000000119
},
"cycle_letters": {
"acc": 0.0,
"acc_stderr": 0.0
},
"random_insertion": {
"acc": 0.0004,
"acc_stderr": 0.00019996999474889126
},
"reversed_words": {
"acc": 0.0,
"acc_stderr": 0.0
}
},
"versions": {
"anagrams1": 0,
"anagrams2": 0,
"cycle_letters": 0,
"random_insertion": 0,
"reversed_words": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=mosaicml/mpt-7b,trust_remote_code=True",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/mpt/mpt-7b/mpt-7b_xcopa_0-shot.json deleted 100644 → 0
View file @ 465c695b
{
"results": {
"xcopa_et": {
"acc": 0.474,
"acc_stderr": 0.02235279165091416
},
"xcopa_ht": {
"acc": 0.498,
"acc_stderr": 0.02238289498648353
},
"xcopa_id": {
"acc": 0.568,
"acc_stderr": 0.02217510926561316
},
"xcopa_it": {
"acc": 0.594,
"acc_stderr": 0.02198396209008634
},
"xcopa_qu": {
"acc": 0.484,
"acc_stderr": 0.0223716109825804
},
"xcopa_sw": {
"acc": 0.516,
"acc_stderr": 0.0223716109825804
},
"xcopa_ta": {
"acc": 0.54,
"acc_stderr": 0.022311333245289663
},
"xcopa_th": {
"acc": 0.542,
"acc_stderr": 0.02230396677426995
},
"xcopa_tr": {
"acc": 0.516,
"acc_stderr": 0.022371610982580396
},
"xcopa_vi": {
"acc": 0.536,
"acc_stderr": 0.022324981738385246
},
"xcopa_zh": {
"acc": 0.632,
"acc_stderr": 0.02158898256835354
}
},
"versions": {
"xcopa_et": 0,
"xcopa_ht": 0,
"xcopa_id": 0,
"xcopa_it": 0,
"xcopa_qu": 0,
"xcopa_sw": 0,
"xcopa_ta": 0,
"xcopa_th": 0,
"xcopa_tr": 0,
"xcopa_vi": 0,
"xcopa_zh": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=mosaicml/mpt-7b,trust_remote_code=True,dtype=bfloat16",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/mpt/mpt-7b/mpt-7b_xnli_0-shot.json deleted 100644 → 0
View file @ 465c695b
{
"results": {
"xnli_ar": {
"acc": 0.3331337325349301,
"acc_stderr": 0.006659676975732312
},
"xnli_bg": {
"acc": 0.36826347305389223,
"acc_stderr": 0.00681509322031267
},
"xnli_de": {
"acc": 0.46447105788423154,
"acc_stderr": 0.007046854204317796
},
"xnli_el": {
"acc": 0.36187624750499003,
"acc_stderr": 0.006789799946197319
},
"xnli_en": {
"acc": 0.543313373253493,
"acc_stderr": 0.007038155029004149
},
"xnli_es": {
"acc": 0.4564870259481038,
"acc_stderr": 0.007037909229199955
},
"xnli_fr": {
"acc": 0.4880239520958084,
"acc_stderr": 0.007062685615595022
},
"xnli_hi": {
"acc": 0.3473053892215569,
"acc_stderr": 0.006727214239733754
},
"xnli_ru": {
"acc": 0.444311377245509,
"acc_stderr": 0.007020757195791274
},
"xnli_sw": {
"acc": 0.3341317365269461,
"acc_stderr": 0.006664652441694265
},
"xnli_th": {
"acc": 0.36127744510978044,
"acc_stderr": 0.006787362347422093
},
"xnli_tr": {
"acc": 0.37684630738522956,
"acc_stderr": 0.006847061089041557
},
"xnli_ur": {
"acc": 0.33632734530938124,
"acc_stderr": 0.006675480563072364
},
"xnli_vi": {
"acc": 0.37325349301397204,
"acc_stderr": 0.0068339592620100505
},
"xnli_zh": {
"acc": 0.3534930139720559,
"acc_stderr": 0.006754629196407293
}
},
"versions": {
"xnli_ar": 0,
"xnli_bg": 0,
"xnli_de": 0,
"xnli_el": 0,
"xnli_en": 0,
"xnli_es": 0,
"xnli_fr": 0,
"xnli_hi": 0,
"xnli_ru": 0,
"xnli_sw": 0,
"xnli_th": 0,
"xnli_tr": 0,
"xnli_ur": 0,
"xnli_vi": 0,
"xnli_zh": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=mosaicml/mpt-7b,trust_remote_code=True,dtype=bfloat16",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/mpt/mpt-7b/mpt-7b_xstory_cloze_0-shot.json deleted 100644 → 0
View file @ 465c695b
{
"results": {
"xstory_cloze_ar": {
"acc": 0.48510919920582396,
"acc_stderr": 0.012861417842074004
},
"xstory_cloze_en": {
"acc": 0.7789543348775645,
"acc_stderr": 0.010678457581809239
},
"xstory_cloze_es": {
"acc": 0.6604897418927862,
"acc_stderr": 0.01218627614665943
},
"xstory_cloze_eu": {
"acc": 0.5109199205823958,
"acc_stderr": 0.012864056278255038
},
"xstory_cloze_hi": {
"acc": 0.5168762409000662,
"acc_stderr": 0.012859793919977606
},
"xstory_cloze_id": {
"acc": 0.5519523494374586,
"acc_stderr": 0.012797478885304742
},
"xstory_cloze_my": {
"acc": 0.48378557246856385,
"acc_stderr": 0.01286035780505586
},
"xstory_cloze_ru": {
"acc": 0.5724685638649901,
"acc_stderr": 0.012731259626982528
},
"xstory_cloze_sw": {
"acc": 0.4990072799470549,
"acc_stderr": 0.012867099955422935
},
"xstory_cloze_te": {
"acc": 0.5294506949040371,
"acc_stderr": 0.012844785490017004
},
"xstory_cloze_zh": {
"acc": 0.5956320317670417,
"acc_stderr": 0.012629580396570932
}
},
"versions": {
"xstory_cloze_ar": 0,
"xstory_cloze_en": 0,
"xstory_cloze_es": 0,
"xstory_cloze_eu": 0,
"xstory_cloze_hi": 0,
"xstory_cloze_id": 0,
"xstory_cloze_my": 0,
"xstory_cloze_ru": 0,
"xstory_cloze_sw": 0,
"xstory_cloze_te": 0,
"xstory_cloze_zh": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=mosaicml/mpt-7b,trust_remote_code=True,dtype=bfloat16",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/mpt/mpt-7b/mpt-7b_xwinograd_0-shot.json deleted 100644 → 0
View file @ 465c695b
{
"results": {
"xwinograd_en": {
"acc": 0.8666666666666667,
"acc_stderr": 0.0070514325016347275
},
"xwinograd_fr": {
"acc": 0.6626506024096386,
"acc_stderr": 0.05221260262032129
},
"xwinograd_jp": {
"acc": 0.602711157455683,
"acc_stderr": 0.015809751560314552
},
"xwinograd_pt": {
"acc": 0.6692015209125475,
"acc_stderr": 0.02906762615931534
},
"xwinograd_ru": {
"acc": 0.6952380952380952,
"acc_stderr": 0.025976599352305375
},
"xwinograd_zh": {
"acc": 0.7162698412698413,
"acc_stderr": 0.02010051064884106
}
},
"versions": {
"xwinograd_en": 0,
"xwinograd_fr": 0,
"xwinograd_jp": 0,
"xwinograd_pt": 0,
"xwinograd_ru": 0,
"xwinograd_zh": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=mosaicml/mpt-7b,trust_remote_code=True,dtype=bfloat16",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/opt/opt-1.3b/README.md deleted 100644 → 0
View file @ 465c695b
# opt-1.3b
## opt-1.3b.json
| Task |Version| Metric |Value| |Stderr|
|-------------------------------------------------|------:|--------|----:|---|-----:|
|arc_challenge | 0|acc |23.12|± | 1.23|
| | |acc_norm|29.44|± | 1.33|
|arc_easy | 0|acc |57.03|± | 1.02|
| | |acc_norm|50.93|± | 1.03|
|hendrycksTest-abstract_algebra | 0|acc |26.00|± | 4.41|
| | |acc_norm|25.00|± | 4.35|
|hendrycksTest-anatomy | 0|acc |22.96|± | 3.63|
| | |acc_norm|21.48|± | 3.55|
|hendrycksTest-astronomy | 0|acc |23.68|± | 3.46|
| | |acc_norm|34.21|± | 3.86|
|hendrycksTest-business_ethics | 0|acc |34.00|± | 4.76|
| | |acc_norm|33.00|± | 4.73|
|hendrycksTest-clinical_knowledge | 0|acc |19.25|± | 2.43|
| | |acc_norm|25.28|± | 2.67|
|hendrycksTest-college_biology | 0|acc |26.39|± | 3.69|
| | |acc_norm|27.78|± | 3.75|
|hendrycksTest-college_chemistry | 0|acc |28.00|± | 4.51|
| | |acc_norm|34.00|± | 4.76|
|hendrycksTest-college_computer_science | 0|acc |33.00|± | 4.73|
| | |acc_norm|30.00|± | 4.61|
|hendrycksTest-college_mathematics | 0|acc |19.00|± | 3.94|
| | |acc_norm|30.00|± | 4.61|
|hendrycksTest-college_medicine | 0|acc |17.92|± | 2.92|
| | |acc_norm|23.70|± | 3.24|
|hendrycksTest-college_physics | 0|acc |27.45|± | 4.44|
| | |acc_norm|30.39|± | 4.58|
|hendrycksTest-computer_security | 0|acc |29.00|± | 4.56|
| | |acc_norm|38.00|± | 4.88|
|hendrycksTest-conceptual_physics | 0|acc |21.70|± | 2.69|
| | |acc_norm|20.43|± | 2.64|
|hendrycksTest-econometrics | 0|acc |24.56|± | 4.05|
| | |acc_norm|23.68|± | 4.00|
|hendrycksTest-electrical_engineering | 0|acc |25.52|± | 3.63|
| | |acc_norm|28.97|± | 3.78|
|hendrycksTest-elementary_mathematics | 0|acc |19.58|± | 2.04|
| | |acc_norm|24.87|± | 2.23|
|hendrycksTest-formal_logic | 0|acc |29.37|± | 4.07|
| | |acc_norm|26.98|± | 3.97|
|hendrycksTest-global_facts | 0|acc |16.00|± | 3.68|
| | |acc_norm|18.00|± | 3.86|
|hendrycksTest-high_school_biology | 0|acc |20.97|± | 2.32|
| | |acc_norm|26.77|± | 2.52|
|hendrycksTest-high_school_chemistry | 0|acc |24.63|± | 3.03|
| | |acc_norm|30.54|± | 3.24|
|hendrycksTest-high_school_computer_science | 0|acc |27.00|± | 4.46|
| | |acc_norm|32.00|± | 4.69|
|hendrycksTest-high_school_european_history | 0|acc |24.24|± | 3.35|
| | |acc_norm|27.27|± | 3.48|
|hendrycksTest-high_school_geography | 0|acc |22.22|± | 2.96|
| | |acc_norm|28.28|± | 3.21|
|hendrycksTest-high_school_government_and_politics| 0|acc |20.73|± | 2.93|
| | |acc_norm|23.83|± | 3.07|
|hendrycksTest-high_school_macroeconomics | 0|acc |29.23|± | 2.31|
| | |acc_norm|29.23|± | 2.31|
|hendrycksTest-high_school_mathematics | 0|acc |21.85|± | 2.52|
| | |acc_norm|28.89|± | 2.76|
|hendrycksTest-high_school_microeconomics | 0|acc |21.43|± | 2.67|
| | |acc_norm|30.25|± | 2.98|
|hendrycksTest-high_school_physics | 0|acc |22.52|± | 3.41|
| | |acc_norm|25.17|± | 3.54|
|hendrycksTest-high_school_psychology | 0|acc |22.57|± | 1.79|
| | |acc_norm|24.22|± | 1.84|
|hendrycksTest-high_school_statistics | 0|acc |25.46|± | 2.97|
| | |acc_norm|27.78|± | 3.05|
|hendrycksTest-high_school_us_history | 0|acc |25.00|± | 3.04|
| | |acc_norm|25.49|± | 3.06|
|hendrycksTest-high_school_world_history | 0|acc |26.58|± | 2.88|
| | |acc_norm|27.85|± | 2.92|
|hendrycksTest-human_aging | 0|acc |35.43|± | 3.21|
| | |acc_norm|29.15|± | 3.05|
|hendrycksTest-human_sexuality | 0|acc |40.46|± | 4.30|
| | |acc_norm|31.30|± | 4.07|
|hendrycksTest-international_law | 0|acc |17.36|± | 3.46|
| | |acc_norm|47.93|± | 4.56|
|hendrycksTest-jurisprudence | 0|acc |28.70|± | 4.37|
| | |acc_norm|39.81|± | 4.73|
|hendrycksTest-logical_fallacies | 0|acc |19.02|± | 3.08|
| | |acc_norm|28.83|± | 3.56|
|hendrycksTest-machine_learning | 0|acc |27.68|± | 4.25|
| | |acc_norm|27.68|± | 4.25|
|hendrycksTest-management | 0|acc |20.39|± | 3.99|
| | |acc_norm|27.18|± | 4.41|
|hendrycksTest-marketing | 0|acc |28.21|± | 2.95|
| | |acc_norm|33.76|± | 3.10|
|hendrycksTest-medical_genetics | 0|acc |27.00|± | 4.46|
| | |acc_norm|36.00|± | 4.82|
|hendrycksTest-miscellaneous | 0|acc |28.35|± | 1.61|
| | |acc_norm|28.74|± | 1.62|
|hendrycksTest-moral_disputes | 0|acc |27.17|± | 2.39|
| | |acc_norm|30.35|± | 2.48|
|hendrycksTest-moral_scenarios | 0|acc |23.80|± | 1.42|
| | |acc_norm|27.26|± | 1.49|
|hendrycksTest-nutrition | 0|acc |29.41|± | 2.61|
| | |acc_norm|39.54|± | 2.80|
|hendrycksTest-philosophy | 0|acc |23.79|± | 2.42|
| | |acc_norm|30.87|± | 2.62|
|hendrycksTest-prehistory | 0|acc |24.07|± | 2.38|
| | |acc_norm|21.60|± | 2.29|
|hendrycksTest-professional_accounting | 0|acc |25.89|± | 2.61|
| | |acc_norm|26.24|± | 2.62|
|hendrycksTest-professional_law | 0|acc |26.01|± | 1.12|
| | |acc_norm|28.03|± | 1.15|
|hendrycksTest-professional_medicine | 0|acc |24.63|± | 2.62|
| | |acc_norm|22.43|± | 2.53|
|hendrycksTest-professional_psychology | 0|acc |23.69|± | 1.72|
| | |acc_norm|25.49|± | 1.76|
|hendrycksTest-public_relations | 0|acc |25.45|± | 4.17|
| | |acc_norm|19.09|± | 3.76|
|hendrycksTest-security_studies | 0|acc |32.24|± | 2.99|
| | |acc_norm|26.53|± | 2.83|
|hendrycksTest-sociology | 0|acc |33.83|± | 3.35|
| | |acc_norm|34.33|± | 3.36|
|hendrycksTest-us_foreign_policy | 0|acc |32.00|± | 4.69|
| | |acc_norm|27.00|± | 4.46|
|hendrycksTest-virology | 0|acc |34.34|± | 3.70|
| | |acc_norm|30.12|± | 3.57|
|hendrycksTest-world_religions | 0|acc |34.50|± | 3.65|
| | |acc_norm|33.92|± | 3.63|
|lambada_openai | 0|ppl | 6.64|± | 0.17|
| | |acc |57.93|± | 0.69|
|logiqa | 0|acc |22.27|± | 1.63|
| | |acc_norm|27.19|± | 1.75|
|piqa | 0|acc |71.71|± | 1.05|
| | |acc_norm|72.47|± | 1.04|
|sciq | 0|acc |84.50|± | 1.15|
| | |acc_norm|76.50|± | 1.34|
|winogrande | 0|acc |59.75|± | 1.38|
|wsc | 0|acc |38.46|± | 4.79|
results/opt/opt-1.3b/opt-1.3b.json deleted 100644 → 0
View file @ 465c695b
{
"results": {
"hendrycksTest-high_school_chemistry": {
"acc": 0.24630541871921183,
"acc_stderr": 0.03031509928561773,
"acc_norm": 0.3054187192118227,
"acc_norm_stderr": 0.03240661565868408
},
"hendrycksTest-international_law": {
"acc": 0.17355371900826447,
"acc_stderr": 0.0345727283691767,
"acc_norm": 0.4793388429752066,
"acc_norm_stderr": 0.04560456086387235
},
"hendrycksTest-abstract_algebra": {
"acc": 0.26,
"acc_stderr": 0.044084400227680794,
"acc_norm": 0.25,
"acc_norm_stderr": 0.04351941398892446
},
"hendrycksTest-anatomy": {
"acc": 0.22962962962962963,
"acc_stderr": 0.03633384414073465,
"acc_norm": 0.21481481481481482,
"acc_norm_stderr": 0.03547854198560826
},
"hendrycksTest-elementary_mathematics": {
"acc": 0.19576719576719576,
"acc_stderr": 0.0204357309715418,
"acc_norm": 0.24867724867724866,
"acc_norm_stderr": 0.022261817692400158
},
"hendrycksTest-high_school_microeconomics": {
"acc": 0.21428571428571427,
"acc_stderr": 0.02665353159671549,
"acc_norm": 0.3025210084033613,
"acc_norm_stderr": 0.02983796238829193
},
"hendrycksTest-college_chemistry": {
"acc": 0.28,
"acc_stderr": 0.045126085985421276,
"acc_norm": 0.34,
"acc_norm_stderr": 0.04760952285695236
},
"hendrycksTest-college_biology": {
"acc": 0.2638888888888889,
"acc_stderr": 0.03685651095897532,
"acc_norm": 0.2777777777777778,
"acc_norm_stderr": 0.03745554791462457
},
"hendrycksTest-high_school_psychology": {
"acc": 0.22568807339449543,
"acc_stderr": 0.017923087667803057,
"acc_norm": 0.24220183486238533,
"acc_norm_stderr": 0.01836817630659862
},
"hendrycksTest-philosophy": {
"acc": 0.2379421221864952,
"acc_stderr": 0.024185150647818707,
"acc_norm": 0.3086816720257235,
"acc_norm_stderr": 0.026236965881153256
},
"hendrycksTest-professional_law": {
"acc": 0.26010430247718386,
"acc_stderr": 0.011204382887823836,
"acc_norm": 0.2803129074315515,
"acc_norm_stderr": 0.011471555944958614
},
"hendrycksTest-college_computer_science": {
"acc": 0.33,
"acc_stderr": 0.047258156262526045,
"acc_norm": 0.3,
"acc_norm_stderr": 0.046056618647183814
},
"hendrycksTest-business_ethics": {
"acc": 0.34,
"acc_stderr": 0.04760952285695235,
"acc_norm": 0.33,
"acc_norm_stderr": 0.04725815626252604
},
"hendrycksTest-machine_learning": {
"acc": 0.2767857142857143,
"acc_stderr": 0.04246624336697626,
"acc_norm": 0.2767857142857143,
"acc_norm_stderr": 0.04246624336697624
},
"hendrycksTest-public_relations": {
"acc": 0.2545454545454545,
"acc_stderr": 0.041723430387053825,
"acc_norm": 0.19090909090909092,
"acc_norm_stderr": 0.03764425585984925
},
"hendrycksTest-logical_fallacies": {
"acc": 0.1901840490797546,
"acc_stderr": 0.030833491146281224,
"acc_norm": 0.2883435582822086,
"acc_norm_stderr": 0.035590395316173425
},
"hendrycksTest-professional_psychology": {
"acc": 0.2369281045751634,
"acc_stderr": 0.017201662169789782,
"acc_norm": 0.2549019607843137,
"acc_norm_stderr": 0.017630827375148383
},
"hendrycksTest-nutrition": {
"acc": 0.29411764705882354,
"acc_stderr": 0.02609016250427904,
"acc_norm": 0.3954248366013072,
"acc_norm_stderr": 0.02799672318063145
},
"arc_challenge": {
"acc": 0.23122866894197952,
"acc_stderr": 0.012320858834772276,
"acc_norm": 0.29436860068259385,
"acc_norm_stderr": 0.013318528460539426
},
"hendrycksTest-college_medicine": {
"acc": 0.1791907514450867,
"acc_stderr": 0.02924251305906328,
"acc_norm": 0.23699421965317918,
"acc_norm_stderr": 0.03242414757483098
},
"hendrycksTest-professional_accounting": {
"acc": 0.25886524822695034,
"acc_stderr": 0.026129572527180848,
"acc_norm": 0.2624113475177305,
"acc_norm_stderr": 0.026244920349843
},
"hendrycksTest-jurisprudence": {
"acc": 0.28703703703703703,
"acc_stderr": 0.043733130409147614,
"acc_norm": 0.39814814814814814,
"acc_norm_stderr": 0.047323326159788154
},
"winogrande": {
"acc": 0.5974743488555643,
"acc_stderr": 0.01378286683170305
},
"hendrycksTest-high_school_us_history": {
"acc": 0.25,
"acc_stderr": 0.03039153369274154,
"acc_norm": 0.2549019607843137,
"acc_norm_stderr": 0.030587591351604246
},
"hendrycksTest-econometrics": {
"acc": 0.24561403508771928,
"acc_stderr": 0.04049339297748142,
"acc_norm": 0.23684210526315788,
"acc_norm_stderr": 0.03999423879281336
},
"hendrycksTest-professional_medicine": {
"acc": 0.24632352941176472,
"acc_stderr": 0.02617343857052,
"acc_norm": 0.22426470588235295,
"acc_norm_stderr": 0.02533684856333237
},
"hendrycksTest-clinical_knowledge": {
"acc": 0.19245283018867926,
"acc_stderr": 0.024262979839372277,
"acc_norm": 0.2528301886792453,
"acc_norm_stderr": 0.02674989977124124
},
"hendrycksTest-high_school_geography": {
"acc": 0.2222222222222222,
"acc_stderr": 0.029620227874790482,
"acc_norm": 0.2828282828282828,
"acc_norm_stderr": 0.03208779558786751
},
"hendrycksTest-high_school_physics": {
"acc": 0.2251655629139073,
"acc_stderr": 0.034104352820089376,
"acc_norm": 0.25165562913907286,
"acc_norm_stderr": 0.035433042343899844
},
"hendrycksTest-astronomy": {
"acc": 0.23684210526315788,
"acc_stderr": 0.03459777606810537,
"acc_norm": 0.34210526315789475,
"acc_norm_stderr": 0.038607315993160925
},
"hendrycksTest-medical_genetics": {
"acc": 0.27,
"acc_stderr": 0.0446196043338474,
"acc_norm": 0.36,
"acc_norm_stderr": 0.04824181513244218
},
"hendrycksTest-us_foreign_policy": {
"acc": 0.32,
"acc_stderr": 0.046882617226215034,
"acc_norm": 0.27,
"acc_norm_stderr": 0.04461960433384741
},
"hendrycksTest-formal_logic": {
"acc": 0.29365079365079366,
"acc_stderr": 0.040735243221471276,
"acc_norm": 0.2698412698412698,
"acc_norm_stderr": 0.039701582732351734
},
"hendrycksTest-conceptual_physics": {
"acc": 0.2170212765957447,
"acc_stderr": 0.026947483121496217,
"acc_norm": 0.20425531914893616,
"acc_norm_stderr": 0.026355158413349417
},
"hendrycksTest-electrical_engineering": {
"acc": 0.25517241379310346,
"acc_stderr": 0.03632984052707842,
"acc_norm": 0.2896551724137931,
"acc_norm_stderr": 0.03780019230438014
},
"hendrycksTest-virology": {
"acc": 0.3433734939759036,
"acc_stderr": 0.03696584317010601,
"acc_norm": 0.30120481927710846,
"acc_norm_stderr": 0.0357160923005348
},
"piqa": {
"acc": 0.7170837867247007,
"acc_stderr": 0.010508949177489683,
"acc_norm": 0.7247007616974973,
"acc_norm_stderr": 0.01042142927736953
},
"hendrycksTest-high_school_statistics": {
"acc": 0.25462962962962965,
"acc_stderr": 0.02971127586000535,
"acc_norm": 0.2777777777777778,
"acc_norm_stderr": 0.03054674526495318
},
"hendrycksTest-college_physics": {
"acc": 0.27450980392156865,
"acc_stderr": 0.04440521906179327,
"acc_norm": 0.30392156862745096,
"acc_norm_stderr": 0.045766654032077636
},
"hendrycksTest-high_school_biology": {
"acc": 0.20967741935483872,
"acc_stderr": 0.02315787934908352,
"acc_norm": 0.267741935483871,
"acc_norm_stderr": 0.02518900666021238
},
"hendrycksTest-world_religions": {
"acc": 0.34502923976608185,
"acc_stderr": 0.036459813773888065,
"acc_norm": 0.3391812865497076,
"acc_norm_stderr": 0.036310534964889056
},
"hendrycksTest-marketing": {
"acc": 0.28205128205128205,
"acc_stderr": 0.02948036054954119,
"acc_norm": 0.33760683760683763,
"acc_norm_stderr": 0.030980296992618558
},
"hendrycksTest-security_studies": {
"acc": 0.3224489795918367,
"acc_stderr": 0.029923100563683913,
"acc_norm": 0.2653061224489796,
"acc_norm_stderr": 0.028263889943784603
},
"arc_easy": {
"acc": 0.5702861952861953,
"acc_stderr": 0.010157908005763676,
"acc_norm": 0.5092592592592593,
"acc_norm_stderr": 0.01025802414786065
},
"hendrycksTest-high_school_world_history": {
"acc": 0.26582278481012656,
"acc_stderr": 0.028756799629658335,
"acc_norm": 0.27848101265822783,
"acc_norm_stderr": 0.029178682304842555
},
"hendrycksTest-human_aging": {
"acc": 0.3542600896860987,
"acc_stderr": 0.03210062154134986,
"acc_norm": 0.2914798206278027,
"acc_norm_stderr": 0.030500283176545902
},
"sciq": {
"acc": 0.845,
"acc_stderr": 0.01145015747079947,
"acc_norm": 0.765,
"acc_norm_stderr": 0.013414729030247121
},
"hendrycksTest-moral_disputes": {
"acc": 0.27167630057803466,
"acc_stderr": 0.02394851290546836,
"acc_norm": 0.30346820809248554,
"acc_norm_stderr": 0.024752411960917212
},
"hendrycksTest-management": {
"acc": 0.20388349514563106,
"acc_stderr": 0.03989139859531771,
"acc_norm": 0.27184466019417475,
"acc_norm_stderr": 0.044052680241409216
},
"hendrycksTest-computer_security": {
"acc": 0.29,
"acc_stderr": 0.045604802157206845,
"acc_norm": 0.38,
"acc_norm_stderr": 0.04878317312145633
},
"hendrycksTest-moral_scenarios": {
"acc": 0.23798882681564246,
"acc_stderr": 0.014242630070574915,
"acc_norm": 0.27262569832402234,
"acc_norm_stderr": 0.014893391735249588
},
"wsc": {
"acc": 0.38461538461538464,
"acc_stderr": 0.0479366886807504
},
"hendrycksTest-high_school_government_and_politics": {
"acc": 0.20725388601036268,
"acc_stderr": 0.029252823291803627,
"acc_norm": 0.23834196891191708,
"acc_norm_stderr": 0.030748905363909895
},
"hendrycksTest-human_sexuality": {
"acc": 0.40458015267175573,
"acc_stderr": 0.043046937953806645,
"acc_norm": 0.31297709923664124,
"acc_norm_stderr": 0.04066962905677698
},
"hendrycksTest-high_school_european_history": {
"acc": 0.24242424242424243,
"acc_stderr": 0.03346409881055953,
"acc_norm": 0.2727272727272727,
"acc_norm_stderr": 0.0347769116216366
},
"hendrycksTest-high_school_macroeconomics": {
"acc": 0.2923076923076923,
"acc_stderr": 0.023060438380857733,
"acc_norm": 0.2923076923076923,
"acc_norm_stderr": 0.02306043838085774
},
"hendrycksTest-college_mathematics": {
"acc": 0.19,
"acc_stderr": 0.03942772444036623,
"acc_norm": 0.3,
"acc_norm_stderr": 0.046056618647183814
},
"hendrycksTest-miscellaneous": {
"acc": 0.2835249042145594,
"acc_stderr": 0.016117318166832272,
"acc_norm": 0.28735632183908044,
"acc_norm_stderr": 0.0161824107306827
},
"logiqa": {
"acc": 0.2227342549923195,
"acc_stderr": 0.01632005404616512,
"acc_norm": 0.271889400921659,
"acc_norm_stderr": 0.01745171600943683
},
"hendrycksTest-prehistory": {
"acc": 0.24074074074074073,
"acc_stderr": 0.023788583551658537,
"acc_norm": 0.21604938271604937,
"acc_norm_stderr": 0.022899162918445796
},
"hendrycksTest-high_school_mathematics": {
"acc": 0.21851851851851853,
"acc_stderr": 0.025195752251823793,
"acc_norm": 0.28888888888888886,
"acc_norm_stderr": 0.027634907264178544
},
"lambada_openai": {
"ppl": 6.644056379058006,
"ppl_stderr": 0.1717099929921861,
"acc": 0.5792742091985251,
"acc_stderr": 0.00687786642328006
},
"hendrycksTest-global_facts": {
"acc": 0.16,
"acc_stderr": 0.03684529491774708,
"acc_norm": 0.18,
"acc_norm_stderr": 0.03861229196653695
},
"hendrycksTest-high_school_computer_science": {
"acc": 0.27,
"acc_stderr": 0.04461960433384741,
"acc_norm": 0.32,
"acc_norm_stderr": 0.04688261722621503
},
"hendrycksTest-sociology": {
"acc": 0.3383084577114428,
"acc_stderr": 0.033455630703391914,
"acc_norm": 0.34328358208955223,
"acc_norm_stderr": 0.03357379665433431
}
},
"versions": {
"hendrycksTest-high_school_chemistry": 0,
"hendrycksTest-international_law": 0,
"hendrycksTest-abstract_algebra": 0,
"hendrycksTest-anatomy": 0,
"hendrycksTest-elementary_mathematics": 0,
"hendrycksTest-high_school_microeconomics": 0,
"hendrycksTest-college_chemistry": 0,
"hendrycksTest-college_biology": 0,
"hendrycksTest-high_school_psychology": 0,
"hendrycksTest-philosophy": 0,
"hendrycksTest-professional_law": 0,
"hendrycksTest-college_computer_science": 0,
"hendrycksTest-business_ethics": 0,
"hendrycksTest-machine_learning": 0,
"hendrycksTest-public_relations": 0,
"hendrycksTest-logical_fallacies": 0,
"hendrycksTest-professional_psychology": 0,
"hendrycksTest-nutrition": 0,
"arc_challenge": 0,
"hendrycksTest-college_medicine": 0,
"hendrycksTest-professional_accounting": 0,
"hendrycksTest-jurisprudence": 0,
"winogrande": 0,
"hendrycksTest-high_school_us_history": 0,
"hendrycksTest-econometrics": 0,
"hendrycksTest-professional_medicine": 0,
"hendrycksTest-clinical_knowledge": 0,
"hendrycksTest-high_school_geography": 0,
"hendrycksTest-high_school_physics": 0,
"hendrycksTest-astronomy": 0,
"hendrycksTest-medical_genetics": 0,
"hendrycksTest-us_foreign_policy": 0,
"hendrycksTest-formal_logic": 0,
"hendrycksTest-conceptual_physics": 0,
"hendrycksTest-electrical_engineering": 0,
"hendrycksTest-virology": 0,
"piqa": 0,
"hendrycksTest-high_school_statistics": 0,
"hendrycksTest-college_physics": 0,
"hendrycksTest-high_school_biology": 0,
"hendrycksTest-world_religions": 0,
"hendrycksTest-marketing": 0,
"hendrycksTest-security_studies": 0,
"arc_easy": 0,
"hendrycksTest-high_school_world_history": 0,
"hendrycksTest-human_aging": 0,
"sciq": 0,
"hendrycksTest-moral_disputes": 0,
"hendrycksTest-management": 0,
"hendrycksTest-computer_security": 0,
"hendrycksTest-moral_scenarios": 0,
"wsc": 0,
"hendrycksTest-high_school_government_and_politics": 0,
"hendrycksTest-human_sexuality": 0,
"hendrycksTest-high_school_european_history": 0,
"hendrycksTest-high_school_macroeconomics": 0,
"hendrycksTest-college_mathematics": 0,
"hendrycksTest-miscellaneous": 0,
"logiqa": 0,
"hendrycksTest-prehistory": 0,
"hendrycksTest-high_school_mathematics": 0,
"lambada_openai": 0,
"hendrycksTest-global_facts": 0,
"hendrycksTest-high_school_computer_science": 0,
"hendrycksTest-sociology": 0
},
"config": {
"model": "hf-causal",
"model_args": "pretrained=facebook/opt-1.3b,use_accelerate=True",
"num_fewshot": 0,
"batch_size": 16,
"device": "cuda",
"no_cache": false,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/opt/opt-125m/README.md deleted 100644 → 0
View file @ 465c695b
# opt-125m
## opt-125m.json
| Task |Version| Metric |Value| |Stderr|
|-------------------------------------------------|------:|--------|----:|---|-----:|
|arc_challenge | 0|acc |18.94|± | 1.15|
| | |acc_norm|22.78|± | 1.23|
|arc_easy | 0|acc |43.52|± | 1.02|
| | |acc_norm|39.98|± | 1.01|
|hendrycksTest-abstract_algebra | 0|acc |17.00|± | 3.78|
| | |acc_norm|20.00|± | 4.02|
|hendrycksTest-anatomy | 0|acc |24.44|± | 3.71|
| | |acc_norm|21.48|± | 3.55|
|hendrycksTest-astronomy | 0|acc |20.39|± | 3.28|
| | |acc_norm|33.55|± | 3.84|
|hendrycksTest-business_ethics | 0|acc |35.00|± | 4.79|
| | |acc_norm|23.00|± | 4.23|
|hendrycksTest-clinical_knowledge | 0|acc |18.87|± | 2.41|
| | |acc_norm|27.92|± | 2.76|
|hendrycksTest-college_biology | 0|acc |22.92|± | 3.51|
| | |acc_norm|23.61|± | 3.55|
|hendrycksTest-college_chemistry | 0|acc |28.00|± | 4.51|
| | |acc_norm|27.00|± | 4.46|
|hendrycksTest-college_computer_science | 0|acc |27.00|± | 4.46|
| | |acc_norm|22.00|± | 4.16|
|hendrycksTest-college_mathematics | 0|acc |16.00|± | 3.68|
| | |acc_norm|25.00|± | 4.35|
|hendrycksTest-college_medicine | 0|acc |24.28|± | 3.27|
| | |acc_norm|26.59|± | 3.37|
|hendrycksTest-college_physics | 0|acc |28.43|± | 4.49|
| | |acc_norm|23.53|± | 4.22|
|hendrycksTest-computer_security | 0|acc |31.00|± | 4.65|
| | |acc_norm|33.00|± | 4.73|
|hendrycksTest-conceptual_physics | 0|acc |26.38|± | 2.88|
| | |acc_norm|17.45|± | 2.48|
|hendrycksTest-econometrics | 0|acc |31.58|± | 4.37|
| | |acc_norm|28.95|± | 4.27|
|hendrycksTest-electrical_engineering | 0|acc |24.14|± | 3.57|
| | |acc_norm|30.34|± | 3.83|
|hendrycksTest-elementary_mathematics | 0|acc |24.87|± | 2.23|
| | |acc_norm|25.13|± | 2.23|
|hendrycksTest-formal_logic | 0|acc |30.95|± | 4.13|
| | |acc_norm|28.57|± | 4.04|
|hendrycksTest-global_facts | 0|acc |19.00|± | 3.94|
| | |acc_norm|22.00|± | 4.16|
|hendrycksTest-high_school_biology | 0|acc |24.84|± | 2.46|
| | |acc_norm|29.03|± | 2.58|
|hendrycksTest-high_school_chemistry | 0|acc |17.24|± | 2.66|
| | |acc_norm|24.63|± | 3.03|
|hendrycksTest-high_school_computer_science | 0|acc |21.00|± | 4.09|
| | |acc_norm|25.00|± | 4.35|
|hendrycksTest-high_school_european_history | 0|acc |24.85|± | 3.37|
| | |acc_norm|31.52|± | 3.63|
|hendrycksTest-high_school_geography | 0|acc |22.73|± | 2.99|
| | |acc_norm|28.79|± | 3.23|
|hendrycksTest-high_school_government_and_politics| 0|acc |24.87|± | 3.12|
| | |acc_norm|30.05|± | 3.31|
|hendrycksTest-high_school_macroeconomics | 0|acc |26.15|± | 2.23|
| | |acc_norm|24.87|± | 2.19|
|hendrycksTest-high_school_mathematics | 0|acc |17.04|± | 2.29|
| | |acc_norm|23.33|± | 2.58|
|hendrycksTest-high_school_microeconomics | 0|acc |23.53|± | 2.76|
| | |acc_norm|29.83|± | 2.97|
|hendrycksTest-high_school_physics | 0|acc |22.52|± | 3.41|
| | |acc_norm|21.19|± | 3.34|
|hendrycksTest-high_school_psychology | 0|acc |25.14|± | 1.86|
| | |acc_norm|25.69|± | 1.87|
|hendrycksTest-high_school_statistics | 0|acc |24.54|± | 2.93|
| | |acc_norm|27.78|± | 3.05|
|hendrycksTest-high_school_us_history | 0|acc |24.51|± | 3.02|
| | |acc_norm|26.47|± | 3.10|
|hendrycksTest-high_school_world_history | 0|acc |26.16|± | 2.86|
| | |acc_norm|28.69|± | 2.94|
|hendrycksTest-human_aging | 0|acc |36.32|± | 3.23|
| | |acc_norm|26.01|± | 2.94|
|hendrycksTest-human_sexuality | 0|acc |32.82|± | 4.12|
| | |acc_norm|32.06|± | 4.09|
|hendrycksTest-international_law | 0|acc |14.88|± | 3.25|
| | |acc_norm|38.02|± | 4.43|
|hendrycksTest-jurisprudence | 0|acc |15.74|± | 3.52|
| | |acc_norm|37.04|± | 4.67|
|hendrycksTest-logical_fallacies | 0|acc |26.38|± | 3.46|
| | |acc_norm|30.06|± | 3.60|
|hendrycksTest-machine_learning | 0|acc |24.11|± | 4.06|
| | |acc_norm|27.68|± | 4.25|
|hendrycksTest-management | 0|acc |17.48|± | 3.76|
| | |acc_norm|27.18|± | 4.41|
|hendrycksTest-marketing | 0|acc |33.76|± | 3.10|
| | |acc_norm|35.90|± | 3.14|
|hendrycksTest-medical_genetics | 0|acc |29.00|± | 4.56|
| | |acc_norm|38.00|± | 4.88|
|hendrycksTest-miscellaneous | 0|acc |27.46|± | 1.60|
| | |acc_norm|26.18|± | 1.57|
|hendrycksTest-moral_disputes | 0|acc |28.32|± | 2.43|
| | |acc_norm|29.48|± | 2.45|
|hendrycksTest-moral_scenarios | 0|acc |23.80|± | 1.42|
| | |acc_norm|27.26|± | 1.49|
|hendrycksTest-nutrition | 0|acc |28.76|± | 2.59|
| | |acc_norm|34.97|± | 2.73|
|hendrycksTest-philosophy | 0|acc |20.90|± | 2.31|
| | |acc_norm|31.19|± | 2.63|
|hendrycksTest-prehistory | 0|acc |27.47|± | 2.48|
| | |acc_norm|21.91|± | 2.30|
|hendrycksTest-professional_accounting | 0|acc |25.89|± | 2.61|
| | |acc_norm|26.24|± | 2.62|
|hendrycksTest-professional_law | 0|acc |25.23|± | 1.11|
| | |acc_norm|27.38|± | 1.14|
|hendrycksTest-professional_medicine | 0|acc |25.00|± | 2.63|
| | |acc_norm|22.79|± | 2.55|
|hendrycksTest-professional_psychology | 0|acc |23.86|± | 1.72|
| | |acc_norm|24.84|± | 1.75|
|hendrycksTest-public_relations | 0|acc |29.09|± | 4.35|
| | |acc_norm|25.45|± | 4.17|
|hendrycksTest-security_studies | 0|acc |33.47|± | 3.02|
| | |acc_norm|25.71|± | 2.80|
|hendrycksTest-sociology | 0|acc |27.86|± | 3.17|
| | |acc_norm|24.88|± | 3.06|
|hendrycksTest-us_foreign_policy | 0|acc |28.00|± | 4.51|
| | |acc_norm|34.00|± | 4.76|
|hendrycksTest-virology | 0|acc |31.93|± | 3.63|
| | |acc_norm|27.71|± | 3.48|
|hendrycksTest-world_religions | 0|acc |28.65|± | 3.47|
| | |acc_norm|32.16|± | 3.58|
|lambada_openai | 0|ppl |26.02|± | 0.94|
| | |acc |37.90|± | 0.68|
|logiqa | 0|acc |22.73|± | 1.64|
| | |acc_norm|27.96|± | 1.76|
|piqa | 0|acc |63.00|± | 1.13|
| | |acc_norm|62.02|± | 1.13|
|sciq | 0|acc |75.10|± | 1.37|
| | |acc_norm|66.90|± | 1.49|
|winogrande | 0|acc |50.28|± | 1.41|
|wsc | 0|acc |36.54|± | 4.74|
results/opt/opt-125m/opt-125m.json deleted 100644 → 0
View file @ 465c695b
{
"results": {
"lambada_openai": {
"ppl": 26.02103896885129,
"ppl_stderr": 0.9418773107503278,
"acc": 0.3790025228022511,
"acc_stderr": 0.006758931440268226
},
"hendrycksTest-college_biology": {
"acc": 0.22916666666666666,
"acc_stderr": 0.03514697467862388,
"acc_norm": 0.2361111111111111,
"acc_norm_stderr": 0.03551446610810826
},
"hendrycksTest-professional_law": {
"acc": 0.2522816166883963,
"acc_stderr": 0.01109278905687524,
"acc_norm": 0.2737940026075619,
"acc_norm_stderr": 0.011388612167979395
},
"hendrycksTest-virology": {
"acc": 0.3192771084337349,
"acc_stderr": 0.03629335329947861,
"acc_norm": 0.27710843373493976,
"acc_norm_stderr": 0.03484331592680587
},
"winogrande": {
"acc": 0.5027624309392266,
"acc_stderr": 0.014052271211616445
},
"hendrycksTest-high_school_microeconomics": {
"acc": 0.23529411764705882,
"acc_stderr": 0.027553614467863804,
"acc_norm": 0.29831932773109243,
"acc_norm_stderr": 0.029719142876342863
},
"hendrycksTest-college_physics": {
"acc": 0.28431372549019607,
"acc_stderr": 0.04488482852329017,
"acc_norm": 0.23529411764705882,
"acc_norm_stderr": 0.04220773659171453
},
"hendrycksTest-computer_security": {
"acc": 0.31,
"acc_stderr": 0.04648231987117316,
"acc_norm": 0.33,
"acc_norm_stderr": 0.047258156262526045
},
"hendrycksTest-conceptual_physics": {
"acc": 0.26382978723404255,
"acc_stderr": 0.02880998985410297,
"acc_norm": 0.17446808510638298,
"acc_norm_stderr": 0.024809442335503966
},
"hendrycksTest-high_school_world_history": {
"acc": 0.2616033755274262,
"acc_stderr": 0.028609516716994934,
"acc_norm": 0.2869198312236287,
"acc_norm_stderr": 0.02944377302259469
},
"hendrycksTest-sociology": {
"acc": 0.27860696517412936,
"acc_stderr": 0.031700561834973086,
"acc_norm": 0.24875621890547264,
"acc_norm_stderr": 0.030567675938916707
},
"sciq": {
"acc": 0.751,
"acc_stderr": 0.013681600278702308,
"acc_norm": 0.669,
"acc_norm_stderr": 0.014888272588203941
},
"hendrycksTest-professional_accounting": {
"acc": 0.25886524822695034,
"acc_stderr": 0.026129572527180848,
"acc_norm": 0.2624113475177305,
"acc_norm_stderr": 0.026244920349843007
},
"hendrycksTest-high_school_biology": {
"acc": 0.24838709677419354,
"acc_stderr": 0.024580028921481003,
"acc_norm": 0.2903225806451613,
"acc_norm_stderr": 0.025822106119415905
},
"hendrycksTest-college_computer_science": {
"acc": 0.27,
"acc_stderr": 0.0446196043338474,
"acc_norm": 0.22,
"acc_norm_stderr": 0.04163331998932269
},
"hendrycksTest-econometrics": {
"acc": 0.3157894736842105,
"acc_stderr": 0.04372748290278007,
"acc_norm": 0.2894736842105263,
"acc_norm_stderr": 0.042663394431593935
},
"piqa": {
"acc": 0.6300326441784548,
"acc_stderr": 0.011264415223415284,
"acc_norm": 0.6202393906420022,
"acc_norm_stderr": 0.011323483504715843
},
"hendrycksTest-formal_logic": {
"acc": 0.30952380952380953,
"acc_stderr": 0.04134913018303316,
"acc_norm": 0.2857142857142857,
"acc_norm_stderr": 0.04040610178208841
},
"hendrycksTest-astronomy": {
"acc": 0.20394736842105263,
"acc_stderr": 0.03279000406310051,
"acc_norm": 0.3355263157894737,
"acc_norm_stderr": 0.038424985593952694
},
"hendrycksTest-high_school_european_history": {
"acc": 0.24848484848484848,
"acc_stderr": 0.033744026441394036,
"acc_norm": 0.3151515151515151,
"acc_norm_stderr": 0.0362773057502241
},
"hendrycksTest-high_school_geography": {
"acc": 0.22727272727272727,
"acc_stderr": 0.02985751567338641,
"acc_norm": 0.2878787878787879,
"acc_norm_stderr": 0.03225883512300993
},
"hendrycksTest-moral_disputes": {
"acc": 0.2832369942196532,
"acc_stderr": 0.02425790170532337,
"acc_norm": 0.2947976878612717,
"acc_norm_stderr": 0.024547617794803835
},
"hendrycksTest-high_school_macroeconomics": {
"acc": 0.26153846153846155,
"acc_stderr": 0.022282141204204426,
"acc_norm": 0.24871794871794872,
"acc_norm_stderr": 0.021916957709213796
},
"arc_easy": {
"acc": 0.4351851851851852,
"acc_stderr": 0.010173216430370906,
"acc_norm": 0.3998316498316498,
"acc_norm_stderr": 0.010051788039412935
},
"hendrycksTest-marketing": {
"acc": 0.33760683760683763,
"acc_stderr": 0.030980296992618558,
"acc_norm": 0.358974358974359,
"acc_norm_stderr": 0.03142616993791924
},
"hendrycksTest-nutrition": {
"acc": 0.2875816993464052,
"acc_stderr": 0.02591780611714716,
"acc_norm": 0.34967320261437906,
"acc_norm_stderr": 0.027305308076274702
},
"hendrycksTest-management": {
"acc": 0.17475728155339806,
"acc_stderr": 0.037601780060266196,
"acc_norm": 0.27184466019417475,
"acc_norm_stderr": 0.044052680241409216
},
"hendrycksTest-prehistory": {
"acc": 0.27469135802469136,
"acc_stderr": 0.024836057868294677,
"acc_norm": 0.2191358024691358,
"acc_norm_stderr": 0.02301670564026219
},
"hendrycksTest-public_relations": {
"acc": 0.2909090909090909,
"acc_stderr": 0.04350271442923243,
"acc_norm": 0.2545454545454545,
"acc_norm_stderr": 0.041723430387053825
},
"hendrycksTest-high_school_chemistry": {
"acc": 0.1724137931034483,
"acc_stderr": 0.026577672183036586,
"acc_norm": 0.24630541871921183,
"acc_norm_stderr": 0.030315099285617732
},
"hendrycksTest-high_school_psychology": {
"acc": 0.25137614678899084,
"acc_stderr": 0.01859920636028741,
"acc_norm": 0.25688073394495414,
"acc_norm_stderr": 0.018732492928342472
},
"hendrycksTest-us_foreign_policy": {
"acc": 0.28,
"acc_stderr": 0.04512608598542127,
"acc_norm": 0.34,
"acc_norm_stderr": 0.047609522856952365
},
"hendrycksTest-elementary_mathematics": {
"acc": 0.24867724867724866,
"acc_stderr": 0.02226181769240015,
"acc_norm": 0.25132275132275134,
"acc_norm_stderr": 0.022340482339643895
},
"hendrycksTest-electrical_engineering": {
"acc": 0.2413793103448276,
"acc_stderr": 0.03565998174135302,
"acc_norm": 0.30344827586206896,
"acc_norm_stderr": 0.038312260488503336
},
"hendrycksTest-moral_scenarios": {
"acc": 0.23798882681564246,
"acc_stderr": 0.014242630070574915,
"acc_norm": 0.27262569832402234,
"acc_norm_stderr": 0.014893391735249588
},
"hendrycksTest-logical_fallacies": {
"acc": 0.26380368098159507,
"acc_stderr": 0.03462419931615625,
"acc_norm": 0.3006134969325153,
"acc_norm_stderr": 0.0360251131880677
},
"hendrycksTest-professional_medicine": {
"acc": 0.25,
"acc_stderr": 0.026303648393696036,
"acc_norm": 0.22794117647058823,
"acc_norm_stderr": 0.025483081468029804
},
"hendrycksTest-global_facts": {
"acc": 0.19,
"acc_stderr": 0.039427724440366255,
"acc_norm": 0.22,
"acc_norm_stderr": 0.041633319989322674
},
"hendrycksTest-abstract_algebra": {
"acc": 0.17,
"acc_stderr": 0.0377525168068637,
"acc_norm": 0.2,
"acc_norm_stderr": 0.04020151261036844
},
"hendrycksTest-clinical_knowledge": {
"acc": 0.18867924528301888,
"acc_stderr": 0.02407999513006222,
"acc_norm": 0.2792452830188679,
"acc_norm_stderr": 0.027611163402399715
},
"hendrycksTest-high_school_computer_science": {
"acc": 0.21,
"acc_stderr": 0.04093601807403326,
"acc_norm": 0.25,
"acc_norm_stderr": 0.04351941398892446
},
"hendrycksTest-college_mathematics": {
"acc": 0.16,
"acc_stderr": 0.03684529491774709,
"acc_norm": 0.25,
"acc_norm_stderr": 0.04351941398892446
},
"hendrycksTest-high_school_physics": {
"acc": 0.2251655629139073,
"acc_stderr": 0.03410435282008936,
"acc_norm": 0.2119205298013245,
"acc_norm_stderr": 0.03336767086567978
},
"logiqa": {
"acc": 0.22734254992319508,
"acc_stderr": 0.016439067675117734,
"acc_norm": 0.27956989247311825,
"acc_norm_stderr": 0.01760290918682245
},
"hendrycksTest-anatomy": {
"acc": 0.24444444444444444,
"acc_stderr": 0.037125378336148665,
"acc_norm": 0.21481481481481482,
"acc_norm_stderr": 0.03547854198560826
},
"hendrycksTest-college_chemistry": {
"acc": 0.28,
"acc_stderr": 0.045126085985421276,
"acc_norm": 0.27,
"acc_norm_stderr": 0.044619604333847394
},
"hendrycksTest-professional_psychology": {
"acc": 0.238562091503268,
"acc_stderr": 0.0172423858287796,
"acc_norm": 0.24836601307189543,
"acc_norm_stderr": 0.01747948700136476
},
"hendrycksTest-security_studies": {
"acc": 0.3346938775510204,
"acc_stderr": 0.030209235226242307,
"acc_norm": 0.2571428571428571,
"acc_norm_stderr": 0.027979823538744546
},
"hendrycksTest-international_law": {
"acc": 0.1487603305785124,
"acc_stderr": 0.03248470083807193,
"acc_norm": 0.38016528925619836,
"acc_norm_stderr": 0.04431324501968431
},
"hendrycksTest-human_aging": {
"acc": 0.3632286995515695,
"acc_stderr": 0.032277904428505,
"acc_norm": 0.2600896860986547,
"acc_norm_stderr": 0.029442495585857473
},
"hendrycksTest-jurisprudence": {
"acc": 0.1574074074074074,
"acc_stderr": 0.035207039905179614,
"acc_norm": 0.37037037037037035,
"acc_norm_stderr": 0.04668408033024931
},
"hendrycksTest-philosophy": {
"acc": 0.2090032154340836,
"acc_stderr": 0.023093140398374224,
"acc_norm": 0.31189710610932475,
"acc_norm_stderr": 0.02631185807185416
},
"hendrycksTest-miscellaneous": {
"acc": 0.27458492975734355,
"acc_stderr": 0.015959829933084032,
"acc_norm": 0.26181353767560667,
"acc_norm_stderr": 0.015720838678445266
},
"hendrycksTest-business_ethics": {
"acc": 0.35,
"acc_stderr": 0.047937248544110196,
"acc_norm": 0.23,
"acc_norm_stderr": 0.04229525846816506
},
"wsc": {
"acc": 0.36538461538461536,
"acc_stderr": 0.0474473339327792
},
"hendrycksTest-high_school_mathematics": {
"acc": 0.17037037037037037,
"acc_stderr": 0.022922554863074974,
"acc_norm": 0.23333333333333334,
"acc_norm_stderr": 0.02578787422095931
},
"hendrycksTest-medical_genetics": {
"acc": 0.29,
"acc_stderr": 0.045604802157206845,
"acc_norm": 0.38,
"acc_norm_stderr": 0.04878317312145633
},
"hendrycksTest-high_school_us_history": {
"acc": 0.24509803921568626,
"acc_stderr": 0.030190282453501936,
"acc_norm": 0.2647058823529412,
"acc_norm_stderr": 0.03096451792692341
},
"hendrycksTest-human_sexuality": {
"acc": 0.3282442748091603,
"acc_stderr": 0.04118438565806298,
"acc_norm": 0.32061068702290074,
"acc_norm_stderr": 0.04093329229834278
},
"arc_challenge": {
"acc": 0.189419795221843,
"acc_stderr": 0.011450705115910769,
"acc_norm": 0.22781569965870307,
"acc_norm_stderr": 0.012256708602326914
},
"hendrycksTest-high_school_government_and_politics": {
"acc": 0.24870466321243523,
"acc_stderr": 0.031195840877700304,
"acc_norm": 0.3005181347150259,
"acc_norm_stderr": 0.03308818594415751
},
"hendrycksTest-machine_learning": {
"acc": 0.24107142857142858,
"acc_stderr": 0.04059867246952687,
"acc_norm": 0.2767857142857143,
"acc_norm_stderr": 0.04246624336697624
},
"hendrycksTest-world_religions": {
"acc": 0.28654970760233917,
"acc_stderr": 0.03467826685703826,
"acc_norm": 0.3216374269005848,
"acc_norm_stderr": 0.03582529442573122
},
"hendrycksTest-college_medicine": {
"acc": 0.24277456647398843,
"acc_stderr": 0.0326926380614177,
"acc_norm": 0.2658959537572254,
"acc_norm_stderr": 0.03368762932259431
},
"hendrycksTest-high_school_statistics": {
"acc": 0.24537037037037038,
"acc_stderr": 0.029346665094372948,
"acc_norm": 0.2777777777777778,
"acc_norm_stderr": 0.03054674526495318
}
},
"versions": {
"lambada_openai": 0,
"hendrycksTest-college_biology": 0,
"hendrycksTest-professional_law": 0,
"hendrycksTest-virology": 0,
"winogrande": 0,
"hendrycksTest-high_school_microeconomics": 0,
"hendrycksTest-college_physics": 0,
"hendrycksTest-computer_security": 0,
"hendrycksTest-conceptual_physics": 0,
"hendrycksTest-high_school_world_history": 0,
"hendrycksTest-sociology": 0,
"sciq": 0,
"hendrycksTest-professional_accounting": 0,
"hendrycksTest-high_school_biology": 0,
"hendrycksTest-college_computer_science": 0,
"hendrycksTest-econometrics": 0,
"piqa": 0,
"hendrycksTest-formal_logic": 0,
"hendrycksTest-astronomy": 0,
"hendrycksTest-high_school_european_history": 0,
"hendrycksTest-high_school_geography": 0,
"hendrycksTest-moral_disputes": 0,
"hendrycksTest-high_school_macroeconomics": 0,
"arc_easy": 0,
"hendrycksTest-marketing": 0,
"hendrycksTest-nutrition": 0,
"hendrycksTest-management": 0,
"hendrycksTest-prehistory": 0,
"hendrycksTest-public_relations": 0,
"hendrycksTest-high_school_chemistry": 0,
"hendrycksTest-high_school_psychology": 0,
"hendrycksTest-us_foreign_policy": 0,
"hendrycksTest-elementary_mathematics": 0,
"hendrycksTest-electrical_engineering": 0,
"hendrycksTest-moral_scenarios": 0,
"hendrycksTest-logical_fallacies": 0,
"hendrycksTest-professional_medicine": 0,
"hendrycksTest-global_facts": 0,
"hendrycksTest-abstract_algebra": 0,
"hendrycksTest-clinical_knowledge": 0,
"hendrycksTest-high_school_computer_science": 0,
"hendrycksTest-college_mathematics": 0,
"hendrycksTest-high_school_physics": 0,
"logiqa": 0,
"hendrycksTest-anatomy": 0,
"hendrycksTest-college_chemistry": 0,
"hendrycksTest-professional_psychology": 0,
"hendrycksTest-security_studies": 0,
"hendrycksTest-international_law": 0,
"hendrycksTest-human_aging": 0,
"hendrycksTest-jurisprudence": 0,
"hendrycksTest-philosophy": 0,
"hendrycksTest-miscellaneous": 0,
"hendrycksTest-business_ethics": 0,
"wsc": 0,
"hendrycksTest-high_school_mathematics": 0,
"hendrycksTest-medical_genetics": 0,
"hendrycksTest-high_school_us_history": 0,
"hendrycksTest-human_sexuality": 0,
"arc_challenge": 0,
"hendrycksTest-high_school_government_and_politics": 0,
"hendrycksTest-machine_learning": 0,
"hendrycksTest-world_religions": 0,
"hendrycksTest-college_medicine": 0,
"hendrycksTest-high_school_statistics": 0
},
"config": {
"model": "gpt2",
"model_args": "pretrained=facebook/opt-125m",
"num_fewshot": 0,
"batch_size": 64,
"device": "cuda",
"no_cache": false,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment