Skip to content

GitLab

"kubernetes/helm/vscode:/vscode.git/clone" did not exist on "7813c3316f3f2412fed504a8c7b712ac38032d6c"
Unverified Commit 546fd5cd authored by Lintang Sutawika's avatar Lintang Sutawika Committed by GitHub
Browse files

Merge pull request #686 from EleutherAI/cleanup 

[Refactor] Cleanup for `big-refactor`
parents 465c695b 540d468e
Hide whitespace changes
Inline Side-by-side
Showing with 0 additions and 2460 deletions
results/llama/llama-7B/llama-7B_bbh_3-shot.json deleted 100644 → 0
View file @ 465c695b
{
"results": {
"bigbench_tracking_shuffled_objects_five_objects": {
"multiple_choice_grade": 0.1824,
"multiple_choice_grade_stderr": 0.010927017514830547
},
"bigbench_logical_deduction_seven_objects": {
"multiple_choice_grade": 0.24571428571428572,
"multiple_choice_grade_stderr": 0.01628337995683342
},
"bigbench_date_understanding": {
"multiple_choice_grade": 0.6205962059620597,
"multiple_choice_grade_stderr": 0.02529481360676469
},
"bigbench_navigate": {
"multiple_choice_grade": 0.495,
"multiple_choice_grade_stderr": 0.015818508944436645
},
"bigbench_geometric_shapes": {
"multiple_choice_grade": 0.17827298050139276,
"multiple_choice_grade_stderr": 0.02022856303248108,
"exact_str_match": 0.0,
"exact_str_match_stderr": 0.0
},
"bigbench_dyck_languages": {
"multiple_choice_grade": 0.154,
"multiple_choice_grade_stderr": 0.011419913065098684
},
"bigbench_temporal_sequences": {
"multiple_choice_grade": 0.272,
"multiple_choice_grade_stderr": 0.014078856992462611
},
"bigbench_snarks": {
"multiple_choice_grade": 0.5082872928176796,
"multiple_choice_grade_stderr": 0.03726268022638988
},
"bigbench_disambiguation_qa": {
"multiple_choice_grade": 0.35271317829457366,
"multiple_choice_grade_stderr": 0.029805242804674153
},
"bigbench_tracking_shuffled_objects_seven_objects": {
"multiple_choice_grade": 0.13714285714285715,
"multiple_choice_grade_stderr": 0.008225477923226985
},
"bigbench_ruin_names": {
"multiple_choice_grade": 0.29910714285714285,
"multiple_choice_grade_stderr": 0.021656359273376977
},
"bigbench_movie_recommendation": {
"multiple_choice_grade": 0.404,
"multiple_choice_grade_stderr": 0.021966635293832918
},
"bigbench_salient_translation_error_detection": {
"multiple_choice_grade": 0.1653306613226453,
"multiple_choice_grade_stderr": 0.011764848862417502
},
"bigbench_logical_deduction_five_objects": {
"multiple_choice_grade": 0.29,
"multiple_choice_grade_stderr": 0.020313179231745183
},
"bigbench_causal_judgement": {
"multiple_choice_grade": 0.4842105263157895,
"multiple_choice_grade_stderr": 0.036351509398643456
},
"bigbench_hyperbaton": {
"multiple_choice_grade": 0.49508,
"multiple_choice_grade_stderr": 0.0022359820804999713
},
"bigbench_sports_understanding": {
"multiple_choice_grade": 0.5,
"multiple_choice_grade_stderr": 0.015931324696929153
},
"bigbench_logical_deduction_three_objects": {
"multiple_choice_grade": 0.3933333333333333,
"multiple_choice_grade_stderr": 0.028250090846760875
},
"bigbench_tracking_shuffled_objects_three_objects": {
"multiple_choice_grade": 0.3933333333333333,
"multiple_choice_grade_stderr": 0.028250090846760875
},
"bigbench_formal_fallacies_syllogisms_negation": {
"multiple_choice_grade": 0.5134507042253521,
"multiple_choice_grade_stderr": 0.004194535955193854
},
"bigbench_reasoning_about_colored_objects": {
"multiple_choice_grade": 0.346,
"multiple_choice_grade_stderr": 0.010639483037236658
}
},
"versions": {
"bigbench_tracking_shuffled_objects_five_objects": 0,
"bigbench_logical_deduction_seven_objects": 0,
"bigbench_date_understanding": 0,
"bigbench_navigate": 0,
"bigbench_geometric_shapes": 0,
"bigbench_dyck_languages": 0,
"bigbench_temporal_sequences": 0,
"bigbench_snarks": 0,
"bigbench_disambiguation_qa": 0,
"bigbench_tracking_shuffled_objects_seven_objects": 0,
"bigbench_ruin_names": 0,
"bigbench_movie_recommendation": 0,
"bigbench_salient_translation_error_detection": 0,
"bigbench_logical_deduction_five_objects": 0,
"bigbench_causal_judgement": 0,
"bigbench_hyperbaton": 0,
"bigbench_sports_understanding": 0,
"bigbench_logical_deduction_three_objects": 0,
"bigbench_tracking_shuffled_objects_three_objects": 0,
"bigbench_formal_fallacies_syllogisms_negation": 0,
"bigbench_reasoning_about_colored_objects": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B,use_accelerate=True",
"num_fewshot": 3,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/llama/llama-7B/llama-7B_blimp_0-shot.json deleted 100644 → 0
View file @ 465c695b
{
"results": {
"blimp_wh_vs_that_no_gap_long_distance": {
"acc": 0.812,
"acc_stderr": 0.01236158601510377
},
"blimp_ellipsis_n_bar_1": {
"acc": 0.668,
"acc_stderr": 0.014899597242811478
},
"blimp_distractor_agreement_relative_clause": {
"acc": 0.314,
"acc_stderr": 0.014683991951087966
},
"blimp_determiner_noun_agreement_with_adjective_1": {
"acc": 0.577,
"acc_stderr": 0.01563058909047635
},
"blimp_principle_A_reconstruction": {
"acc": 0.673,
"acc_stderr": 0.014842213153411245
},
"blimp_determiner_noun_agreement_2": {
"acc": 0.598,
"acc_stderr": 0.015512467135715077
},
"blimp_npi_present_1": {
"acc": 0.534,
"acc_stderr": 0.015782683329937618
},
"blimp_existential_there_quantifiers_2": {
"acc": 0.674,
"acc_stderr": 0.01483050720454104
},
"blimp_existential_there_subject_raising": {
"acc": 0.696,
"acc_stderr": 0.014553205687950438
},
"blimp_tough_vs_raising_1": {
"acc": 0.298,
"acc_stderr": 0.014470846741134717
},
"blimp_wh_questions_subject_gap_long_distance": {
"acc": 0.746,
"acc_stderr": 0.01377220656516854
},
"blimp_left_branch_island_echo_question": {
"acc": 0.835,
"acc_stderr": 0.011743632866916164
},
"blimp_only_npi_licensor_present": {
"acc": 0.814,
"acc_stderr": 0.012310790208412803
},
"blimp_adjunct_island": {
"acc": 0.539,
"acc_stderr": 0.01577110420128319
},
"blimp_coordinate_structure_constraint_object_extraction": {
"acc": 0.629,
"acc_stderr": 0.015283736211823188
},
"blimp_irregular_plural_subject_verb_agreement_1": {
"acc": 0.653,
"acc_stderr": 0.01506047203170662
},
"blimp_passive_2": {
"acc": 0.703,
"acc_stderr": 0.0144568322948011
},
"blimp_drop_argument": {
"acc": 0.701,
"acc_stderr": 0.014484778521220461
},
"blimp_wh_vs_that_with_gap_long_distance": {
"acc": 0.227,
"acc_stderr": 0.013253174964763925
},
"blimp_existential_there_quantifiers_1": {
"acc": 0.683,
"acc_stderr": 0.014721675438880224
},
"blimp_coordinate_structure_constraint_complex_left_branch": {
"acc": 0.682,
"acc_stderr": 0.014734079309311901
},
"blimp_ellipsis_n_bar_2": {
"acc": 0.794,
"acc_stderr": 0.012795613612786548
},
"blimp_sentential_subject_island": {
"acc": 0.606,
"acc_stderr": 0.01545972195749338
},
"blimp_determiner_noun_agreement_with_adj_irregular_2": {
"acc": 0.591,
"acc_stderr": 0.015555094373257946
},
"blimp_npi_present_2": {
"acc": 0.53,
"acc_stderr": 0.015790799515836763
},
"blimp_wh_questions_subject_gap": {
"acc": 0.72,
"acc_stderr": 0.014205696104091519
},
"blimp_determiner_noun_agreement_irregular_1": {
"acc": 0.572,
"acc_stderr": 0.01565442624502927
},
"blimp_superlative_quantifiers_1": {
"acc": 0.612,
"acc_stderr": 0.015417317979911076
},
"blimp_left_branch_island_simple_question": {
"acc": 0.74,
"acc_stderr": 0.013877773329774164
},
"blimp_irregular_past_participle_adjectives": {
"acc": 0.429,
"acc_stderr": 0.01565899754787024
},
"blimp_principle_A_domain_2": {
"acc": 0.646,
"acc_stderr": 0.015129868238451772
},
"blimp_regular_plural_subject_verb_agreement_1": {
"acc": 0.645,
"acc_stderr": 0.015139491543780529
},
"blimp_principle_A_case_1": {
"acc": 0.985,
"acc_stderr": 0.003845749574502997
},
"blimp_principle_A_case_2": {
"acc": 0.554,
"acc_stderr": 0.015726771166750357
},
"blimp_matrix_question_npi_licensor_present": {
"acc": 0.117,
"acc_stderr": 0.010169287802713327
},
"blimp_determiner_noun_agreement_with_adj_irregular_1": {
"acc": 0.563,
"acc_stderr": 0.015693223928730377
},
"blimp_principle_A_domain_1": {
"acc": 0.962,
"acc_stderr": 0.006049181150584942
},
"blimp_superlative_quantifiers_2": {
"acc": 0.561,
"acc_stderr": 0.01570113134540077
},
"blimp_wh_island": {
"acc": 0.275,
"acc_stderr": 0.014127086556490531
},
"blimp_only_npi_scope": {
"acc": 0.266,
"acc_stderr": 0.013979965645145156
},
"blimp_regular_plural_subject_verb_agreement_2": {
"acc": 0.705,
"acc_stderr": 0.014428554438445512
},
"blimp_complex_NP_island": {
"acc": 0.416,
"acc_stderr": 0.015594460144140603
},
"blimp_sentential_negation_npi_scope": {
"acc": 0.588,
"acc_stderr": 0.015572363292015093
},
"blimp_transitive": {
"acc": 0.698,
"acc_stderr": 0.014526080235459543
},
"blimp_wh_vs_that_with_gap": {
"acc": 0.239,
"acc_stderr": 0.013493000446937587
},
"blimp_wh_questions_object_gap": {
"acc": 0.67,
"acc_stderr": 0.014876872027456736
},
"blimp_sentential_negation_npi_licensor_present": {
"acc": 0.94,
"acc_stderr": 0.007513751157474913
},
"blimp_expletive_it_object_raising": {
"acc": 0.659,
"acc_stderr": 0.0149981313484027
},
"blimp_determiner_noun_agreement_1": {
"acc": 0.636,
"acc_stderr": 0.015222868840522017
},
"blimp_anaphor_gender_agreement": {
"acc": 0.448,
"acc_stderr": 0.01573351656634784
},
"blimp_irregular_plural_subject_verb_agreement_2": {
"acc": 0.7,
"acc_stderr": 0.014498627873361425
},
"blimp_passive_1": {
"acc": 0.702,
"acc_stderr": 0.014470846741134705
},
"blimp_determiner_noun_agreement_irregular_2": {
"acc": 0.602,
"acc_stderr": 0.015486634102858913
},
"blimp_irregular_past_participle_verbs": {
"acc": 0.725,
"acc_stderr": 0.014127086556490523
},
"blimp_existential_there_object_raising": {
"acc": 0.788,
"acc_stderr": 0.012931481864938055
},
"blimp_determiner_noun_agreement_with_adj_2": {
"acc": 0.54,
"acc_stderr": 0.015768596914394375
},
"blimp_principle_A_domain_3": {
"acc": 0.501,
"acc_stderr": 0.015819268290576817
},
"blimp_causative": {
"acc": 0.508,
"acc_stderr": 0.015817274929209008
},
"blimp_tough_vs_raising_2": {
"acc": 0.768,
"acc_stderr": 0.01335493745228157
},
"blimp_wh_vs_that_no_gap": {
"acc": 0.848,
"acc_stderr": 0.011358918303475294
},
"blimp_anaphor_number_agreement": {
"acc": 0.659,
"acc_stderr": 0.014998131348402704
},
"blimp_principle_A_c_command": {
"acc": 0.39,
"acc_stderr": 0.01543172505386661
},
"blimp_distractor_agreement_relational_noun": {
"acc": 0.441,
"acc_stderr": 0.015708779894242676
},
"blimp_intransitive": {
"acc": 0.592,
"acc_stderr": 0.015549205052920673
},
"blimp_animate_subject_passive": {
"acc": 0.626,
"acc_stderr": 0.015308767369006363
},
"blimp_animate_subject_trans": {
"acc": 0.761,
"acc_stderr": 0.01349300044693759
},
"blimp_inchoative": {
"acc": 0.42,
"acc_stderr": 0.015615500115072957
}
},
"versions": {
"blimp_wh_vs_that_no_gap_long_distance": 0,
"blimp_ellipsis_n_bar_1": 0,
"blimp_distractor_agreement_relative_clause": 0,
"blimp_determiner_noun_agreement_with_adjective_1": 0,
"blimp_principle_A_reconstruction": 0,
"blimp_determiner_noun_agreement_2": 0,
"blimp_npi_present_1": 0,
"blimp_existential_there_quantifiers_2": 0,
"blimp_existential_there_subject_raising": 0,
"blimp_tough_vs_raising_1": 0,
"blimp_wh_questions_subject_gap_long_distance": 0,
"blimp_left_branch_island_echo_question": 0,
"blimp_only_npi_licensor_present": 0,
"blimp_adjunct_island": 0,
"blimp_coordinate_structure_constraint_object_extraction": 0,
"blimp_irregular_plural_subject_verb_agreement_1": 0,
"blimp_passive_2": 0,
"blimp_drop_argument": 0,
"blimp_wh_vs_that_with_gap_long_distance": 0,
"blimp_existential_there_quantifiers_1": 0,
"blimp_coordinate_structure_constraint_complex_left_branch": 0,
"blimp_ellipsis_n_bar_2": 0,
"blimp_sentential_subject_island": 0,
"blimp_determiner_noun_agreement_with_adj_irregular_2": 0,
"blimp_npi_present_2": 0,
"blimp_wh_questions_subject_gap": 0,
"blimp_determiner_noun_agreement_irregular_1": 0,
"blimp_superlative_quantifiers_1": 0,
"blimp_left_branch_island_simple_question": 0,
"blimp_irregular_past_participle_adjectives": 0,
"blimp_principle_A_domain_2": 0,
"blimp_regular_plural_subject_verb_agreement_1": 0,
"blimp_principle_A_case_1": 0,
"blimp_principle_A_case_2": 0,
"blimp_matrix_question_npi_licensor_present": 0,
"blimp_determiner_noun_agreement_with_adj_irregular_1": 0,
"blimp_principle_A_domain_1": 0,
"blimp_superlative_quantifiers_2": 0,
"blimp_wh_island": 0,
"blimp_only_npi_scope": 0,
"blimp_regular_plural_subject_verb_agreement_2": 0,
"blimp_complex_NP_island": 0,
"blimp_sentential_negation_npi_scope": 0,
"blimp_transitive": 0,
"blimp_wh_vs_that_with_gap": 0,
"blimp_wh_questions_object_gap": 0,
"blimp_sentential_negation_npi_licensor_present": 0,
"blimp_expletive_it_object_raising": 0,
"blimp_determiner_noun_agreement_1": 0,
"blimp_anaphor_gender_agreement": 0,
"blimp_irregular_plural_subject_verb_agreement_2": 0,
"blimp_passive_1": 0,
"blimp_determiner_noun_agreement_irregular_2": 0,
"blimp_irregular_past_participle_verbs": 0,
"blimp_existential_there_object_raising": 0,
"blimp_determiner_noun_agreement_with_adj_2": 0,
"blimp_principle_A_domain_3": 0,
"blimp_causative": 0,
"blimp_tough_vs_raising_2": 0,
"blimp_wh_vs_that_no_gap": 0,
"blimp_anaphor_number_agreement": 0,
"blimp_principle_A_c_command": 0,
"blimp_distractor_agreement_relational_noun": 0,
"blimp_intransitive": 0,
"blimp_animate_subject_passive": 0,
"blimp_animate_subject_trans": 0,
"blimp_inchoative": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda:0",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/llama/llama-7B/llama-7B_common_sense_reasoning_0-shot.json deleted 100644 → 0
View file @ 465c695b
{
"results": {
"piqa": {
"acc": 0.7818280739934712,
"acc_stderr": 0.009636081958374381,
"acc_norm": 0.7742110990206746,
"acc_norm_stderr": 0.00975498067091731
},
"wsc273": {
"acc": 0.8095238095238095,
"acc_stderr": 0.023809523809523777
},
"arc_easy": {
"acc": 0.6738215488215489,
"acc_stderr": 0.009619849417035172,
"acc_norm": 0.5248316498316499,
"acc_norm_stderr": 0.010247123122159281
},
"hellaswag": {
"acc": 0.563931487751444,
"acc_stderr": 0.004948824501355491,
"acc_norm": 0.7298346942840072,
"acc_norm_stderr": 0.004431375549911366
},
"winogrande": {
"acc": 0.6692975532754538,
"acc_stderr": 0.013222435887002705
},
"prost": {
"acc": 0.2568851409052092,
"acc_stderr": 0.003192056839011391,
"acc_norm": 0.28031596925704527,
"acc_norm_stderr": 0.0032814667207950675
},
"swag": {
"acc": 0.5546835949215235,
"acc_stderr": 0.0035138865053857436,
"acc_norm": 0.6687493751874438,
"acc_norm_stderr": 0.003327673972187388
},
"boolq": {
"acc": 0.7305810397553517,
"acc_stderr": 0.007759626474907443
},
"arc_challenge": {
"acc": 0.3822525597269625,
"acc_stderr": 0.014200454049979293,
"acc_norm": 0.4138225255972696,
"acc_norm_stderr": 0.014392730009221009
},
"mc_taco": {
"em": 0.11261261261261261,
"f1": 0.4827075067316446
},
"copa": {
"acc": 0.84,
"acc_stderr": 0.036845294917747094
},
"openbookqa": {
"acc": 0.282,
"acc_stderr": 0.020143572847290774,
"acc_norm": 0.424,
"acc_norm_stderr": 0.022122993778135404
}
},
"versions": {
"piqa": 0,
"wsc273": 0,
"arc_easy": 0,
"hellaswag": 0,
"winogrande": 0,
"prost": 0,
"swag": 0,
"boolq": 1,
"arc_challenge": 0,
"mc_taco": 0,
"copa": 0,
"openbookqa": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B,use_accelerate=True",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda:0",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/llama/llama-7B/llama-7B_glue_0-shot.json deleted 100644 → 0
View file @ 465c695b
{
"results": {
"mrpc": {
"acc": 0.6838235294117647,
"acc_stderr": 0.023048336668420204,
"f1": 0.8122270742358079,
"f1_stderr": 0.01624762253426993
},
"sst": {
"acc": 0.5298165137614679,
"acc_stderr": 0.016911703415318852
},
"wnli": {
"acc": 0.5633802816901409,
"acc_stderr": 0.0592793555841297
},
"mnli_mismatched": {
"acc": 0.3572009764035802,
"acc_stderr": 0.0048327582938812235
},
"qnli": {
"acc": 0.49569833424858134,
"acc_stderr": 0.006765160168388141
},
"rte": {
"acc": 0.5306859205776173,
"acc_stderr": 0.03003973059219781
},
"qqp": {
"acc": 0.3683650754390304,
"acc_stderr": 0.002398975385820536,
"f1": 0.5380844713755992,
"f1_stderr": 0.0025560675394743124
},
"mnli": {
"acc": 0.34396332144676517,
"acc_stderr": 0.0047950937299233165
},
"cola": {
"mcc": 0.0,
"mcc_stderr": 0.0
}
},
"versions": {
"mrpc": 0,
"sst": 0,
"wnli": 1,
"mnli_mismatched": 0,
"qnli": 0,
"rte": 0,
"qqp": 0,
"mnli": 0,
"cola": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B,use_accelerate=True",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda:0",
"no_cache": false,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/llama/llama-7B/llama-7B_gsm8k_8-shot.json deleted 100644 → 0
View file @ 465c695b
{
"results": {
"gsm8k": {
"acc": 0.0803639120545868,
"acc_stderr": 0.007488258573239077
}
},
"versions": {
"gsm8k": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B,use_accelerate=True",
"num_fewshot": 8,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/llama/llama-7B/llama-7B_human_alignment_0-shot.json deleted 100644 → 0
View file @ 465c695b
{
"results": {
"ethics_virtue": {
"acc": 0.20984924623115578,
"acc_stderr": 0.005773721023799748,
"em": 0.0
},
"crows_pairs_french_race_color": {
"likelihood_difference": 12.048913043478262,
"likelihood_difference_stderr": 0.7332463392189781,
"pct_stereotype": 0.4326086956521739,
"pct_stereotype_stderr": 0.023125046645341776
},
"ethics_utilitarianism_original": {
"acc": 0.9586106489184693,
"acc_stderr": 0.002872952014248801
},
"crows_pairs_english_nationality": {
"likelihood_difference": 6.762586805555555,
"likelihood_difference_stderr": 0.5868865852525466,
"pct_stereotype": 0.5370370370370371,
"pct_stereotype_stderr": 0.03400603625538272
},
"crows_pairs_english_socioeconomic": {
"likelihood_difference": 6.401644736842106,
"likelihood_difference_stderr": 0.5420413190484897,
"pct_stereotype": 0.5684210526315789,
"pct_stereotype_stderr": 0.03602751443822843
},
"crows_pairs_french_socioeconomic": {
"likelihood_difference": 9.80843431122449,
"likelihood_difference_stderr": 1.0151042209820862,
"pct_stereotype": 0.5204081632653061,
"pct_stereotype_stderr": 0.03577590557703757
},
"crows_pairs_english_religion": {
"likelihood_difference": 7.219594594594595,
"likelihood_difference_stderr": 0.759154104063707,
"pct_stereotype": 0.6666666666666666,
"pct_stereotype_stderr": 0.04494665749754944
},
"ethics_justice": {
"acc": 0.4996301775147929,
"acc_stderr": 0.009617160470756728,
"em": 0.0014792899408284023
},
"crows_pairs_english_autre": {
"likelihood_difference": 11.011363636363637,
"likelihood_difference_stderr": 5.8907614264514025,
"pct_stereotype": 0.45454545454545453,
"pct_stereotype_stderr": 0.15745916432444335
},
"toxigen": {
"acc": 0.4308510638297872,
"acc_stderr": 0.016160089171486036,
"acc_norm": 0.4319148936170213,
"acc_norm_stderr": 0.016164899004911828
},
"crows_pairs_french_autre": {
"likelihood_difference": 7.512019230769231,
"likelihood_difference_stderr": 2.0958404773406696,
"pct_stereotype": 0.6153846153846154,
"pct_stereotype_stderr": 0.14044168141158106
},
"ethics_cm": {
"acc": 0.5691119691119692,
"acc_stderr": 0.007945870163705206
},
"crows_pairs_english_gender": {
"likelihood_difference": 7.9173828125,
"likelihood_difference_stderr": 0.5501949212762886,
"pct_stereotype": 0.53125,
"pct_stereotype_stderr": 0.0279398950447155
},
"crows_pairs_english_race_color": {
"likelihood_difference": 6.246493602362205,
"likelihood_difference_stderr": 0.3239007651371134,
"pct_stereotype": 0.46653543307086615,
"pct_stereotype_stderr": 0.022155988267174086
},
"crows_pairs_english_age": {
"likelihood_difference": 5.9423076923076925,
"likelihood_difference_stderr": 0.7902909296461826,
"pct_stereotype": 0.5164835164835165,
"pct_stereotype_stderr": 0.05267597952306975
},
"ethics_utilitarianism": {
"acc": 0.4981281198003328,
"acc_stderr": 0.007211571268099885
},
"crows_pairs_english_sexual_orientation": {
"likelihood_difference": 8.304771505376344,
"likelihood_difference_stderr": 0.8427804261467623,
"pct_stereotype": 0.6236559139784946,
"pct_stereotype_stderr": 0.05050927755267201
},
"ethics_deontology": {
"acc": 0.5058398220244716,
"acc_stderr": 0.008338557598970859,
"em": 0.002224694104560623
},
"crows_pairs_french_religion": {
"likelihood_difference": 9.585326086956522,
"likelihood_difference_stderr": 0.8749663998788697,
"pct_stereotype": 0.43478260869565216,
"pct_stereotype_stderr": 0.04642922286356426
},
"crows_pairs_french_gender": {
"likelihood_difference": 11.798968068535826,
"likelihood_difference_stderr": 0.8713501661430004,
"pct_stereotype": 0.5202492211838006,
"pct_stereotype_stderr": 0.0279279188851323
},
"crows_pairs_french_nationality": {
"likelihood_difference": 10.416501976284586,
"likelihood_difference_stderr": 0.9065784742122508,
"pct_stereotype": 0.40711462450592883,
"pct_stereotype_stderr": 0.030948774049323072
},
"crows_pairs_english_physical_appearance": {
"likelihood_difference": 4.512586805555555,
"likelihood_difference_stderr": 0.6931576110749077,
"pct_stereotype": 0.5,
"pct_stereotype_stderr": 0.05933908290969268
},
"crows_pairs_french_age": {
"likelihood_difference": 11.939583333333333,
"likelihood_difference_stderr": 1.5376984338772959,
"pct_stereotype": 0.35555555555555557,
"pct_stereotype_stderr": 0.05074011803597719
},
"crows_pairs_english_disability": {
"likelihood_difference": 9.669711538461538,
"likelihood_difference_stderr": 1.1386178272217904,
"pct_stereotype": 0.6615384615384615,
"pct_stereotype_stderr": 0.05914829422780653
},
"crows_pairs_french_sexual_orientation": {
"likelihood_difference": 7.605769230769231,
"likelihood_difference_stderr": 0.7938984905689758,
"pct_stereotype": 0.6703296703296703,
"pct_stereotype_stderr": 0.04955219508596587
},
"crows_pairs_french_physical_appearance": {
"likelihood_difference": 7.045138888888889,
"likelihood_difference_stderr": 0.9484318157143898,
"pct_stereotype": 0.5555555555555556,
"pct_stereotype_stderr": 0.05897165471491952
},
"crows_pairs_french_disability": {
"likelihood_difference": 10.147727272727273,
"likelihood_difference_stderr": 1.3907137676702652,
"pct_stereotype": 0.42424242424242425,
"pct_stereotype_stderr": 0.06130137276858363
}
},
"versions": {
"ethics_virtue": 0,
"crows_pairs_french_race_color": 0,
"ethics_utilitarianism_original": 0,
"crows_pairs_english_nationality": 0,
"crows_pairs_english_socioeconomic": 0,
"crows_pairs_french_socioeconomic": 0,
"crows_pairs_english_religion": 0,
"ethics_justice": 0,
"crows_pairs_english_autre": 0,
"toxigen": 0,
"crows_pairs_french_autre": 0,
"ethics_cm": 0,
"crows_pairs_english_gender": 0,
"crows_pairs_english_race_color": 0,
"crows_pairs_english_age": 0,
"ethics_utilitarianism": 0,
"crows_pairs_english_sexual_orientation": 0,
"ethics_deontology": 0,
"crows_pairs_french_religion": 0,
"crows_pairs_french_gender": 0,
"crows_pairs_french_nationality": 0,
"crows_pairs_english_physical_appearance": 0,
"crows_pairs_french_age": 0,
"crows_pairs_english_disability": 0,
"crows_pairs_french_sexual_orientation": 0,
"crows_pairs_french_physical_appearance": 0,
"crows_pairs_french_disability": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B,use_accelerate=True",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda:0",
"no_cache": false,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/llama/llama-7B/llama-7B_lambada_0-shot.json deleted 100644 → 0
View file @ 465c695b
{
"results": {
"lambada_openai_mt_it": {
"ppl": 3653680.5734235523,
"ppl_stderr": 197082.9860932525,
"acc": 0.0,
"acc_stderr": 0.0
},
"lambada_standard": {
"ppl": 2460346.8572795168,
"ppl_stderr": 81216.56551688322,
"acc": 0.0,
"acc_stderr": 0.0
},
"lambada_openai_mt_es": {
"ppl": 3818890.4545065593,
"ppl_stderr": 197999.05318216747,
"acc": 0.0,
"acc_stderr": 0.0
},
"lambada_openai": {
"ppl": 2817465.092468485,
"ppl_stderr": 138319.08822004002,
"acc": 0.0,
"acc_stderr": 0.0
},
"lambada_openai_mt_fr": {
"ppl": 2111186.115467981,
"ppl_stderr": 111724.42842108487,
"acc": 0.0,
"acc_stderr": 0.0
},
"lambada_openai_mt_de": {
"ppl": 1805613.6770583114,
"ppl_stderr": 97892.78908113715,
"acc": 0.0,
"acc_stderr": 0.0
},
"lambada_standard_cloze": {
"ppl": 6710057.2411258025,
"ppl_stderr": 169833.90998542923,
"acc": 0.0,
"acc_stderr": 0.0
},
"lambada_openai_mt_en": {
"ppl": 2817465.092468485,
"ppl_stderr": 138319.08822004002,
"acc": 0.0,
"acc_stderr": 0.0
},
"lambada_openai_cloze": {
"ppl": 255777.71115985065,
"ppl_stderr": 11345.7709705634,
"acc": 0.00038812342324859306,
"acc_stderr": 0.00027441806845051746
}
},
"versions": {
"lambada_openai_mt_it": 0,
"lambada_standard": 0,
"lambada_openai_mt_es": 0,
"lambada_openai": 0,
"lambada_openai_mt_fr": 0,
"lambada_openai_mt_de": 0,
"lambada_standard_cloze": 0,
"lambada_openai_mt_en": 0,
"lambada_openai_cloze": 0
},
"config": {
"model": "hf-causal",
"model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/llama/llama-7B/llama-7B_mathematical_reasoning_0-shot.json deleted 100644 → 0
View file @ 465c695b
{
"results": {
"mathqa": {
"acc": 0.26767169179229483,
"acc_stderr": 0.008105031808599684,
"acc_norm": 0.27872696817420434,
"acc_norm_stderr": 0.008208048863665952
},
"math_asdiv": {
"acc": 0.0,
"acc_stderr": 0.0
},
"gsm8k": {
"acc": 0.0,
"acc_stderr": 0.0
},
"math_num_theory": {
"acc": 0.007407407407407408,
"acc_stderr": 0.003693382168437238
},
"math_precalc": {
"acc": 0.003663003663003663,
"acc_stderr": 0.002587757368193461
},
"drop": {
"em": 0.04268036912751678,
"em_stderr": 0.002070056585023236,
"f1": 0.1215950083892614,
"f1_stderr": 0.0024765528531984883
},
"math_geometry": {
"acc": 0.008350730688935281,
"acc_stderr": 0.004162242110295851
},
"math_counting_and_prob": {
"acc": 0.016877637130801686,
"acc_stderr": 0.0059228268948526815
},
"math_intermediate_algebra": {
"acc": 0.006644518272425249,
"acc_stderr": 0.0027050844483854013
},
"math_prealgebra": {
"acc": 0.012629161882893225,
"acc_stderr": 0.003785888218263002
},
"math_algebra": {
"acc": 0.016849199663016005,
"acc_stderr": 0.0037372948497597248
}
},
"versions": {
"mathqa": 0,
"math_asdiv": 0,
"gsm8k": 0,
"math_num_theory": 1,
"math_precalc": 1,
"drop": 1,
"math_geometry": 1,
"math_counting_and_prob": 1,
"math_intermediate_algebra": 1,
"math_prealgebra": 1,
"math_algebra": 1
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B,use_accelerate=True",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda:0",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/llama/llama-7B/llama-7B_mathematical_reasoning_few_shot_5-shot.json deleted 100644 → 0
View file @ 465c695b
{
"results": {
"mathqa": {
"acc": 0.2820770519262982,
"acc_stderr": 0.008238030326915545,
"acc_norm": 0.2877721943048576,
"acc_norm_stderr": 0.008287708494779904
},
"math_prealgebra": {
"acc": 0.001148105625717566,
"acc_stderr": 0.0011481056257175704
},
"math_geometry": {
"acc": 0.0,
"acc_stderr": 0.0
},
"math_intermediate_algebra": {
"acc": 0.0,
"acc_stderr": 0.0
},
"math_counting_and_prob": {
"acc": 0.0,
"acc_stderr": 0.0
},
"math_num_theory": {
"acc": 0.0,
"acc_stderr": 0.0
},
"gsm8k": {
"acc": 0.0,
"acc_stderr": 0.0
},
"drop": {
"em": 0.012374161073825503,
"em_stderr": 0.0011321233703992673,
"f1": 0.020981543624161086,
"f1_stderr": 0.001252441423790731
},
"math_precalc": {
"acc": 0.0,
"acc_stderr": 0.0
},
"math_algebra": {
"acc": 0.0,
"acc_stderr": 0.0
}
},
"versions": {
"mathqa": 0,
"math_prealgebra": 1,
"math_geometry": 1,
"math_intermediate_algebra": 1,
"math_counting_and_prob": 1,
"math_num_theory": 1,
"gsm8k": 0,
"drop": 1,
"math_precalc": 1,
"math_algebra": 1
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B,use_accelerate=True",
"num_fewshot": 5,
"batch_size": "auto",
"device": "cuda:0",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/llama/llama-7B/llama-7B_mmlu_5-shot.json deleted 100644 → 0
View file @ 465c695b
{
"results": {
"hendrycksTest-high_school_geography": {
"acc": 0.4292929292929293,
"acc_stderr": 0.035265527246011986,
"acc_norm": 0.36363636363636365,
"acc_norm_stderr": 0.03427308652999934
},
"hendrycksTest-philosophy": {
"acc": 0.40192926045016075,
"acc_stderr": 0.027846476005930477,
"acc_norm": 0.3536977491961415,
"acc_norm_stderr": 0.02715520810320088
},
"hendrycksTest-world_religions": {
"acc": 0.6257309941520468,
"acc_stderr": 0.03711601185389481,
"acc_norm": 0.5146198830409356,
"acc_norm_stderr": 0.038331852752130254
},
"hendrycksTest-college_biology": {
"acc": 0.3194444444444444,
"acc_stderr": 0.03899073687357335,
"acc_norm": 0.2916666666666667,
"acc_norm_stderr": 0.03800968060554858
},
"hendrycksTest-electrical_engineering": {
"acc": 0.3586206896551724,
"acc_stderr": 0.03996629574876719,
"acc_norm": 0.32413793103448274,
"acc_norm_stderr": 0.03900432069185554
},
"hendrycksTest-global_facts": {
"acc": 0.32,
"acc_stderr": 0.046882617226215034,
"acc_norm": 0.29,
"acc_norm_stderr": 0.045604802157206824
},
"hendrycksTest-high_school_government_and_politics": {
"acc": 0.48186528497409326,
"acc_stderr": 0.03606065001832917,
"acc_norm": 0.37305699481865284,
"acc_norm_stderr": 0.03490205592048573
},
"hendrycksTest-moral_scenarios": {
"acc": 0.2759776536312849,
"acc_stderr": 0.014950103002475353,
"acc_norm": 0.27262569832402234,
"acc_norm_stderr": 0.014893391735249588
},
"hendrycksTest-econometrics": {
"acc": 0.2894736842105263,
"acc_stderr": 0.04266339443159394,
"acc_norm": 0.2631578947368421,
"acc_norm_stderr": 0.0414243971948936
},
"hendrycksTest-international_law": {
"acc": 0.3884297520661157,
"acc_stderr": 0.04449270350068382,
"acc_norm": 0.5785123966942148,
"acc_norm_stderr": 0.045077322787750874
},
"hendrycksTest-us_foreign_policy": {
"acc": 0.56,
"acc_stderr": 0.049888765156985884,
"acc_norm": 0.45,
"acc_norm_stderr": 0.05
},
"hendrycksTest-high_school_macroeconomics": {
"acc": 0.31794871794871793,
"acc_stderr": 0.02361088430892786,
"acc_norm": 0.30256410256410254,
"acc_norm_stderr": 0.023290888053772742
},
"hendrycksTest-virology": {
"acc": 0.39759036144578314,
"acc_stderr": 0.038099730845402184,
"acc_norm": 0.2891566265060241,
"acc_norm_stderr": 0.035294868015111155
},
"hendrycksTest-high_school_mathematics": {
"acc": 0.22592592592592592,
"acc_stderr": 0.025497532639609542,
"acc_norm": 0.3074074074074074,
"acc_norm_stderr": 0.02813325257881564
},
"hendrycksTest-clinical_knowledge": {
"acc": 0.3886792452830189,
"acc_stderr": 0.03000048544867599,
"acc_norm": 0.38113207547169814,
"acc_norm_stderr": 0.029890609686286627
},
"hendrycksTest-professional_psychology": {
"acc": 0.3839869281045752,
"acc_stderr": 0.01967580813528152,
"acc_norm": 0.29901960784313725,
"acc_norm_stderr": 0.01852175621542302
},
"hendrycksTest-formal_logic": {
"acc": 0.30952380952380953,
"acc_stderr": 0.04134913018303316,
"acc_norm": 0.3492063492063492,
"acc_norm_stderr": 0.042639068927951315
},
"hendrycksTest-management": {
"acc": 0.4854368932038835,
"acc_stderr": 0.04948637324026637,
"acc_norm": 0.36893203883495146,
"acc_norm_stderr": 0.0477761518115674
},
"hendrycksTest-human_sexuality": {
"acc": 0.5114503816793893,
"acc_stderr": 0.043841400240780176,
"acc_norm": 0.366412213740458,
"acc_norm_stderr": 0.042258754519696386
},
"hendrycksTest-high_school_world_history": {
"acc": 0.3924050632911392,
"acc_stderr": 0.03178471874564729,
"acc_norm": 0.33755274261603374,
"acc_norm_stderr": 0.030781549102026216
},
"hendrycksTest-medical_genetics": {
"acc": 0.44,
"acc_stderr": 0.04988876515698589,
"acc_norm": 0.4,
"acc_norm_stderr": 0.04923659639173309
},
"hendrycksTest-computer_security": {
"acc": 0.37,
"acc_stderr": 0.048523658709391,
"acc_norm": 0.44,
"acc_norm_stderr": 0.04988876515698589
},
"hendrycksTest-miscellaneous": {
"acc": 0.5836526181353767,
"acc_stderr": 0.0176279480304303,
"acc_norm": 0.3895274584929757,
"acc_norm_stderr": 0.017438082556264597
},
"hendrycksTest-public_relations": {
"acc": 0.39090909090909093,
"acc_stderr": 0.046737523336702384,
"acc_norm": 0.22727272727272727,
"acc_norm_stderr": 0.040139645540727735
},
"hendrycksTest-college_physics": {
"acc": 0.23529411764705882,
"acc_stderr": 0.04220773659171453,
"acc_norm": 0.3235294117647059,
"acc_norm_stderr": 0.046550104113196177
},
"hendrycksTest-professional_accounting": {
"acc": 0.30141843971631205,
"acc_stderr": 0.02737412888263115,
"acc_norm": 0.29432624113475175,
"acc_norm_stderr": 0.027187127011503793
},
"hendrycksTest-logical_fallacies": {
"acc": 0.3803680981595092,
"acc_stderr": 0.03814269893261837,
"acc_norm": 0.3496932515337423,
"acc_norm_stderr": 0.037466683254700206
},
"hendrycksTest-business_ethics": {
"acc": 0.53,
"acc_stderr": 0.050161355804659205,
"acc_norm": 0.46,
"acc_norm_stderr": 0.05009082659620332
},
"hendrycksTest-high_school_chemistry": {
"acc": 0.2512315270935961,
"acc_stderr": 0.030516530732694436,
"acc_norm": 0.2955665024630542,
"acc_norm_stderr": 0.03210494433751458
},
"hendrycksTest-astronomy": {
"acc": 0.45394736842105265,
"acc_stderr": 0.04051646342874143,
"acc_norm": 0.4605263157894737,
"acc_norm_stderr": 0.04056242252249033
},
"hendrycksTest-high_school_us_history": {
"acc": 0.4264705882352941,
"acc_stderr": 0.03471157907953424,
"acc_norm": 0.3137254901960784,
"acc_norm_stderr": 0.032566854844603886
},
"hendrycksTest-college_chemistry": {
"acc": 0.33,
"acc_stderr": 0.047258156262526045,
"acc_norm": 0.3,
"acc_norm_stderr": 0.046056618647183814
},
"hendrycksTest-abstract_algebra": {
"acc": 0.23,
"acc_stderr": 0.042295258468165065,
"acc_norm": 0.26,
"acc_norm_stderr": 0.0440844002276808
},
"hendrycksTest-moral_disputes": {
"acc": 0.36416184971098264,
"acc_stderr": 0.025906632631016117,
"acc_norm": 0.33236994219653176,
"acc_norm_stderr": 0.02536116874968821
},
"hendrycksTest-college_computer_science": {
"acc": 0.33,
"acc_stderr": 0.04725815626252605,
"acc_norm": 0.28,
"acc_norm_stderr": 0.04512608598542128
},
"hendrycksTest-professional_law": {
"acc": 0.2966101694915254,
"acc_stderr": 0.011665946586082849,
"acc_norm": 0.28552803129074317,
"acc_norm_stderr": 0.011535751586665664
},
"hendrycksTest-college_mathematics": {
"acc": 0.32,
"acc_stderr": 0.046882617226215034,
"acc_norm": 0.32,
"acc_norm_stderr": 0.04688261722621505
},
"hendrycksTest-high_school_microeconomics": {
"acc": 0.3865546218487395,
"acc_stderr": 0.0316314580755238,
"acc_norm": 0.36554621848739494,
"acc_norm_stderr": 0.0312821770636846
},
"hendrycksTest-high_school_european_history": {
"acc": 0.40606060606060607,
"acc_stderr": 0.03834816355401181,
"acc_norm": 0.3696969696969697,
"acc_norm_stderr": 0.03769430314512568
},
"hendrycksTest-high_school_biology": {
"acc": 0.3580645161290323,
"acc_stderr": 0.027273890594300642,
"acc_norm": 0.3580645161290323,
"acc_norm_stderr": 0.02727389059430063
},
"hendrycksTest-security_studies": {
"acc": 0.40816326530612246,
"acc_stderr": 0.03146465712827424,
"acc_norm": 0.31020408163265306,
"acc_norm_stderr": 0.029613459872484375
},
"hendrycksTest-high_school_psychology": {
"acc": 0.46605504587155966,
"acc_stderr": 0.02138786335035399,
"acc_norm": 0.30825688073394497,
"acc_norm_stderr": 0.01979836669836726
},
"hendrycksTest-conceptual_physics": {
"acc": 0.3276595744680851,
"acc_stderr": 0.030683020843231004,
"acc_norm": 0.2170212765957447,
"acc_norm_stderr": 0.026947483121496228
},
"hendrycksTest-human_aging": {
"acc": 0.3721973094170404,
"acc_stderr": 0.03244305283008731,
"acc_norm": 0.25112107623318386,
"acc_norm_stderr": 0.02910522083322462
},
"hendrycksTest-prehistory": {
"acc": 0.4012345679012346,
"acc_stderr": 0.0272725828498398,
"acc_norm": 0.2777777777777778,
"acc_norm_stderr": 0.02492200116888633
},
"hendrycksTest-sociology": {
"acc": 0.47761194029850745,
"acc_stderr": 0.035319879302087305,
"acc_norm": 0.42786069651741293,
"acc_norm_stderr": 0.03498541988407795
},
"hendrycksTest-marketing": {
"acc": 0.6111111111111112,
"acc_stderr": 0.031937057262002924,
"acc_norm": 0.5042735042735043,
"acc_norm_stderr": 0.03275489264382132
},
"hendrycksTest-high_school_computer_science": {
"acc": 0.41,
"acc_stderr": 0.049431107042371025,
"acc_norm": 0.34,
"acc_norm_stderr": 0.047609522856952365
},
"hendrycksTest-machine_learning": {
"acc": 0.30357142857142855,
"acc_stderr": 0.04364226155841044,
"acc_norm": 0.26785714285714285,
"acc_norm_stderr": 0.04203277291467762
},
"hendrycksTest-elementary_mathematics": {
"acc": 0.3201058201058201,
"acc_stderr": 0.024026846392873506,
"acc_norm": 0.291005291005291,
"acc_norm_stderr": 0.023393826500484865
},
"hendrycksTest-nutrition": {
"acc": 0.3954248366013072,
"acc_stderr": 0.027996723180631435,
"acc_norm": 0.43790849673202614,
"acc_norm_stderr": 0.028408302020332694
},
"hendrycksTest-anatomy": {
"acc": 0.3851851851851852,
"acc_stderr": 0.042039210401562783,
"acc_norm": 0.2814814814814815,
"acc_norm_stderr": 0.03885004245800254
},
"hendrycksTest-jurisprudence": {
"acc": 0.4351851851851852,
"acc_stderr": 0.04792898170907062,
"acc_norm": 0.5,
"acc_norm_stderr": 0.04833682445228318
},
"hendrycksTest-college_medicine": {
"acc": 0.37572254335260113,
"acc_stderr": 0.036928207672648664,
"acc_norm": 0.3063583815028902,
"acc_norm_stderr": 0.03514942551267439
},
"hendrycksTest-high_school_statistics": {
"acc": 0.3425925925925926,
"acc_stderr": 0.03236585252602156,
"acc_norm": 0.3425925925925926,
"acc_norm_stderr": 0.03236585252602156
},
"hendrycksTest-high_school_physics": {
"acc": 0.2052980132450331,
"acc_stderr": 0.03297986648473834,
"acc_norm": 0.271523178807947,
"acc_norm_stderr": 0.036313298039696525
},
"hendrycksTest-professional_medicine": {
"acc": 0.3382352941176471,
"acc_stderr": 0.028739328513983576,
"acc_norm": 0.27941176470588236,
"acc_norm_stderr": 0.027257202606114948
}
},
"versions": {
"hendrycksTest-high_school_geography": 0,
"hendrycksTest-philosophy": 0,
"hendrycksTest-world_religions": 0,
"hendrycksTest-college_biology": 0,
"hendrycksTest-electrical_engineering": 0,
"hendrycksTest-global_facts": 0,
"hendrycksTest-high_school_government_and_politics": 0,
"hendrycksTest-moral_scenarios": 0,
"hendrycksTest-econometrics": 0,
"hendrycksTest-international_law": 0,
"hendrycksTest-us_foreign_policy": 0,
"hendrycksTest-high_school_macroeconomics": 0,
"hendrycksTest-virology": 0,
"hendrycksTest-high_school_mathematics": 0,
"hendrycksTest-clinical_knowledge": 0,
"hendrycksTest-professional_psychology": 0,
"hendrycksTest-formal_logic": 0,
"hendrycksTest-management": 0,
"hendrycksTest-human_sexuality": 0,
"hendrycksTest-high_school_world_history": 0,
"hendrycksTest-medical_genetics": 0,
"hendrycksTest-computer_security": 0,
"hendrycksTest-miscellaneous": 0,
"hendrycksTest-public_relations": 0,
"hendrycksTest-college_physics": 0,
"hendrycksTest-professional_accounting": 0,
"hendrycksTest-logical_fallacies": 0,
"hendrycksTest-business_ethics": 0,
"hendrycksTest-high_school_chemistry": 0,
"hendrycksTest-astronomy": 0,
"hendrycksTest-high_school_us_history": 0,
"hendrycksTest-college_chemistry": 0,
"hendrycksTest-abstract_algebra": 0,
"hendrycksTest-moral_disputes": 0,
"hendrycksTest-college_computer_science": 0,
"hendrycksTest-professional_law": 0,
"hendrycksTest-college_mathematics": 0,
"hendrycksTest-high_school_microeconomics": 0,
"hendrycksTest-high_school_european_history": 0,
"hendrycksTest-high_school_biology": 0,
"hendrycksTest-security_studies": 0,
"hendrycksTest-high_school_psychology": 0,
"hendrycksTest-conceptual_physics": 0,
"hendrycksTest-human_aging": 0,
"hendrycksTest-prehistory": 0,
"hendrycksTest-sociology": 0,
"hendrycksTest-marketing": 0,
"hendrycksTest-high_school_computer_science": 0,
"hendrycksTest-machine_learning": 0,
"hendrycksTest-elementary_mathematics": 0,
"hendrycksTest-nutrition": 0,
"hendrycksTest-anatomy": 0,
"hendrycksTest-jurisprudence": 0,
"hendrycksTest-college_medicine": 0,
"hendrycksTest-high_school_statistics": 0,
"hendrycksTest-high_school_physics": 0,
"hendrycksTest-professional_medicine": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B,use_accelerate=True",
"num_fewshot": 5,
"batch_size": "auto",
"device": "cuda:0",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/llama/llama-7B/llama-7B_pawsx_0-shot.json deleted 100644 → 0
View file @ 465c695b
{
"results": {
"pawsx_en": {
"acc": 0.6185,
"acc_stderr": 0.010864524561478635
},
"pawsx_es": {
"acc": 0.561,
"acc_stderr": 0.011099599116647334
},
"pawsx_zh": {
"acc": 0.491,
"acc_stderr": 0.01118132420626029
},
"pawsx_fr": {
"acc": 0.5295,
"acc_stderr": 0.01116365480451166
},
"pawsx_de": {
"acc": 0.5465,
"acc_stderr": 0.011134669525078668
},
"pawsx_ko": {
"acc": 0.497,
"acc_stderr": 0.011182934722804558
},
"pawsx_ja": {
"acc": 0.567,
"acc_stderr": 0.011082279027990133
}
},
"versions": {
"pawsx_en": 0,
"pawsx_es": 0,
"pawsx_zh": 0,
"pawsx_fr": 0,
"pawsx_de": 0,
"pawsx_ko": 0,
"pawsx_ja": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/llama/llama-7B/llama-7B_question_answering_0-shot.json deleted 100644 → 0
View file @ 465c695b
{
"results": {
"webqs": {
"acc": 0.0,
"acc_stderr": 0.0
},
"truthfulqa_mc": {
"mc1": 0.21052631578947367,
"mc1_stderr": 0.014271740645964186,
"mc2": 0.3413779054949544,
"mc2_stderr": 0.013147086422190785
},
"headqa_en": {
"acc": 0.3242159008023341,
"acc_stderr": 0.008940599111132593,
"acc_norm": 0.35922684172137126,
"acc_norm_stderr": 0.009163935584608705
},
"triviaqa": {
"acc": 0.0,
"acc_stderr": 0.0
},
"headqa_es": {
"acc": 0.2826404084609774,
"acc_stderr": 0.00860064580960105,
"acc_norm": 0.3242159008023341,
"acc_norm_stderr": 0.008940599111132597
},
"logiqa": {
"acc": 0.21812596006144394,
"acc_stderr": 0.01619814925841932,
"acc_norm": 0.30261136712749614,
"acc_norm_stderr": 0.018018696598158843
},
"squad2": {
"exact": 9.416322749094585,
"f1": 19.449005859216243,
"HasAns_exact": 18.488529014844804,
"HasAns_f1": 38.58266642484388,
"NoAns_exact": 0.3700588730025231,
"NoAns_f1": 0.3700588730025231,
"best_exact": 50.07159100480081,
"best_f1": 50.0801228586961
}
},
"versions": {
"webqs": 0,
"truthfulqa_mc": 1,
"headqa_en": 0,
"triviaqa": 1,
"headqa_es": 0,
"logiqa": 0,
"squad2": 1
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B,use_accelerate=True",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda:0",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/llama/llama-7B/llama-7B_reading_comprehension_0-shot.json deleted 100644 → 0
View file @ 465c695b
{
"results": {
"coqa": {
"f1": 0.752079907334399,
"f1_stderr": 0.01529202913294504,
"em": 0.6266666666666667,
"em_stderr": 0.01881953816383566
},
"drop": {
"em": 0.035864093959731544,
"em_stderr": 0.001904314663911949,
"f1": 0.11349412751677862,
"f1_stderr": 0.002340148712955784
},
"race": {
"acc": 0.3990430622009569,
"acc_stderr": 0.015155885289809327
}
},
"versions": {
"coqa": 1,
"drop": 1,
"race": 1
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B,use_accelerate=True",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda:0",
"no_cache": false,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/llama/llama-7B/llama-7B_unscramble_0-shot.json deleted 100644 → 0
View file @ 465c695b
{
"results": {
"anagrams1": {
"acc": 0.0,
"acc_stderr": 0.0
},
"anagrams2": {
"acc": 0.0,
"acc_stderr": 0.0
},
"cycle_letters": {
"acc": 0.0,
"acc_stderr": 0.0
},
"random_insertion": {
"acc": 0.0,
"acc_stderr": 0.0
},
"reversed_words": {
"acc": 0.0,
"acc_stderr": 0.0
}
},
"versions": {
"anagrams1": 0,
"anagrams2": 0,
"cycle_letters": 0,
"random_insertion": 0,
"reversed_words": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/llama/llama-7B/llama-7B_xcopa_0-shot.json deleted 100644 → 0
View file @ 465c695b
{
"results": {
"xcopa_sw": {
"acc": 0.508,
"acc_stderr": 0.022380208834928025
},
"xcopa_zh": {
"acc": 0.562,
"acc_stderr": 0.022210326363977417
},
"xcopa_et": {
"acc": 0.488,
"acc_stderr": 0.02237662679792717
},
"xcopa_th": {
"acc": 0.558,
"acc_stderr": 0.022231970696321122
},
"xcopa_tr": {
"acc": 0.556,
"acc_stderr": 0.022242244375731017
},
"xcopa_qu": {
"acc": 0.514,
"acc_stderr": 0.022374298166353185
},
"xcopa_id": {
"acc": 0.546,
"acc_stderr": 0.02228814759117695
},
"xcopa_ta": {
"acc": 0.552,
"acc_stderr": 0.02226169729227013
},
"xcopa_ht": {
"acc": 0.51,
"acc_stderr": 0.02237859698923078
},
"xcopa_vi": {
"acc": 0.516,
"acc_stderr": 0.0223716109825804
},
"xcopa_it": {
"acc": 0.62,
"acc_stderr": 0.021728881438701705
}
},
"versions": {
"xcopa_sw": 0,
"xcopa_zh": 0,
"xcopa_et": 0,
"xcopa_th": 0,
"xcopa_tr": 0,
"xcopa_qu": 0,
"xcopa_id": 0,
"xcopa_ta": 0,
"xcopa_ht": 0,
"xcopa_vi": 0,
"xcopa_it": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/llama/llama-7B/llama-7B_xnli_0-shot.json deleted 100644 → 0
View file @ 465c695b
{
"results": {
"xnli_hi": {
"acc": 0.36107784431137724,
"acc_stderr": 0.006786547275949383
},
"xnli_vi": {
"acc": 0.35588822355289423,
"acc_stderr": 0.006764908277770048
},
"xnli_fr": {
"acc": 0.4375249500998004,
"acc_stderr": 0.007009346470228512
},
"xnli_tr": {
"acc": 0.35588822355289423,
"acc_stderr": 0.006764908277770052
},
"xnli_el": {
"acc": 0.34930139720558884,
"acc_stderr": 0.00673619394518908
},
"xnli_zh": {
"acc": 0.36227544910179643,
"acc_stderr": 0.00679141867023231
},
"xnli_es": {
"acc": 0.406187624750499,
"acc_stderr": 0.006939248658213527
},
"xnli_sw": {
"acc": 0.33712574850299404,
"acc_stderr": 0.006679377985812511
},
"xnli_th": {
"acc": 0.34510978043912177,
"acc_stderr": 0.0067171859376095565
},
"xnli_bg": {
"acc": 0.36986027944111777,
"acc_stderr": 0.006821215321725033
},
"xnli_ru": {
"acc": 0.3936127744510978,
"acc_stderr": 0.006902939997053364
},
"xnli_ur": {
"acc": 0.3339321357285429,
"acc_stderr": 0.006663660032909966
},
"xnli_ar": {
"acc": 0.33572854291417165,
"acc_stderr": 0.006672543485924257
},
"xnli_en": {
"acc": 0.5105788423153692,
"acc_stderr": 0.007063131001466571
},
"xnli_de": {
"acc": 0.4477045908183633,
"acc_stderr": 0.007025964880868372
}
},
"versions": {
"xnli_hi": 0,
"xnli_vi": 0,
"xnli_fr": 0,
"xnli_tr": 0,
"xnli_el": 0,
"xnli_zh": 0,
"xnli_es": 0,
"xnli_sw": 0,
"xnli_th": 0,
"xnli_bg": 0,
"xnli_ru": 0,
"xnli_ur": 0,
"xnli_ar": 0,
"xnli_en": 0,
"xnli_de": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/llama/llama-7B/llama-7B_xstory_cloze_0-shot.json deleted 100644 → 0
View file @ 465c695b
{
"results": {
"xstory_cloze_id": {
"acc": 0.5208471211118465,
"acc_stderr": 0.012855936282881265
},
"xstory_cloze_en": {
"acc": 0.7478491065519524,
"acc_stderr": 0.011175031901561881
},
"xstory_cloze_ar": {
"acc": 0.4831237590999338,
"acc_stderr": 0.012859793919977602
},
"xstory_cloze_sw": {
"acc": 0.5036399735274653,
"acc_stderr": 0.012866784348289232
},
"xstory_cloze_my": {
"acc": 0.4870946393117141,
"acc_stderr": 0.012862838605728472
},
"xstory_cloze_hi": {
"acc": 0.5268034414295168,
"acc_stderr": 0.012848623899505763
},
"xstory_cloze_te": {
"acc": 0.528788881535407,
"acc_stderr": 0.012845779070719484
},
"xstory_cloze_es": {
"acc": 0.6512243547319656,
"acc_stderr": 0.012264502012981189
},
"xstory_cloze_zh": {
"acc": 0.5433487756452681,
"acc_stderr": 0.012818676452481954
},
"xstory_cloze_eu": {
"acc": 0.500992720052945,
"acc_stderr": 0.012867099955422926
},
"xstory_cloze_ru": {
"acc": 0.613500992720053,
"acc_stderr": 0.012531219943771484
}
},
"versions": {
"xstory_cloze_id": 0,
"xstory_cloze_en": 0,
"xstory_cloze_ar": 0,
"xstory_cloze_sw": 0,
"xstory_cloze_my": 0,
"xstory_cloze_hi": 0,
"xstory_cloze_te": 0,
"xstory_cloze_es": 0,
"xstory_cloze_zh": 0,
"xstory_cloze_eu": 0,
"xstory_cloze_ru": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/llama/llama-7B/llama-7B_xwinograd_0-shot.json deleted 100644 → 0
View file @ 465c695b
{
"results": {
"xwinograd_en": {
"acc": 0.8494623655913979,
"acc_stderr": 0.007417824483418801
},
"xwinograd_fr": {
"acc": 0.7228915662650602,
"acc_stderr": 0.049425892997830914
},
"xwinograd_jp": {
"acc": 0.5891553701772679,
"acc_stderr": 0.01589538213585439
},
"xwinograd_zh": {
"acc": 0.6369047619047619,
"acc_stderr": 0.02144191312603829
},
"xwinograd_ru": {
"acc": 0.6444444444444445,
"acc_stderr": 0.02701354947053879
},
"xwinograd_pt": {
"acc": 0.7072243346007605,
"acc_stderr": 0.02811223926963042
}
},
"versions": {
"xwinograd_en": 0,
"xwinograd_fr": 0,
"xwinograd_jp": 0,
"xwinograd_zh": 0,
"xwinograd_ru": 0,
"xwinograd_pt": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/mpt/mpt-7b/README.md deleted 100644 → 0
View file @ 465c695b
# mpt-7b
## mpt-7b_anli_0-shot.json
| Task |Version|Metric|Value| |Stderr|
|-------|------:|------|----:|---|-----:|
|anli_r1| 0|acc | 33.2|± | 1.49|
|anli_r2| 0|acc | 33.6|± | 1.49|
|anli_r3| 0|acc | 34.5|± | 1.37|
## mpt-7b_arithmetic_5-shot.json
| Task |Version|Metric|Value| |Stderr|
|--------------|------:|------|----:|---|-----:|
|arithmetic_1dc| 0|acc | 8.10|± | 0.61|
|arithmetic_2da| 0|acc |91.80|± | 0.61|
|arithmetic_2dm| 0|acc |25.60|± | 0.98|
|arithmetic_2ds| 0|acc |78.75|± | 0.91|
|arithmetic_3da| 0|acc |29.15|± | 1.02|
|arithmetic_3ds| 0|acc |42.80|± | 1.11|
|arithmetic_4da| 0|acc | 2.60|± | 0.36|
|arithmetic_4ds| 0|acc | 2.60|± | 0.36|
|arithmetic_5da| 0|acc | 0.45|± | 0.15|
|arithmetic_5ds| 0|acc | 0.20|± | 0.10|
## mpt-7b_bbh_3-shot.json
| Task |Version| Metric |Value| |Stderr|
|------------------------------------------------|------:|---------------------|----:|---|-----:|
|bigbench_causal_judgement | 0|multiple_choice_grade|56.32|± | 3.61|
|bigbench_date_understanding | 0|multiple_choice_grade|58.27|± | 2.57|
|bigbench_disambiguation_qa | 0|multiple_choice_grade|36.43|± | 3.00|
|bigbench_dyck_languages | 0|multiple_choice_grade|12.30|± | 1.04|
|bigbench_formal_fallacies_syllogisms_negation | 0|multiple_choice_grade|49.92|± | 0.42|
|bigbench_geometric_shapes | 0|multiple_choice_grade|20.33|± | 2.13|
| | |exact_str_match |12.26|± | 1.73|
|bigbench_hyperbaton | 0|multiple_choice_grade|49.36|± | 0.22|
|bigbench_logical_deduction_five_objects | 0|multiple_choice_grade|24.00|± | 1.91|
|bigbench_logical_deduction_seven_objects | 0|multiple_choice_grade|16.57|± | 1.41|
|bigbench_logical_deduction_three_objects | 0|multiple_choice_grade|38.67|± | 2.82|
|bigbench_movie_recommendation | 0|multiple_choice_grade|43.80|± | 2.22|
|bigbench_navigate | 0|multiple_choice_grade|48.60|± | 1.58|
|bigbench_reasoning_about_colored_objects | 0|multiple_choice_grade|29.85|± | 1.02|
|bigbench_ruin_names | 0|multiple_choice_grade|29.69|± | 2.16|
|bigbench_salient_translation_error_detection | 0|multiple_choice_grade|17.94|± | 1.22|
|bigbench_snarks | 0|multiple_choice_grade|53.04|± | 3.72|
|bigbench_sports_understanding | 0|multiple_choice_grade|49.49|± | 1.59|
|bigbench_temporal_sequences | 0|multiple_choice_grade|29.60|± | 1.44|
|bigbench_tracking_shuffled_objects_five_objects | 0|multiple_choice_grade|19.44|± | 1.12|
|bigbench_tracking_shuffled_objects_seven_objects| 0|multiple_choice_grade|13.43|± | 0.82|
|bigbench_tracking_shuffled_objects_three_objects| 0|multiple_choice_grade|38.67|± | 2.82|
## mpt-7b_blimp_0-shot.json
| Task |Version|Metric|Value| |Stderr|
|---------------------------------------------------------|------:|------|----:|---|-----:|
|blimp_adjunct_island | 0|acc | 87.8|± | 1.04|
|blimp_anaphor_gender_agreement | 0|acc | 99.5|± | 0.22|
|blimp_anaphor_number_agreement | 0|acc | 99.5|± | 0.22|
|blimp_animate_subject_passive | 0|acc | 77.5|± | 1.32|
|blimp_animate_subject_trans | 0|acc | 88.4|± | 1.01|
|blimp_causative | 0|acc | 74.7|± | 1.38|
|blimp_complex_NP_island | 0|acc | 52.8|± | 1.58|
|blimp_coordinate_structure_constraint_complex_left_branch| 0|acc | 77.9|± | 1.31|
|blimp_coordinate_structure_constraint_object_extraction | 0|acc | 84.8|± | 1.14|
|blimp_determiner_noun_agreement_1 | 0|acc | 99.2|± | 0.28|
|blimp_determiner_noun_agreement_2 | 0|acc | 97.4|± | 0.50|
|blimp_determiner_noun_agreement_irregular_1 | 0|acc | 93.6|± | 0.77|
|blimp_determiner_noun_agreement_irregular_2 | 0|acc | 92.8|± | 0.82|
|blimp_determiner_noun_agreement_with_adj_2 | 0|acc | 93.5|± | 0.78|
|blimp_determiner_noun_agreement_with_adj_irregular_1 | 0|acc | 88.3|± | 1.02|
|blimp_determiner_noun_agreement_with_adj_irregular_2 | 0|acc | 91.9|± | 0.86|
|blimp_determiner_noun_agreement_with_adjective_1 | 0|acc | 97.2|± | 0.52|
|blimp_distractor_agreement_relational_noun | 0|acc | 88.9|± | 0.99|
|blimp_distractor_agreement_relative_clause | 0|acc | 74.3|± | 1.38|
|blimp_drop_argument | 0|acc | 78.7|± | 1.30|
|blimp_ellipsis_n_bar_1 | 0|acc | 79.0|± | 1.29|
|blimp_ellipsis_n_bar_2 | 0|acc | 92.8|± | 0.82|
|blimp_existential_there_object_raising | 0|acc | 83.4|± | 1.18|
|blimp_existential_there_quantifiers_1 | 0|acc | 98.8|± | 0.34|
|blimp_existential_there_quantifiers_2 | 0|acc | 27.0|± | 1.40|
|blimp_existential_there_subject_raising | 0|acc | 88.8|± | 1.00|
|blimp_expletive_it_object_raising | 0|acc | 80.0|± | 1.27|
|blimp_inchoative | 0|acc | 67.3|± | 1.48|
|blimp_intransitive | 0|acc | 83.2|± | 1.18|
|blimp_irregular_past_participle_adjectives | 0|acc | 97.2|± | 0.52|
|blimp_irregular_past_participle_verbs | 0|acc | 88.5|± | 1.01|
|blimp_irregular_plural_subject_verb_agreement_1 | 0|acc | 92.0|± | 0.86|
|blimp_irregular_plural_subject_verb_agreement_2 | 0|acc | 90.8|± | 0.91|
|blimp_left_branch_island_echo_question | 0|acc | 43.2|± | 1.57|
|blimp_left_branch_island_simple_question | 0|acc | 89.7|± | 0.96|
|blimp_matrix_question_npi_licensor_present | 0|acc | 70.5|± | 1.44|
|blimp_npi_present_1 | 0|acc | 57.9|± | 1.56|
|blimp_npi_present_2 | 0|acc | 68.8|± | 1.47|
|blimp_only_npi_licensor_present | 0|acc | 91.6|± | 0.88|
|blimp_only_npi_scope | 0|acc | 73.2|± | 1.40|
|blimp_passive_1 | 0|acc | 88.7|± | 1.00|
|blimp_passive_2 | 0|acc | 89.5|± | 0.97|
|blimp_principle_A_c_command | 0|acc | 75.0|± | 1.37|
|blimp_principle_A_case_1 | 0|acc |100.0|± | 0.00|
|blimp_principle_A_case_2 | 0|acc | 94.0|± | 0.75|
|blimp_principle_A_domain_1 | 0|acc | 99.7|± | 0.17|
|blimp_principle_A_domain_2 | 0|acc | 82.8|± | 1.19|
|blimp_principle_A_domain_3 | 0|acc | 76.1|± | 1.35|
|blimp_principle_A_reconstruction | 0|acc | 41.0|± | 1.56|
|blimp_regular_plural_subject_verb_agreement_1 | 0|acc | 97.1|± | 0.53|
|blimp_regular_plural_subject_verb_agreement_2 | 0|acc | 90.7|± | 0.92|
|blimp_sentential_negation_npi_licensor_present | 0|acc | 98.9|± | 0.33|
|blimp_sentential_negation_npi_scope | 0|acc | 73.3|± | 1.40|
|blimp_sentential_subject_island | 0|acc | 39.9|± | 1.55|
|blimp_superlative_quantifiers_1 | 0|acc | 82.2|± | 1.21|
|blimp_superlative_quantifiers_2 | 0|acc | 89.7|± | 0.96|
|blimp_tough_vs_raising_1 | 0|acc | 69.0|± | 1.46|
|blimp_tough_vs_raising_2 | 0|acc | 82.9|± | 1.19|
|blimp_transitive | 0|acc | 87.2|± | 1.06|
|blimp_wh_island | 0|acc | 81.3|± | 1.23|
|blimp_wh_questions_object_gap | 0|acc | 76.3|± | 1.35|
|blimp_wh_questions_subject_gap | 0|acc | 89.0|± | 0.99|
|blimp_wh_questions_subject_gap_long_distance | 0|acc | 89.3|± | 0.98|
|blimp_wh_vs_that_no_gap | 0|acc | 94.6|± | 0.72|
|blimp_wh_vs_that_no_gap_long_distance | 0|acc | 95.1|± | 0.68|
|blimp_wh_vs_that_with_gap | 0|acc | 32.1|± | 1.48|
|blimp_wh_vs_that_with_gap_long_distance | 0|acc | 29.2|± | 1.44|
## mpt-7b_common_sense_reasoning_0-shot.json
| Task |Version| Metric |Value| |Stderr|
|-------------|------:|--------|----:|---|-----:|
|arc_challenge| 0|acc |40.61|± | 1.44|
| | |acc_norm|41.81|± | 1.44|
|arc_easy | 0|acc |74.87|± | 0.89|
| | |acc_norm|70.29|± | 0.94|
|boolq | 1|acc |73.52|± | 0.77|
|copa | 0|acc |85.00|± | 3.59|
|hellaswag | 0|acc |57.24|± | 0.49|
| | |acc_norm|76.12|± | 0.43|
|mc_taco | 0|em |13.51| | |
| | |f1 |45.48| | |
|openbookqa | 0|acc |32.00|± | 2.09|
| | |acc_norm|42.60|± | 2.21|
|piqa | 0|acc |79.16|± | 0.95|
| | |acc_norm|80.41|± | 0.93|
|prost | 0|acc |25.73|± | 0.32|
| | |acc_norm|30.12|± | 0.34|
|swag | 0|acc |56.17|± | 0.35|
| | |acc_norm|75.80|± | 0.30|
|winogrande | 0|acc |68.67|± | 1.30|
|wsc273 | 0|acc |85.71|± | 2.12|
## mpt-7b_glue_0-shot.json
| Task |Version|Metric|Value| |Stderr|
|---------------|------:|------|----:|---|-----:|
|cola | 0|mcc |-4.41|± | 3.12|
|mnli | 0|acc |37.83|± | 0.49|
|mnli_mismatched| 0|acc |37.49|± | 0.49|
|mrpc | 0|acc |62.99|± | 2.39|
| | |f1 |75.61|± | 1.93|
|qnli | 0|acc |51.35|± | 0.68|
|qqp | 0|acc |50.36|± | 0.25|
| | |f1 |54.14|± | 0.29|
|rte | 0|acc |63.90|± | 2.89|
|sst | 0|acc |76.83|± | 1.43|
|wnli | 1|acc |47.89|± | 5.97|
## mpt-7b_human_alignment_0-shot.json
| Task |Version| Metric |Value | |Stderr|
|---------------------------------------|------:|---------------------|-----:|---|-----:|
|crows_pairs_english_age | 0|likelihood_difference|415.11|± | 38.32|
| | |pct_stereotype | 73.63|± | 4.64|
|crows_pairs_english_autre | 0|likelihood_difference|505.68|± |177.03|
| | |pct_stereotype | 72.73|± | 14.08|
|crows_pairs_english_disability | 0|likelihood_difference|601.92|± | 63.31|
| | |pct_stereotype | 76.92|± | 5.27|
|crows_pairs_english_gender | 0|likelihood_difference|268.24|± | 17.01|
| | |pct_stereotype | 63.75|± | 2.69|
|crows_pairs_english_nationality | 0|likelihood_difference|349.83|± | 21.51|
| | |pct_stereotype | 61.57|± | 3.32|
|crows_pairs_english_physical_appearance| 0|likelihood_difference|373.78|± | 33.85|
| | |pct_stereotype | 72.22|± | 5.32|
|crows_pairs_english_race_color | 0|likelihood_difference|336.20|± | 14.10|
| | |pct_stereotype | 57.28|± | 2.20|
|crows_pairs_english_religion | 0|likelihood_difference|366.44|± | 33.86|
| | |pct_stereotype | 72.97|± | 4.23|
|crows_pairs_english_sexual_orientation | 0|likelihood_difference|463.04|± | 45.75|
| | |pct_stereotype | 82.80|± | 3.93|
|crows_pairs_english_socioeconomic | 0|likelihood_difference|406.51|± | 23.52|
| | |pct_stereotype | 67.89|± | 3.40|
|crows_pairs_french_age | 0|likelihood_difference|360.97|± | 36.15|
| | |pct_stereotype | 42.22|± | 5.24|
|crows_pairs_french_autre | 0|likelihood_difference|269.23|± | 92.30|
| | |pct_stereotype | 61.54|± | 14.04|
|crows_pairs_french_disability | 0|likelihood_difference|495.83|± | 42.69|
| | |pct_stereotype | 63.64|± | 5.97|
|crows_pairs_french_gender | 0|likelihood_difference|321.38|± | 17.59|
| | |pct_stereotype | 51.09|± | 2.79|
|crows_pairs_french_nationality | 0|likelihood_difference|388.34|± | 21.84|
| | |pct_stereotype | 34.39|± | 2.99|
|crows_pairs_french_physical_appearance | 0|likelihood_difference|322.74|± | 43.29|
| | |pct_stereotype | 59.72|± | 5.82|
|crows_pairs_french_race_color | 0|likelihood_difference|316.14|± | 16.56|
| | |pct_stereotype | 43.70|± | 2.32|
|crows_pairs_french_religion | 0|likelihood_difference|356.74|± | 33.68|
| | |pct_stereotype | 62.61|± | 4.53|
|crows_pairs_french_sexual_orientation | 0|likelihood_difference|479.12|± | 40.10|
| | |pct_stereotype | 78.02|± | 4.36|
|crows_pairs_french_socioeconomic | 0|likelihood_difference|399.39|± | 26.31|
| | |pct_stereotype | 65.82|± | 3.40|
|ethics_cm | 0|acc | 54.59|± | 0.80|
|ethics_deontology | 0|acc | 50.25|± | 0.83|
| | |em | 0.44| | |
|ethics_justice | 0|acc | 51.96|± | 0.96|
| | |em | 1.18| | |
|ethics_utilitarianism | 0|acc | 57.49|± | 0.71|
|ethics_utilitarianism_original | 0|acc | 99.56|± | 0.10|
|ethics_virtue | 0|acc | 80.40|± | 0.56|
| | |em | 12.56| | |
|toxigen | 0|acc | 43.19|± | 1.62|
| | |acc_norm | 43.19|± | 1.62|
## mpt-7b_lambada_0-shot.json
| Task |Version|Metric|Value | |Stderr|
|----------------------|------:|------|-----:|---|-----:|
|lambada_openai | 0|ppl | 3.87|± | 0.08|
| | |acc | 68.35|± | 0.65|
|lambada_openai_cloze | 0|ppl | 26.56|± | 0.70|
| | |acc | 39.65|± | 0.68|
|lambada_openai_mt_de | 0|ppl | 70.12|± | 4.04|
| | |acc | 33.77|± | 0.66|
|lambada_openai_mt_en | 0|ppl | 3.87|± | 0.08|
| | |acc | 68.35|± | 0.65|
|lambada_openai_mt_es | 0|ppl | 67.23|± | 3.69|
| | |acc | 36.95|± | 0.67|
|lambada_openai_mt_fr | 0|ppl | 42.93|± | 2.37|
| | |acc | 43.02|± | 0.69|
|lambada_openai_mt_it | 0|ppl | 65.76|± | 3.87|
| | |acc | 39.20|± | 0.68|
|lambada_standard | 0|ppl | 4.92|± | 0.11|
| | |acc | 61.91|± | 0.68|
|lambada_standard_cloze| 0|ppl |109.10|± | 3.04|
| | |acc | 16.75|± | 0.52|
## mpt-7b_mmlu_5-shot.json
| Task |Version| Metric |Value| |Stderr|
|-------------------------------------------------|------:|--------|----:|---|-----:|
|hendrycksTest-abstract_algebra | 0|acc |18.00|± | 3.86|
| | |acc_norm|21.00|± | 4.09|
|hendrycksTest-anatomy | 0|acc |38.52|± | 4.20|
| | |acc_norm|37.78|± | 4.19|
|hendrycksTest-astronomy | 0|acc |39.47|± | 3.98|
| | |acc_norm|42.11|± | 4.02|
|hendrycksTest-business_ethics | 0|acc |49.00|± | 5.02|
| | |acc_norm|48.00|± | 5.02|
|hendrycksTest-clinical_knowledge | 0|acc |33.21|± | 2.90|
| | |acc_norm|37.74|± | 2.98|
|hendrycksTest-college_biology | 0|acc |38.19|± | 4.06|
| | |acc_norm|35.42|± | 4.00|
|hendrycksTest-college_chemistry | 0|acc |39.00|± | 4.90|
| | |acc_norm|41.00|± | 4.94|
|hendrycksTest-college_computer_science | 0|acc |34.00|± | 4.76|
| | |acc_norm|32.00|± | 4.69|
|hendrycksTest-college_mathematics | 0|acc |27.00|± | 4.46|
| | |acc_norm|33.00|± | 4.73|
|hendrycksTest-college_medicine | 0|acc |36.42|± | 3.67|
| | |acc_norm|34.68|± | 3.63|
|hendrycksTest-college_physics | 0|acc |30.39|± | 4.58|
| | |acc_norm|33.33|± | 4.69|
|hendrycksTest-computer_security | 0|acc |41.00|± | 4.94|
| | |acc_norm|41.00|± | 4.94|
|hendrycksTest-conceptual_physics | 0|acc |32.77|± | 3.07|
| | |acc_norm|25.53|± | 2.85|
|hendrycksTest-econometrics | 0|acc |27.19|± | 4.19|
| | |acc_norm|23.68|± | 4.00|
|hendrycksTest-electrical_engineering | 0|acc |36.55|± | 4.01|
| | |acc_norm|33.79|± | 3.94|
|hendrycksTest-elementary_mathematics | 0|acc |29.89|± | 2.36|
| | |acc_norm|28.84|± | 2.33|
|hendrycksTest-formal_logic | 0|acc |30.95|± | 4.13|
| | |acc_norm|28.57|± | 4.04|
|hendrycksTest-global_facts | 0|acc |35.00|± | 4.79|
| | |acc_norm|33.00|± | 4.73|
|hendrycksTest-high_school_biology | 0|acc |36.45|± | 2.74|
| | |acc_norm|39.03|± | 2.78|
|hendrycksTest-high_school_chemistry | 0|acc |21.18|± | 2.87|
| | |acc_norm|21.67|± | 2.90|
|hendrycksTest-high_school_computer_science | 0|acc |43.00|± | 4.98|
| | |acc_norm|41.00|± | 4.94|
|hendrycksTest-high_school_european_history | 0|acc |38.18|± | 3.79|
| | |acc_norm|37.58|± | 3.78|
|hendrycksTest-high_school_geography | 0|acc |38.38|± | 3.46|
| | |acc_norm|40.40|± | 3.50|
|hendrycksTest-high_school_government_and_politics| 0|acc |41.45|± | 3.56|
| | |acc_norm|41.45|± | 3.56|
|hendrycksTest-high_school_macroeconomics | 0|acc |34.87|± | 2.42|
| | |acc_norm|29.74|± | 2.32|
|hendrycksTest-high_school_mathematics | 0|acc |29.26|± | 2.77|
| | |acc_norm|30.37|± | 2.80|
|hendrycksTest-high_school_microeconomics | 0|acc |33.61|± | 3.07|
| | |acc_norm|36.97|± | 3.14|
|hendrycksTest-high_school_physics | 0|acc |27.81|± | 3.66|
| | |acc_norm|27.81|± | 3.66|
|hendrycksTest-high_school_psychology | 0|acc |46.97|± | 2.14|
| | |acc_norm|44.59|± | 2.13|
|hendrycksTest-high_school_statistics | 0|acc |32.87|± | 3.20|
| | |acc_norm|32.41|± | 3.19|
|hendrycksTest-high_school_us_history | 0|acc |34.31|± | 3.33|
| | |acc_norm|31.37|± | 3.26|
|hendrycksTest-high_school_world_history | 0|acc |29.54|± | 2.97|
| | |acc_norm|28.69|± | 2.94|
|hendrycksTest-human_aging | 0|acc |33.63|± | 3.17|
| | |acc_norm|32.74|± | 3.15|
|hendrycksTest-human_sexuality | 0|acc |27.48|± | 3.92|
| | |acc_norm|32.82|± | 4.12|
|hendrycksTest-international_law | 0|acc |37.19|± | 4.41|
| | |acc_norm|49.59|± | 4.56|
|hendrycksTest-jurisprudence | 0|acc |34.26|± | 4.59|
| | |acc_norm|39.81|± | 4.73|
|hendrycksTest-logical_fallacies | 0|acc |38.04|± | 3.81|
| | |acc_norm|36.81|± | 3.79|
|hendrycksTest-machine_learning | 0|acc |26.79|± | 4.20|
| | |acc_norm|24.11|± | 4.06|
|hendrycksTest-management | 0|acc |42.72|± | 4.90|
| | |acc_norm|39.81|± | 4.85|
|hendrycksTest-marketing | 0|acc |55.13|± | 3.26|
| | |acc_norm|55.13|± | 3.26|
|hendrycksTest-medical_genetics | 0|acc |39.00|± | 4.90|
| | |acc_norm|38.00|± | 4.88|
|hendrycksTest-miscellaneous | 0|acc |55.56|± | 1.78|
| | |acc_norm|55.68|± | 1.78|
|hendrycksTest-moral_disputes | 0|acc |32.08|± | 2.51|
| | |acc_norm|30.06|± | 2.47|
|hendrycksTest-moral_scenarios | 0|acc |26.03|± | 1.47|
| | |acc_norm|27.26|± | 1.49|
|hendrycksTest-nutrition | 0|acc |34.31|± | 2.72|
| | |acc_norm|40.20|± | 2.81|
|hendrycksTest-philosophy | 0|acc |37.62|± | 2.75|
| | |acc_norm|36.98|± | 2.74|
|hendrycksTest-prehistory | 0|acc |33.64|± | 2.63|
| | |acc_norm|30.56|± | 2.56|
|hendrycksTest-professional_accounting | 0|acc |30.50|± | 2.75|
| | |acc_norm|29.08|± | 2.71|
|hendrycksTest-professional_law | 0|acc |25.95|± | 1.12|
| | |acc_norm|28.42|± | 1.15|
|hendrycksTest-professional_medicine | 0|acc |29.41|± | 2.77|
| | |acc_norm|31.62|± | 2.82|
|hendrycksTest-professional_psychology | 0|acc |31.54|± | 1.88|
| | |acc_norm|30.23|± | 1.86|
|hendrycksTest-public_relations | 0|acc |41.82|± | 4.72|
| | |acc_norm|42.73|± | 4.74|
|hendrycksTest-security_studies | 0|acc |28.16|± | 2.88|
| | |acc_norm|24.08|± | 2.74|
|hendrycksTest-sociology | 0|acc |34.33|± | 3.36|
| | |acc_norm|36.82|± | 3.41|
|hendrycksTest-us_foreign_policy | 0|acc |38.00|± | 4.88|
| | |acc_norm|39.00|± | 4.90|
|hendrycksTest-virology | 0|acc |32.53|± | 3.65|
| | |acc_norm|32.53|± | 3.65|
|hendrycksTest-world_religions | 0|acc |54.39|± | 3.82|
| | |acc_norm|57.89|± | 3.79|
## mpt-7b_pawsx_0-shot.json
| Task |Version|Metric|Value| |Stderr|
|--------|------:|------|----:|---|-----:|
|pawsx_de| 0|acc |61.40|± | 1.09|
|pawsx_en| 0|acc |70.35|± | 1.02|
|pawsx_es| 0|acc |64.95|± | 1.07|
|pawsx_fr| 0|acc |62.85|± | 1.08|
|pawsx_ja| 0|acc |49.30|± | 1.12|
|pawsx_ko| 0|acc |53.65|± | 1.12|
|pawsx_zh| 0|acc |56.25|± | 1.11|
## mpt-7b_reading_comprehension_0-shot.json
|Task|Version|Metric|Value| |Stderr|
|----|------:|------|----:|---|-----:|
|coqa| 1|f1 |76.51|± | 1.48|
| | |em |63.02|± | 1.87|
|drop| 1|em | 3.43|± | 0.19|
| | |f1 |13.39|± | 0.25|
|race| 1|acc |38.66|± | 1.51|
## mpt-7b_superglue_0-shot.json
| Task |Version|Metric|Value| |Stderr|
|-------|------:|------|----:|---|-----:|
|boolq | 1|acc |73.82|± | 0.77|
|cb | 1|acc |41.07|± | 6.63|
| | |f1 |21.27| | |
|copa | 0|acc |84.00|± | 3.68|
|multirc| 1|acc | 0.84|± | 0.30|
|record | 0|f1 |90.10|± | 0.29|
| | |em |89.30|± | 0.31|
|wic | 0|acc |48.43|± | 1.98|
|wsc | 0|acc |63.46|± | 4.74|
## mpt-7b_unscramble_0-shot.json
| Task |Version|Metric|Value| |Stderr|
|----------------|------:|------|----:|---|-----:|
|anagrams1 | 0|acc | 0.00|± | 0.00|
|anagrams2 | 0|acc | 0.01|± | 0.01|
|cycle_letters | 0|acc | 0.00|± | 0.00|
|random_insertion| 0|acc | 0.04|± | 0.02|
|reversed_words | 0|acc | 0.00|± | 0.00|
## mpt-7b_xcopa_0-shot.json
| Task |Version|Metric|Value| |Stderr|
|--------|------:|------|----:|---|-----:|
|xcopa_et| 0|acc | 47.4|± | 2.24|
|xcopa_ht| 0|acc | 49.8|± | 2.24|
|xcopa_id| 0|acc | 56.8|± | 2.22|
|xcopa_it| 0|acc | 59.4|± | 2.20|
|xcopa_qu| 0|acc | 48.4|± | 2.24|
|xcopa_sw| 0|acc | 51.6|± | 2.24|
|xcopa_ta| 0|acc | 54.0|± | 2.23|
|xcopa_th| 0|acc | 54.2|± | 2.23|
|xcopa_tr| 0|acc | 51.6|± | 2.24|
|xcopa_vi| 0|acc | 53.6|± | 2.23|
|xcopa_zh| 0|acc | 63.2|± | 2.16|
## mpt-7b_xnli_0-shot.json
| Task |Version|Metric|Value| |Stderr|
|-------|------:|------|----:|---|-----:|
|xnli_ar| 0|acc |33.31|± | 0.67|
|xnli_bg| 0|acc |36.83|± | 0.68|
|xnli_de| 0|acc |46.45|± | 0.70|
|xnli_el| 0|acc |36.19|± | 0.68|
|xnli_en| 0|acc |54.33|± | 0.70|
|xnli_es| 0|acc |45.65|± | 0.70|
|xnli_fr| 0|acc |48.80|± | 0.71|
|xnli_hi| 0|acc |34.73|± | 0.67|
|xnli_ru| 0|acc |44.43|± | 0.70|
|xnli_sw| 0|acc |33.41|± | 0.67|
|xnli_th| 0|acc |36.13|± | 0.68|
|xnli_tr| 0|acc |37.68|± | 0.68|
|xnli_ur| 0|acc |33.63|± | 0.67|
|xnli_vi| 0|acc |37.33|± | 0.68|
|xnli_zh| 0|acc |35.35|± | 0.68|
## mpt-7b_xstory_cloze_0-shot.json
| Task |Version|Metric|Value| |Stderr|
|---------------|------:|------|----:|---|-----:|
|xstory_cloze_ar| 0|acc |48.51|± | 1.29|
|xstory_cloze_en| 0|acc |77.90|± | 1.07|
|xstory_cloze_es| 0|acc |66.05|± | 1.22|
|xstory_cloze_eu| 0|acc |51.09|± | 1.29|
|xstory_cloze_hi| 0|acc |51.69|± | 1.29|
|xstory_cloze_id| 0|acc |55.20|± | 1.28|
|xstory_cloze_my| 0|acc |48.38|± | 1.29|
|xstory_cloze_ru| 0|acc |57.25|± | 1.27|
|xstory_cloze_sw| 0|acc |49.90|± | 1.29|
|xstory_cloze_te| 0|acc |52.95|± | 1.28|
|xstory_cloze_zh| 0|acc |59.56|± | 1.26|
## mpt-7b_xwinograd_0-shot.json
| Task |Version|Metric|Value| |Stderr|
|------------|------:|------|----:|---|-----:|
|xwinograd_en| 0|acc |86.67|± | 0.71|
|xwinograd_fr| 0|acc |66.27|± | 5.22|
|xwinograd_jp| 0|acc |60.27|± | 1.58|
|xwinograd_pt| 0|acc |66.92|± | 2.91|
|xwinograd_ru| 0|acc |69.52|± | 2.60|
|xwinograd_zh| 0|acc |71.63|± | 2.01|
results/mpt/mpt-7b/mpt-7b_anli_0-shot.json deleted 100644 → 0
View file @ 465c695b
{
"results": {
"anli_r1": {
"acc": 0.332,
"acc_stderr": 0.014899597242811485
},
"anli_r2": {
"acc": 0.336,
"acc_stderr": 0.014944140233795027
},
"anli_r3": {
"acc": 0.345,
"acc_stderr": 0.013728421539454881
}
},
"versions": {
"anli_r1": 0,
"anli_r2": 0,
"anli_r3": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=mosaicml/mpt-7b,trust_remote_code=True",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment