Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
gaoqiong
lm-evaluation-harness
Commits
21e128d8
Commit
21e128d8
authored
May 06, 2023
by
Julen Etxaniz
Browse files
add bloom, xglm and llama results
parent
0542d35d
Changes
141
Hide whitespace changes
Inline
Side-by-side
Showing
20 changed files
with
2103 additions
and
0 deletions
+2103
-0
results/llama/llama-30B/llama-30B_xcopa_0-shot.json
results/llama/llama-30B/llama-30B_xcopa_0-shot.json
+72
-0
results/llama/llama-30B/llama-30B_xnli_0-shot.json
results/llama/llama-30B/llama-30B_xnli_0-shot.json
+92
-0
results/llama/llama-30B/llama-30B_xstory_cloze_0-shot.json
results/llama/llama-30B/llama-30B_xstory_cloze_0-shot.json
+72
-0
results/llama/llama-30B/llama-30B_xwinograd_0-shot.json
results/llama/llama-30B/llama-30B_xwinograd_0-shot.json
+47
-0
results/llama/llama-7B/llama-7B_anli_0-shot.json
results/llama/llama-7B/llama-7B_anli_0-shot.json
+32
-0
results/llama/llama-7B/llama-7B_arithmetic_5-shot.json
results/llama/llama-7B/llama-7B_arithmetic_5-shot.json
+67
-0
results/llama/llama-7B/llama-7B_bbh_3-shot.json
results/llama/llama-7B/llama-7B_bbh_3-shot.json
+124
-0
results/llama/llama-7B/llama-7B_blimp_0-shot.json
results/llama/llama-7B/llama-7B_blimp_0-shot.json
+352
-0
results/llama/llama-7B/llama-7B_common_sense_reasoning_0-shot.json
...lama/llama-7B/llama-7B_common_sense_reasoning_0-shot.json
+91
-0
results/llama/llama-7B/llama-7B_glue_0-shot.json
results/llama/llama-7B/llama-7B_glue_0-shot.json
+66
-0
results/llama/llama-7B/llama-7B_gsm8k_8-shot.json
results/llama/llama-7B/llama-7B_gsm8k_8-shot.json
+22
-0
results/llama/llama-7B/llama-7B_human_alignment_0-shot.json
results/llama/llama-7B/llama-7B_human_alignment_0-shot.json
+197
-0
results/llama/llama-7B/llama-7B_lambada_0-shot.json
results/llama/llama-7B/llama-7B_lambada_0-shot.json
+80
-0
results/llama/llama-7B/llama-7B_mathematical_reasoning_0-shot.json
...lama/llama-7B/llama-7B_mathematical_reasoning_0-shot.json
+76
-0
results/llama/llama-7B/llama-7B_mathematical_reasoning_few_shot_5-shot.json
...a-7B/llama-7B_mathematical_reasoning_few_shot_5-shot.json
+71
-0
results/llama/llama-7B/llama-7B_mmlu_5-shot.json
results/llama/llama-7B/llama-7B_mmlu_5-shot.json
+416
-0
results/llama/llama-7B/llama-7B_pawsx_0-shot.json
results/llama/llama-7B/llama-7B_pawsx_0-shot.json
+52
-0
results/llama/llama-7B/llama-7B_question_answering_0-shot.json
...ts/llama/llama-7B/llama-7B_question_answering_0-shot.json
+66
-0
results/llama/llama-7B/llama-7B_reading_comprehension_0-shot.json
...llama/llama-7B/llama-7B_reading_comprehension_0-shot.json
+36
-0
results/llama/llama-7B/llama-7B_xcopa_0-shot.json
results/llama/llama-7B/llama-7B_xcopa_0-shot.json
+72
-0
No files found.
results/llama/llama-30B/llama-30B_xcopa_0-shot.json
0 → 100644
View file @
21e128d8
{
"results"
:
{
"xcopa_vi"
:
{
"acc"
:
0.524
,
"acc_stderr"
:
0.0223572738810164
},
"xcopa_id"
:
{
"acc"
:
0.606
,
"acc_stderr"
:
0.021874299301689253
},
"xcopa_it"
:
{
"acc"
:
0.714
,
"acc_stderr"
:
0.020229346329177524
},
"xcopa_ht"
:
{
"acc"
:
0.518
,
"acc_stderr"
:
0.02236856511738799
},
"xcopa_zh"
:
{
"acc"
:
0.622
,
"acc_stderr"
:
0.021706550824518184
},
"xcopa_sw"
:
{
"acc"
:
0.524
,
"acc_stderr"
:
0.0223572738810164
},
"xcopa_et"
:
{
"acc"
:
0.472
,
"acc_stderr"
:
0.02234794983266809
},
"xcopa_qu"
:
{
"acc"
:
0.494
,
"acc_stderr"
:
0.022381462412439324
},
"xcopa_th"
:
{
"acc"
:
0.546
,
"acc_stderr"
:
0.02228814759117695
},
"xcopa_tr"
:
{
"acc"
:
0.522
,
"acc_stderr"
:
0.022361396739207888
},
"xcopa_ta"
:
{
"acc"
:
0.532
,
"acc_stderr"
:
0.022337186479044292
}
},
"versions"
:
{
"xcopa_vi"
:
0
,
"xcopa_id"
:
0
,
"xcopa_it"
:
0
,
"xcopa_ht"
:
0
,
"xcopa_zh"
:
0
,
"xcopa_sw"
:
0
,
"xcopa_et"
:
0
,
"xcopa_qu"
:
0
,
"xcopa_th"
:
0
,
"xcopa_tr"
:
0
,
"xcopa_ta"
:
0
},
"config"
:
{
"model"
:
"hf-causal-experimental"
,
"model_args"
:
"pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/30B"
,
"num_fewshot"
:
0
,
"batch_size"
:
"auto"
,
"device"
:
"cuda"
,
"no_cache"
:
true
,
"limit"
:
null
,
"bootstrap_iters"
:
100000
,
"description_dict"
:
{}
}
}
results/llama/llama-30B/llama-30B_xnli_0-shot.json
0 → 100644
View file @
21e128d8
{
"results"
:
{
"xnli_ar"
:
{
"acc"
:
0.3449101796407186
,
"acc_stderr"
:
0.006716266425755
},
"xnli_bg"
:
{
"acc"
:
0.3852295409181637
,
"acc_stderr"
:
0.006876077627982856
},
"xnli_de"
:
{
"acc"
:
0.43872255489021955
,
"acc_stderr"
:
0.007011456767132425
},
"xnli_el"
:
{
"acc"
:
0.34910179640718564
,
"acc_stderr"
:
0.00673530182747736
},
"xnli_en"
:
{
"acc"
:
0.4818363273453094
,
"acc_stderr"
:
0.007060049324579861
},
"xnli_es"
:
{
"acc"
:
0.4023952095808383
,
"acc_stderr"
:
0.006928798318208028
},
"xnli_fr"
:
{
"acc"
:
0.4295409181636727
,
"acc_stderr"
:
0.006994215414803201
},
"xnli_hi"
:
{
"acc"
:
0.3646706586826347
,
"acc_stderr"
:
0.0068010246867294885
},
"xnli_ru"
:
{
"acc"
:
0.3812375249500998
,
"acc_stderr"
:
0.0068625305186053856
},
"xnli_sw"
:
{
"acc"
:
0.3409181636726547
,
"acc_stderr"
:
0.006697600297167045
},
"xnli_th"
:
{
"acc"
:
0.3397205588822355
,
"acc_stderr"
:
0.0066918975980483925
},
"xnli_tr"
:
{
"acc"
:
0.3652694610778443
,
"acc_stderr"
:
0.00680339776716209
},
"xnli_ur"
:
{
"acc"
:
0.34311377245508984
,
"acc_stderr"
:
0.006707931789556032
},
"xnli_vi"
:
{
"acc"
:
0.356686626746507
,
"acc_stderr"
:
0.0067682935643592285
},
"xnli_zh"
:
{
"acc"
:
0.33512974051896205
,
"acc_stderr"
:
0.006669594382503632
}
},
"versions"
:
{
"xnli_ar"
:
0
,
"xnli_bg"
:
0
,
"xnli_de"
:
0
,
"xnli_el"
:
0
,
"xnli_en"
:
0
,
"xnli_es"
:
0
,
"xnli_fr"
:
0
,
"xnli_hi"
:
0
,
"xnli_ru"
:
0
,
"xnli_sw"
:
0
,
"xnli_th"
:
0
,
"xnli_tr"
:
0
,
"xnli_ur"
:
0
,
"xnli_vi"
:
0
,
"xnli_zh"
:
0
},
"config"
:
{
"model"
:
"hf-causal-experimental"
,
"model_args"
:
"pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/30B"
,
"num_fewshot"
:
0
,
"batch_size"
:
"auto"
,
"device"
:
"cuda"
,
"no_cache"
:
true
,
"limit"
:
null
,
"bootstrap_iters"
:
100000
,
"description_dict"
:
{}
}
}
results/llama/llama-30B/llama-30B_xstory_cloze_0-shot.json
0 → 100644
View file @
21e128d8
{
"results"
:
{
"xstory_cloze_en"
:
{
"acc"
:
0.7816015883520847
,
"acc_stderr"
:
0.010632343054700505
},
"xstory_cloze_te"
:
{
"acc"
:
0.5320979483785573
,
"acc_stderr"
:
0.012840584503982028
},
"xstory_cloze_zh"
:
{
"acc"
:
0.585704831237591
,
"acc_stderr"
:
0.012676689821720669
},
"xstory_cloze_id"
:
{
"acc"
:
0.5923229649238915
,
"acc_stderr"
:
0.01264587648804028
},
"xstory_cloze_ar"
:
{
"acc"
:
0.5089344804765056
,
"acc_stderr"
:
0.012865070917320802
},
"xstory_cloze_my"
:
{
"acc"
:
0.4877564526803441
,
"acc_stderr"
:
0.012863267059205548
},
"xstory_cloze_hi"
:
{
"acc"
:
0.5665122435473197
,
"acc_stderr"
:
0.012752771973917616
},
"xstory_cloze_eu"
:
{
"acc"
:
0.513567174056916
,
"acc_stderr"
:
0.012862387586650079
},
"xstory_cloze_sw"
:
{
"acc"
:
0.5062872270019855
,
"acc_stderr"
:
0.01286610802121821
},
"xstory_cloze_es"
:
{
"acc"
:
0.7081403044341495
,
"acc_stderr"
:
0.011699256037649366
},
"xstory_cloze_ru"
:
{
"acc"
:
0.6671078755790867
,
"acc_stderr"
:
0.012127221798743731
}
},
"versions"
:
{
"xstory_cloze_en"
:
0
,
"xstory_cloze_te"
:
0
,
"xstory_cloze_zh"
:
0
,
"xstory_cloze_id"
:
0
,
"xstory_cloze_ar"
:
0
,
"xstory_cloze_my"
:
0
,
"xstory_cloze_hi"
:
0
,
"xstory_cloze_eu"
:
0
,
"xstory_cloze_sw"
:
0
,
"xstory_cloze_es"
:
0
,
"xstory_cloze_ru"
:
0
},
"config"
:
{
"model"
:
"hf-causal-experimental"
,
"model_args"
:
"pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/30B"
,
"num_fewshot"
:
0
,
"batch_size"
:
"auto"
,
"device"
:
"cuda"
,
"no_cache"
:
true
,
"limit"
:
null
,
"bootstrap_iters"
:
100000
,
"description_dict"
:
{}
}
}
results/llama/llama-30B/llama-30B_xwinograd_0-shot.json
0 → 100644
View file @
21e128d8
{
"results"
:
{
"xwinograd_en"
:
{
"acc"
:
0.873978494623656
,
"acc_stderr"
:
0.006884218449880497
},
"xwinograd_ru"
:
{
"acc"
:
0.6698412698412698
,
"acc_stderr"
:
0.026538875646287704
},
"xwinograd_jp"
:
{
"acc"
:
0.6736183524504692
,
"acc_stderr"
:
0.015149108150588548
},
"xwinograd_pt"
:
{
"acc"
:
0.7680608365019012
,
"acc_stderr"
:
0.02607559386030469
},
"xwinograd_zh"
:
{
"acc"
:
0.7123015873015873
,
"acc_stderr"
:
0.02018443961183448
},
"xwinograd_fr"
:
{
"acc"
:
0.7349397590361446
,
"acc_stderr"
:
0.04874064133109369
}
},
"versions"
:
{
"xwinograd_en"
:
0
,
"xwinograd_ru"
:
0
,
"xwinograd_jp"
:
0
,
"xwinograd_pt"
:
0
,
"xwinograd_zh"
:
0
,
"xwinograd_fr"
:
0
},
"config"
:
{
"model"
:
"hf-causal-experimental"
,
"model_args"
:
"pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/30B"
,
"num_fewshot"
:
0
,
"batch_size"
:
"auto"
,
"device"
:
"cuda"
,
"no_cache"
:
true
,
"limit"
:
null
,
"bootstrap_iters"
:
100000
,
"description_dict"
:
{}
}
}
results/llama/llama-7B/llama-7B_anli_0-shot.json
0 → 100644
View file @
21e128d8
{
"results"
:
{
"anli_r1"
:
{
"acc"
:
0.348
,
"acc_stderr"
:
0.015070604603768408
},
"anli_r2"
:
{
"acc"
:
0.337
,
"acc_stderr"
:
0.014955087918653593
},
"anli_r3"
:
{
"acc"
:
0.36583333333333334
,
"acc_stderr"
:
0.01391021206270117
}
},
"versions"
:
{
"anli_r1"
:
0
,
"anli_r2"
:
0
,
"anli_r3"
:
0
},
"config"
:
{
"model"
:
"hf-causal-experimental"
,
"model_args"
:
"pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B"
,
"num_fewshot"
:
0
,
"batch_size"
:
"auto"
,
"device"
:
"cuda"
,
"no_cache"
:
true
,
"limit"
:
null
,
"bootstrap_iters"
:
100000
,
"description_dict"
:
{}
}
}
results/llama/llama-7B/llama-7B_arithmetic_5-shot.json
0 → 100644
View file @
21e128d8
{
"results"
:
{
"arithmetic_3ds"
:
{
"acc"
:
0.0
,
"acc_stderr"
:
0.0
},
"arithmetic_1dc"
:
{
"acc"
:
0.0
,
"acc_stderr"
:
0.0
},
"arithmetic_2da"
:
{
"acc"
:
0.0
,
"acc_stderr"
:
0.0
},
"arithmetic_4ds"
:
{
"acc"
:
0.0
,
"acc_stderr"
:
0.0
},
"arithmetic_3da"
:
{
"acc"
:
0.0
,
"acc_stderr"
:
0.0
},
"arithmetic_2ds"
:
{
"acc"
:
0.0
,
"acc_stderr"
:
0.0
},
"arithmetic_4da"
:
{
"acc"
:
0.0
,
"acc_stderr"
:
0.0
},
"arithmetic_5ds"
:
{
"acc"
:
0.0
,
"acc_stderr"
:
0.0
},
"arithmetic_2dm"
:
{
"acc"
:
0.0
,
"acc_stderr"
:
0.0
},
"arithmetic_5da"
:
{
"acc"
:
0.0
,
"acc_stderr"
:
0.0
}
},
"versions"
:
{
"arithmetic_3ds"
:
0
,
"arithmetic_1dc"
:
0
,
"arithmetic_2da"
:
0
,
"arithmetic_4ds"
:
0
,
"arithmetic_3da"
:
0
,
"arithmetic_2ds"
:
0
,
"arithmetic_4da"
:
0
,
"arithmetic_5ds"
:
0
,
"arithmetic_2dm"
:
0
,
"arithmetic_5da"
:
0
},
"config"
:
{
"model"
:
"hf-causal-experimental"
,
"model_args"
:
"pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B,use_accelerate=True"
,
"num_fewshot"
:
5
,
"batch_size"
:
"auto"
,
"device"
:
"cuda:0"
,
"no_cache"
:
false
,
"limit"
:
null
,
"bootstrap_iters"
:
100000
,
"description_dict"
:
{}
}
}
results/llama/llama-7B/llama-7B_bbh_3-shot.json
0 → 100644
View file @
21e128d8
{
"results"
:
{
"bigbench_tracking_shuffled_objects_five_objects"
:
{
"multiple_choice_grade"
:
0.1824
,
"multiple_choice_grade_stderr"
:
0.010927017514830547
},
"bigbench_logical_deduction_seven_objects"
:
{
"multiple_choice_grade"
:
0.24571428571428572
,
"multiple_choice_grade_stderr"
:
0.01628337995683342
},
"bigbench_date_understanding"
:
{
"multiple_choice_grade"
:
0.6205962059620597
,
"multiple_choice_grade_stderr"
:
0.02529481360676469
},
"bigbench_navigate"
:
{
"multiple_choice_grade"
:
0.495
,
"multiple_choice_grade_stderr"
:
0.015818508944436645
},
"bigbench_geometric_shapes"
:
{
"multiple_choice_grade"
:
0.17827298050139276
,
"multiple_choice_grade_stderr"
:
0.02022856303248108
,
"exact_str_match"
:
0.0
,
"exact_str_match_stderr"
:
0.0
},
"bigbench_dyck_languages"
:
{
"multiple_choice_grade"
:
0.154
,
"multiple_choice_grade_stderr"
:
0.011419913065098684
},
"bigbench_temporal_sequences"
:
{
"multiple_choice_grade"
:
0.272
,
"multiple_choice_grade_stderr"
:
0.014078856992462611
},
"bigbench_snarks"
:
{
"multiple_choice_grade"
:
0.5082872928176796
,
"multiple_choice_grade_stderr"
:
0.03726268022638988
},
"bigbench_disambiguation_qa"
:
{
"multiple_choice_grade"
:
0.35271317829457366
,
"multiple_choice_grade_stderr"
:
0.029805242804674153
},
"bigbench_tracking_shuffled_objects_seven_objects"
:
{
"multiple_choice_grade"
:
0.13714285714285715
,
"multiple_choice_grade_stderr"
:
0.008225477923226985
},
"bigbench_ruin_names"
:
{
"multiple_choice_grade"
:
0.29910714285714285
,
"multiple_choice_grade_stderr"
:
0.021656359273376977
},
"bigbench_movie_recommendation"
:
{
"multiple_choice_grade"
:
0.404
,
"multiple_choice_grade_stderr"
:
0.021966635293832918
},
"bigbench_salient_translation_error_detection"
:
{
"multiple_choice_grade"
:
0.1653306613226453
,
"multiple_choice_grade_stderr"
:
0.011764848862417502
},
"bigbench_logical_deduction_five_objects"
:
{
"multiple_choice_grade"
:
0.29
,
"multiple_choice_grade_stderr"
:
0.020313179231745183
},
"bigbench_causal_judgement"
:
{
"multiple_choice_grade"
:
0.4842105263157895
,
"multiple_choice_grade_stderr"
:
0.036351509398643456
},
"bigbench_hyperbaton"
:
{
"multiple_choice_grade"
:
0.49508
,
"multiple_choice_grade_stderr"
:
0.0022359820804999713
},
"bigbench_sports_understanding"
:
{
"multiple_choice_grade"
:
0.5
,
"multiple_choice_grade_stderr"
:
0.015931324696929153
},
"bigbench_logical_deduction_three_objects"
:
{
"multiple_choice_grade"
:
0.3933333333333333
,
"multiple_choice_grade_stderr"
:
0.028250090846760875
},
"bigbench_tracking_shuffled_objects_three_objects"
:
{
"multiple_choice_grade"
:
0.3933333333333333
,
"multiple_choice_grade_stderr"
:
0.028250090846760875
},
"bigbench_formal_fallacies_syllogisms_negation"
:
{
"multiple_choice_grade"
:
0.5134507042253521
,
"multiple_choice_grade_stderr"
:
0.004194535955193854
},
"bigbench_reasoning_about_colored_objects"
:
{
"multiple_choice_grade"
:
0.346
,
"multiple_choice_grade_stderr"
:
0.010639483037236658
}
},
"versions"
:
{
"bigbench_tracking_shuffled_objects_five_objects"
:
0
,
"bigbench_logical_deduction_seven_objects"
:
0
,
"bigbench_date_understanding"
:
0
,
"bigbench_navigate"
:
0
,
"bigbench_geometric_shapes"
:
0
,
"bigbench_dyck_languages"
:
0
,
"bigbench_temporal_sequences"
:
0
,
"bigbench_snarks"
:
0
,
"bigbench_disambiguation_qa"
:
0
,
"bigbench_tracking_shuffled_objects_seven_objects"
:
0
,
"bigbench_ruin_names"
:
0
,
"bigbench_movie_recommendation"
:
0
,
"bigbench_salient_translation_error_detection"
:
0
,
"bigbench_logical_deduction_five_objects"
:
0
,
"bigbench_causal_judgement"
:
0
,
"bigbench_hyperbaton"
:
0
,
"bigbench_sports_understanding"
:
0
,
"bigbench_logical_deduction_three_objects"
:
0
,
"bigbench_tracking_shuffled_objects_three_objects"
:
0
,
"bigbench_formal_fallacies_syllogisms_negation"
:
0
,
"bigbench_reasoning_about_colored_objects"
:
0
},
"config"
:
{
"model"
:
"hf-causal-experimental"
,
"model_args"
:
"pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B,use_accelerate=True"
,
"num_fewshot"
:
3
,
"batch_size"
:
"auto"
,
"device"
:
"cuda"
,
"no_cache"
:
true
,
"limit"
:
null
,
"bootstrap_iters"
:
100000
,
"description_dict"
:
{}
}
}
results/llama/llama-7B/llama-7B_blimp_0-shot.json
0 → 100644
View file @
21e128d8
{
"results"
:
{
"blimp_wh_vs_that_no_gap_long_distance"
:
{
"acc"
:
0.812
,
"acc_stderr"
:
0.01236158601510377
},
"blimp_ellipsis_n_bar_1"
:
{
"acc"
:
0.668
,
"acc_stderr"
:
0.014899597242811478
},
"blimp_distractor_agreement_relative_clause"
:
{
"acc"
:
0.314
,
"acc_stderr"
:
0.014683991951087966
},
"blimp_determiner_noun_agreement_with_adjective_1"
:
{
"acc"
:
0.577
,
"acc_stderr"
:
0.01563058909047635
},
"blimp_principle_A_reconstruction"
:
{
"acc"
:
0.673
,
"acc_stderr"
:
0.014842213153411245
},
"blimp_determiner_noun_agreement_2"
:
{
"acc"
:
0.598
,
"acc_stderr"
:
0.015512467135715077
},
"blimp_npi_present_1"
:
{
"acc"
:
0.534
,
"acc_stderr"
:
0.015782683329937618
},
"blimp_existential_there_quantifiers_2"
:
{
"acc"
:
0.674
,
"acc_stderr"
:
0.01483050720454104
},
"blimp_existential_there_subject_raising"
:
{
"acc"
:
0.696
,
"acc_stderr"
:
0.014553205687950438
},
"blimp_tough_vs_raising_1"
:
{
"acc"
:
0.298
,
"acc_stderr"
:
0.014470846741134717
},
"blimp_wh_questions_subject_gap_long_distance"
:
{
"acc"
:
0.746
,
"acc_stderr"
:
0.01377220656516854
},
"blimp_left_branch_island_echo_question"
:
{
"acc"
:
0.835
,
"acc_stderr"
:
0.011743632866916164
},
"blimp_only_npi_licensor_present"
:
{
"acc"
:
0.814
,
"acc_stderr"
:
0.012310790208412803
},
"blimp_adjunct_island"
:
{
"acc"
:
0.539
,
"acc_stderr"
:
0.01577110420128319
},
"blimp_coordinate_structure_constraint_object_extraction"
:
{
"acc"
:
0.629
,
"acc_stderr"
:
0.015283736211823188
},
"blimp_irregular_plural_subject_verb_agreement_1"
:
{
"acc"
:
0.653
,
"acc_stderr"
:
0.01506047203170662
},
"blimp_passive_2"
:
{
"acc"
:
0.703
,
"acc_stderr"
:
0.0144568322948011
},
"blimp_drop_argument"
:
{
"acc"
:
0.701
,
"acc_stderr"
:
0.014484778521220461
},
"blimp_wh_vs_that_with_gap_long_distance"
:
{
"acc"
:
0.227
,
"acc_stderr"
:
0.013253174964763925
},
"blimp_existential_there_quantifiers_1"
:
{
"acc"
:
0.683
,
"acc_stderr"
:
0.014721675438880224
},
"blimp_coordinate_structure_constraint_complex_left_branch"
:
{
"acc"
:
0.682
,
"acc_stderr"
:
0.014734079309311901
},
"blimp_ellipsis_n_bar_2"
:
{
"acc"
:
0.794
,
"acc_stderr"
:
0.012795613612786548
},
"blimp_sentential_subject_island"
:
{
"acc"
:
0.606
,
"acc_stderr"
:
0.01545972195749338
},
"blimp_determiner_noun_agreement_with_adj_irregular_2"
:
{
"acc"
:
0.591
,
"acc_stderr"
:
0.015555094373257946
},
"blimp_npi_present_2"
:
{
"acc"
:
0.53
,
"acc_stderr"
:
0.015790799515836763
},
"blimp_wh_questions_subject_gap"
:
{
"acc"
:
0.72
,
"acc_stderr"
:
0.014205696104091519
},
"blimp_determiner_noun_agreement_irregular_1"
:
{
"acc"
:
0.572
,
"acc_stderr"
:
0.01565442624502927
},
"blimp_superlative_quantifiers_1"
:
{
"acc"
:
0.612
,
"acc_stderr"
:
0.015417317979911076
},
"blimp_left_branch_island_simple_question"
:
{
"acc"
:
0.74
,
"acc_stderr"
:
0.013877773329774164
},
"blimp_irregular_past_participle_adjectives"
:
{
"acc"
:
0.429
,
"acc_stderr"
:
0.01565899754787024
},
"blimp_principle_A_domain_2"
:
{
"acc"
:
0.646
,
"acc_stderr"
:
0.015129868238451772
},
"blimp_regular_plural_subject_verb_agreement_1"
:
{
"acc"
:
0.645
,
"acc_stderr"
:
0.015139491543780529
},
"blimp_principle_A_case_1"
:
{
"acc"
:
0.985
,
"acc_stderr"
:
0.003845749574502997
},
"blimp_principle_A_case_2"
:
{
"acc"
:
0.554
,
"acc_stderr"
:
0.015726771166750357
},
"blimp_matrix_question_npi_licensor_present"
:
{
"acc"
:
0.117
,
"acc_stderr"
:
0.010169287802713327
},
"blimp_determiner_noun_agreement_with_adj_irregular_1"
:
{
"acc"
:
0.563
,
"acc_stderr"
:
0.015693223928730377
},
"blimp_principle_A_domain_1"
:
{
"acc"
:
0.962
,
"acc_stderr"
:
0.006049181150584942
},
"blimp_superlative_quantifiers_2"
:
{
"acc"
:
0.561
,
"acc_stderr"
:
0.01570113134540077
},
"blimp_wh_island"
:
{
"acc"
:
0.275
,
"acc_stderr"
:
0.014127086556490531
},
"blimp_only_npi_scope"
:
{
"acc"
:
0.266
,
"acc_stderr"
:
0.013979965645145156
},
"blimp_regular_plural_subject_verb_agreement_2"
:
{
"acc"
:
0.705
,
"acc_stderr"
:
0.014428554438445512
},
"blimp_complex_NP_island"
:
{
"acc"
:
0.416
,
"acc_stderr"
:
0.015594460144140603
},
"blimp_sentential_negation_npi_scope"
:
{
"acc"
:
0.588
,
"acc_stderr"
:
0.015572363292015093
},
"blimp_transitive"
:
{
"acc"
:
0.698
,
"acc_stderr"
:
0.014526080235459543
},
"blimp_wh_vs_that_with_gap"
:
{
"acc"
:
0.239
,
"acc_stderr"
:
0.013493000446937587
},
"blimp_wh_questions_object_gap"
:
{
"acc"
:
0.67
,
"acc_stderr"
:
0.014876872027456736
},
"blimp_sentential_negation_npi_licensor_present"
:
{
"acc"
:
0.94
,
"acc_stderr"
:
0.007513751157474913
},
"blimp_expletive_it_object_raising"
:
{
"acc"
:
0.659
,
"acc_stderr"
:
0.0149981313484027
},
"blimp_determiner_noun_agreement_1"
:
{
"acc"
:
0.636
,
"acc_stderr"
:
0.015222868840522017
},
"blimp_anaphor_gender_agreement"
:
{
"acc"
:
0.448
,
"acc_stderr"
:
0.01573351656634784
},
"blimp_irregular_plural_subject_verb_agreement_2"
:
{
"acc"
:
0.7
,
"acc_stderr"
:
0.014498627873361425
},
"blimp_passive_1"
:
{
"acc"
:
0.702
,
"acc_stderr"
:
0.014470846741134705
},
"blimp_determiner_noun_agreement_irregular_2"
:
{
"acc"
:
0.602
,
"acc_stderr"
:
0.015486634102858913
},
"blimp_irregular_past_participle_verbs"
:
{
"acc"
:
0.725
,
"acc_stderr"
:
0.014127086556490523
},
"blimp_existential_there_object_raising"
:
{
"acc"
:
0.788
,
"acc_stderr"
:
0.012931481864938055
},
"blimp_determiner_noun_agreement_with_adj_2"
:
{
"acc"
:
0.54
,
"acc_stderr"
:
0.015768596914394375
},
"blimp_principle_A_domain_3"
:
{
"acc"
:
0.501
,
"acc_stderr"
:
0.015819268290576817
},
"blimp_causative"
:
{
"acc"
:
0.508
,
"acc_stderr"
:
0.015817274929209008
},
"blimp_tough_vs_raising_2"
:
{
"acc"
:
0.768
,
"acc_stderr"
:
0.01335493745228157
},
"blimp_wh_vs_that_no_gap"
:
{
"acc"
:
0.848
,
"acc_stderr"
:
0.011358918303475294
},
"blimp_anaphor_number_agreement"
:
{
"acc"
:
0.659
,
"acc_stderr"
:
0.014998131348402704
},
"blimp_principle_A_c_command"
:
{
"acc"
:
0.39
,
"acc_stderr"
:
0.01543172505386661
},
"blimp_distractor_agreement_relational_noun"
:
{
"acc"
:
0.441
,
"acc_stderr"
:
0.015708779894242676
},
"blimp_intransitive"
:
{
"acc"
:
0.592
,
"acc_stderr"
:
0.015549205052920673
},
"blimp_animate_subject_passive"
:
{
"acc"
:
0.626
,
"acc_stderr"
:
0.015308767369006363
},
"blimp_animate_subject_trans"
:
{
"acc"
:
0.761
,
"acc_stderr"
:
0.01349300044693759
},
"blimp_inchoative"
:
{
"acc"
:
0.42
,
"acc_stderr"
:
0.015615500115072957
}
},
"versions"
:
{
"blimp_wh_vs_that_no_gap_long_distance"
:
0
,
"blimp_ellipsis_n_bar_1"
:
0
,
"blimp_distractor_agreement_relative_clause"
:
0
,
"blimp_determiner_noun_agreement_with_adjective_1"
:
0
,
"blimp_principle_A_reconstruction"
:
0
,
"blimp_determiner_noun_agreement_2"
:
0
,
"blimp_npi_present_1"
:
0
,
"blimp_existential_there_quantifiers_2"
:
0
,
"blimp_existential_there_subject_raising"
:
0
,
"blimp_tough_vs_raising_1"
:
0
,
"blimp_wh_questions_subject_gap_long_distance"
:
0
,
"blimp_left_branch_island_echo_question"
:
0
,
"blimp_only_npi_licensor_present"
:
0
,
"blimp_adjunct_island"
:
0
,
"blimp_coordinate_structure_constraint_object_extraction"
:
0
,
"blimp_irregular_plural_subject_verb_agreement_1"
:
0
,
"blimp_passive_2"
:
0
,
"blimp_drop_argument"
:
0
,
"blimp_wh_vs_that_with_gap_long_distance"
:
0
,
"blimp_existential_there_quantifiers_1"
:
0
,
"blimp_coordinate_structure_constraint_complex_left_branch"
:
0
,
"blimp_ellipsis_n_bar_2"
:
0
,
"blimp_sentential_subject_island"
:
0
,
"blimp_determiner_noun_agreement_with_adj_irregular_2"
:
0
,
"blimp_npi_present_2"
:
0
,
"blimp_wh_questions_subject_gap"
:
0
,
"blimp_determiner_noun_agreement_irregular_1"
:
0
,
"blimp_superlative_quantifiers_1"
:
0
,
"blimp_left_branch_island_simple_question"
:
0
,
"blimp_irregular_past_participle_adjectives"
:
0
,
"blimp_principle_A_domain_2"
:
0
,
"blimp_regular_plural_subject_verb_agreement_1"
:
0
,
"blimp_principle_A_case_1"
:
0
,
"blimp_principle_A_case_2"
:
0
,
"blimp_matrix_question_npi_licensor_present"
:
0
,
"blimp_determiner_noun_agreement_with_adj_irregular_1"
:
0
,
"blimp_principle_A_domain_1"
:
0
,
"blimp_superlative_quantifiers_2"
:
0
,
"blimp_wh_island"
:
0
,
"blimp_only_npi_scope"
:
0
,
"blimp_regular_plural_subject_verb_agreement_2"
:
0
,
"blimp_complex_NP_island"
:
0
,
"blimp_sentential_negation_npi_scope"
:
0
,
"blimp_transitive"
:
0
,
"blimp_wh_vs_that_with_gap"
:
0
,
"blimp_wh_questions_object_gap"
:
0
,
"blimp_sentential_negation_npi_licensor_present"
:
0
,
"blimp_expletive_it_object_raising"
:
0
,
"blimp_determiner_noun_agreement_1"
:
0
,
"blimp_anaphor_gender_agreement"
:
0
,
"blimp_irregular_plural_subject_verb_agreement_2"
:
0
,
"blimp_passive_1"
:
0
,
"blimp_determiner_noun_agreement_irregular_2"
:
0
,
"blimp_irregular_past_participle_verbs"
:
0
,
"blimp_existential_there_object_raising"
:
0
,
"blimp_determiner_noun_agreement_with_adj_2"
:
0
,
"blimp_principle_A_domain_3"
:
0
,
"blimp_causative"
:
0
,
"blimp_tough_vs_raising_2"
:
0
,
"blimp_wh_vs_that_no_gap"
:
0
,
"blimp_anaphor_number_agreement"
:
0
,
"blimp_principle_A_c_command"
:
0
,
"blimp_distractor_agreement_relational_noun"
:
0
,
"blimp_intransitive"
:
0
,
"blimp_animate_subject_passive"
:
0
,
"blimp_animate_subject_trans"
:
0
,
"blimp_inchoative"
:
0
},
"config"
:
{
"model"
:
"hf-causal-experimental"
,
"model_args"
:
"pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B"
,
"num_fewshot"
:
0
,
"batch_size"
:
"auto"
,
"device"
:
"cuda:0"
,
"no_cache"
:
true
,
"limit"
:
null
,
"bootstrap_iters"
:
100000
,
"description_dict"
:
{}
}
}
results/llama/llama-7B/llama-7B_common_sense_reasoning_0-shot.json
0 → 100644
View file @
21e128d8
{
"results"
:
{
"piqa"
:
{
"acc"
:
0.7818280739934712
,
"acc_stderr"
:
0.009636081958374381
,
"acc_norm"
:
0.7742110990206746
,
"acc_norm_stderr"
:
0.00975498067091731
},
"wsc273"
:
{
"acc"
:
0.8095238095238095
,
"acc_stderr"
:
0.023809523809523777
},
"arc_easy"
:
{
"acc"
:
0.6738215488215489
,
"acc_stderr"
:
0.009619849417035172
,
"acc_norm"
:
0.5248316498316499
,
"acc_norm_stderr"
:
0.010247123122159281
},
"hellaswag"
:
{
"acc"
:
0.563931487751444
,
"acc_stderr"
:
0.004948824501355491
,
"acc_norm"
:
0.7298346942840072
,
"acc_norm_stderr"
:
0.004431375549911366
},
"winogrande"
:
{
"acc"
:
0.6692975532754538
,
"acc_stderr"
:
0.013222435887002705
},
"prost"
:
{
"acc"
:
0.2568851409052092
,
"acc_stderr"
:
0.003192056839011391
,
"acc_norm"
:
0.28031596925704527
,
"acc_norm_stderr"
:
0.0032814667207950675
},
"swag"
:
{
"acc"
:
0.5546835949215235
,
"acc_stderr"
:
0.0035138865053857436
,
"acc_norm"
:
0.6687493751874438
,
"acc_norm_stderr"
:
0.003327673972187388
},
"boolq"
:
{
"acc"
:
0.7305810397553517
,
"acc_stderr"
:
0.007759626474907443
},
"arc_challenge"
:
{
"acc"
:
0.3822525597269625
,
"acc_stderr"
:
0.014200454049979293
,
"acc_norm"
:
0.4138225255972696
,
"acc_norm_stderr"
:
0.014392730009221009
},
"mc_taco"
:
{
"em"
:
0.11261261261261261
,
"f1"
:
0.4827075067316446
},
"copa"
:
{
"acc"
:
0.84
,
"acc_stderr"
:
0.036845294917747094
},
"openbookqa"
:
{
"acc"
:
0.282
,
"acc_stderr"
:
0.020143572847290774
,
"acc_norm"
:
0.424
,
"acc_norm_stderr"
:
0.022122993778135404
}
},
"versions"
:
{
"piqa"
:
0
,
"wsc273"
:
0
,
"arc_easy"
:
0
,
"hellaswag"
:
0
,
"winogrande"
:
0
,
"prost"
:
0
,
"swag"
:
0
,
"boolq"
:
1
,
"arc_challenge"
:
0
,
"mc_taco"
:
0
,
"copa"
:
0
,
"openbookqa"
:
0
},
"config"
:
{
"model"
:
"hf-causal-experimental"
,
"model_args"
:
"pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B,use_accelerate=True"
,
"num_fewshot"
:
0
,
"batch_size"
:
"auto"
,
"device"
:
"cuda:0"
,
"no_cache"
:
true
,
"limit"
:
null
,
"bootstrap_iters"
:
100000
,
"description_dict"
:
{}
}
}
results/llama/llama-7B/llama-7B_glue_0-shot.json
0 → 100644
View file @
21e128d8
{
"results"
:
{
"mrpc"
:
{
"acc"
:
0.6838235294117647
,
"acc_stderr"
:
0.023048336668420204
,
"f1"
:
0.8122270742358079
,
"f1_stderr"
:
0.01624762253426993
},
"sst"
:
{
"acc"
:
0.5298165137614679
,
"acc_stderr"
:
0.016911703415318852
},
"wnli"
:
{
"acc"
:
0.5633802816901409
,
"acc_stderr"
:
0.0592793555841297
},
"mnli_mismatched"
:
{
"acc"
:
0.3572009764035802
,
"acc_stderr"
:
0.0048327582938812235
},
"qnli"
:
{
"acc"
:
0.49569833424858134
,
"acc_stderr"
:
0.006765160168388141
},
"rte"
:
{
"acc"
:
0.5306859205776173
,
"acc_stderr"
:
0.03003973059219781
},
"qqp"
:
{
"acc"
:
0.3683650754390304
,
"acc_stderr"
:
0.002398975385820536
,
"f1"
:
0.5380844713755992
,
"f1_stderr"
:
0.0025560675394743124
},
"mnli"
:
{
"acc"
:
0.34396332144676517
,
"acc_stderr"
:
0.0047950937299233165
},
"cola"
:
{
"mcc"
:
0.0
,
"mcc_stderr"
:
0.0
}
},
"versions"
:
{
"mrpc"
:
0
,
"sst"
:
0
,
"wnli"
:
1
,
"mnli_mismatched"
:
0
,
"qnli"
:
0
,
"rte"
:
0
,
"qqp"
:
0
,
"mnli"
:
0
,
"cola"
:
0
},
"config"
:
{
"model"
:
"hf-causal-experimental"
,
"model_args"
:
"pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B,use_accelerate=True"
,
"num_fewshot"
:
0
,
"batch_size"
:
"auto"
,
"device"
:
"cuda:0"
,
"no_cache"
:
false
,
"limit"
:
null
,
"bootstrap_iters"
:
100000
,
"description_dict"
:
{}
}
}
results/llama/llama-7B/llama-7B_gsm8k_8-shot.json
0 → 100644
View file @
21e128d8
{
"results"
:
{
"gsm8k"
:
{
"acc"
:
0.0803639120545868
,
"acc_stderr"
:
0.007488258573239077
}
},
"versions"
:
{
"gsm8k"
:
0
},
"config"
:
{
"model"
:
"hf-causal-experimental"
,
"model_args"
:
"pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B,use_accelerate=True"
,
"num_fewshot"
:
8
,
"batch_size"
:
"auto"
,
"device"
:
"cuda"
,
"no_cache"
:
true
,
"limit"
:
null
,
"bootstrap_iters"
:
100000
,
"description_dict"
:
{}
}
}
results/llama/llama-7B/llama-7B_human_alignment_0-shot.json
0 → 100644
View file @
21e128d8
{
"results"
:
{
"ethics_virtue"
:
{
"acc"
:
0.20984924623115578
,
"acc_stderr"
:
0.005773721023799748
,
"em"
:
0.0
},
"crows_pairs_french_race_color"
:
{
"likelihood_difference"
:
12.048913043478262
,
"likelihood_difference_stderr"
:
0.7332463392189781
,
"pct_stereotype"
:
0.4326086956521739
,
"pct_stereotype_stderr"
:
0.023125046645341776
},
"ethics_utilitarianism_original"
:
{
"acc"
:
0.9586106489184693
,
"acc_stderr"
:
0.002872952014248801
},
"crows_pairs_english_nationality"
:
{
"likelihood_difference"
:
6.762586805555555
,
"likelihood_difference_stderr"
:
0.5868865852525466
,
"pct_stereotype"
:
0.5370370370370371
,
"pct_stereotype_stderr"
:
0.03400603625538272
},
"crows_pairs_english_socioeconomic"
:
{
"likelihood_difference"
:
6.401644736842106
,
"likelihood_difference_stderr"
:
0.5420413190484897
,
"pct_stereotype"
:
0.5684210526315789
,
"pct_stereotype_stderr"
:
0.03602751443822843
},
"crows_pairs_french_socioeconomic"
:
{
"likelihood_difference"
:
9.80843431122449
,
"likelihood_difference_stderr"
:
1.0151042209820862
,
"pct_stereotype"
:
0.5204081632653061
,
"pct_stereotype_stderr"
:
0.03577590557703757
},
"crows_pairs_english_religion"
:
{
"likelihood_difference"
:
7.219594594594595
,
"likelihood_difference_stderr"
:
0.759154104063707
,
"pct_stereotype"
:
0.6666666666666666
,
"pct_stereotype_stderr"
:
0.04494665749754944
},
"ethics_justice"
:
{
"acc"
:
0.4996301775147929
,
"acc_stderr"
:
0.009617160470756728
,
"em"
:
0.0014792899408284023
},
"crows_pairs_english_autre"
:
{
"likelihood_difference"
:
11.011363636363637
,
"likelihood_difference_stderr"
:
5.8907614264514025
,
"pct_stereotype"
:
0.45454545454545453
,
"pct_stereotype_stderr"
:
0.15745916432444335
},
"toxigen"
:
{
"acc"
:
0.4308510638297872
,
"acc_stderr"
:
0.016160089171486036
,
"acc_norm"
:
0.4319148936170213
,
"acc_norm_stderr"
:
0.016164899004911828
},
"crows_pairs_french_autre"
:
{
"likelihood_difference"
:
7.512019230769231
,
"likelihood_difference_stderr"
:
2.0958404773406696
,
"pct_stereotype"
:
0.6153846153846154
,
"pct_stereotype_stderr"
:
0.14044168141158106
},
"ethics_cm"
:
{
"acc"
:
0.5691119691119692
,
"acc_stderr"
:
0.007945870163705206
},
"crows_pairs_english_gender"
:
{
"likelihood_difference"
:
7.9173828125
,
"likelihood_difference_stderr"
:
0.5501949212762886
,
"pct_stereotype"
:
0.53125
,
"pct_stereotype_stderr"
:
0.0279398950447155
},
"crows_pairs_english_race_color"
:
{
"likelihood_difference"
:
6.246493602362205
,
"likelihood_difference_stderr"
:
0.3239007651371134
,
"pct_stereotype"
:
0.46653543307086615
,
"pct_stereotype_stderr"
:
0.022155988267174086
},
"crows_pairs_english_age"
:
{
"likelihood_difference"
:
5.9423076923076925
,
"likelihood_difference_stderr"
:
0.7902909296461826
,
"pct_stereotype"
:
0.5164835164835165
,
"pct_stereotype_stderr"
:
0.05267597952306975
},
"ethics_utilitarianism"
:
{
"acc"
:
0.4981281198003328
,
"acc_stderr"
:
0.007211571268099885
},
"crows_pairs_english_sexual_orientation"
:
{
"likelihood_difference"
:
8.304771505376344
,
"likelihood_difference_stderr"
:
0.8427804261467623
,
"pct_stereotype"
:
0.6236559139784946
,
"pct_stereotype_stderr"
:
0.05050927755267201
},
"ethics_deontology"
:
{
"acc"
:
0.5058398220244716
,
"acc_stderr"
:
0.008338557598970859
,
"em"
:
0.002224694104560623
},
"crows_pairs_french_religion"
:
{
"likelihood_difference"
:
9.585326086956522
,
"likelihood_difference_stderr"
:
0.8749663998788697
,
"pct_stereotype"
:
0.43478260869565216
,
"pct_stereotype_stderr"
:
0.04642922286356426
},
"crows_pairs_french_gender"
:
{
"likelihood_difference"
:
11.798968068535826
,
"likelihood_difference_stderr"
:
0.8713501661430004
,
"pct_stereotype"
:
0.5202492211838006
,
"pct_stereotype_stderr"
:
0.0279279188851323
},
"crows_pairs_french_nationality"
:
{
"likelihood_difference"
:
10.416501976284586
,
"likelihood_difference_stderr"
:
0.9065784742122508
,
"pct_stereotype"
:
0.40711462450592883
,
"pct_stereotype_stderr"
:
0.030948774049323072
},
"crows_pairs_english_physical_appearance"
:
{
"likelihood_difference"
:
4.512586805555555
,
"likelihood_difference_stderr"
:
0.6931576110749077
,
"pct_stereotype"
:
0.5
,
"pct_stereotype_stderr"
:
0.05933908290969268
},
"crows_pairs_french_age"
:
{
"likelihood_difference"
:
11.939583333333333
,
"likelihood_difference_stderr"
:
1.5376984338772959
,
"pct_stereotype"
:
0.35555555555555557
,
"pct_stereotype_stderr"
:
0.05074011803597719
},
"crows_pairs_english_disability"
:
{
"likelihood_difference"
:
9.669711538461538
,
"likelihood_difference_stderr"
:
1.1386178272217904
,
"pct_stereotype"
:
0.6615384615384615
,
"pct_stereotype_stderr"
:
0.05914829422780653
},
"crows_pairs_french_sexual_orientation"
:
{
"likelihood_difference"
:
7.605769230769231
,
"likelihood_difference_stderr"
:
0.7938984905689758
,
"pct_stereotype"
:
0.6703296703296703
,
"pct_stereotype_stderr"
:
0.04955219508596587
},
"crows_pairs_french_physical_appearance"
:
{
"likelihood_difference"
:
7.045138888888889
,
"likelihood_difference_stderr"
:
0.9484318157143898
,
"pct_stereotype"
:
0.5555555555555556
,
"pct_stereotype_stderr"
:
0.05897165471491952
},
"crows_pairs_french_disability"
:
{
"likelihood_difference"
:
10.147727272727273
,
"likelihood_difference_stderr"
:
1.3907137676702652
,
"pct_stereotype"
:
0.42424242424242425
,
"pct_stereotype_stderr"
:
0.06130137276858363
}
},
"versions"
:
{
"ethics_virtue"
:
0
,
"crows_pairs_french_race_color"
:
0
,
"ethics_utilitarianism_original"
:
0
,
"crows_pairs_english_nationality"
:
0
,
"crows_pairs_english_socioeconomic"
:
0
,
"crows_pairs_french_socioeconomic"
:
0
,
"crows_pairs_english_religion"
:
0
,
"ethics_justice"
:
0
,
"crows_pairs_english_autre"
:
0
,
"toxigen"
:
0
,
"crows_pairs_french_autre"
:
0
,
"ethics_cm"
:
0
,
"crows_pairs_english_gender"
:
0
,
"crows_pairs_english_race_color"
:
0
,
"crows_pairs_english_age"
:
0
,
"ethics_utilitarianism"
:
0
,
"crows_pairs_english_sexual_orientation"
:
0
,
"ethics_deontology"
:
0
,
"crows_pairs_french_religion"
:
0
,
"crows_pairs_french_gender"
:
0
,
"crows_pairs_french_nationality"
:
0
,
"crows_pairs_english_physical_appearance"
:
0
,
"crows_pairs_french_age"
:
0
,
"crows_pairs_english_disability"
:
0
,
"crows_pairs_french_sexual_orientation"
:
0
,
"crows_pairs_french_physical_appearance"
:
0
,
"crows_pairs_french_disability"
:
0
},
"config"
:
{
"model"
:
"hf-causal-experimental"
,
"model_args"
:
"pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B,use_accelerate=True"
,
"num_fewshot"
:
0
,
"batch_size"
:
"auto"
,
"device"
:
"cuda:0"
,
"no_cache"
:
false
,
"limit"
:
null
,
"bootstrap_iters"
:
100000
,
"description_dict"
:
{}
}
}
results/llama/llama-7B/llama-7B_lambada_0-shot.json
0 → 100644
View file @
21e128d8
{
"results"
:
{
"lambada_openai_mt_it"
:
{
"ppl"
:
3653680.5734235523
,
"ppl_stderr"
:
197082.9860932525
,
"acc"
:
0.0
,
"acc_stderr"
:
0.0
},
"lambada_standard"
:
{
"ppl"
:
2460346.8572795168
,
"ppl_stderr"
:
81216.56551688322
,
"acc"
:
0.0
,
"acc_stderr"
:
0.0
},
"lambada_openai_mt_es"
:
{
"ppl"
:
3818890.4545065593
,
"ppl_stderr"
:
197999.05318216747
,
"acc"
:
0.0
,
"acc_stderr"
:
0.0
},
"lambada_openai"
:
{
"ppl"
:
2817465.092468485
,
"ppl_stderr"
:
138319.08822004002
,
"acc"
:
0.0
,
"acc_stderr"
:
0.0
},
"lambada_openai_mt_fr"
:
{
"ppl"
:
2111186.115467981
,
"ppl_stderr"
:
111724.42842108487
,
"acc"
:
0.0
,
"acc_stderr"
:
0.0
},
"lambada_openai_mt_de"
:
{
"ppl"
:
1805613.6770583114
,
"ppl_stderr"
:
97892.78908113715
,
"acc"
:
0.0
,
"acc_stderr"
:
0.0
},
"lambada_standard_cloze"
:
{
"ppl"
:
6710057.2411258025
,
"ppl_stderr"
:
169833.90998542923
,
"acc"
:
0.0
,
"acc_stderr"
:
0.0
},
"lambada_openai_mt_en"
:
{
"ppl"
:
2817465.092468485
,
"ppl_stderr"
:
138319.08822004002
,
"acc"
:
0.0
,
"acc_stderr"
:
0.0
},
"lambada_openai_cloze"
:
{
"ppl"
:
255777.71115985065
,
"ppl_stderr"
:
11345.7709705634
,
"acc"
:
0.00038812342324859306
,
"acc_stderr"
:
0.00027441806845051746
}
},
"versions"
:
{
"lambada_openai_mt_it"
:
0
,
"lambada_standard"
:
0
,
"lambada_openai_mt_es"
:
0
,
"lambada_openai"
:
0
,
"lambada_openai_mt_fr"
:
0
,
"lambada_openai_mt_de"
:
0
,
"lambada_standard_cloze"
:
0
,
"lambada_openai_mt_en"
:
0
,
"lambada_openai_cloze"
:
0
},
"config"
:
{
"model"
:
"hf-causal"
,
"model_args"
:
"pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B"
,
"num_fewshot"
:
0
,
"batch_size"
:
"auto"
,
"device"
:
"cuda"
,
"no_cache"
:
true
,
"limit"
:
null
,
"bootstrap_iters"
:
100000
,
"description_dict"
:
{}
}
}
results/llama/llama-7B/llama-7B_mathematical_reasoning_0-shot.json
0 → 100644
View file @
21e128d8
{
"results"
:
{
"mathqa"
:
{
"acc"
:
0.26767169179229483
,
"acc_stderr"
:
0.008105031808599684
,
"acc_norm"
:
0.27872696817420434
,
"acc_norm_stderr"
:
0.008208048863665952
},
"math_asdiv"
:
{
"acc"
:
0.0
,
"acc_stderr"
:
0.0
},
"gsm8k"
:
{
"acc"
:
0.0
,
"acc_stderr"
:
0.0
},
"math_num_theory"
:
{
"acc"
:
0.007407407407407408
,
"acc_stderr"
:
0.003693382168437238
},
"math_precalc"
:
{
"acc"
:
0.003663003663003663
,
"acc_stderr"
:
0.002587757368193461
},
"drop"
:
{
"em"
:
0.04268036912751678
,
"em_stderr"
:
0.002070056585023236
,
"f1"
:
0.1215950083892614
,
"f1_stderr"
:
0.0024765528531984883
},
"math_geometry"
:
{
"acc"
:
0.008350730688935281
,
"acc_stderr"
:
0.004162242110295851
},
"math_counting_and_prob"
:
{
"acc"
:
0.016877637130801686
,
"acc_stderr"
:
0.0059228268948526815
},
"math_intermediate_algebra"
:
{
"acc"
:
0.006644518272425249
,
"acc_stderr"
:
0.0027050844483854013
},
"math_prealgebra"
:
{
"acc"
:
0.012629161882893225
,
"acc_stderr"
:
0.003785888218263002
},
"math_algebra"
:
{
"acc"
:
0.016849199663016005
,
"acc_stderr"
:
0.0037372948497597248
}
},
"versions"
:
{
"mathqa"
:
0
,
"math_asdiv"
:
0
,
"gsm8k"
:
0
,
"math_num_theory"
:
1
,
"math_precalc"
:
1
,
"drop"
:
1
,
"math_geometry"
:
1
,
"math_counting_and_prob"
:
1
,
"math_intermediate_algebra"
:
1
,
"math_prealgebra"
:
1
,
"math_algebra"
:
1
},
"config"
:
{
"model"
:
"hf-causal-experimental"
,
"model_args"
:
"pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B,use_accelerate=True"
,
"num_fewshot"
:
0
,
"batch_size"
:
"auto"
,
"device"
:
"cuda:0"
,
"no_cache"
:
true
,
"limit"
:
null
,
"bootstrap_iters"
:
100000
,
"description_dict"
:
{}
}
}
results/llama/llama-7B/llama-7B_mathematical_reasoning_few_shot_5-shot.json
0 → 100644
View file @
21e128d8
{
"results"
:
{
"mathqa"
:
{
"acc"
:
0.2820770519262982
,
"acc_stderr"
:
0.008238030326915545
,
"acc_norm"
:
0.2877721943048576
,
"acc_norm_stderr"
:
0.008287708494779904
},
"math_prealgebra"
:
{
"acc"
:
0.001148105625717566
,
"acc_stderr"
:
0.0011481056257175704
},
"math_geometry"
:
{
"acc"
:
0.0
,
"acc_stderr"
:
0.0
},
"math_intermediate_algebra"
:
{
"acc"
:
0.0
,
"acc_stderr"
:
0.0
},
"math_counting_and_prob"
:
{
"acc"
:
0.0
,
"acc_stderr"
:
0.0
},
"math_num_theory"
:
{
"acc"
:
0.0
,
"acc_stderr"
:
0.0
},
"gsm8k"
:
{
"acc"
:
0.0
,
"acc_stderr"
:
0.0
},
"drop"
:
{
"em"
:
0.012374161073825503
,
"em_stderr"
:
0.0011321233703992673
,
"f1"
:
0.020981543624161086
,
"f1_stderr"
:
0.001252441423790731
},
"math_precalc"
:
{
"acc"
:
0.0
,
"acc_stderr"
:
0.0
},
"math_algebra"
:
{
"acc"
:
0.0
,
"acc_stderr"
:
0.0
}
},
"versions"
:
{
"mathqa"
:
0
,
"math_prealgebra"
:
1
,
"math_geometry"
:
1
,
"math_intermediate_algebra"
:
1
,
"math_counting_and_prob"
:
1
,
"math_num_theory"
:
1
,
"gsm8k"
:
0
,
"drop"
:
1
,
"math_precalc"
:
1
,
"math_algebra"
:
1
},
"config"
:
{
"model"
:
"hf-causal-experimental"
,
"model_args"
:
"pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B,use_accelerate=True"
,
"num_fewshot"
:
5
,
"batch_size"
:
"auto"
,
"device"
:
"cuda:0"
,
"no_cache"
:
true
,
"limit"
:
null
,
"bootstrap_iters"
:
100000
,
"description_dict"
:
{}
}
}
results/llama/llama-7B/llama-7B_mmlu_5-shot.json
0 → 100644
View file @
21e128d8
{
"results"
:
{
"hendrycksTest-high_school_geography"
:
{
"acc"
:
0.4292929292929293
,
"acc_stderr"
:
0.035265527246011986
,
"acc_norm"
:
0.36363636363636365
,
"acc_norm_stderr"
:
0.03427308652999934
},
"hendrycksTest-philosophy"
:
{
"acc"
:
0.40192926045016075
,
"acc_stderr"
:
0.027846476005930477
,
"acc_norm"
:
0.3536977491961415
,
"acc_norm_stderr"
:
0.02715520810320088
},
"hendrycksTest-world_religions"
:
{
"acc"
:
0.6257309941520468
,
"acc_stderr"
:
0.03711601185389481
,
"acc_norm"
:
0.5146198830409356
,
"acc_norm_stderr"
:
0.038331852752130254
},
"hendrycksTest-college_biology"
:
{
"acc"
:
0.3194444444444444
,
"acc_stderr"
:
0.03899073687357335
,
"acc_norm"
:
0.2916666666666667
,
"acc_norm_stderr"
:
0.03800968060554858
},
"hendrycksTest-electrical_engineering"
:
{
"acc"
:
0.3586206896551724
,
"acc_stderr"
:
0.03996629574876719
,
"acc_norm"
:
0.32413793103448274
,
"acc_norm_stderr"
:
0.03900432069185554
},
"hendrycksTest-global_facts"
:
{
"acc"
:
0.32
,
"acc_stderr"
:
0.046882617226215034
,
"acc_norm"
:
0.29
,
"acc_norm_stderr"
:
0.045604802157206824
},
"hendrycksTest-high_school_government_and_politics"
:
{
"acc"
:
0.48186528497409326
,
"acc_stderr"
:
0.03606065001832917
,
"acc_norm"
:
0.37305699481865284
,
"acc_norm_stderr"
:
0.03490205592048573
},
"hendrycksTest-moral_scenarios"
:
{
"acc"
:
0.2759776536312849
,
"acc_stderr"
:
0.014950103002475353
,
"acc_norm"
:
0.27262569832402234
,
"acc_norm_stderr"
:
0.014893391735249588
},
"hendrycksTest-econometrics"
:
{
"acc"
:
0.2894736842105263
,
"acc_stderr"
:
0.04266339443159394
,
"acc_norm"
:
0.2631578947368421
,
"acc_norm_stderr"
:
0.0414243971948936
},
"hendrycksTest-international_law"
:
{
"acc"
:
0.3884297520661157
,
"acc_stderr"
:
0.04449270350068382
,
"acc_norm"
:
0.5785123966942148
,
"acc_norm_stderr"
:
0.045077322787750874
},
"hendrycksTest-us_foreign_policy"
:
{
"acc"
:
0.56
,
"acc_stderr"
:
0.049888765156985884
,
"acc_norm"
:
0.45
,
"acc_norm_stderr"
:
0.05
},
"hendrycksTest-high_school_macroeconomics"
:
{
"acc"
:
0.31794871794871793
,
"acc_stderr"
:
0.02361088430892786
,
"acc_norm"
:
0.30256410256410254
,
"acc_norm_stderr"
:
0.023290888053772742
},
"hendrycksTest-virology"
:
{
"acc"
:
0.39759036144578314
,
"acc_stderr"
:
0.038099730845402184
,
"acc_norm"
:
0.2891566265060241
,
"acc_norm_stderr"
:
0.035294868015111155
},
"hendrycksTest-high_school_mathematics"
:
{
"acc"
:
0.22592592592592592
,
"acc_stderr"
:
0.025497532639609542
,
"acc_norm"
:
0.3074074074074074
,
"acc_norm_stderr"
:
0.02813325257881564
},
"hendrycksTest-clinical_knowledge"
:
{
"acc"
:
0.3886792452830189
,
"acc_stderr"
:
0.03000048544867599
,
"acc_norm"
:
0.38113207547169814
,
"acc_norm_stderr"
:
0.029890609686286627
},
"hendrycksTest-professional_psychology"
:
{
"acc"
:
0.3839869281045752
,
"acc_stderr"
:
0.01967580813528152
,
"acc_norm"
:
0.29901960784313725
,
"acc_norm_stderr"
:
0.01852175621542302
},
"hendrycksTest-formal_logic"
:
{
"acc"
:
0.30952380952380953
,
"acc_stderr"
:
0.04134913018303316
,
"acc_norm"
:
0.3492063492063492
,
"acc_norm_stderr"
:
0.042639068927951315
},
"hendrycksTest-management"
:
{
"acc"
:
0.4854368932038835
,
"acc_stderr"
:
0.04948637324026637
,
"acc_norm"
:
0.36893203883495146
,
"acc_norm_stderr"
:
0.0477761518115674
},
"hendrycksTest-human_sexuality"
:
{
"acc"
:
0.5114503816793893
,
"acc_stderr"
:
0.043841400240780176
,
"acc_norm"
:
0.366412213740458
,
"acc_norm_stderr"
:
0.042258754519696386
},
"hendrycksTest-high_school_world_history"
:
{
"acc"
:
0.3924050632911392
,
"acc_stderr"
:
0.03178471874564729
,
"acc_norm"
:
0.33755274261603374
,
"acc_norm_stderr"
:
0.030781549102026216
},
"hendrycksTest-medical_genetics"
:
{
"acc"
:
0.44
,
"acc_stderr"
:
0.04988876515698589
,
"acc_norm"
:
0.4
,
"acc_norm_stderr"
:
0.04923659639173309
},
"hendrycksTest-computer_security"
:
{
"acc"
:
0.37
,
"acc_stderr"
:
0.048523658709391
,
"acc_norm"
:
0.44
,
"acc_norm_stderr"
:
0.04988876515698589
},
"hendrycksTest-miscellaneous"
:
{
"acc"
:
0.5836526181353767
,
"acc_stderr"
:
0.0176279480304303
,
"acc_norm"
:
0.3895274584929757
,
"acc_norm_stderr"
:
0.017438082556264597
},
"hendrycksTest-public_relations"
:
{
"acc"
:
0.39090909090909093
,
"acc_stderr"
:
0.046737523336702384
,
"acc_norm"
:
0.22727272727272727
,
"acc_norm_stderr"
:
0.040139645540727735
},
"hendrycksTest-college_physics"
:
{
"acc"
:
0.23529411764705882
,
"acc_stderr"
:
0.04220773659171453
,
"acc_norm"
:
0.3235294117647059
,
"acc_norm_stderr"
:
0.046550104113196177
},
"hendrycksTest-professional_accounting"
:
{
"acc"
:
0.30141843971631205
,
"acc_stderr"
:
0.02737412888263115
,
"acc_norm"
:
0.29432624113475175
,
"acc_norm_stderr"
:
0.027187127011503793
},
"hendrycksTest-logical_fallacies"
:
{
"acc"
:
0.3803680981595092
,
"acc_stderr"
:
0.03814269893261837
,
"acc_norm"
:
0.3496932515337423
,
"acc_norm_stderr"
:
0.037466683254700206
},
"hendrycksTest-business_ethics"
:
{
"acc"
:
0.53
,
"acc_stderr"
:
0.050161355804659205
,
"acc_norm"
:
0.46
,
"acc_norm_stderr"
:
0.05009082659620332
},
"hendrycksTest-high_school_chemistry"
:
{
"acc"
:
0.2512315270935961
,
"acc_stderr"
:
0.030516530732694436
,
"acc_norm"
:
0.2955665024630542
,
"acc_norm_stderr"
:
0.03210494433751458
},
"hendrycksTest-astronomy"
:
{
"acc"
:
0.45394736842105265
,
"acc_stderr"
:
0.04051646342874143
,
"acc_norm"
:
0.4605263157894737
,
"acc_norm_stderr"
:
0.04056242252249033
},
"hendrycksTest-high_school_us_history"
:
{
"acc"
:
0.4264705882352941
,
"acc_stderr"
:
0.03471157907953424
,
"acc_norm"
:
0.3137254901960784
,
"acc_norm_stderr"
:
0.032566854844603886
},
"hendrycksTest-college_chemistry"
:
{
"acc"
:
0.33
,
"acc_stderr"
:
0.047258156262526045
,
"acc_norm"
:
0.3
,
"acc_norm_stderr"
:
0.046056618647183814
},
"hendrycksTest-abstract_algebra"
:
{
"acc"
:
0.23
,
"acc_stderr"
:
0.042295258468165065
,
"acc_norm"
:
0.26
,
"acc_norm_stderr"
:
0.0440844002276808
},
"hendrycksTest-moral_disputes"
:
{
"acc"
:
0.36416184971098264
,
"acc_stderr"
:
0.025906632631016117
,
"acc_norm"
:
0.33236994219653176
,
"acc_norm_stderr"
:
0.02536116874968821
},
"hendrycksTest-college_computer_science"
:
{
"acc"
:
0.33
,
"acc_stderr"
:
0.04725815626252605
,
"acc_norm"
:
0.28
,
"acc_norm_stderr"
:
0.04512608598542128
},
"hendrycksTest-professional_law"
:
{
"acc"
:
0.2966101694915254
,
"acc_stderr"
:
0.011665946586082849
,
"acc_norm"
:
0.28552803129074317
,
"acc_norm_stderr"
:
0.011535751586665664
},
"hendrycksTest-college_mathematics"
:
{
"acc"
:
0.32
,
"acc_stderr"
:
0.046882617226215034
,
"acc_norm"
:
0.32
,
"acc_norm_stderr"
:
0.04688261722621505
},
"hendrycksTest-high_school_microeconomics"
:
{
"acc"
:
0.3865546218487395
,
"acc_stderr"
:
0.0316314580755238
,
"acc_norm"
:
0.36554621848739494
,
"acc_norm_stderr"
:
0.0312821770636846
},
"hendrycksTest-high_school_european_history"
:
{
"acc"
:
0.40606060606060607
,
"acc_stderr"
:
0.03834816355401181
,
"acc_norm"
:
0.3696969696969697
,
"acc_norm_stderr"
:
0.03769430314512568
},
"hendrycksTest-high_school_biology"
:
{
"acc"
:
0.3580645161290323
,
"acc_stderr"
:
0.027273890594300642
,
"acc_norm"
:
0.3580645161290323
,
"acc_norm_stderr"
:
0.02727389059430063
},
"hendrycksTest-security_studies"
:
{
"acc"
:
0.40816326530612246
,
"acc_stderr"
:
0.03146465712827424
,
"acc_norm"
:
0.31020408163265306
,
"acc_norm_stderr"
:
0.029613459872484375
},
"hendrycksTest-high_school_psychology"
:
{
"acc"
:
0.46605504587155966
,
"acc_stderr"
:
0.02138786335035399
,
"acc_norm"
:
0.30825688073394497
,
"acc_norm_stderr"
:
0.01979836669836726
},
"hendrycksTest-conceptual_physics"
:
{
"acc"
:
0.3276595744680851
,
"acc_stderr"
:
0.030683020843231004
,
"acc_norm"
:
0.2170212765957447
,
"acc_norm_stderr"
:
0.026947483121496228
},
"hendrycksTest-human_aging"
:
{
"acc"
:
0.3721973094170404
,
"acc_stderr"
:
0.03244305283008731
,
"acc_norm"
:
0.25112107623318386
,
"acc_norm_stderr"
:
0.02910522083322462
},
"hendrycksTest-prehistory"
:
{
"acc"
:
0.4012345679012346
,
"acc_stderr"
:
0.0272725828498398
,
"acc_norm"
:
0.2777777777777778
,
"acc_norm_stderr"
:
0.02492200116888633
},
"hendrycksTest-sociology"
:
{
"acc"
:
0.47761194029850745
,
"acc_stderr"
:
0.035319879302087305
,
"acc_norm"
:
0.42786069651741293
,
"acc_norm_stderr"
:
0.03498541988407795
},
"hendrycksTest-marketing"
:
{
"acc"
:
0.6111111111111112
,
"acc_stderr"
:
0.031937057262002924
,
"acc_norm"
:
0.5042735042735043
,
"acc_norm_stderr"
:
0.03275489264382132
},
"hendrycksTest-high_school_computer_science"
:
{
"acc"
:
0.41
,
"acc_stderr"
:
0.049431107042371025
,
"acc_norm"
:
0.34
,
"acc_norm_stderr"
:
0.047609522856952365
},
"hendrycksTest-machine_learning"
:
{
"acc"
:
0.30357142857142855
,
"acc_stderr"
:
0.04364226155841044
,
"acc_norm"
:
0.26785714285714285
,
"acc_norm_stderr"
:
0.04203277291467762
},
"hendrycksTest-elementary_mathematics"
:
{
"acc"
:
0.3201058201058201
,
"acc_stderr"
:
0.024026846392873506
,
"acc_norm"
:
0.291005291005291
,
"acc_norm_stderr"
:
0.023393826500484865
},
"hendrycksTest-nutrition"
:
{
"acc"
:
0.3954248366013072
,
"acc_stderr"
:
0.027996723180631435
,
"acc_norm"
:
0.43790849673202614
,
"acc_norm_stderr"
:
0.028408302020332694
},
"hendrycksTest-anatomy"
:
{
"acc"
:
0.3851851851851852
,
"acc_stderr"
:
0.042039210401562783
,
"acc_norm"
:
0.2814814814814815
,
"acc_norm_stderr"
:
0.03885004245800254
},
"hendrycksTest-jurisprudence"
:
{
"acc"
:
0.4351851851851852
,
"acc_stderr"
:
0.04792898170907062
,
"acc_norm"
:
0.5
,
"acc_norm_stderr"
:
0.04833682445228318
},
"hendrycksTest-college_medicine"
:
{
"acc"
:
0.37572254335260113
,
"acc_stderr"
:
0.036928207672648664
,
"acc_norm"
:
0.3063583815028902
,
"acc_norm_stderr"
:
0.03514942551267439
},
"hendrycksTest-high_school_statistics"
:
{
"acc"
:
0.3425925925925926
,
"acc_stderr"
:
0.03236585252602156
,
"acc_norm"
:
0.3425925925925926
,
"acc_norm_stderr"
:
0.03236585252602156
},
"hendrycksTest-high_school_physics"
:
{
"acc"
:
0.2052980132450331
,
"acc_stderr"
:
0.03297986648473834
,
"acc_norm"
:
0.271523178807947
,
"acc_norm_stderr"
:
0.036313298039696525
},
"hendrycksTest-professional_medicine"
:
{
"acc"
:
0.3382352941176471
,
"acc_stderr"
:
0.028739328513983576
,
"acc_norm"
:
0.27941176470588236
,
"acc_norm_stderr"
:
0.027257202606114948
}
},
"versions"
:
{
"hendrycksTest-high_school_geography"
:
0
,
"hendrycksTest-philosophy"
:
0
,
"hendrycksTest-world_religions"
:
0
,
"hendrycksTest-college_biology"
:
0
,
"hendrycksTest-electrical_engineering"
:
0
,
"hendrycksTest-global_facts"
:
0
,
"hendrycksTest-high_school_government_and_politics"
:
0
,
"hendrycksTest-moral_scenarios"
:
0
,
"hendrycksTest-econometrics"
:
0
,
"hendrycksTest-international_law"
:
0
,
"hendrycksTest-us_foreign_policy"
:
0
,
"hendrycksTest-high_school_macroeconomics"
:
0
,
"hendrycksTest-virology"
:
0
,
"hendrycksTest-high_school_mathematics"
:
0
,
"hendrycksTest-clinical_knowledge"
:
0
,
"hendrycksTest-professional_psychology"
:
0
,
"hendrycksTest-formal_logic"
:
0
,
"hendrycksTest-management"
:
0
,
"hendrycksTest-human_sexuality"
:
0
,
"hendrycksTest-high_school_world_history"
:
0
,
"hendrycksTest-medical_genetics"
:
0
,
"hendrycksTest-computer_security"
:
0
,
"hendrycksTest-miscellaneous"
:
0
,
"hendrycksTest-public_relations"
:
0
,
"hendrycksTest-college_physics"
:
0
,
"hendrycksTest-professional_accounting"
:
0
,
"hendrycksTest-logical_fallacies"
:
0
,
"hendrycksTest-business_ethics"
:
0
,
"hendrycksTest-high_school_chemistry"
:
0
,
"hendrycksTest-astronomy"
:
0
,
"hendrycksTest-high_school_us_history"
:
0
,
"hendrycksTest-college_chemistry"
:
0
,
"hendrycksTest-abstract_algebra"
:
0
,
"hendrycksTest-moral_disputes"
:
0
,
"hendrycksTest-college_computer_science"
:
0
,
"hendrycksTest-professional_law"
:
0
,
"hendrycksTest-college_mathematics"
:
0
,
"hendrycksTest-high_school_microeconomics"
:
0
,
"hendrycksTest-high_school_european_history"
:
0
,
"hendrycksTest-high_school_biology"
:
0
,
"hendrycksTest-security_studies"
:
0
,
"hendrycksTest-high_school_psychology"
:
0
,
"hendrycksTest-conceptual_physics"
:
0
,
"hendrycksTest-human_aging"
:
0
,
"hendrycksTest-prehistory"
:
0
,
"hendrycksTest-sociology"
:
0
,
"hendrycksTest-marketing"
:
0
,
"hendrycksTest-high_school_computer_science"
:
0
,
"hendrycksTest-machine_learning"
:
0
,
"hendrycksTest-elementary_mathematics"
:
0
,
"hendrycksTest-nutrition"
:
0
,
"hendrycksTest-anatomy"
:
0
,
"hendrycksTest-jurisprudence"
:
0
,
"hendrycksTest-college_medicine"
:
0
,
"hendrycksTest-high_school_statistics"
:
0
,
"hendrycksTest-high_school_physics"
:
0
,
"hendrycksTest-professional_medicine"
:
0
},
"config"
:
{
"model"
:
"hf-causal-experimental"
,
"model_args"
:
"pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B,use_accelerate=True"
,
"num_fewshot"
:
5
,
"batch_size"
:
"auto"
,
"device"
:
"cuda:0"
,
"no_cache"
:
true
,
"limit"
:
null
,
"bootstrap_iters"
:
100000
,
"description_dict"
:
{}
}
}
results/llama/llama-7B/llama-7B_pawsx_0-shot.json
0 → 100644
View file @
21e128d8
{
"results"
:
{
"pawsx_en"
:
{
"acc"
:
0.6185
,
"acc_stderr"
:
0.010864524561478635
},
"pawsx_es"
:
{
"acc"
:
0.561
,
"acc_stderr"
:
0.011099599116647334
},
"pawsx_zh"
:
{
"acc"
:
0.491
,
"acc_stderr"
:
0.01118132420626029
},
"pawsx_fr"
:
{
"acc"
:
0.5295
,
"acc_stderr"
:
0.01116365480451166
},
"pawsx_de"
:
{
"acc"
:
0.5465
,
"acc_stderr"
:
0.011134669525078668
},
"pawsx_ko"
:
{
"acc"
:
0.497
,
"acc_stderr"
:
0.011182934722804558
},
"pawsx_ja"
:
{
"acc"
:
0.567
,
"acc_stderr"
:
0.011082279027990133
}
},
"versions"
:
{
"pawsx_en"
:
0
,
"pawsx_es"
:
0
,
"pawsx_zh"
:
0
,
"pawsx_fr"
:
0
,
"pawsx_de"
:
0
,
"pawsx_ko"
:
0
,
"pawsx_ja"
:
0
},
"config"
:
{
"model"
:
"hf-causal-experimental"
,
"model_args"
:
"pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B"
,
"num_fewshot"
:
0
,
"batch_size"
:
"auto"
,
"device"
:
"cuda"
,
"no_cache"
:
true
,
"limit"
:
null
,
"bootstrap_iters"
:
100000
,
"description_dict"
:
{}
}
}
results/llama/llama-7B/llama-7B_question_answering_0-shot.json
0 → 100644
View file @
21e128d8
{
"results"
:
{
"webqs"
:
{
"acc"
:
0.0
,
"acc_stderr"
:
0.0
},
"truthfulqa_mc"
:
{
"mc1"
:
0.21052631578947367
,
"mc1_stderr"
:
0.014271740645964186
,
"mc2"
:
0.3413779054949544
,
"mc2_stderr"
:
0.013147086422190785
},
"headqa_en"
:
{
"acc"
:
0.3242159008023341
,
"acc_stderr"
:
0.008940599111132593
,
"acc_norm"
:
0.35922684172137126
,
"acc_norm_stderr"
:
0.009163935584608705
},
"triviaqa"
:
{
"acc"
:
0.0
,
"acc_stderr"
:
0.0
},
"headqa_es"
:
{
"acc"
:
0.2826404084609774
,
"acc_stderr"
:
0.00860064580960105
,
"acc_norm"
:
0.3242159008023341
,
"acc_norm_stderr"
:
0.008940599111132597
},
"logiqa"
:
{
"acc"
:
0.21812596006144394
,
"acc_stderr"
:
0.01619814925841932
,
"acc_norm"
:
0.30261136712749614
,
"acc_norm_stderr"
:
0.018018696598158843
},
"squad2"
:
{
"exact"
:
9.416322749094585
,
"f1"
:
19.449005859216243
,
"HasAns_exact"
:
18.488529014844804
,
"HasAns_f1"
:
38.58266642484388
,
"NoAns_exact"
:
0.3700588730025231
,
"NoAns_f1"
:
0.3700588730025231
,
"best_exact"
:
50.07159100480081
,
"best_f1"
:
50.0801228586961
}
},
"versions"
:
{
"webqs"
:
0
,
"truthfulqa_mc"
:
1
,
"headqa_en"
:
0
,
"triviaqa"
:
1
,
"headqa_es"
:
0
,
"logiqa"
:
0
,
"squad2"
:
1
},
"config"
:
{
"model"
:
"hf-causal-experimental"
,
"model_args"
:
"pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B,use_accelerate=True"
,
"num_fewshot"
:
0
,
"batch_size"
:
"auto"
,
"device"
:
"cuda:0"
,
"no_cache"
:
true
,
"limit"
:
null
,
"bootstrap_iters"
:
100000
,
"description_dict"
:
{}
}
}
results/llama/llama-7B/llama-7B_reading_comprehension_0-shot.json
0 → 100644
View file @
21e128d8
{
"results"
:
{
"coqa"
:
{
"f1"
:
0.752079907334399
,
"f1_stderr"
:
0.01529202913294504
,
"em"
:
0.6266666666666667
,
"em_stderr"
:
0.01881953816383566
},
"drop"
:
{
"em"
:
0.035864093959731544
,
"em_stderr"
:
0.001904314663911949
,
"f1"
:
0.11349412751677862
,
"f1_stderr"
:
0.002340148712955784
},
"race"
:
{
"acc"
:
0.3990430622009569
,
"acc_stderr"
:
0.015155885289809327
}
},
"versions"
:
{
"coqa"
:
1
,
"drop"
:
1
,
"race"
:
1
},
"config"
:
{
"model"
:
"hf-causal-experimental"
,
"model_args"
:
"pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B,use_accelerate=True"
,
"num_fewshot"
:
0
,
"batch_size"
:
"auto"
,
"device"
:
"cuda:0"
,
"no_cache"
:
false
,
"limit"
:
null
,
"bootstrap_iters"
:
100000
,
"description_dict"
:
{}
}
}
results/llama/llama-7B/llama-7B_xcopa_0-shot.json
0 → 100644
View file @
21e128d8
{
"results"
:
{
"xcopa_sw"
:
{
"acc"
:
0.508
,
"acc_stderr"
:
0.022380208834928025
},
"xcopa_zh"
:
{
"acc"
:
0.562
,
"acc_stderr"
:
0.022210326363977417
},
"xcopa_et"
:
{
"acc"
:
0.488
,
"acc_stderr"
:
0.02237662679792717
},
"xcopa_th"
:
{
"acc"
:
0.558
,
"acc_stderr"
:
0.022231970696321122
},
"xcopa_tr"
:
{
"acc"
:
0.556
,
"acc_stderr"
:
0.022242244375731017
},
"xcopa_qu"
:
{
"acc"
:
0.514
,
"acc_stderr"
:
0.022374298166353185
},
"xcopa_id"
:
{
"acc"
:
0.546
,
"acc_stderr"
:
0.02228814759117695
},
"xcopa_ta"
:
{
"acc"
:
0.552
,
"acc_stderr"
:
0.02226169729227013
},
"xcopa_ht"
:
{
"acc"
:
0.51
,
"acc_stderr"
:
0.02237859698923078
},
"xcopa_vi"
:
{
"acc"
:
0.516
,
"acc_stderr"
:
0.0223716109825804
},
"xcopa_it"
:
{
"acc"
:
0.62
,
"acc_stderr"
:
0.021728881438701705
}
},
"versions"
:
{
"xcopa_sw"
:
0
,
"xcopa_zh"
:
0
,
"xcopa_et"
:
0
,
"xcopa_th"
:
0
,
"xcopa_tr"
:
0
,
"xcopa_qu"
:
0
,
"xcopa_id"
:
0
,
"xcopa_ta"
:
0
,
"xcopa_ht"
:
0
,
"xcopa_vi"
:
0
,
"xcopa_it"
:
0
},
"config"
:
{
"model"
:
"hf-causal-experimental"
,
"model_args"
:
"pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B"
,
"num_fewshot"
:
0
,
"batch_size"
:
"auto"
,
"device"
:
"cuda"
,
"no_cache"
:
true
,
"limit"
:
null
,
"bootstrap_iters"
:
100000
,
"description_dict"
:
{}
}
}
Prev
1
2
3
4
5
6
7
8
Next
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment
