Skip to content

GitLab

Unverified Commit 24e3e3fa authored by Lintang Sutawika's avatar Lintang Sutawika Committed by GitHub
Browse files

Merge pull request #590 from gakada/big-refactor-merge 

Merge master into big-refactor
parents 3fc5bedc b7c3580a
Expand all Show whitespace changes
Inline Side-by-side
Showing with 2549 additions and 0 deletions
results/bloom/bloom-7b1/bloom-7b1_pawsx_0-shot.json 0 → 100644
View file @ 24e3e3fa
{
"results": {
"pawsx_zh": {
"acc": 0.4735,
"acc_stderr": 0.011167418260963935
},
"pawsx_de": {
"acc": 0.5285,
"acc_stderr": 0.011164954236428803
},
"pawsx_en": {
"acc": 0.613,
"acc_stderr": 0.010893798117218195
},
"pawsx_ko": {
"acc": 0.451,
"acc_stderr": 0.01112930504188632
},
"pawsx_fr": {
"acc": 0.509,
"acc_stderr": 0.011181324206260283
},
"pawsx_es": {
"acc": 0.5935,
"acc_stderr": 0.010985864536294245
},
"pawsx_ja": {
"acc": 0.4545,
"acc_stderr": 0.01113673598700373
}
},
"versions": {
"pawsx_zh": 0,
"pawsx_de": 0,
"pawsx_en": 0,
"pawsx_ko": 0,
"pawsx_fr": 0,
"pawsx_es": 0,
"pawsx_ja": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=bigscience/bloom-7b1",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/bloom/bloom-7b1/bloom-7b1_question_answering_0-shot.json 0 → 100644
View file @ 24e3e3fa
{
"results": {
"webqs": {
"acc": 0.022637795275590553,
"acc_stderr": 0.0033005770276179373
},
"headqa_en": {
"acc": 0.31181619256017507,
"acc_stderr": 0.008848039223989218,
"acc_norm": 0.35557986870897157,
"acc_norm_stderr": 0.009143208309033068
},
"squad2": {
"exact": 7.816053230017687,
"f1": 12.640343596838946,
"HasAns_exact": 14.84480431848853,
"HasAns_f1": 24.507219892926596,
"NoAns_exact": 0.8074011774600505,
"NoAns_f1": 0.8074011774600505,
"best_exact": 50.07159100480081,
"best_f1": 50.07159100480081
},
"truthfulqa_mc": {
"mc1": 0.22399020807833536,
"mc1_stderr": 0.014594964329474202,
"mc2": 0.38898018897492265,
"mc2_stderr": 0.014014176010735629
},
"triviaqa": {
"acc": 0.055246176964554056,
"acc_stderr": 0.0021480319949071717
},
"headqa_es": {
"acc": 0.29540481400437635,
"acc_stderr": 0.008714131357853837,
"acc_norm": 0.34318016046681254,
"acc_norm_stderr": 0.009068379779817705
},
"logiqa": {
"acc": 0.20276497695852536,
"acc_stderr": 0.015770046635584564,
"acc_norm": 0.28110599078341014,
"acc_norm_stderr": 0.017632374626460005
}
},
"versions": {
"webqs": 0,
"headqa_en": 0,
"squad2": 1,
"truthfulqa_mc": 1,
"triviaqa": 1,
"headqa_es": 0,
"logiqa": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=bigscience/bloom-7b1,use_accelerate=True",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda:0",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/bloom/bloom-7b1/bloom-7b1_reading_comprehension_0-shot.json 0 → 100644
View file @ 24e3e3fa
{
"results": {
"coqa": {
"f1": 0.6882976860781418,
"f1_stderr": 0.016322647326969194,
"em": 0.5386666666666665,
"em_stderr": 0.01995482540089559
},
"drop": {
"em": 0.02569211409395973,
"em_stderr": 0.0016202710827118362,
"f1": 0.09853712248322138,
"f1_stderr": 0.0021424507419289577
},
"race": {
"acc": 0.36555023923444974,
"acc_stderr": 0.014904654247182307
}
},
"versions": {
"coqa": 1,
"race": 1,
"drop": 1
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=bigscience/bloom-7b1,use_accelerate=True",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda:0",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/bloom/bloom-7b1/bloom-7b1_xcopa_0-shot.json 0 → 100644
View file @ 24e3e3fa
{
"results": {
"xcopa_ta": {
"acc": 0.592,
"acc_stderr": 0.02200091089387719
},
"xcopa_id": {
"acc": 0.698,
"acc_stderr": 0.02055326917420918
},
"xcopa_tr": {
"acc": 0.512,
"acc_stderr": 0.02237662679792717
},
"xcopa_th": {
"acc": 0.554,
"acc_stderr": 0.022252153078595897
},
"xcopa_ht": {
"acc": 0.508,
"acc_stderr": 0.022380208834928035
},
"xcopa_qu": {
"acc": 0.508,
"acc_stderr": 0.022380208834928035
},
"xcopa_sw": {
"acc": 0.516,
"acc_stderr": 0.0223716109825804
},
"xcopa_it": {
"acc": 0.528,
"acc_stderr": 0.022347949832668086
},
"xcopa_zh": {
"acc": 0.652,
"acc_stderr": 0.021323728632807498
},
"xcopa_et": {
"acc": 0.482,
"acc_stderr": 0.02236856511738799
},
"xcopa_vi": {
"acc": 0.708,
"acc_stderr": 0.02035437548053008
}
},
"versions": {
"xcopa_ta": 0,
"xcopa_id": 0,
"xcopa_tr": 0,
"xcopa_th": 0,
"xcopa_ht": 0,
"xcopa_qu": 0,
"xcopa_sw": 0,
"xcopa_it": 0,
"xcopa_zh": 0,
"xcopa_et": 0,
"xcopa_vi": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=bigscience/bloom-7b1",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/bloom/bloom-7b1/bloom-7b1_xnli_0-shot.json 0 → 100644
View file @ 24e3e3fa
{
"results": {
"xnli_ar": {
"acc": 0.3383233532934132,
"acc_stderr": 0.006685184166851475
},
"xnli_bg": {
"acc": 0.3970059880239521,
"acc_stderr": 0.006913206227417221
},
"xnli_de": {
"acc": 0.39860279441117763,
"acc_stderr": 0.0069179171504068675
},
"xnli_el": {
"acc": 0.35748502994011977,
"acc_stderr": 0.006771658365506411
},
"xnli_en": {
"acc": 0.539121756487026,
"acc_stderr": 0.007043053978003474
},
"xnli_es": {
"acc": 0.4870259481037924,
"acc_stderr": 0.007062333678954121
},
"xnli_fr": {
"acc": 0.49680638722554893,
"acc_stderr": 0.00706456831954508
},
"xnli_hi": {
"acc": 0.46506986027944114,
"acc_stderr": 0.007047451825220883
},
"xnli_ru": {
"acc": 0.4305389221556886,
"acc_stderr": 0.006996208063220089
},
"xnli_sw": {
"acc": 0.37924151696606784,
"acc_stderr": 0.006855572898852684
},
"xnli_th": {
"acc": 0.3499001996007984,
"acc_stderr": 0.00673886250800537
},
"xnli_tr": {
"acc": 0.3508982035928144,
"acc_stderr": 0.00674328417575373
},
"xnli_ur": {
"acc": 0.42095808383233535,
"acc_stderr": 0.006975878576227378
},
"xnli_vi": {
"acc": 0.47045908183632734,
"acc_stderr": 0.007052371383794704
},
"xnli_zh": {
"acc": 0.35429141716566864,
"acc_stderr": 0.006758076124936785
}
},
"versions": {
"xnli_ar": 0,
"xnli_bg": 0,
"xnli_de": 0,
"xnli_el": 0,
"xnli_en": 0,
"xnli_es": 0,
"xnli_fr": 0,
"xnli_hi": 0,
"xnli_ru": 0,
"xnli_sw": 0,
"xnli_th": 0,
"xnli_tr": 0,
"xnli_ur": 0,
"xnli_vi": 0,
"xnli_zh": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=bigscience/bloom-7b1",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/bloom/bloom-7b1/bloom-7b1_xstory_cloze_0-shot.json 0 → 100644
View file @ 24e3e3fa
{
"results": {
"xstory_cloze_ru": {
"acc": 0.5274652547981469,
"acc_stderr": 0.012847698270388222
},
"xstory_cloze_eu": {
"acc": 0.57180675049636,
"acc_stderr": 0.012733742799515155
},
"xstory_cloze_en": {
"acc": 0.7074784910655195,
"acc_stderr": 0.011707038572975033
},
"xstory_cloze_ar": {
"acc": 0.585704831237591,
"acc_stderr": 0.012676689821720669
},
"xstory_cloze_es": {
"acc": 0.6611515552614163,
"acc_stderr": 0.012180490758739058
},
"xstory_cloze_hi": {
"acc": 0.6055592322964924,
"acc_stderr": 0.01257710651393614
},
"xstory_cloze_my": {
"acc": 0.48974189278623426,
"acc_stderr": 0.012864417047980468
},
"xstory_cloze_sw": {
"acc": 0.5393778954334878,
"acc_stderr": 0.012827159238891916
},
"xstory_cloze_zh": {
"acc": 0.6187954996690933,
"acc_stderr": 0.01249867885093408
},
"xstory_cloze_id": {
"acc": 0.6446062210456651,
"acc_stderr": 0.01231724793041837
},
"xstory_cloze_te": {
"acc": 0.5744540039708802,
"acc_stderr": 0.012723670419166324
}
},
"versions": {
"xstory_cloze_ru": 0,
"xstory_cloze_eu": 0,
"xstory_cloze_en": 0,
"xstory_cloze_ar": 0,
"xstory_cloze_es": 0,
"xstory_cloze_hi": 0,
"xstory_cloze_my": 0,
"xstory_cloze_sw": 0,
"xstory_cloze_zh": 0,
"xstory_cloze_id": 0,
"xstory_cloze_te": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=bigscience/bloom-7b1",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/bloom/bloom-7b1/bloom-7b1_xwinograd_0-shot.json 0 → 100644
View file @ 24e3e3fa
{
"results": {
"xwinograd_fr": {
"acc": 0.7108433734939759,
"acc_stderr": 0.050066428050419186
},
"xwinograd_ru": {
"acc": 0.5682539682539682,
"acc_stderr": 0.027952495861671634
},
"xwinograd_en": {
"acc": 0.821505376344086,
"acc_stderr": 0.00794327709606643
},
"xwinograd_pt": {
"acc": 0.7680608365019012,
"acc_stderr": 0.026075593860304693
},
"xwinograd_jp": {
"acc": 0.5849843587069864,
"acc_stderr": 0.015919213413834392
},
"xwinograd_zh": {
"acc": 0.7440476190476191,
"acc_stderr": 0.019457899684028012
}
},
"versions": {
"xwinograd_fr": 0,
"xwinograd_ru": 0,
"xwinograd_en": 0,
"xwinograd_pt": 0,
"xwinograd_jp": 0,
"xwinograd_zh": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=bigscience/bloom-7b1",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/llama/llama-13B/README.md 0 → 100644
View file @ 24e3e3fa
This diff is collapsed.
results/llama/llama-13B/llama-13B_arithmetic_5-shot.json 0 → 100644
View file @ 24e3e3fa
{
"results": {
"arithmetic_2dm": {
"acc": 0.0,
"acc_stderr": 0.0
},
"arithmetic_5ds": {
"acc": 0.0,
"acc_stderr": 0.0
},
"arithmetic_4da": {
"acc": 0.0,
"acc_stderr": 0.0
},
"arithmetic_3da": {
"acc": 0.0,
"acc_stderr": 0.0
},
"arithmetic_4ds": {
"acc": 0.0,
"acc_stderr": 0.0
},
"arithmetic_5da": {
"acc": 0.0,
"acc_stderr": 0.0
},
"arithmetic_2da": {
"acc": 0.0,
"acc_stderr": 0.0
},
"arithmetic_3ds": {
"acc": 0.0,
"acc_stderr": 0.0
},
"arithmetic_1dc": {
"acc": 0.0,
"acc_stderr": 0.0
},
"arithmetic_2ds": {
"acc": 0.0,
"acc_stderr": 0.0
}
},
"versions": {
"arithmetic_2dm": 0,
"arithmetic_5ds": 0,
"arithmetic_4da": 0,
"arithmetic_3da": 0,
"arithmetic_4ds": 0,
"arithmetic_5da": 0,
"arithmetic_2da": 0,
"arithmetic_3ds": 0,
"arithmetic_1dc": 0,
"arithmetic_2ds": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/13B,use_accelerate=True",
"num_fewshot": 5,
"batch_size": "auto",
"device": "cuda:0",
"no_cache": false,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/llama/llama-13B/llama-13B_bbh_3-shot.json 0 → 100644
View file @ 24e3e3fa
{
"results": {
"bigbench_sports_understanding": {
"multiple_choice_grade": 0.5811359026369168,
"multiple_choice_grade_stderr": 0.015720172474974117
},
"bigbench_salient_translation_error_detection": {
"multiple_choice_grade": 0.1933867735470942,
"multiple_choice_grade_stderr": 0.012508305339715512
},
"bigbench_date_understanding": {
"multiple_choice_grade": 0.6395663956639567,
"multiple_choice_grade_stderr": 0.025028311208714224
},
"bigbench_navigate": {
"multiple_choice_grade": 0.517,
"multiple_choice_grade_stderr": 0.015810153729833434
},
"bigbench_dyck_languages": {
"multiple_choice_grade": 0.201,
"multiple_choice_grade_stderr": 0.012679107214617324
},
"bigbench_movie_recommendation": {
"multiple_choice_grade": 0.436,
"multiple_choice_grade_stderr": 0.022198954641476802
},
"bigbench_snarks": {
"multiple_choice_grade": 0.4696132596685083,
"multiple_choice_grade_stderr": 0.03719891321680327
},
"bigbench_disambiguation_qa": {
"multiple_choice_grade": 0.4573643410852713,
"multiple_choice_grade_stderr": 0.03107554499047266
},
"bigbench_reasoning_about_colored_objects": {
"multiple_choice_grade": 0.3705,
"multiple_choice_grade_stderr": 0.010801537464907349
},
"bigbench_geometric_shapes": {
"multiple_choice_grade": 0.23119777158774374,
"multiple_choice_grade_stderr": 0.02228217728550543,
"exact_str_match": 0.0,
"exact_str_match_stderr": 0.0
},
"bigbench_tracking_shuffled_objects_five_objects": {
"multiple_choice_grade": 0.2144,
"multiple_choice_grade_stderr": 0.011612665292522431
},
"bigbench_formal_fallacies_syllogisms_negation": {
"multiple_choice_grade": 0.5113380281690141,
"multiple_choice_grade_stderr": 0.004194975590734721
},
"bigbench_tracking_shuffled_objects_three_objects": {
"multiple_choice_grade": 0.4166666666666667,
"multiple_choice_grade_stderr": 0.028511310643917567
},
"bigbench_hyperbaton": {
"multiple_choice_grade": 0.5038,
"multiple_choice_grade_stderr": 0.0022360257592931206
},
"bigbench_temporal_sequences": {
"multiple_choice_grade": 0.28,
"multiple_choice_grade_stderr": 0.014205696104091493
},
"bigbench_logical_deduction_three_objects": {
"multiple_choice_grade": 0.4166666666666667,
"multiple_choice_grade_stderr": 0.028511310643917567
},
"bigbench_causal_judgement": {
"multiple_choice_grade": 0.49473684210526314,
"multiple_choice_grade_stderr": 0.036367633377878836
},
"bigbench_tracking_shuffled_objects_seven_objects": {
"multiple_choice_grade": 0.14457142857142857,
"multiple_choice_grade_stderr": 0.008408881015830339
},
"bigbench_logical_deduction_seven_objects": {
"multiple_choice_grade": 0.22285714285714286,
"multiple_choice_grade_stderr": 0.015740739118727993
},
"bigbench_logical_deduction_five_objects": {
"multiple_choice_grade": 0.3,
"multiple_choice_grade_stderr": 0.020514426225628046
},
"bigbench_ruin_names": {
"multiple_choice_grade": 0.34598214285714285,
"multiple_choice_grade_stderr": 0.02249924183068251
}
},
"versions": {
"bigbench_sports_understanding": 0,
"bigbench_salient_translation_error_detection": 0,
"bigbench_date_understanding": 0,
"bigbench_navigate": 0,
"bigbench_dyck_languages": 0,
"bigbench_movie_recommendation": 0,
"bigbench_snarks": 0,
"bigbench_disambiguation_qa": 0,
"bigbench_reasoning_about_colored_objects": 0,
"bigbench_geometric_shapes": 0,
"bigbench_tracking_shuffled_objects_five_objects": 0,
"bigbench_formal_fallacies_syllogisms_negation": 0,
"bigbench_tracking_shuffled_objects_three_objects": 0,
"bigbench_hyperbaton": 0,
"bigbench_temporal_sequences": 0,
"bigbench_logical_deduction_three_objects": 0,
"bigbench_causal_judgement": 0,
"bigbench_tracking_shuffled_objects_seven_objects": 0,
"bigbench_logical_deduction_seven_objects": 0,
"bigbench_logical_deduction_five_objects": 0,
"bigbench_ruin_names": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/13B,use_accelerate=True",
"num_fewshot": 3,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/llama/llama-13B/llama-13B_blimp_0-shot.json 0 → 100644
View file @ 24e3e3fa
This diff is collapsed.
results/llama/llama-13B/llama-13B_common_sense_reasoning_0-shot.json 0 → 100644
View file @ 24e3e3fa
{
"results": {
"arc_challenge": {
"acc": 0.439419795221843,
"acc_stderr": 0.014503747823580122,
"acc_norm": 0.4462457337883959,
"acc_norm_stderr": 0.014526705548539982
},
"prost": {
"acc": 0.2688941076003416,
"acc_stderr": 0.0032393206239968247,
"acc_norm": 0.3052412467976089,
"acc_norm_stderr": 0.003364432149066356
},
"swag": {
"acc": 0.5673298010596821,
"acc_stderr": 0.003502894135944166,
"acc_norm": 0.6934919524142757,
"acc_norm_stderr": 0.0032596605453371346
},
"arc_easy": {
"acc": 0.7457912457912458,
"acc_stderr": 0.008934537681141528,
"acc_norm": 0.5989057239057239,
"acc_norm_stderr": 0.010057051106534378
},
"boolq": {
"acc": 0.6850152905198776,
"acc_stderr": 0.00812432724981665
},
"wsc273": {
"acc": 0.8608058608058609,
"acc_stderr": 0.020988366070851
},
"mc_taco": {
"em": 0.10960960960960961,
"f1": 0.4753174430074593
},
"piqa": {
"acc": 0.7883569096844396,
"acc_stderr": 0.009530351270479397,
"acc_norm": 0.7910772578890098,
"acc_norm_stderr": 0.009485227030105093
},
"hellaswag": {
"acc": 0.5910177255526787,
"acc_stderr": 0.004906411984476791,
"acc_norm": 0.7623979286994622,
"acc_norm_stderr": 0.004247442237702478
},
"winogrande": {
"acc": 0.7016574585635359,
"acc_stderr": 0.012858885010030434
},
"copa": {
"acc": 0.9,
"acc_stderr": 0.030151134457776348
},
"openbookqa": {
"acc": 0.306,
"acc_stderr": 0.020629569998345403,
"acc_norm": 0.422,
"acc_norm_stderr": 0.022109039310618552
}
},
"versions": {
"arc_challenge": 0,
"prost": 0,
"swag": 0,
"arc_easy": 0,
"boolq": 1,
"wsc273": 0,
"mc_taco": 0,
"piqa": 0,
"hellaswag": 0,
"winogrande": 0,
"copa": 0,
"openbookqa": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/13B,use_accelerate=True",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda:0",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/llama/llama-13B/llama-13B_glue_0-shot.json 0 → 100644
View file @ 24e3e3fa
{
"results": {
"mnli_mismatched": {
"acc": 0.45351912123677784,
"acc_stderr": 0.005020956265665617
},
"wnli": {
"acc": 0.4647887323943662,
"acc_stderr": 0.0596130578497224
},
"sst": {
"acc": 0.6536697247706422,
"acc_stderr": 0.01612186710508361
},
"cola": {
"mcc": 0.0,
"mcc_stderr": 0.0
},
"mnli": {
"acc": 0.43555781966377993,
"acc_stderr": 0.005005063722742048
},
"qnli": {
"acc": 0.4995423759838916,
"acc_stderr": 0.006765407718154766
},
"mrpc": {
"acc": 0.6862745098039216,
"acc_stderr": 0.022999936277943434,
"f1": 0.8134110787172011,
"f1_stderr": 0.01621238238910757
},
"rte": {
"acc": 0.6534296028880866,
"acc_stderr": 0.02864445699455754
},
"qqp": {
"acc": 0.3679198614889933,
"acc_stderr": 0.0023983700314094665,
"f1": 0.5365853658536586,
"f1_stderr": 0.0025607085094365924
}
},
"versions": {
"mnli_mismatched": 0,
"wnli": 1,
"sst": 0,
"cola": 0,
"mnli": 0,
"qnli": 0,
"mrpc": 0,
"rte": 0,
"qqp": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/13B,use_accelerate=True",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda:0",
"no_cache": false,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/llama/llama-13B/llama-13B_gsm8k_8-shot.json 0 → 100644
View file @ 24e3e3fa
{
"results": {
"gsm8k": {
"acc": 0.13570887035633056,
"acc_stderr": 0.009433577908567345
}
},
"versions": {
"gsm8k": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/13B,use_accelerate=True",
"num_fewshot": 8,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/llama/llama-13B/llama-13B_human_alignment_0-shot.json 0 → 100644
View file @ 24e3e3fa
This diff is collapsed.
results/llama/llama-13B/llama-13B_lambada_0-shot.json 0 → 100644
View file @ 24e3e3fa
This diff is collapsed.
results/llama/llama-13B/llama-13B_mmlu_5-shot.json 0 → 100644
View file @ 24e3e3fa
This diff is collapsed.
results/llama/llama-13B/llama-13B_pawsx_0-shot.json 0 → 100644
View file @ 24e3e3fa
{
"results": {
"pawsx_fr": {
"acc": 0.545,
"acc_stderr": 0.011137752231145222
},
"pawsx_en": {
"acc": 0.537,
"acc_stderr": 0.011152474561478174
},
"pawsx_ko": {
"acc": 0.4705,
"acc_stderr": 0.011163654804511664
},
"pawsx_ja": {
"acc": 0.45,
"acc_stderr": 0.011127079848413735
},
"pawsx_es": {
"acc": 0.521,
"acc_stderr": 0.011173268141438304
},
"pawsx_de": {
"acc": 0.5295,
"acc_stderr": 0.011163654804511655
},
"pawsx_zh": {
"acc": 0.452,
"acc_stderr": 0.01113148485052578
}
},
"versions": {
"pawsx_fr": 0,
"pawsx_en": 0,
"pawsx_ko": 0,
"pawsx_ja": 0,
"pawsx_es": 0,
"pawsx_de": 0,
"pawsx_zh": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/13B",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment