Skip to content

GitLab

Unverified Commit 29f12dd9 authored by Lintang Sutawika's avatar Lintang Sutawika Committed by GitHub
Browse files

Merge branch 'big-refactor' into benchmark-scripts

parents e37698df 4168c05f
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 0 additions and 2194 deletions
results/bloom/bloom-7b1/bloom-7b1_common_sense_reasoning_0-shot.json deleted 100644 → 0
View file @ e37698df
{
"results": {
"copa": {
"acc": 0.72,
"acc_stderr": 0.04512608598542127
},
"winogrande": {
"acc": 0.6432517758484609,
"acc_stderr": 0.013463393958028726
},
"piqa": {
"acc": 0.7274211099020674,
"acc_stderr": 0.010389256803296021,
"acc_norm": 0.7366702937976061,
"acc_norm_stderr": 0.010276185322196764
},
"arc_challenge": {
"acc": 0.3037542662116041,
"acc_stderr": 0.013438909184778757,
"acc_norm": 0.33532423208191126,
"acc_norm_stderr": 0.013796182947785564
},
"arc_easy": {
"acc": 0.6494107744107744,
"acc_stderr": 0.009791003829831557,
"acc_norm": 0.5732323232323232,
"acc_norm_stderr": 0.010149141043955626
},
"boolq": {
"acc": 0.6287461773700306,
"acc_stderr": 0.008450174658715903
},
"wsc273": {
"acc": 0.8131868131868132,
"acc_stderr": 0.023632761722644544
},
"openbookqa": {
"acc": 0.252,
"acc_stderr": 0.019435727282249536,
"acc_norm": 0.358,
"acc_norm_stderr": 0.021461434862859122
},
"prost": {
"acc": 0.26184884713919726,
"acc_stderr": 0.003211967450351038,
"acc_norm": 0.30572160546541416,
"acc_norm_stderr": 0.003365914208405272
},
"mc_taco": {
"em": 0.13588588588588588,
"f1": 0.5052611696967991
},
"hellaswag": {
"acc": 0.4623580959968134,
"acc_stderr": 0.0049756211474061025,
"acc_norm": 0.5967934674367655,
"acc_norm_stderr": 0.0048953903414456264
},
"swag": {
"acc": 0.5024992502249325,
"acc_stderr": 0.0035350478846161142,
"acc_norm": 0.6825952214335699,
"acc_norm_stderr": 0.0032909332559412758
}
},
"versions": {
"copa": 0,
"winogrande": 0,
"piqa": 0,
"arc_challenge": 0,
"arc_easy": 0,
"boolq": 1,
"wsc273": 0,
"openbookqa": 0,
"prost": 0,
"mc_taco": 0,
"hellaswag": 0,
"swag": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=bigscience/bloom-7b1,use_accelerate=True",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda:0",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/bloom/bloom-7b1/bloom-7b1_gsm8k_8-shot.json deleted 100644 → 0
View file @ e37698df
{
"results": {
"gsm8k": {
"acc": 0.018953752843062926,
"acc_stderr": 0.0037560783410314704
}
},
"versions": {
"gsm8k": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=bigscience/bloom-7b1,use_accelerate=True",
"num_fewshot": 8,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/bloom/bloom-7b1/bloom-7b1_mathematical_reasoning_few_shot_5-shot.json deleted 100644 → 0
View file @ e37698df
{
"results": {
"mathqa": {
"acc": 0.26566164154103855,
"acc_stderr": 0.008085616216226046,
"acc_norm": 0.26532663316582916,
"acc_norm_stderr": 0.008082359462649721
},
"math_prealgebra": {
"acc": 0.0,
"acc_stderr": 0.0
},
"drop": {
"em": 0.02506291946308725,
"em_stderr": 0.0016008246934367681,
"f1": 0.05092911073825512,
"f1_stderr": 0.0017766603696206904
},
"math_precalc": {
"acc": 0.0,
"acc_stderr": 0.0
},
"math_geometry": {
"acc": 0.0,
"acc_stderr": 0.0
},
"gsm8k": {
"acc": 0.0,
"acc_stderr": 0.0
},
"math_counting_and_prob": {
"acc": 0.0,
"acc_stderr": 0.0
},
"math_num_theory": {
"acc": 0.0,
"acc_stderr": 0.0
},
"math_algebra": {
"acc": 0.0,
"acc_stderr": 0.0
},
"math_intermediate_algebra": {
"acc": 0.0,
"acc_stderr": 0.0
}
},
"versions": {
"mathqa": 0,
"math_prealgebra": 1,
"drop": 1,
"math_precalc": 1,
"math_geometry": 1,
"gsm8k": 0,
"math_counting_and_prob": 1,
"math_num_theory": 1,
"math_algebra": 1,
"math_intermediate_algebra": 1
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=bigscience/bloom-7b1,use_accelerate=True",
"num_fewshot": 5,
"batch_size": "auto",
"device": "cuda:0",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/bloom/bloom-7b1/bloom-7b1_pawsx_0-shot.json deleted 100644 → 0
View file @ e37698df
{
"results": {
"pawsx_zh": {
"acc": 0.4735,
"acc_stderr": 0.011167418260963935
},
"pawsx_de": {
"acc": 0.5285,
"acc_stderr": 0.011164954236428803
},
"pawsx_en": {
"acc": 0.613,
"acc_stderr": 0.010893798117218195
},
"pawsx_ko": {
"acc": 0.451,
"acc_stderr": 0.01112930504188632
},
"pawsx_fr": {
"acc": 0.509,
"acc_stderr": 0.011181324206260283
},
"pawsx_es": {
"acc": 0.5935,
"acc_stderr": 0.010985864536294245
},
"pawsx_ja": {
"acc": 0.4545,
"acc_stderr": 0.01113673598700373
}
},
"versions": {
"pawsx_zh": 0,
"pawsx_de": 0,
"pawsx_en": 0,
"pawsx_ko": 0,
"pawsx_fr": 0,
"pawsx_es": 0,
"pawsx_ja": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=bigscience/bloom-7b1",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/bloom/bloom-7b1/bloom-7b1_question_answering_0-shot.json deleted 100644 → 0
View file @ e37698df
{
"results": {
"webqs": {
"acc": 0.022637795275590553,
"acc_stderr": 0.0033005770276179373
},
"headqa_en": {
"acc": 0.31181619256017507,
"acc_stderr": 0.008848039223989218,
"acc_norm": 0.35557986870897157,
"acc_norm_stderr": 0.009143208309033068
},
"squad2": {
"exact": 7.816053230017687,
"f1": 12.640343596838946,
"HasAns_exact": 14.84480431848853,
"HasAns_f1": 24.507219892926596,
"NoAns_exact": 0.8074011774600505,
"NoAns_f1": 0.8074011774600505,
"best_exact": 50.07159100480081,
"best_f1": 50.07159100480081
},
"truthfulqa_mc": {
"mc1": 0.22399020807833536,
"mc1_stderr": 0.014594964329474202,
"mc2": 0.38898018897492265,
"mc2_stderr": 0.014014176010735629
},
"triviaqa": {
"acc": 0.055246176964554056,
"acc_stderr": 0.0021480319949071717
},
"headqa_es": {
"acc": 0.29540481400437635,
"acc_stderr": 0.008714131357853837,
"acc_norm": 0.34318016046681254,
"acc_norm_stderr": 0.009068379779817705
},
"logiqa": {
"acc": 0.20276497695852536,
"acc_stderr": 0.015770046635584564,
"acc_norm": 0.28110599078341014,
"acc_norm_stderr": 0.017632374626460005
}
},
"versions": {
"webqs": 0,
"headqa_en": 0,
"squad2": 1,
"truthfulqa_mc": 1,
"triviaqa": 1,
"headqa_es": 0,
"logiqa": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=bigscience/bloom-7b1,use_accelerate=True",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda:0",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/bloom/bloom-7b1/bloom-7b1_reading_comprehension_0-shot.json deleted 100644 → 0
View file @ e37698df
{
"results": {
"coqa": {
"f1": 0.6882976860781418,
"f1_stderr": 0.016322647326969194,
"em": 0.5386666666666665,
"em_stderr": 0.01995482540089559
},
"drop": {
"em": 0.02569211409395973,
"em_stderr": 0.0016202710827118362,
"f1": 0.09853712248322138,
"f1_stderr": 0.0021424507419289577
},
"race": {
"acc": 0.36555023923444974,
"acc_stderr": 0.014904654247182307
}
},
"versions": {
"coqa": 1,
"race": 1,
"drop": 1
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=bigscience/bloom-7b1,use_accelerate=True",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda:0",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/bloom/bloom-7b1/bloom-7b1_xcopa_0-shot.json deleted 100644 → 0
View file @ e37698df
{
"results": {
"xcopa_ta": {
"acc": 0.592,
"acc_stderr": 0.02200091089387719
},
"xcopa_id": {
"acc": 0.698,
"acc_stderr": 0.02055326917420918
},
"xcopa_tr": {
"acc": 0.512,
"acc_stderr": 0.02237662679792717
},
"xcopa_th": {
"acc": 0.554,
"acc_stderr": 0.022252153078595897
},
"xcopa_ht": {
"acc": 0.508,
"acc_stderr": 0.022380208834928035
},
"xcopa_qu": {
"acc": 0.508,
"acc_stderr": 0.022380208834928035
},
"xcopa_sw": {
"acc": 0.516,
"acc_stderr": 0.0223716109825804
},
"xcopa_it": {
"acc": 0.528,
"acc_stderr": 0.022347949832668086
},
"xcopa_zh": {
"acc": 0.652,
"acc_stderr": 0.021323728632807498
},
"xcopa_et": {
"acc": 0.482,
"acc_stderr": 0.02236856511738799
},
"xcopa_vi": {
"acc": 0.708,
"acc_stderr": 0.02035437548053008
}
},
"versions": {
"xcopa_ta": 0,
"xcopa_id": 0,
"xcopa_tr": 0,
"xcopa_th": 0,
"xcopa_ht": 0,
"xcopa_qu": 0,
"xcopa_sw": 0,
"xcopa_it": 0,
"xcopa_zh": 0,
"xcopa_et": 0,
"xcopa_vi": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=bigscience/bloom-7b1",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/bloom/bloom-7b1/bloom-7b1_xnli_0-shot.json deleted 100644 → 0
View file @ e37698df
{
"results": {
"xnli_ar": {
"acc": 0.3383233532934132,
"acc_stderr": 0.006685184166851475
},
"xnli_bg": {
"acc": 0.3970059880239521,
"acc_stderr": 0.006913206227417221
},
"xnli_de": {
"acc": 0.39860279441117763,
"acc_stderr": 0.0069179171504068675
},
"xnli_el": {
"acc": 0.35748502994011977,
"acc_stderr": 0.006771658365506411
},
"xnli_en": {
"acc": 0.539121756487026,
"acc_stderr": 0.007043053978003474
},
"xnli_es": {
"acc": 0.4870259481037924,
"acc_stderr": 0.007062333678954121
},
"xnli_fr": {
"acc": 0.49680638722554893,
"acc_stderr": 0.00706456831954508
},
"xnli_hi": {
"acc": 0.46506986027944114,
"acc_stderr": 0.007047451825220883
},
"xnli_ru": {
"acc": 0.4305389221556886,
"acc_stderr": 0.006996208063220089
},
"xnli_sw": {
"acc": 0.37924151696606784,
"acc_stderr": 0.006855572898852684
},
"xnli_th": {
"acc": 0.3499001996007984,
"acc_stderr": 0.00673886250800537
},
"xnli_tr": {
"acc": 0.3508982035928144,
"acc_stderr": 0.00674328417575373
},
"xnli_ur": {
"acc": 0.42095808383233535,
"acc_stderr": 0.006975878576227378
},
"xnli_vi": {
"acc": 0.47045908183632734,
"acc_stderr": 0.007052371383794704
},
"xnli_zh": {
"acc": 0.35429141716566864,
"acc_stderr": 0.006758076124936785
}
},
"versions": {
"xnli_ar": 0,
"xnli_bg": 0,
"xnli_de": 0,
"xnli_el": 0,
"xnli_en": 0,
"xnli_es": 0,
"xnli_fr": 0,
"xnli_hi": 0,
"xnli_ru": 0,
"xnli_sw": 0,
"xnli_th": 0,
"xnli_tr": 0,
"xnli_ur": 0,
"xnli_vi": 0,
"xnli_zh": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=bigscience/bloom-7b1",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/bloom/bloom-7b1/bloom-7b1_xstory_cloze_0-shot.json deleted 100644 → 0
View file @ e37698df
{
"results": {
"xstory_cloze_ru": {
"acc": 0.5274652547981469,
"acc_stderr": 0.012847698270388222
},
"xstory_cloze_eu": {
"acc": 0.57180675049636,
"acc_stderr": 0.012733742799515155
},
"xstory_cloze_en": {
"acc": 0.7074784910655195,
"acc_stderr": 0.011707038572975033
},
"xstory_cloze_ar": {
"acc": 0.585704831237591,
"acc_stderr": 0.012676689821720669
},
"xstory_cloze_es": {
"acc": 0.6611515552614163,
"acc_stderr": 0.012180490758739058
},
"xstory_cloze_hi": {
"acc": 0.6055592322964924,
"acc_stderr": 0.01257710651393614
},
"xstory_cloze_my": {
"acc": 0.48974189278623426,
"acc_stderr": 0.012864417047980468
},
"xstory_cloze_sw": {
"acc": 0.5393778954334878,
"acc_stderr": 0.012827159238891916
},
"xstory_cloze_zh": {
"acc": 0.6187954996690933,
"acc_stderr": 0.01249867885093408
},
"xstory_cloze_id": {
"acc": 0.6446062210456651,
"acc_stderr": 0.01231724793041837
},
"xstory_cloze_te": {
"acc": 0.5744540039708802,
"acc_stderr": 0.012723670419166324
}
},
"versions": {
"xstory_cloze_ru": 0,
"xstory_cloze_eu": 0,
"xstory_cloze_en": 0,
"xstory_cloze_ar": 0,
"xstory_cloze_es": 0,
"xstory_cloze_hi": 0,
"xstory_cloze_my": 0,
"xstory_cloze_sw": 0,
"xstory_cloze_zh": 0,
"xstory_cloze_id": 0,
"xstory_cloze_te": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=bigscience/bloom-7b1",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/bloom/bloom-7b1/bloom-7b1_xwinograd_0-shot.json deleted 100644 → 0
View file @ e37698df
{
"results": {
"xwinograd_fr": {
"acc": 0.7108433734939759,
"acc_stderr": 0.050066428050419186
},
"xwinograd_ru": {
"acc": 0.5682539682539682,
"acc_stderr": 0.027952495861671634
},
"xwinograd_en": {
"acc": 0.821505376344086,
"acc_stderr": 0.00794327709606643
},
"xwinograd_pt": {
"acc": 0.7680608365019012,
"acc_stderr": 0.026075593860304693
},
"xwinograd_jp": {
"acc": 0.5849843587069864,
"acc_stderr": 0.015919213413834392
},
"xwinograd_zh": {
"acc": 0.7440476190476191,
"acc_stderr": 0.019457899684028012
}
},
"versions": {
"xwinograd_fr": 0,
"xwinograd_ru": 0,
"xwinograd_en": 0,
"xwinograd_pt": 0,
"xwinograd_jp": 0,
"xwinograd_zh": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=bigscience/bloom-7b1",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/llama/llama-13B/README.md deleted 100644 → 0
View file @ e37698df
This diff is collapsed.
results/llama/llama-13B/llama-13B_arithmetic_5-shot.json deleted 100644 → 0
View file @ e37698df
{
"results": {
"arithmetic_2dm": {
"acc": 0.0,
"acc_stderr": 0.0
},
"arithmetic_5ds": {
"acc": 0.0,
"acc_stderr": 0.0
},
"arithmetic_4da": {
"acc": 0.0,
"acc_stderr": 0.0
},
"arithmetic_3da": {
"acc": 0.0,
"acc_stderr": 0.0
},
"arithmetic_4ds": {
"acc": 0.0,
"acc_stderr": 0.0
},
"arithmetic_5da": {
"acc": 0.0,
"acc_stderr": 0.0
},
"arithmetic_2da": {
"acc": 0.0,
"acc_stderr": 0.0
},
"arithmetic_3ds": {
"acc": 0.0,
"acc_stderr": 0.0
},
"arithmetic_1dc": {
"acc": 0.0,
"acc_stderr": 0.0
},
"arithmetic_2ds": {
"acc": 0.0,
"acc_stderr": 0.0
}
},
"versions": {
"arithmetic_2dm": 0,
"arithmetic_5ds": 0,
"arithmetic_4da": 0,
"arithmetic_3da": 0,
"arithmetic_4ds": 0,
"arithmetic_5da": 0,
"arithmetic_2da": 0,
"arithmetic_3ds": 0,
"arithmetic_1dc": 0,
"arithmetic_2ds": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/13B,use_accelerate=True",
"num_fewshot": 5,
"batch_size": "auto",
"device": "cuda:0",
"no_cache": false,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/llama/llama-13B/llama-13B_bbh_3-shot.json deleted 100644 → 0
View file @ e37698df
This diff is collapsed.
results/llama/llama-13B/llama-13B_blimp_0-shot.json deleted 100644 → 0
View file @ e37698df
This diff is collapsed.
results/llama/llama-13B/llama-13B_common_sense_reasoning_0-shot.json deleted 100644 → 0
View file @ e37698df
{
"results": {
"arc_challenge": {
"acc": 0.439419795221843,
"acc_stderr": 0.014503747823580122,
"acc_norm": 0.4462457337883959,
"acc_norm_stderr": 0.014526705548539982
},
"prost": {
"acc": 0.2688941076003416,
"acc_stderr": 0.0032393206239968247,
"acc_norm": 0.3052412467976089,
"acc_norm_stderr": 0.003364432149066356
},
"swag": {
"acc": 0.5673298010596821,
"acc_stderr": 0.003502894135944166,
"acc_norm": 0.6934919524142757,
"acc_norm_stderr": 0.0032596605453371346
},
"arc_easy": {
"acc": 0.7457912457912458,
"acc_stderr": 0.008934537681141528,
"acc_norm": 0.5989057239057239,
"acc_norm_stderr": 0.010057051106534378
},
"boolq": {
"acc": 0.6850152905198776,
"acc_stderr": 0.00812432724981665
},
"wsc273": {
"acc": 0.8608058608058609,
"acc_stderr": 0.020988366070851
},
"mc_taco": {
"em": 0.10960960960960961,
"f1": 0.4753174430074593
},
"piqa": {
"acc": 0.7883569096844396,
"acc_stderr": 0.009530351270479397,
"acc_norm": 0.7910772578890098,
"acc_norm_stderr": 0.009485227030105093
},
"hellaswag": {
"acc": 0.5910177255526787,
"acc_stderr": 0.004906411984476791,
"acc_norm": 0.7623979286994622,
"acc_norm_stderr": 0.004247442237702478
},
"winogrande": {
"acc": 0.7016574585635359,
"acc_stderr": 0.012858885010030434
},
"copa": {
"acc": 0.9,
"acc_stderr": 0.030151134457776348
},
"openbookqa": {
"acc": 0.306,
"acc_stderr": 0.020629569998345403,
"acc_norm": 0.422,
"acc_norm_stderr": 0.022109039310618552
}
},
"versions": {
"arc_challenge": 0,
"prost": 0,
"swag": 0,
"arc_easy": 0,
"boolq": 1,
"wsc273": 0,
"mc_taco": 0,
"piqa": 0,
"hellaswag": 0,
"winogrande": 0,
"copa": 0,
"openbookqa": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/13B,use_accelerate=True",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda:0",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/llama/llama-13B/llama-13B_glue_0-shot.json deleted 100644 → 0
View file @ e37698df
This diff is collapsed.
results/llama/llama-13B/llama-13B_gsm8k_8-shot.json deleted 100644 → 0
View file @ e37698df
{
"results": {
"gsm8k": {
"acc": 0.13570887035633056,
"acc_stderr": 0.009433577908567345
}
},
"versions": {
"gsm8k": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/13B,use_accelerate=True",
"num_fewshot": 8,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/llama/llama-13B/llama-13B_lambada_0-shot.json deleted 100644 → 0
View file @ e37698df
This diff is collapsed.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment