Skip to content

GitLab

Commit 4a0b0d6e authored by lintangsutawika's avatar lintangsutawika
Browse files

Merge branch 'gakada-big-refactor-merge' into big-refactor

parents 6ae376e3 c490f165
Hide whitespace changes
Inline Side-by-side
Showing with 1381 additions and 0 deletions
results/xglm/xglm-4.5B/xglm-4.5B_xcopa_0-shot.json 0 → 100644
View file @ 4a0b0d6e
{
"results": {
"xcopa_vi": {
"acc": 0.66,
"acc_stderr": 0.021206117013673066
},
"xcopa_sw": {
"acc": 0.562,
"acc_stderr": 0.022210326363977413
},
"xcopa_ta": {
"acc": 0.556,
"acc_stderr": 0.02224224437573102
},
"xcopa_qu": {
"acc": 0.5,
"acc_stderr": 0.022383074051792257
},
"xcopa_zh": {
"acc": 0.616,
"acc_stderr": 0.02177236946554719
},
"xcopa_ht": {
"acc": 0.512,
"acc_stderr": 0.02237662679792717
},
"xcopa_it": {
"acc": 0.616,
"acc_stderr": 0.02177236946554719
},
"xcopa_th": {
"acc": 0.552,
"acc_stderr": 0.02226169729227013
},
"xcopa_et": {
"acc": 0.55,
"acc_stderr": 0.022270877485360437
},
"xcopa_tr": {
"acc": 0.572,
"acc_stderr": 0.022149790663861933
},
"xcopa_id": {
"acc": 0.67,
"acc_stderr": 0.021049612166134803
}
},
"versions": {
"xcopa_vi": 0,
"xcopa_sw": 0,
"xcopa_ta": 0,
"xcopa_qu": 0,
"xcopa_zh": 0,
"xcopa_ht": 0,
"xcopa_it": 0,
"xcopa_th": 0,
"xcopa_et": 0,
"xcopa_tr": 0,
"xcopa_id": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=facebook/xglm-4.5B,use_accelerate=True",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/xglm/xglm-4.5B/xglm-4.5B_xnli_0-shot.json 0 → 100644
View file @ 4a0b0d6e
{
"results": {
"xnli_ar": {
"acc": 0.33592814371257484,
"acc_stderr": 0.006673523847107451
},
"xnli_bg": {
"acc": 0.4560878243512974,
"acc_stderr": 0.00703741421319664
},
"xnli_de": {
"acc": 0.47105788423153694,
"acc_stderr": 0.007052867041612212
},
"xnli_el": {
"acc": 0.39840319361277443,
"acc_stderr": 0.006917332484904377
},
"xnli_en": {
"acc": 0.5363273453093812,
"acc_stderr": 0.007046041494004668
},
"xnli_es": {
"acc": 0.47684630738522954,
"acc_stderr": 0.00705713367164529
},
"xnli_fr": {
"acc": 0.47305389221556887,
"acc_stderr": 0.007054445696700673
},
"xnli_hi": {
"acc": 0.4249500998003992,
"acc_stderr": 0.0069846752434985565
},
"xnli_ru": {
"acc": 0.4614770459081836,
"acc_stderr": 0.0070437128985425335
},
"xnli_sw": {
"acc": 0.39580838323353296,
"acc_stderr": 0.006909622580687441
},
"xnli_th": {
"acc": 0.3968063872255489,
"acc_stderr": 0.0069126119585257075
},
"xnli_tr": {
"acc": 0.4485029940119761,
"acc_stderr": 0.007027142119481902
},
"xnli_ur": {
"acc": 0.37465069860279443,
"acc_stderr": 0.006839102174587385
},
"xnli_vi": {
"acc": 0.45868263473053894,
"acc_stderr": 0.007040550421310631
},
"xnli_zh": {
"acc": 0.34770459081836325,
"acc_stderr": 0.006729020595578698
}
},
"versions": {
"xnli_ar": 0,
"xnli_bg": 0,
"xnli_de": 0,
"xnli_el": 0,
"xnli_en": 0,
"xnli_es": 0,
"xnli_fr": 0,
"xnli_hi": 0,
"xnli_ru": 0,
"xnli_sw": 0,
"xnli_th": 0,
"xnli_tr": 0,
"xnli_ur": 0,
"xnli_vi": 0,
"xnli_zh": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=facebook/xglm-4.5B,use_accelerate=True",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/xglm/xglm-4.5B/xglm-4.5B_xstory_cloze_0-shot.json 0 → 100644
View file @ 4a0b0d6e
{
"results": {
"xstory_cloze_es": {
"acc": 0.628060886829914,
"acc_stderr": 0.012437936235202006
},
"xstory_cloze_en": {
"acc": 0.6915949702183984,
"acc_stderr": 0.011884972073313783
},
"xstory_cloze_my": {
"acc": 0.5076108537392455,
"acc_stderr": 0.012865634571114483
},
"xstory_cloze_eu": {
"acc": 0.5373924553275976,
"acc_stderr": 0.012831093347016567
},
"xstory_cloze_hi": {
"acc": 0.5684976836532097,
"acc_stderr": 0.012745810046098417
},
"xstory_cloze_ar": {
"acc": 0.5367306419589676,
"acc_stderr": 0.012832359240206969
},
"xstory_cloze_zh": {
"acc": 0.5817339510258107,
"acc_stderr": 0.012694045150564695
},
"xstory_cloze_sw": {
"acc": 0.5506287227001986,
"acc_stderr": 0.012800991591293387
},
"xstory_cloze_ru": {
"acc": 0.6273990734612839,
"acc_stderr": 0.012442436359907104
},
"xstory_cloze_te": {
"acc": 0.5704831237590999,
"acc_stderr": 0.012738639381353988
},
"xstory_cloze_id": {
"acc": 0.6042356055592323,
"acc_stderr": 0.012584415320654347
}
},
"versions": {
"xstory_cloze_es": 0,
"xstory_cloze_en": 0,
"xstory_cloze_my": 0,
"xstory_cloze_eu": 0,
"xstory_cloze_hi": 0,
"xstory_cloze_ar": 0,
"xstory_cloze_zh": 0,
"xstory_cloze_sw": 0,
"xstory_cloze_ru": 0,
"xstory_cloze_te": 0,
"xstory_cloze_id": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=facebook/xglm-4.5B,use_accelerate=True",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/xglm/xglm-4.5B/xglm-4.5B_xwinograd_0-shot.json 0 → 100644
View file @ 4a0b0d6e
{
"results": {
"xwinograd_fr": {
"acc": 0.6024096385542169,
"acc_stderr": 0.054045178247868135
},
"xwinograd_jp": {
"acc": 0.6266944734098019,
"acc_stderr": 0.015627063885651445
},
"xwinograd_pt": {
"acc": 0.6463878326996197,
"acc_stderr": 0.02953653465680206
},
"xwinograd_ru": {
"acc": 0.6222222222222222,
"acc_stderr": 0.027360632861056396
},
"xwinograd_zh": {
"acc": 0.7063492063492064,
"acc_stderr": 0.020306792341159757
},
"xwinograd_en": {
"acc": 0.7625806451612903,
"acc_stderr": 0.008826385416032015
}
},
"versions": {
"xwinograd_fr": 0,
"xwinograd_jp": 0,
"xwinograd_pt": 0,
"xwinograd_ru": 0,
"xwinograd_zh": 0,
"xwinograd_en": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=facebook/xglm-4.5B,use_accelerate=True",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/xglm/xglm-564M/README.md 0 → 100644
View file @ 4a0b0d6e
# xglm-564M
## xglm-564M_common_sense_reasoning_0-shot.json
| Task |Version| Metric |Value| |Stderr|
|-------------|------:|--------|----:|---|-----:|
|arc_challenge| 0|acc |19.97|± | 1.17|
| | |acc_norm|24.23|± | 1.25|
|arc_easy | 0|acc |45.71|± | 1.02|
| | |acc_norm|41.20|± | 1.01|
|boolq | 1|acc |53.33|± | 0.87|
|copa | 0|acc |69.00|± | 4.65|
|hellaswag | 0|acc |30.78|± | 0.46|
| | |acc_norm|35.20|± | 0.48|
|mc_taco | 0|em |14.04| | |
| | |f1 |40.42| | |
|openbookqa | 0|acc |15.00|± | 1.60|
| | |acc_norm|28.80|± | 2.03|
|piqa | 0|acc |65.13|± | 1.11|
| | |acc_norm|64.85|± | 1.11|
|prost | 0|acc |24.04|± | 0.31|
| | |acc_norm|31.05|± | 0.34|
|swag | 0|acc |41.11|± | 0.35|
| | |acc_norm|54.26|± | 0.35|
|winogrande | 0|acc |52.49|± | 1.40|
|wsc273 | 0|acc |58.61|± | 2.99|
## xglm-564M_gsm8k_8-shot.json
|Task |Version|Metric|Value| |Stderr|
|-----|------:|------|----:|---|-----:|
|gsm8k| 0|acc | 1.21|± | 0.3|
## xglm-564M_lambada_openai_0-shot.json
| Task |Version|Metric| Value | |Stderr|
|--------------------|------:|------|------:|---|-----:|
|lambada_openai | 0|ppl | 28.57|± | 1.03|
| | |acc | 35.94|± | 0.67|
|lambada_openai_cloze| 0|ppl |6898.44|± |322.93|
| | |acc | 0.04|± | 0.03|
## xglm-564M_mathematical_reasoning_few_shot_5-shot.json
| Task |Version| Metric |Value| |Stderr|
|-------------------------|------:|--------|----:|---|-----:|
|drop | 1|em | 0.38|± | 0.06|
| | |f1 | 3.06|± | 0.11|
|gsm8k | 0|acc | 0.83|± | 0.25|
|math_algebra | 1|acc | 0.00|± | 0.00|
|math_counting_and_prob | 1|acc | 0.21|± | 0.21|
|math_geometry | 1|acc | 0.00|± | 0.00|
|math_intermediate_algebra| 1|acc | 0.00|± | 0.00|
|math_num_theory | 1|acc | 0.00|± | 0.00|
|math_prealgebra | 1|acc | 0.23|± | 0.16|
|math_precalc | 1|acc | 0.00|± | 0.00|
|mathqa | 0|acc |21.11|± | 0.75|
| | |acc_norm|21.17|± | 0.75|
## xglm-564M_pawsx_0-shot.json
| Task |Version|Metric|Value| |Stderr|
|--------|------:|------|----:|---|-----:|
|pawsx_de| 0|acc |49.10|± | 1.12|
|pawsx_en| 0|acc |50.65|± | 1.12|
|pawsx_es| 0|acc |52.55|± | 1.12|
|pawsx_fr| 0|acc |50.80|± | 1.12|
|pawsx_ja| 0|acc |44.10|± | 1.11|
|pawsx_ko| 0|acc |46.25|± | 1.12|
|pawsx_zh| 0|acc |47.80|± | 1.12|
## xglm-564M_xcopa_0-shot.json
| Task |Version|Metric|Value| |Stderr|
|--------|------:|------|----:|---|-----:|
|xcopa_et| 0|acc | 55.6|± | 2.22|
|xcopa_ht| 0|acc | 55.0|± | 2.23|
|xcopa_id| 0|acc | 57.2|± | 2.21|
|xcopa_it| 0|acc | 53.8|± | 2.23|
|xcopa_qu| 0|acc | 49.2|± | 2.24|
|xcopa_sw| 0|acc | 53.2|± | 2.23|
|xcopa_ta| 0|acc | 56.2|± | 2.22|
|xcopa_th| 0|acc | 55.2|± | 2.23|
|xcopa_tr| 0|acc | 54.4|± | 2.23|
|xcopa_vi| 0|acc | 58.4|± | 2.21|
|xcopa_zh| 0|acc | 55.6|± | 2.22|
## xglm-564M_xnli_0-shot.json
| Task |Version|Metric|Value| |Stderr|
|-------|------:|------|----:|---|-----:|
|xnli_ar| 0|acc |33.41|± | 0.67|
|xnli_bg| 0|acc |41.30|± | 0.70|
|xnli_de| 0|acc |44.49|± | 0.70|
|xnli_el| 0|acc |39.56|± | 0.69|
|xnli_en| 0|acc |48.28|± | 0.71|
|xnli_es| 0|acc |42.04|± | 0.70|
|xnli_fr| 0|acc |45.49|± | 0.70|
|xnli_hi| 0|acc |38.68|± | 0.69|
|xnli_ru| 0|acc |44.63|± | 0.70|
|xnli_sw| 0|acc |36.07|± | 0.68|
|xnli_th| 0|acc |38.78|± | 0.69|
|xnli_tr| 0|acc |40.20|± | 0.69|
|xnli_ur| 0|acc |34.47|± | 0.67|
|xnli_vi| 0|acc |38.48|± | 0.69|
|xnli_zh| 0|acc |33.51|± | 0.67|
## xglm-564M_xstory_cloze_0-shot.json
| Task |Version|Metric|Value| |Stderr|
|---------------|------:|------|----:|---|-----:|
|xstory_cloze_ar| 0|acc |50.10|± | 1.29|
|xstory_cloze_en| 0|acc |60.56|± | 1.26|
|xstory_cloze_es| 0|acc |55.06|± | 1.28|
|xstory_cloze_eu| 0|acc |53.14|± | 1.28|
|xstory_cloze_hi| 0|acc |52.28|± | 1.29|
|xstory_cloze_id| 0|acc |54.00|± | 1.28|
|xstory_cloze_my| 0|acc |51.49|± | 1.29|
|xstory_cloze_ru| 0|acc |56.19|± | 1.28|
|xstory_cloze_sw| 0|acc |53.08|± | 1.28|
|xstory_cloze_te| 0|acc |55.86|± | 1.28|
|xstory_cloze_zh| 0|acc |53.28|± | 1.28|
## xglm-564M_xwinograd_0-shot.json
| Task |Version|Metric|Value| |Stderr|
|------------|------:|------|----:|---|-----:|
|xwinograd_en| 0|acc |62.62|± | 1.00|
|xwinograd_fr| 0|acc |57.83|± | 5.45|
|xwinograd_jp| 0|acc |54.54|± | 1.61|
|xwinograd_pt| 0|acc |58.56|± | 3.04|
|xwinograd_ru| 0|acc |59.05|± | 2.78|
|xwinograd_zh| 0|acc |65.67|± | 2.12|
results/xglm/xglm-564M/xglm-564M_common_sense_reasoning_0-shot.json 0 → 100644
View file @ 4a0b0d6e
{
"results": {
"copa": {
"acc": 0.69,
"acc_stderr": 0.04648231987117316
},
"mc_taco": {
"em": 0.1403903903903904,
"f1": 0.40416372323956457
},
"prost": {
"acc": 0.24044619982920581,
"acc_stderr": 0.003122204753095043,
"acc_norm": 0.3104718189581554,
"acc_norm_stderr": 0.0033803389886508614
},
"piqa": {
"acc": 0.6512513601741022,
"acc_stderr": 0.011119263056159583,
"acc_norm": 0.6485310119695321,
"acc_norm_stderr": 0.011139207691931193
},
"wsc273": {
"acc": 0.5860805860805861,
"acc_stderr": 0.029864284486117058
},
"winogrande": {
"acc": 0.5248618784530387,
"acc_stderr": 0.01403510288362775
},
"swag": {
"acc": 0.4111266620013996,
"acc_stderr": 0.0034788002208961154,
"acc_norm": 0.5425872238328502,
"acc_norm_stderr": 0.003522245713261821
},
"boolq": {
"acc": 0.5333333333333333,
"acc_stderr": 0.008725599880049204
},
"hellaswag": {
"acc": 0.30780720971917946,
"acc_stderr": 0.004606429684604547,
"acc_norm": 0.3520215096594304,
"acc_norm_stderr": 0.004766245539606642
},
"arc_challenge": {
"acc": 0.19965870307167236,
"acc_stderr": 0.011681625756888688,
"acc_norm": 0.24232081911262798,
"acc_norm_stderr": 0.012521593295800113
},
"arc_easy": {
"acc": 0.45707070707070707,
"acc_stderr": 0.010221897564256037,
"acc_norm": 0.41203703703703703,
"acc_norm_stderr": 0.010099765857562767
},
"openbookqa": {
"acc": 0.15,
"acc_stderr": 0.015984712135164926,
"acc_norm": 0.288,
"acc_norm_stderr": 0.020271503835075214
}
},
"versions": {
"copa": 0,
"mc_taco": 0,
"prost": 0,
"piqa": 0,
"wsc273": 0,
"winogrande": 0,
"swag": 0,
"boolq": 1,
"hellaswag": 0,
"arc_challenge": 0,
"arc_easy": 0,
"openbookqa": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=facebook/xglm-564M,use_accelerate=True",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda:0",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/xglm/xglm-564M/xglm-564M_gsm8k_8-shot.json 0 → 100644
View file @ 4a0b0d6e
{
"results": {
"gsm8k": {
"acc": 0.012130401819560273,
"acc_stderr": 0.003015294242890939
}
},
"versions": {
"gsm8k": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=facebook/xglm-564M,use_accelerate=True",
"num_fewshot": 8,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/xglm/xglm-564M/xglm-564M_lambada_openai_0-shot.json 0 → 100644
View file @ 4a0b0d6e
{
"results": {
"lambada_openai": {
"ppl": 28.56712020387803,
"ppl_stderr": 1.029833227972257,
"acc": 0.35940228992819717,
"acc_stderr": 0.006684904136033
},
"lambada_openai_cloze": {
"ppl": 6898.435163550819,
"ppl_stderr": 322.9340159515305,
"acc": 0.00038812342324859306,
"acc_stderr": 0.00027441806845050347
}
},
"versions": {
"lambada_openai": 0,
"lambada_openai_cloze": 0
},
"config": {
"model": "hf-causal",
"model_args": "pretrained=facebook/xglm-564M",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/xglm/xglm-564M/xglm-564M_mathematical_reasoning_few_shot_5-shot.json 0 → 100644
View file @ 4a0b0d6e
{
"results": {
"gsm8k": {
"acc": 0.008339651250947688,
"acc_stderr": 0.0025049422268605386
},
"math_prealgebra": {
"acc": 0.002296211251435132,
"acc_stderr": 0.001622733136934626
},
"math_precalc": {
"acc": 0.0,
"acc_stderr": 0.0
},
"math_geometry": {
"acc": 0.0,
"acc_stderr": 0.0
},
"math_num_theory": {
"acc": 0.0,
"acc_stderr": 0.0
},
"drop": {
"em": 0.0037751677852348995,
"em_stderr": 0.0006280387809484274,
"f1": 0.03062604865771817,
"f1_stderr": 0.0011423873658349746
},
"math_algebra": {
"acc": 0.0,
"acc_stderr": 0.0
},
"math_intermediate_algebra": {
"acc": 0.0,
"acc_stderr": 0.0
},
"math_counting_and_prob": {
"acc": 0.002109704641350211,
"acc_stderr": 0.002109704641350211
},
"mathqa": {
"acc": 0.21105527638190955,
"acc_stderr": 0.007470023801451704,
"acc_norm": 0.2117252931323283,
"acc_norm_stderr": 0.007478693896187993
}
},
"versions": {
"gsm8k": 0,
"math_prealgebra": 1,
"math_precalc": 1,
"math_geometry": 1,
"math_num_theory": 1,
"drop": 1,
"math_algebra": 1,
"math_intermediate_algebra": 1,
"mathqa": 0,
"math_counting_and_prob": 1
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=facebook/xglm-564M,use_accelerate=True",
"num_fewshot": 5,
"batch_size": "auto",
"device": "cuda:0",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/xglm/xglm-564M/xglm-564M_pawsx_0-shot.json 0 → 100644
View file @ 4a0b0d6e
{
"results": {
"pawsx_de": {
"acc": 0.491,
"acc_stderr": 0.01118132420626029
},
"pawsx_ko": {
"acc": 0.4625,
"acc_stderr": 0.011151639095992287
},
"pawsx_en": {
"acc": 0.5065,
"acc_stderr": 0.0111821910061423
},
"pawsx_es": {
"acc": 0.5255,
"acc_stderr": 0.011168582883330072
},
"pawsx_fr": {
"acc": 0.508,
"acc_stderr": 0.011181704488030002
},
"pawsx_ja": {
"acc": 0.441,
"acc_stderr": 0.011105006104468738
},
"pawsx_zh": {
"acc": 0.478,
"acc_stderr": 0.011172305500884872
}
},
"versions": {
"pawsx_de": 0,
"pawsx_ko": 0,
"pawsx_en": 0,
"pawsx_es": 0,
"pawsx_fr": 0,
"pawsx_ja": 0,
"pawsx_zh": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=facebook/xglm-564M,use_accelerate=True",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/xglm/xglm-564M/xglm-564M_xcopa_0-shot.json 0 → 100644
View file @ 4a0b0d6e
{
"results": {
"xcopa_id": {
"acc": 0.572,
"acc_stderr": 0.02214979066386193
},
"xcopa_th": {
"acc": 0.552,
"acc_stderr": 0.02226169729227013
},
"xcopa_it": {
"acc": 0.538,
"acc_stderr": 0.022318338119870534
},
"xcopa_ht": {
"acc": 0.55,
"acc_stderr": 0.022270877485360437
},
"xcopa_tr": {
"acc": 0.544,
"acc_stderr": 0.02229623834840705
},
"xcopa_zh": {
"acc": 0.556,
"acc_stderr": 0.02224224437573102
},
"xcopa_vi": {
"acc": 0.584,
"acc_stderr": 0.02206494331392886
},
"xcopa_sw": {
"acc": 0.532,
"acc_stderr": 0.022337186479044296
},
"xcopa_ta": {
"acc": 0.562,
"acc_stderr": 0.022210326363977413
},
"xcopa_qu": {
"acc": 0.492,
"acc_stderr": 0.022380208834928028
},
"xcopa_et": {
"acc": 0.556,
"acc_stderr": 0.02224224437573102
}
},
"versions": {
"xcopa_id": 0,
"xcopa_th": 0,
"xcopa_it": 0,
"xcopa_ht": 0,
"xcopa_tr": 0,
"xcopa_zh": 0,
"xcopa_vi": 0,
"xcopa_sw": 0,
"xcopa_ta": 0,
"xcopa_qu": 0,
"xcopa_et": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=facebook/xglm-564M,use_accelerate=True",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/xglm/xglm-564M/xglm-564M_xnli_0-shot.json 0 → 100644
View file @ 4a0b0d6e
{
"results": {
"xnli_de": {
"acc": 0.44491017964071855,
"acc_stderr": 0.007021700267328247
},
"xnli_sw": {
"acc": 0.36067864271457084,
"acc_stderr": 0.006784913322438225
},
"xnli_ar": {
"acc": 0.3341317365269461,
"acc_stderr": 0.006664652441694282
},
"xnli_ru": {
"acc": 0.44630738522954094,
"acc_stderr": 0.007023860641475025
},
"xnli_fr": {
"acc": 0.4548902195608782,
"acc_stderr": 0.007035901825327945
},
"xnli_zh": {
"acc": 0.33512974051896205,
"acc_stderr": 0.0066695943825036295
},
"xnli_th": {
"acc": 0.38782435129740517,
"acc_stderr": 0.006884621148080227
},
"xnli_el": {
"acc": 0.39560878243512976,
"acc_stderr": 0.0069090210962589015
},
"xnli_ur": {
"acc": 0.3447105788423154,
"acc_stderr": 0.006715345603576113
},
"xnli_hi": {
"acc": 0.38682634730538923,
"acc_stderr": 0.00688135955102393
},
"xnli_es": {
"acc": 0.42035928143712575,
"acc_stderr": 0.006974518775334314
},
"xnli_tr": {
"acc": 0.40199600798403196,
"acc_stderr": 0.006927673254973623
},
"xnli_bg": {
"acc": 0.41297405189620756,
"acc_stderr": 0.006956880058406987
},
"xnli_vi": {
"acc": 0.3848303393213573,
"acc_stderr": 0.0068747449435859675
},
"xnli_en": {
"acc": 0.48283433133732534,
"acc_stderr": 0.0070605478229289675
}
},
"versions": {
"xnli_de": 0,
"xnli_sw": 0,
"xnli_ar": 0,
"xnli_ru": 0,
"xnli_fr": 0,
"xnli_zh": 0,
"xnli_th": 0,
"xnli_el": 0,
"xnli_ur": 0,
"xnli_hi": 0,
"xnli_es": 0,
"xnli_tr": 0,
"xnli_bg": 0,
"xnli_vi": 0,
"xnli_en": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=facebook/xglm-564M,use_accelerate=True",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/xglm/xglm-564M/xglm-564M_xstory_cloze_0-shot.json 0 → 100644
View file @ 4a0b0d6e
{
"results": {
"xstory_cloze_ar": {
"acc": 0.500992720052945,
"acc_stderr": 0.012867099955422925
},
"xstory_cloze_id": {
"acc": 0.5400397088021178,
"acc_stderr": 0.012825802370083988
},
"xstory_cloze_sw": {
"acc": 0.5307743216412971,
"acc_stderr": 0.01284273034058578
},
"xstory_cloze_en": {
"acc": 0.6055592322964924,
"acc_stderr": 0.012577106513936133
},
"xstory_cloze_te": {
"acc": 0.5585704831237591,
"acc_stderr": 0.012778538985880637
},
"xstory_cloze_zh": {
"acc": 0.5327597617471873,
"acc_stderr": 0.012839477563855915
},
"xstory_cloze_my": {
"acc": 0.514890800794176,
"acc_stderr": 0.012861417842074004
},
"xstory_cloze_hi": {
"acc": 0.5228325612177366,
"acc_stderr": 0.01285370238487085
},
"xstory_cloze_ru": {
"acc": 0.5618795499669094,
"acc_stderr": 0.01276820661627776
},
"xstory_cloze_es": {
"acc": 0.5506287227001986,
"acc_stderr": 0.012800991591293383
},
"xstory_cloze_eu": {
"acc": 0.5314361350099271,
"acc_stderr": 0.012841668760976905
}
},
"versions": {
"xstory_cloze_ar": 0,
"xstory_cloze_id": 0,
"xstory_cloze_sw": 0,
"xstory_cloze_en": 0,
"xstory_cloze_te": 0,
"xstory_cloze_zh": 0,
"xstory_cloze_my": 0,
"xstory_cloze_hi": 0,
"xstory_cloze_ru": 0,
"xstory_cloze_es": 0,
"xstory_cloze_eu": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=facebook/xglm-564M,use_accelerate=True",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/xglm/xglm-564M/xglm-564M_xwinograd_0-shot.json 0 → 100644
View file @ 4a0b0d6e
{
"results": {
"xwinograd_pt": {
"acc": 0.5855513307984791,
"acc_stderr": 0.030434573161228055
},
"xwinograd_zh": {
"acc": 0.6567460317460317,
"acc_stderr": 0.0211700809891982
},
"xwinograd_ru": {
"acc": 0.5904761904761905,
"acc_stderr": 0.02775082824017435
},
"xwinograd_fr": {
"acc": 0.5783132530120482,
"acc_stderr": 0.054534284852951115
},
"xwinograd_en": {
"acc": 0.6262365591397849,
"acc_stderr": 0.01003574358830904
},
"xwinograd_jp": {
"acc": 0.5453597497393118,
"acc_stderr": 0.01608765437474968
}
},
"versions": {
"xwinograd_pt": 0,
"xwinograd_zh": 0,
"xwinograd_ru": 0,
"xwinograd_fr": 0,
"xwinograd_en": 0,
"xwinograd_jp": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=facebook/xglm-564M,use_accelerate=True",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/xglm/xglm-7.5B/README.md 0 → 100644
View file @ 4a0b0d6e
# xglm-7.5B
## xglm-7.5B_common_sense_reasoning_0-shot.json
| Task |Version| Metric |Value| |Stderr|
|-------------|------:|--------|----:|---|-----:|
|arc_challenge| 0|acc |28.75|± | 1.32|
| | |acc_norm|31.91|± | 1.36|
|arc_easy | 0|acc |62.37|± | 0.99|
| | |acc_norm|58.63|± | 1.01|
|boolq | 1|acc |60.18|± | 0.86|
|copa | 0|acc |79.00|± | 4.09|
|hellaswag | 0|acc |45.69|± | 0.50|
| | |acc_norm|61.23|± | 0.49|
|mc_taco | 0|em |13.81| | |
| | |f1 |47.92| | |
|openbookqa | 0|acc |25.40|± | 1.95|
| | |acc_norm|35.80|± | 2.15|
|piqa | 0|acc |73.94|± | 1.02|
| | |acc_norm|74.92|± | 1.01|
|prost | 0|acc |25.89|± | 0.32|
| | |acc_norm|26.36|± | 0.32|
|swag | 0|acc |50.51|± | 0.35|
| | |acc_norm|69.23|± | 0.33|
|winogrande | 0|acc |57.85|± | 1.39|
|wsc273 | 0|acc |75.82|± | 2.60|
## xglm-7.5B_gsm8k_8-shot.json
|Task |Version|Metric|Value| |Stderr|
|-----|------:|------|----:|---|-----:|
|gsm8k| 0|acc | 0.15|± | 0.11|
## xglm-7.5B_mathematical_reasoning_few_shot_5-shot.json
| Task |Version| Metric |Value| |Stderr|
|-------------------------|------:|--------|----:|---|-----:|
|drop | 1|em | 5.42|± | 0.23|
| | |f1 | 8.96|± | 0.26|
|gsm8k | 0|acc | 0.23|± | 0.13|
|math_algebra | 1|acc | 0.00|± | 0.00|
|math_counting_and_prob | 1|acc | 0.00|± | 0.00|
|math_geometry | 1|acc | 0.00|± | 0.00|
|math_intermediate_algebra| 1|acc | 0.00|± | 0.00|
|math_num_theory | 1|acc | 0.00|± | 0.00|
|math_prealgebra | 1|acc | 0.00|± | 0.00|
|math_precalc | 1|acc | 0.00|± | 0.00|
|mathqa | 0|acc |23.99|± | 0.78|
| | |acc_norm|23.52|± | 0.78|
## xglm-7.5B_pawsx_0-shot.json
| Task |Version|Metric|Value| |Stderr|
|--------|------:|------|----:|---|-----:|
|pawsx_de| 0|acc |55.90|± | 1.11|
|pawsx_en| 0|acc |58.85|± | 1.10|
|pawsx_es| 0|acc |52.80|± | 1.12|
|pawsx_fr| 0|acc |51.80|± | 1.12|
|pawsx_ja| 0|acc |52.00|± | 1.12|
|pawsx_ko| 0|acc |45.95|± | 1.11|
|pawsx_zh| 0|acc |51.30|± | 1.12|
## xglm-7.5B_xcopa_0-shot.json
| Task |Version|Metric|Value| |Stderr|
|--------|------:|------|----:|---|-----:|
|xcopa_et| 0|acc | 61.2|± | 2.18|
|xcopa_ht| 0|acc | 57.4|± | 2.21|
|xcopa_id| 0|acc | 69.4|± | 2.06|
|xcopa_it| 0|acc | 63.6|± | 2.15|
|xcopa_qu| 0|acc | 48.8|± | 2.24|
|xcopa_sw| 0|acc | 60.0|± | 2.19|
|xcopa_ta| 0|acc | 54.4|± | 2.23|
|xcopa_th| 0|acc | 59.4|± | 2.20|
|xcopa_tr| 0|acc | 58.4|± | 2.21|
|xcopa_vi| 0|acc | 70.2|± | 2.05|
|xcopa_zh| 0|acc | 63.8|± | 2.15|
## xglm-7.5B_xnli_0-shot.json
| Task |Version|Metric|Value| |Stderr|
|-------|------:|------|----:|---|-----:|
|xnli_ar| 0|acc |33.37|± | 0.67|
|xnli_bg| 0|acc |44.89|± | 0.70|
|xnli_de| 0|acc |48.98|± | 0.71|
|xnli_el| 0|acc |40.66|± | 0.69|
|xnli_en| 0|acc |53.85|± | 0.70|
|xnli_es| 0|acc |47.70|± | 0.71|
|xnli_fr| 0|acc |46.95|± | 0.71|
|xnli_hi| 0|acc |47.21|± | 0.71|
|xnli_ru| 0|acc |46.33|± | 0.70|
|xnli_sw| 0|acc |45.83|± | 0.70|
|xnli_th| 0|acc |43.71|± | 0.70|
|xnli_tr| 0|acc |46.27|± | 0.70|
|xnli_ur| 0|acc |42.10|± | 0.70|
|xnli_vi| 0|acc |46.33|± | 0.70|
|xnli_zh| 0|acc |35.37|± | 0.68|
## xglm-7.5B_xstory_cloze_0-shot.json
| Task |Version|Metric|Value| |Stderr|
|---------------|------:|------|----:|---|-----:|
|xstory_cloze_ar| 0|acc |56.19|± | 1.28|
|xstory_cloze_en| 0|acc |69.82|± | 1.18|
|xstory_cloze_es| 0|acc |64.06|± | 1.23|
|xstory_cloze_eu| 0|acc |57.71|± | 1.27|
|xstory_cloze_hi| 0|acc |58.77|± | 1.27|
|xstory_cloze_id| 0|acc |62.94|± | 1.24|
|xstory_cloze_my| 0|acc |57.11|± | 1.27|
|xstory_cloze_ru| 0|acc |63.53|± | 1.24|
|xstory_cloze_sw| 0|acc |59.30|± | 1.26|
|xstory_cloze_te| 0|acc |60.23|± | 1.26|
|xstory_cloze_zh| 0|acc |58.90|± | 1.27|
## xglm-7.5B_xwinograd_0-shot.json
| Task |Version|Metric|Value| |Stderr|
|------------|------:|------|----:|---|-----:|
|xwinograd_en| 0|acc |79.48|± | 0.84|
|xwinograd_fr| 0|acc |65.06|± | 5.27|
|xwinograd_jp| 0|acc |64.96|± | 1.54|
|xwinograd_pt| 0|acc |67.30|± | 2.90|
|xwinograd_ru| 0|acc |63.17|± | 2.72|
|xwinograd_zh| 0|acc |72.82|± | 1.98|
results/xglm/xglm-7.5B/xglm-7.5B_common_sense_reasoning_0-shot.json 0 → 100644
View file @ 4a0b0d6e
{
"results": {
"openbookqa": {
"acc": 0.254,
"acc_stderr": 0.019486596801643385,
"acc_norm": 0.358,
"acc_norm_stderr": 0.02146143486285912
},
"winogrande": {
"acc": 0.5785319652722968,
"acc_stderr": 0.0138780723774976
},
"arc_easy": {
"acc": 0.6237373737373737,
"acc_stderr": 0.009940646221513789,
"acc_norm": 0.5862794612794613,
"acc_norm_stderr": 0.010105878530238135
},
"copa": {
"acc": 0.79,
"acc_stderr": 0.040936018074033256
},
"mc_taco": {
"em": 0.13813813813813813,
"f1": 0.479152974631639
},
"wsc273": {
"acc": 0.7582417582417582,
"acc_stderr": 0.02596031999685269
},
"hellaswag": {
"acc": 0.45688109938259314,
"acc_stderr": 0.004971192387202445,
"acc_norm": 0.6123282214698267,
"acc_norm_stderr": 0.004862232790041574
},
"boolq": {
"acc": 0.6018348623853211,
"acc_stderr": 0.008561755594317445
},
"swag": {
"acc": 0.505148455463361,
"acc_stderr": 0.003534904635576977,
"acc_norm": 0.692292312306308,
"acc_norm_stderr": 0.003263207195550976
},
"piqa": {
"acc": 0.7393906420021763,
"acc_stderr": 0.010241826155811627,
"acc_norm": 0.749183895538629,
"acc_norm_stderr": 0.010113869547069046
},
"prost": {
"acc": 0.2588599487617421,
"acc_stderr": 0.0032000423309913543,
"acc_norm": 0.26361016225448336,
"acc_norm_stderr": 0.0032189046983713983
},
"arc_challenge": {
"acc": 0.28754266211604096,
"acc_stderr": 0.013226719056266129,
"acc_norm": 0.3191126279863481,
"acc_norm_stderr": 0.013621696119173304
}
},
"versions": {
"openbookqa": 0,
"winogrande": 0,
"arc_easy": 0,
"copa": 0,
"mc_taco": 0,
"wsc273": 0,
"hellaswag": 0,
"boolq": 1,
"swag": 0,
"piqa": 0,
"prost": 0,
"arc_challenge": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=facebook/xglm-7.5B,use_accelerate=True",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda:0",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/xglm/xglm-7.5B/xglm-7.5B_gsm8k_8-shot.json 0 → 100644
View file @ 4a0b0d6e
{
"results": {
"gsm8k": {
"acc": 0.001516300227445034,
"acc_stderr": 0.0010717793485492655
}
},
"versions": {
"gsm8k": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=facebook/xglm-7.5B,use_accelerate=True",
"num_fewshot": 8,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/xglm/xglm-7.5B/xglm-7.5B_mathematical_reasoning_few_shot_5-shot.json 0 → 100644
View file @ 4a0b0d6e
{
"results": {
"math_num_theory": {
"acc": 0.0,
"acc_stderr": 0.0
},
"gsm8k": {
"acc": 0.002274450341167551,
"acc_stderr": 0.0013121578148674316
},
"math_geometry": {
"acc": 0.0,
"acc_stderr": 0.0
},
"drop": {
"em": 0.05421560402684564,
"em_stderr": 0.002318984649948223,
"f1": 0.08962458053691245,
"f1_stderr": 0.0026401926224488034
},
"math_prealgebra": {
"acc": 0.0,
"acc_stderr": 0.0
},
"math_counting_and_prob": {
"acc": 0.0,
"acc_stderr": 0.0
},
"math_precalc": {
"acc": 0.0,
"acc_stderr": 0.0
},
"math_intermediate_algebra": {
"acc": 0.0,
"acc_stderr": 0.0
},
"math_algebra": {
"acc": 0.0,
"acc_stderr": 0.0
},
"mathqa": {
"acc": 0.23986599664991626,
"acc_stderr": 0.007816818250028128,
"acc_norm": 0.23517587939698492,
"acc_norm_stderr": 0.0077638612776946255
}
},
"versions": {
"math_num_theory": 1,
"gsm8k": 0,
"math_geometry": 1,
"drop": 1,
"math_prealgebra": 1,
"math_counting_and_prob": 1,
"math_precalc": 1,
"math_intermediate_algebra": 1,
"math_algebra": 1,
"mathqa": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=facebook/xglm-7.5B,use_accelerate=True",
"num_fewshot": 5,
"batch_size": "auto",
"device": "cuda:0",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/xglm/xglm-7.5B/xglm-7.5B_pawsx_0-shot.json 0 → 100644
View file @ 4a0b0d6e
{
"results": {
"pawsx_en": {
"acc": 0.5885,
"acc_stderr": 0.011006563824537298
},
"pawsx_es": {
"acc": 0.528,
"acc_stderr": 0.011165587094621537
},
"pawsx_fr": {
"acc": 0.518,
"acc_stderr": 0.011175886999478619
},
"pawsx_zh": {
"acc": 0.513,
"acc_stderr": 0.01117935548207038
},
"pawsx_ja": {
"acc": 0.52,
"acc_stderr": 0.011174185930778312
},
"pawsx_de": {
"acc": 0.559,
"acc_stderr": 0.011105006104468736
},
"pawsx_ko": {
"acc": 0.4595,
"acc_stderr": 0.011146389370464362
}
},
"versions": {
"pawsx_en": 0,
"pawsx_es": 0,
"pawsx_fr": 0,
"pawsx_zh": 0,
"pawsx_ja": 0,
"pawsx_de": 0,
"pawsx_ko": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=facebook/xglm-7.5B,use_accelerate=True",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
results/xglm/xglm-7.5B/xglm-7.5B_xcopa_0-shot.json 0 → 100644
View file @ 4a0b0d6e
{
"results": {
"xcopa_et": {
"acc": 0.612,
"acc_stderr": 0.021814300984787635
},
"xcopa_th": {
"acc": 0.594,
"acc_stderr": 0.02198396209008634
},
"xcopa_qu": {
"acc": 0.488,
"acc_stderr": 0.02237662679792717
},
"xcopa_ta": {
"acc": 0.544,
"acc_stderr": 0.02229623834840705
},
"xcopa_zh": {
"acc": 0.638,
"acc_stderr": 0.0215136625275824
},
"xcopa_vi": {
"acc": 0.702,
"acc_stderr": 0.02047511809298897
},
"xcopa_sw": {
"acc": 0.6,
"acc_stderr": 0.021930844120728505
},
"xcopa_it": {
"acc": 0.636,
"acc_stderr": 0.021539170637317685
},
"xcopa_tr": {
"acc": 0.584,
"acc_stderr": 0.022064943313928848
},
"xcopa_id": {
"acc": 0.694,
"acc_stderr": 0.0206295699983454
},
"xcopa_ht": {
"acc": 0.574,
"acc_stderr": 0.022136577335085637
}
},
"versions": {
"xcopa_et": 0,
"xcopa_th": 0,
"xcopa_qu": 0,
"xcopa_ta": 0,
"xcopa_zh": 0,
"xcopa_vi": 0,
"xcopa_sw": 0,
"xcopa_it": 0,
"xcopa_tr": 0,
"xcopa_id": 0,
"xcopa_ht": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=facebook/xglm-7.5B,use_accelerate=True",
"num_fewshot": 0,
"batch_size": "auto",
"device": "cuda",
"no_cache": true,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment