add bloom, xglm and llama results

results/llama/llama-7B/llama-7B_xnli_0-shot.json

+{
+  "results": {
+    "xnli_hi": {
+      "acc": 0.36107784431137724,
+      "acc_stderr": 0.006786547275949383
+    },
+    "xnli_vi": {
+      "acc": 0.35588822355289423,
+      "acc_stderr": 0.006764908277770048
+    },
+    "xnli_fr": {
+      "acc": 0.4375249500998004,
+      "acc_stderr": 0.007009346470228512
+    },
+    "xnli_tr": {
+      "acc": 0.35588822355289423,
+      "acc_stderr": 0.006764908277770052
+    },
+    "xnli_el": {
+      "acc": 0.34930139720558884,
+      "acc_stderr": 0.00673619394518908
+    },
+    "xnli_zh": {
+      "acc": 0.36227544910179643,
+      "acc_stderr": 0.00679141867023231
+    },
+    "xnli_es": {
+      "acc": 0.406187624750499,
+      "acc_stderr": 0.006939248658213527
+    },
+    "xnli_sw": {
+      "acc": 0.33712574850299404,
+      "acc_stderr": 0.006679377985812511
+    },
+    "xnli_th": {
+      "acc": 0.34510978043912177,
+      "acc_stderr": 0.0067171859376095565
+    },
+    "xnli_bg": {
+      "acc": 0.36986027944111777,
+      "acc_stderr": 0.006821215321725033
+    },
+    "xnli_ru": {
+      "acc": 0.3936127744510978,
+      "acc_stderr": 0.006902939997053364
+    },
+    "xnli_ur": {
+      "acc": 0.3339321357285429,
+      "acc_stderr": 0.006663660032909966
+    },
+    "xnli_ar": {
+      "acc": 0.33572854291417165,
+      "acc_stderr": 0.006672543485924257
+    },
+    "xnli_en": {
+      "acc": 0.5105788423153692,
+      "acc_stderr": 0.007063131001466571
+    },
+    "xnli_de": {
+      "acc": 0.4477045908183633,
+      "acc_stderr": 0.007025964880868372
+    }
+  },
+  "versions": {
+    "xnli_hi": 0,
+    "xnli_vi": 0,
+    "xnli_fr": 0,
+    "xnli_tr": 0,
+    "xnli_el": 0,
+    "xnli_zh": 0,
+    "xnli_es": 0,
+    "xnli_sw": 0,
+    "xnli_th": 0,
+    "xnli_bg": 0,
+    "xnli_ru": 0,
+    "xnli_ur": 0,
+    "xnli_ar": 0,
+    "xnli_en": 0,
+    "xnli_de": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/llama/llama-7B/llama-7B_xstory_cloze_0-shot.json

+{
+  "results": {
+    "xstory_cloze_id": {
+      "acc": 0.5208471211118465,
+      "acc_stderr": 0.012855936282881265
+    },
+    "xstory_cloze_en": {
+      "acc": 0.7478491065519524,
+      "acc_stderr": 0.011175031901561881
+    },
+    "xstory_cloze_ar": {
+      "acc": 0.4831237590999338,
+      "acc_stderr": 0.012859793919977602
+    },
+    "xstory_cloze_sw": {
+      "acc": 0.5036399735274653,
+      "acc_stderr": 0.012866784348289232
+    },
+    "xstory_cloze_my": {
+      "acc": 0.4870946393117141,
+      "acc_stderr": 0.012862838605728472
+    },
+    "xstory_cloze_hi": {
+      "acc": 0.5268034414295168,
+      "acc_stderr": 0.012848623899505763
+    },
+    "xstory_cloze_te": {
+      "acc": 0.528788881535407,
+      "acc_stderr": 0.012845779070719484
+    },
+    "xstory_cloze_es": {
+      "acc": 0.6512243547319656,
+      "acc_stderr": 0.012264502012981189
+    },
+    "xstory_cloze_zh": {
+      "acc": 0.5433487756452681,
+      "acc_stderr": 0.012818676452481954
+    },
+    "xstory_cloze_eu": {
+      "acc": 0.500992720052945,
+      "acc_stderr": 0.012867099955422926
+    },
+    "xstory_cloze_ru": {
+      "acc": 0.613500992720053,
+      "acc_stderr": 0.012531219943771484
+    }
+  },
+  "versions": {
+    "xstory_cloze_id": 0,
+    "xstory_cloze_en": 0,
+    "xstory_cloze_ar": 0,
+    "xstory_cloze_sw": 0,
+    "xstory_cloze_my": 0,
+    "xstory_cloze_hi": 0,
+    "xstory_cloze_te": 0,
+    "xstory_cloze_es": 0,
+    "xstory_cloze_zh": 0,
+    "xstory_cloze_eu": 0,
+    "xstory_cloze_ru": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/llama/llama-7B/llama-7B_xwinograd_0-shot.json

+{
+  "results": {
+    "xwinograd_en": {
+      "acc": 0.8494623655913979,
+      "acc_stderr": 0.007417824483418801
+    },
+    "xwinograd_fr": {
+      "acc": 0.7228915662650602,
+      "acc_stderr": 0.049425892997830914
+    },
+    "xwinograd_jp": {
+      "acc": 0.5891553701772679,
+      "acc_stderr": 0.01589538213585439
+    },
+    "xwinograd_zh": {
+      "acc": 0.6369047619047619,
+      "acc_stderr": 0.02144191312603829
+    },
+    "xwinograd_ru": {
+      "acc": 0.6444444444444445,
+      "acc_stderr": 0.02701354947053879
+    },
+    "xwinograd_pt": {
+      "acc": 0.7072243346007605,
+      "acc_stderr": 0.02811223926963042
+    }
+  },
+  "versions": {
+    "xwinograd_en": 0,
+    "xwinograd_fr": 0,
+    "xwinograd_jp": 0,
+    "xwinograd_zh": 0,
+    "xwinograd_ru": 0,
+    "xwinograd_pt": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/xglm/xglm-1.7B/xglm-1.7B_common_sense_reasoning_0-shot.json

+{
+  "results": {
+    "copa": {
+      "acc": 0.68,
+      "acc_stderr": 0.046882617226215034
+    },
+    "piqa": {
+      "acc": 0.6969532100108814,
+      "acc_stderr": 0.010722648689531515,
+      "acc_norm": 0.7034820457018498,
+      "acc_norm_stderr": 0.010656078922661134
+    },
+    "prost": {
+      "acc": 0.22694278394534587,
+      "acc_stderr": 0.003060110855833208,
+      "acc_norm": 0.27209649871904357,
+      "acc_norm_stderr": 0.0032514084657504338
+    },
+    "arc_easy": {
+      "acc": 0.5361952861952862,
+      "acc_stderr": 0.01023286555034672,
+      "acc_norm": 0.47895622895622897,
+      "acc_norm_stderr": 0.01025069260202258
+    },
+    "hellaswag": {
+      "acc": 0.3617805218084047,
+      "acc_stderr": 0.004795337009118189,
+      "acc_norm": 0.45797649870543716,
+      "acc_norm_stderr": 0.004972126523031943
+    },
+    "mc_taco": {
+      "em": 0.12912912912912913,
+      "f1": 0.34519977153598014
+    },
+    "winogrande": {
+      "acc": 0.5493291239147593,
+      "acc_stderr": 0.013983928869040239
+    },
+    "wsc273": {
+      "acc": 0.6813186813186813,
+      "acc_stderr": 0.02825328818739863
+    },
+    "swag": {
+      "acc": 0.4596621013695891,
+      "acc_stderr": 0.0035235690445916223,
+      "acc_norm": 0.6219134259722083,
+      "acc_norm_stderr": 0.003428398656668824
+    },
+    "boolq": {
+      "acc": 0.5856269113149847,
+      "acc_stderr": 0.00861586377642113
+    },
+    "openbookqa": {
+      "acc": 0.17,
+      "acc_stderr": 0.016815633531393426,
+      "acc_norm": 0.298,
+      "acc_norm_stderr": 0.02047511809298897
+    },
+    "arc_challenge": {
+      "acc": 0.2098976109215017,
+      "acc_stderr": 0.011900548748047446,
+      "acc_norm": 0.2431740614334471,
+      "acc_norm_stderr": 0.012536554144587089
+    }
+  },
+  "versions": {
+    "copa": 0,
+    "piqa": 0,
+    "prost": 0,
+    "arc_easy": 0,
+    "hellaswag": 0,
+    "mc_taco": 0,
+    "winogrande": 0,
+    "wsc273": 0,
+    "swag": 0,
+    "boolq": 1,
+    "openbookqa": 0,
+    "arc_challenge": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=facebook/xglm-1.7B,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda:0",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/xglm/xglm-1.7B/xglm-1.7B_gsm8k_8-shot.json

+{
+  "results": {
+    "gsm8k": {
+      "acc": 0.009855951478392721,
+      "acc_stderr": 0.00272107657704166
+    }
+  },
+  "versions": {
+    "gsm8k": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=facebook/xglm-1.7B,use_accelerate=True",
+    "num_fewshot": 8,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/xglm/xglm-1.7B/xglm-1.7B_mathematical_reasoning_few_shot_5-shot.json

+{
+  "results": {
+    "math_prealgebra": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "gsm8k": {
+      "acc": 0.008339651250947688,
+      "acc_stderr": 0.002504942226860518
+    },
+    "math_num_theory": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "math_precalc": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "math_algebra": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "math_geometry": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "drop": {
+      "em": 0.006711409395973154,
+      "em_stderr": 0.0008361500895152447,
+      "f1": 0.03435402684563763,
+      "f1_stderr": 0.0012720691502648663
+    },
+    "math_intermediate_algebra": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "math_counting_and_prob": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "mathqa": {
+      "acc": 0.22914572864321608,
+      "acc_stderr": 0.007693830518376538,
+      "acc_norm": 0.21440536013400335,
+      "acc_norm_stderr": 0.0075130739863118485
+    }
+  },
+  "versions": {
+    "math_prealgebra": 1,
+    "gsm8k": 0,
+    "math_num_theory": 1,
+    "math_precalc": 1,
+    "mathqa": 0,
+    "math_algebra": 1,
+    "math_geometry": 1,
+    "drop": 1,
+    "math_intermediate_algebra": 1,
+    "math_counting_and_prob": 1
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=facebook/xglm-1.7B,use_accelerate=True",
+    "num_fewshot": 5,
+    "batch_size": "auto",
+    "device": "cuda:0",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/xglm/xglm-1.7B/xglm-1.7B_pawsx_0-shot.json

+{
+  "results": {
+    "pawsx_fr": {
+      "acc": 0.4735,
+      "acc_stderr": 0.011167418260963935
+    },
+    "pawsx_de": {
+      "acc": 0.5755,
+      "acc_stderr": 0.011054907529701135
+    },
+    "pawsx_ko": {
+      "acc": 0.514,
+      "acc_stderr": 0.011178751372184865
+    },
+    "pawsx_ja": {
+      "acc": 0.461,
+      "acc_stderr": 0.011149065020234333
+    },
+    "pawsx_en": {
+      "acc": 0.5265,
+      "acc_stderr": 0.011167418260963933
+    },
+    "pawsx_es": {
+      "acc": 0.538,
+      "acc_stderr": 0.011150792352341666
+    },
+    "pawsx_zh": {
+      "acc": 0.481,
+      "acc_stderr": 0.011175058879956061
+    }
+  },
+  "versions": {
+    "pawsx_fr": 0,
+    "pawsx_de": 0,
+    "pawsx_ko": 0,
+    "pawsx_ja": 0,
+    "pawsx_en": 0,
+    "pawsx_es": 0,
+    "pawsx_zh": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=facebook/xglm-1.7B,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/xglm/xglm-1.7B/xglm-1.7B_xcopa_0-shot.json

+{
+  "results": {
+    "xcopa_id": {
+      "acc": 0.646,
+      "acc_stderr": 0.02140758204791645
+    },
+    "xcopa_ta": {
+      "acc": 0.552,
+      "acc_stderr": 0.02226169729227013
+    },
+    "xcopa_ht": {
+      "acc": 0.558,
+      "acc_stderr": 0.022231970696321122
+    },
+    "xcopa_it": {
+      "acc": 0.54,
+      "acc_stderr": 0.022311333245289663
+    },
+    "xcopa_tr": {
+      "acc": 0.534,
+      "acc_stderr": 0.02233126442325838
+    },
+    "xcopa_th": {
+      "acc": 0.582,
+      "acc_stderr": 0.022080014812228137
+    },
+    "xcopa_qu": {
+      "acc": 0.522,
+      "acc_stderr": 0.022361396739207878
+    },
+    "xcopa_zh": {
+      "acc": 0.58,
+      "acc_stderr": 0.02209471322976178
+    },
+    "xcopa_vi": {
+      "acc": 0.63,
+      "acc_stderr": 0.02161328916516579
+    },
+    "xcopa_et": {
+      "acc": 0.568,
+      "acc_stderr": 0.022175109265613165
+    },
+    "xcopa_sw": {
+      "acc": 0.566,
+      "acc_stderr": 0.022187215803029008
+    }
+  },
+  "versions": {
+    "xcopa_id": 0,
+    "xcopa_ta": 0,
+    "xcopa_ht": 0,
+    "xcopa_it": 0,
+    "xcopa_tr": 0,
+    "xcopa_th": 0,
+    "xcopa_qu": 0,
+    "xcopa_zh": 0,
+    "xcopa_vi": 0,
+    "xcopa_et": 0,
+    "xcopa_sw": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=facebook/xglm-1.7B,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/xglm/xglm-1.7B/xglm-1.7B_xnli_0-shot.json

+{
+  "results": {
+    "xnli_bg": {
+      "acc": 0.4473053892215569,
+      "acc_stderr": 0.0070253693946827935
+    },
+    "xnli_hi": {
+      "acc": 0.42614770459081835,
+      "acc_stderr": 0.006987223294820979
+    },
+    "xnli_el": {
+      "acc": 0.40099800399201596,
+      "acc_stderr": 0.006924839696959944
+    },
+    "xnli_en": {
+      "acc": 0.49680638722554893,
+      "acc_stderr": 0.00706456831954508
+    },
+    "xnli_ar": {
+      "acc": 0.33512974051896205,
+      "acc_stderr": 0.006669594382503631
+    },
+    "xnli_ru": {
+      "acc": 0.4596806387225549,
+      "acc_stderr": 0.00704170545485625
+    },
+    "xnli_fr": {
+      "acc": 0.45728542914171655,
+      "acc_stderr": 0.007038885597058048
+    },
+    "xnli_tr": {
+      "acc": 0.4295409181636727,
+      "acc_stderr": 0.006994215414803208
+    },
+    "xnli_th": {
+      "acc": 0.4169660678642715,
+      "acc_stderr": 0.006966614137458995
+    },
+    "xnli_vi": {
+      "acc": 0.4502994011976048,
+      "acc_stderr": 0.007029723996054755
+    },
+    "xnli_de": {
+      "acc": 0.4532934131736527,
+      "acc_stderr": 0.0070338214783393326
+    },
+    "xnli_sw": {
+      "acc": 0.41996007984031936,
+      "acc_stderr": 0.006973606391328806
+    },
+    "xnli_zh": {
+      "acc": 0.3377245508982036,
+      "acc_stderr": 0.006682287063203171
+    },
+    "xnli_es": {
+      "acc": 0.436127744510978,
+      "acc_stderr": 0.007006832004922492
+    },
+    "xnli_ur": {
+      "acc": 0.39500998003992016,
+      "acc_stderr": 0.0069072094196003676
+    }
+  },
+  "versions": {
+    "xnli_bg": 0,
+    "xnli_hi": 0,
+    "xnli_el": 0,
+    "xnli_en": 0,
+    "xnli_ar": 0,
+    "xnli_ru": 0,
+    "xnli_fr": 0,
+    "xnli_tr": 0,
+    "xnli_th": 0,
+    "xnli_vi": 0,
+    "xnli_de": 0,
+    "xnli_sw": 0,
+    "xnli_zh": 0,
+    "xnli_es": 0,
+    "xnli_ur": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=facebook/xglm-1.7B,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/xglm/xglm-1.7B/xglm-1.7B_xstory_cloze_0-shot.json

+{
+  "results": {
+    "xstory_cloze_hi": {
+      "acc": 0.5579086697551291,
+      "acc_stderr": 0.012780536370279769
+    },
+    "xstory_cloze_sw": {
+      "acc": 0.5598941098610192,
+      "acc_stderr": 0.012774475160716338
+    },
+    "xstory_cloze_zh": {
+      "acc": 0.5618795499669094,
+      "acc_stderr": 0.012768206616277759
+    },
+    "xstory_cloze_my": {
+      "acc": 0.5380542686962276,
+      "acc_stderr": 0.0128298047203217
+    },
+    "xstory_cloze_eu": {
+      "acc": 0.5612177365982793,
+      "acc_stderr": 0.012770319186938004
+    },
+    "xstory_cloze_id": {
+      "acc": 0.5797485109199206,
+      "acc_stderr": 0.012702405649149104
+    },
+    "xstory_cloze_te": {
+      "acc": 0.5804103242885507,
+      "acc_stderr": 0.012699642268200749
+    },
+    "xstory_cloze_en": {
+      "acc": 0.6432825943084051,
+      "acc_stderr": 0.01232748767711036
+    },
+    "xstory_cloze_es": {
+      "acc": 0.5923229649238915,
+      "acc_stderr": 0.012645876488040282
+    },
+    "xstory_cloze_ar": {
+      "acc": 0.5248180013236268,
+      "acc_stderr": 0.012851264962354841
+    },
+    "xstory_cloze_ru": {
+      "acc": 0.5982792852415619,
+      "acc_stderr": 0.012616114526927917
+    }
+  },
+  "versions": {
+    "xstory_cloze_hi": 0,
+    "xstory_cloze_sw": 0,
+    "xstory_cloze_zh": 0,
+    "xstory_cloze_my": 0,
+    "xstory_cloze_eu": 0,
+    "xstory_cloze_id": 0,
+    "xstory_cloze_te": 0,
+    "xstory_cloze_en": 0,
+    "xstory_cloze_es": 0,
+    "xstory_cloze_ar": 0,
+    "xstory_cloze_ru": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=facebook/xglm-1.7B,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/xglm/xglm-1.7B/xglm-1.7B_xwinograd_0-shot.json

+{
+  "results": {
+    "xwinograd_pt": {
+      "acc": 0.6387832699619772,
+      "acc_stderr": 0.029676320268041578
+    },
+    "xwinograd_zh": {
+      "acc": 0.6984126984126984,
+      "acc_stderr": 0.020463437846223773
+    },
+    "xwinograd_jp": {
+      "acc": 0.6058394160583942,
+      "acc_stderr": 0.015788199459722305
+    },
+    "xwinograd_fr": {
+      "acc": 0.6024096385542169,
+      "acc_stderr": 0.05404517824786813
+    },
+    "xwinograd_en": {
+      "acc": 0.7105376344086022,
+      "acc_stderr": 0.009407441676993788
+    },
+    "xwinograd_ru": {
+      "acc": 0.5968253968253968,
+      "acc_stderr": 0.02768250629102932
+    }
+  },
+  "versions": {
+    "xwinograd_pt": 0,
+    "xwinograd_zh": 0,
+    "xwinograd_jp": 0,
+    "xwinograd_fr": 0,
+    "xwinograd_en": 0,
+    "xwinograd_ru": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=facebook/xglm-1.7B,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/xglm/xglm-2.9B/xglm-2.9B_common_sense_reasoning_0-shot.json

+{
+  "results": {
+    "piqa": {
+      "acc": 0.7127312295973884,
+      "acc_stderr": 0.010557291761528637,
+      "acc_norm": 0.7301414581066377,
+      "acc_norm_stderr": 0.010356595421852193
+    },
+    "openbookqa": {
+      "acc": 0.216,
+      "acc_stderr": 0.01842190906141194,
+      "acc_norm": 0.332,
+      "acc_norm_stderr": 0.021081766571222856
+    },
+    "arc_challenge": {
+      "acc": 0.23464163822525597,
+      "acc_stderr": 0.01238387356076868,
+      "acc_norm": 0.2738907849829352,
+      "acc_norm_stderr": 0.013032004972989505
+    },
+    "arc_easy": {
+      "acc": 0.5664983164983165,
+      "acc_stderr": 0.010168640625454107,
+      "acc_norm": 0.5336700336700336,
+      "acc_norm_stderr": 0.010236494647406476
+    },
+    "boolq": {
+      "acc": 0.6143730886850153,
+      "acc_stderr": 0.008513189460768057
+    },
+    "wsc273": {
+      "acc": 0.7106227106227107,
+      "acc_stderr": 0.027495860234525278
+    },
+    "copa": {
+      "acc": 0.74,
+      "acc_stderr": 0.04408440022768077
+    },
+    "hellaswag": {
+      "acc": 0.4091814379605656,
+      "acc_stderr": 0.004906779523192668,
+      "acc_norm": 0.5370444134634534,
+      "acc_norm_stderr": 0.004976067726432559
+    },
+    "prost": {
+      "acc": 0.21920367207514946,
+      "acc_stderr": 0.003022497462586152,
+      "acc_norm": 0.2664389410760034,
+      "acc_norm_stderr": 0.0032299078734217036
+    },
+    "swag": {
+      "acc": 0.4849045286414076,
+      "acc_stderr": 0.0035334805738792946,
+      "acc_norm": 0.6578026592022393,
+      "acc_norm_stderr": 0.0033544154500719224
+    },
+    "mc_taco": {
+      "em": 0.11936936936936937,
+      "f1": 0.4779548809969738
+    },
+    "winogrande": {
+      "acc": 0.5461720599842147,
+      "acc_stderr": 0.013992441563707058
+    }
+  },
+  "versions": {
+    "piqa": 0,
+    "openbookqa": 0,
+    "arc_challenge": 0,
+    "arc_easy": 0,
+    "boolq": 1,
+    "wsc273": 0,
+    "copa": 0,
+    "hellaswag": 0,
+    "prost": 0,
+    "swag": 0,
+    "mc_taco": 0,
+    "winogrande": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=facebook/xglm-2.9B,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda:0",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/xglm/xglm-2.9B/xglm-2.9B_pawsx_0-shot.json

+{
+  "results": {
+    "pawsx_zh": {
+      "acc": 0.537,
+      "acc_stderr": 0.011152474561478182
+    },
+    "pawsx_de": {
+      "acc": 0.5065,
+      "acc_stderr": 0.0111821910061423
+    },
+    "pawsx_en": {
+      "acc": 0.5475,
+      "acc_stderr": 0.011132557743886095
+    },
+    "pawsx_es": {
+      "acc": 0.5315,
+      "acc_stderr": 0.011160921022883278
+    },
+    "pawsx_fr": {
+      "acc": 0.497,
+      "acc_stderr": 0.011182934722804556
+    },
+    "pawsx_ja": {
+      "acc": 0.5095,
+      "acc_stderr": 0.011181117282805214
+    },
+    "pawsx_ko": {
+      "acc": 0.4675,
+      "acc_stderr": 0.011159486640120933
+    }
+  },
+  "versions": {
+    "pawsx_zh": 0,
+    "pawsx_de": 0,
+    "pawsx_en": 0,
+    "pawsx_es": 0,
+    "pawsx_fr": 0,
+    "pawsx_ja": 0,
+    "pawsx_ko": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=facebook/xglm-2.9B,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/xglm/xglm-2.9B/xglm-2.9B_xcopa_0-shot.json

+{
+  "results": {
+    "xcopa_zh": {
+      "acc": 0.6,
+      "acc_stderr": 0.021930844120728505
+    },
+    "xcopa_ht": {
+      "acc": 0.558,
+      "acc_stderr": 0.02223197069632112
+    },
+    "xcopa_id": {
+      "acc": 0.668,
+      "acc_stderr": 0.021081766571222856
+    },
+    "xcopa_tr": {
+      "acc": 0.566,
+      "acc_stderr": 0.02218721580302901
+    },
+    "xcopa_it": {
+      "acc": 0.602,
+      "acc_stderr": 0.021912377885779967
+    },
+    "xcopa_qu": {
+      "acc": 0.502,
+      "acc_stderr": 0.02238289498648353
+    },
+    "xcopa_sw": {
+      "acc": 0.588,
+      "acc_stderr": 0.022033677993740865
+    },
+    "xcopa_th": {
+      "acc": 0.57,
+      "acc_stderr": 0.02216263442665284
+    },
+    "xcopa_vi": {
+      "acc": 0.652,
+      "acc_stderr": 0.021323728632807494
+    },
+    "xcopa_ta": {
+      "acc": 0.542,
+      "acc_stderr": 0.022303966774269938
+    },
+    "xcopa_et": {
+      "acc": 0.582,
+      "acc_stderr": 0.022080014812228137
+    }
+  },
+  "versions": {
+    "xcopa_zh": 0,
+    "xcopa_ht": 0,
+    "xcopa_id": 0,
+    "xcopa_tr": 0,
+    "xcopa_it": 0,
+    "xcopa_qu": 0,
+    "xcopa_sw": 0,
+    "xcopa_th": 0,
+    "xcopa_vi": 0,
+    "xcopa_ta": 0,
+    "xcopa_et": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=facebook/xglm-2.9B,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/xglm/xglm-2.9B/xglm-2.9B_xnli_0-shot.json

+{
+  "results": {
+    "xnli_ar": {
+      "acc": 0.33652694610778444,
+      "acc_stderr": 0.006676456919028458
+    },
+    "xnli_bg": {
+      "acc": 0.4596806387225549,
+      "acc_stderr": 0.007041705454856254
+    },
+    "xnli_de": {
+      "acc": 0.48323353293413174,
+      "acc_stderr": 0.007060739327060854
+    },
+    "xnli_el": {
+      "acc": 0.41397205588822356,
+      "acc_stderr": 0.00695935771309272
+    },
+    "xnli_en": {
+      "acc": 0.5107784431137724,
+      "acc_stderr": 0.007063070754956929
+    },
+    "xnli_es": {
+      "acc": 0.4666666666666667,
+      "acc_stderr": 0.0070489955857553875
+    },
+    "xnli_fr": {
+      "acc": 0.4502994011976048,
+      "acc_stderr": 0.00702972399605476
+    },
+    "xnli_hi": {
+      "acc": 0.4403193612774451,
+      "acc_stderr": 0.007014206007644934
+    },
+    "xnli_ru": {
+      "acc": 0.4528942115768463,
+      "acc_stderr": 0.007033289986695003
+    },
+    "xnli_sw": {
+      "acc": 0.444311377245509,
+      "acc_stderr": 0.007020757195791273
+    },
+    "xnli_th": {
+      "acc": 0.41976047904191616,
+      "acc_stderr": 0.006973148443615152
+    },
+    "xnli_tr": {
+      "acc": 0.4497005988023952,
+      "acc_stderr": 0.007028873660193274
+    },
+    "xnli_ur": {
+      "acc": 0.40099800399201596,
+      "acc_stderr": 0.006924839696959946
+    },
+    "xnli_vi": {
+      "acc": 0.4598802395209581,
+      "acc_stderr": 0.00704193305036814
+    },
+    "xnli_zh": {
+      "acc": 0.34810379241516964,
+      "acc_stderr": 0.006730821739872395
+    }
+  },
+  "versions": {
+    "xnli_ar": 0,
+    "xnli_bg": 0,
+    "xnli_de": 0,
+    "xnli_el": 0,
+    "xnli_en": 0,
+    "xnli_es": 0,
+    "xnli_fr": 0,
+    "xnli_hi": 0,
+    "xnli_ru": 0,
+    "xnli_sw": 0,
+    "xnli_th": 0,
+    "xnli_tr": 0,
+    "xnli_ur": 0,
+    "xnli_vi": 0,
+    "xnli_zh": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=facebook/xglm-2.9B,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/xglm/xglm-2.9B/xglm-2.9B_xstory_cloze_0-shot.json

+{
+  "results": {
+    "xstory_cloze_es": {
+      "acc": 0.6095301125082727,
+      "acc_stderr": 0.012554600076548371
+    },
+    "xstory_cloze_sw": {
+      "acc": 0.5671740569159497,
+      "acc_stderr": 0.01275047450298583
+    },
+    "xstory_cloze_en": {
+      "acc": 0.6730641958967571,
+      "acc_stderr": 0.012071771683911351
+    },
+    "xstory_cloze_zh": {
+      "acc": 0.5764394440767704,
+      "acc_stderr": 0.01271587138288145
+    },
+    "xstory_cloze_id": {
+      "acc": 0.613500992720053,
+      "acc_stderr": 0.012531219943771486
+    },
+    "xstory_cloze_eu": {
+      "acc": 0.5632031767041694,
+      "acc_stderr": 0.01276391225017363
+    },
+    "xstory_cloze_te": {
+      "acc": 0.600264725347452,
+      "acc_stderr": 0.01260576407762715
+    },
+    "xstory_cloze_ru": {
+      "acc": 0.6221045665122436,
+      "acc_stderr": 0.012477542072994664
+    },
+    "xstory_cloze_hi": {
+      "acc": 0.5751158173395102,
+      "acc_stderr": 0.01272109407352333
+    },
+    "xstory_cloze_ar": {
+      "acc": 0.5387160820648577,
+      "acc_stderr": 0.01282849335327155
+    },
+    "xstory_cloze_my": {
+      "acc": 0.5519523494374586,
+      "acc_stderr": 0.012797478885304733
+    }
+  },
+  "versions": {
+    "xstory_cloze_es": 0,
+    "xstory_cloze_sw": 0,
+    "xstory_cloze_en": 0,
+    "xstory_cloze_zh": 0,
+    "xstory_cloze_id": 0,
+    "xstory_cloze_eu": 0,
+    "xstory_cloze_te": 0,
+    "xstory_cloze_ru": 0,
+    "xstory_cloze_hi": 0,
+    "xstory_cloze_ar": 0,
+    "xstory_cloze_my": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=facebook/xglm-2.9B,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/xglm/xglm-2.9B/xglm-2.9B_xwinograd_0-shot.json

+{
+  "results": {
+    "xwinograd_fr": {
+      "acc": 0.5903614457831325,
+      "acc_stderr": 0.05430658329539147
+    },
+    "xwinograd_en": {
+      "acc": 0.7561290322580645,
+      "acc_stderr": 0.008907584394182084
+    },
+    "xwinograd_zh": {
+      "acc": 0.7162698412698413,
+      "acc_stderr": 0.02010051064884106
+    },
+    "xwinograd_ru": {
+      "acc": 0.6285714285714286,
+      "acc_stderr": 0.027267803028895015
+    },
+    "xwinograd_jp": {
+      "acc": 0.6465067778936392,
+      "acc_stderr": 0.015445228301221376
+    },
+    "xwinograd_pt": {
+      "acc": 0.6615969581749049,
+      "acc_stderr": 0.02923231657730264
+    }
+  },
+  "versions": {
+    "xwinograd_fr": 0,
+    "xwinograd_en": 0,
+    "xwinograd_zh": 0,
+    "xwinograd_ru": 0,
+    "xwinograd_jp": 0,
+    "xwinograd_pt": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=facebook/xglm-2.9B,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/xglm/xglm-4.5B/xglm-4.5B_common_sense_reasoning_0-shot.json

+{
+  "results": {
+    "copa": {
+      "acc": 0.81,
+      "acc_stderr": 0.03942772444036623
+    },
+    "arc_challenge": {
+      "acc": 0.2713310580204778,
+      "acc_stderr": 0.012993807727545796,
+      "acc_norm": 0.2815699658703072,
+      "acc_norm_stderr": 0.01314337673500902
+    },
+    "mc_taco": {
+      "em": 0.1539039039039039,
+      "f1": 0.435083658174568
+    },
+    "prost": {
+      "acc": 0.26430401366353545,
+      "acc_stderr": 0.003221619340165698,
+      "acc_norm": 0.2627561912894962,
+      "acc_norm_stderr": 0.003215549484247182
+    },
+    "piqa": {
+      "acc": 0.7274211099020674,
+      "acc_stderr": 0.01038925680329602,
+      "acc_norm": 0.7295973884657236,
+      "acc_norm_stderr": 0.010363167031620785
+    },
+    "wsc273": {
+      "acc": 0.717948717948718,
+      "acc_stderr": 0.027285147081637318
+    },
+    "winogrande": {
+      "acc": 0.5611681136543015,
+      "acc_stderr": 0.013946933444507032
+    },
+    "boolq": {
+      "acc": 0.6119266055045871,
+      "acc_stderr": 0.008523130584760844
+    },
+    "openbookqa": {
+      "acc": 0.232,
+      "acc_stderr": 0.018896193591952045,
+      "acc_norm": 0.344,
+      "acc_norm_stderr": 0.02126575803797874
+    },
+    "arc_easy": {
+      "acc": 0.6031144781144782,
+      "acc_stderr": 0.010039236800583206,
+      "acc_norm": 0.5723905723905723,
+      "acc_norm_stderr": 0.010151683397430673
+    },
+    "hellaswag": {
+      "acc": 0.43766182035451107,
+      "acc_stderr": 0.004950848456984543,
+      "acc_norm": 0.5823541127265485,
+      "acc_norm_stderr": 0.004921632645102376
+    },
+    "swag": {
+      "acc": 0.49650104968509445,
+      "acc_stderr": 0.0035350054881690377,
+      "acc_norm": 0.6786963910826752,
+      "acc_norm_stderr": 0.0033016139730438565
+    }
+  },
+  "versions": {
+    "copa": 0,
+    "arc_challenge": 0,
+    "mc_taco": 0,
+    "prost": 0,
+    "piqa": 0,
+    "wsc273": 0,
+    "winogrande": 0,
+    "boolq": 1,
+    "openbookqa": 0,
+    "arc_easy": 0,
+    "hellaswag": 0,
+    "swag": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=facebook/xglm-4.5B,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda:0",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/xglm/xglm-4.5B/xglm-4.5B_gsm8k_8-shot.json

+{
+  "results": {
+    "gsm8k": {
+      "acc": 0.000758150113722517,
+      "acc_stderr": 0.0007581501137225263
+    }
+  },
+  "versions": {
+    "gsm8k": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=facebook/xglm-4.5B,use_accelerate=True",
+    "num_fewshot": 8,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/xglm/xglm-4.5B/xglm-4.5B_pawsx_0-shot.json

+{
+  "results": {
+    "pawsx_fr": {
+      "acc": 0.516,
+      "acc_stderr": 0.01117740878887489
+    },
+    "pawsx_en": {
+      "acc": 0.554,
+      "acc_stderr": 0.011117724672834362
+    },
+    "pawsx_de": {
+      "acc": 0.5265,
+      "acc_stderr": 0.011167418260963935
+    },
+    "pawsx_ko": {
+      "acc": 0.491,
+      "acc_stderr": 0.011181324206260288
+    },
+    "pawsx_ja": {
+      "acc": 0.4775,
+      "acc_stderr": 0.011171807357801183
+    },
+    "pawsx_es": {
+      "acc": 0.5105,
+      "acc_stderr": 0.011180669867648658
+    },
+    "pawsx_zh": {
+      "acc": 0.546,
+      "acc_stderr": 0.0111357084193598
+    }
+  },
+  "versions": {
+    "pawsx_fr": 0,
+    "pawsx_en": 0,
+    "pawsx_de": 0,
+    "pawsx_ko": 0,
+    "pawsx_ja": 0,
+    "pawsx_es": 0,
+    "pawsx_zh": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=facebook/xglm-4.5B,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}