add bloom, xglm and llama results

results/xglm/xglm-4.5B/xglm-4.5B_xcopa_0-shot.json

+{
+  "results": {
+    "xcopa_vi": {
+      "acc": 0.66,
+      "acc_stderr": 0.021206117013673066
+    },
+    "xcopa_sw": {
+      "acc": 0.562,
+      "acc_stderr": 0.022210326363977413
+    },
+    "xcopa_ta": {
+      "acc": 0.556,
+      "acc_stderr": 0.02224224437573102
+    },
+    "xcopa_qu": {
+      "acc": 0.5,
+      "acc_stderr": 0.022383074051792257
+    },
+    "xcopa_zh": {
+      "acc": 0.616,
+      "acc_stderr": 0.02177236946554719
+    },
+    "xcopa_ht": {
+      "acc": 0.512,
+      "acc_stderr": 0.02237662679792717
+    },
+    "xcopa_it": {
+      "acc": 0.616,
+      "acc_stderr": 0.02177236946554719
+    },
+    "xcopa_th": {
+      "acc": 0.552,
+      "acc_stderr": 0.02226169729227013
+    },
+    "xcopa_et": {
+      "acc": 0.55,
+      "acc_stderr": 0.022270877485360437
+    },
+    "xcopa_tr": {
+      "acc": 0.572,
+      "acc_stderr": 0.022149790663861933
+    },
+    "xcopa_id": {
+      "acc": 0.67,
+      "acc_stderr": 0.021049612166134803
+    }
+  },
+  "versions": {
+    "xcopa_vi": 0,
+    "xcopa_sw": 0,
+    "xcopa_ta": 0,
+    "xcopa_qu": 0,
+    "xcopa_zh": 0,
+    "xcopa_ht": 0,
+    "xcopa_it": 0,
+    "xcopa_th": 0,
+    "xcopa_et": 0,
+    "xcopa_tr": 0,
+    "xcopa_id": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=facebook/xglm-4.5B,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/xglm/xglm-4.5B/xglm-4.5B_xnli_0-shot.json

+{
+  "results": {
+    "xnli_ar": {
+      "acc": 0.33592814371257484,
+      "acc_stderr": 0.006673523847107451
+    },
+    "xnli_bg": {
+      "acc": 0.4560878243512974,
+      "acc_stderr": 0.00703741421319664
+    },
+    "xnli_de": {
+      "acc": 0.47105788423153694,
+      "acc_stderr": 0.007052867041612212
+    },
+    "xnli_el": {
+      "acc": 0.39840319361277443,
+      "acc_stderr": 0.006917332484904377
+    },
+    "xnli_en": {
+      "acc": 0.5363273453093812,
+      "acc_stderr": 0.007046041494004668
+    },
+    "xnli_es": {
+      "acc": 0.47684630738522954,
+      "acc_stderr": 0.00705713367164529
+    },
+    "xnli_fr": {
+      "acc": 0.47305389221556887,
+      "acc_stderr": 0.007054445696700673
+    },
+    "xnli_hi": {
+      "acc": 0.4249500998003992,
+      "acc_stderr": 0.0069846752434985565
+    },
+    "xnli_ru": {
+      "acc": 0.4614770459081836,
+      "acc_stderr": 0.0070437128985425335
+    },
+    "xnli_sw": {
+      "acc": 0.39580838323353296,
+      "acc_stderr": 0.006909622580687441
+    },
+    "xnli_th": {
+      "acc": 0.3968063872255489,
+      "acc_stderr": 0.0069126119585257075
+    },
+    "xnli_tr": {
+      "acc": 0.4485029940119761,
+      "acc_stderr": 0.007027142119481902
+    },
+    "xnli_ur": {
+      "acc": 0.37465069860279443,
+      "acc_stderr": 0.006839102174587385
+    },
+    "xnli_vi": {
+      "acc": 0.45868263473053894,
+      "acc_stderr": 0.007040550421310631
+    },
+    "xnli_zh": {
+      "acc": 0.34770459081836325,
+      "acc_stderr": 0.006729020595578698
+    }
+  },
+  "versions": {
+    "xnli_ar": 0,
+    "xnli_bg": 0,
+    "xnli_de": 0,
+    "xnli_el": 0,
+    "xnli_en": 0,
+    "xnli_es": 0,
+    "xnli_fr": 0,
+    "xnli_hi": 0,
+    "xnli_ru": 0,
+    "xnli_sw": 0,
+    "xnli_th": 0,
+    "xnli_tr": 0,
+    "xnli_ur": 0,
+    "xnli_vi": 0,
+    "xnli_zh": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=facebook/xglm-4.5B,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/xglm/xglm-4.5B/xglm-4.5B_xstory_cloze_0-shot.json

+{
+  "results": {
+    "xstory_cloze_es": {
+      "acc": 0.628060886829914,
+      "acc_stderr": 0.012437936235202006
+    },
+    "xstory_cloze_en": {
+      "acc": 0.6915949702183984,
+      "acc_stderr": 0.011884972073313783
+    },
+    "xstory_cloze_my": {
+      "acc": 0.5076108537392455,
+      "acc_stderr": 0.012865634571114483
+    },
+    "xstory_cloze_eu": {
+      "acc": 0.5373924553275976,
+      "acc_stderr": 0.012831093347016567
+    },
+    "xstory_cloze_hi": {
+      "acc": 0.5684976836532097,
+      "acc_stderr": 0.012745810046098417
+    },
+    "xstory_cloze_ar": {
+      "acc": 0.5367306419589676,
+      "acc_stderr": 0.012832359240206969
+    },
+    "xstory_cloze_zh": {
+      "acc": 0.5817339510258107,
+      "acc_stderr": 0.012694045150564695
+    },
+    "xstory_cloze_sw": {
+      "acc": 0.5506287227001986,
+      "acc_stderr": 0.012800991591293387
+    },
+    "xstory_cloze_ru": {
+      "acc": 0.6273990734612839,
+      "acc_stderr": 0.012442436359907104
+    },
+    "xstory_cloze_te": {
+      "acc": 0.5704831237590999,
+      "acc_stderr": 0.012738639381353988
+    },
+    "xstory_cloze_id": {
+      "acc": 0.6042356055592323,
+      "acc_stderr": 0.012584415320654347
+    }
+  },
+  "versions": {
+    "xstory_cloze_es": 0,
+    "xstory_cloze_en": 0,
+    "xstory_cloze_my": 0,
+    "xstory_cloze_eu": 0,
+    "xstory_cloze_hi": 0,
+    "xstory_cloze_ar": 0,
+    "xstory_cloze_zh": 0,
+    "xstory_cloze_sw": 0,
+    "xstory_cloze_ru": 0,
+    "xstory_cloze_te": 0,
+    "xstory_cloze_id": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=facebook/xglm-4.5B,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/xglm/xglm-4.5B/xglm-4.5B_xwinograd_0-shot.json

+{
+  "results": {
+    "xwinograd_fr": {
+      "acc": 0.6024096385542169,
+      "acc_stderr": 0.054045178247868135
+    },
+    "xwinograd_jp": {
+      "acc": 0.6266944734098019,
+      "acc_stderr": 0.015627063885651445
+    },
+    "xwinograd_pt": {
+      "acc": 0.6463878326996197,
+      "acc_stderr": 0.02953653465680206
+    },
+    "xwinograd_ru": {
+      "acc": 0.6222222222222222,
+      "acc_stderr": 0.027360632861056396
+    },
+    "xwinograd_zh": {
+      "acc": 0.7063492063492064,
+      "acc_stderr": 0.020306792341159757
+    },
+    "xwinograd_en": {
+      "acc": 0.7625806451612903,
+      "acc_stderr": 0.008826385416032015
+    }
+  },
+  "versions": {
+    "xwinograd_fr": 0,
+    "xwinograd_jp": 0,
+    "xwinograd_pt": 0,
+    "xwinograd_ru": 0,
+    "xwinograd_zh": 0,
+    "xwinograd_en": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=facebook/xglm-4.5B,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/xglm/xglm-564M/xglm-564M_common_sense_reasoning_0-shot.json

+{
+  "results": {
+    "copa": {
+      "acc": 0.69,
+      "acc_stderr": 0.04648231987117316
+    },
+    "mc_taco": {
+      "em": 0.1403903903903904,
+      "f1": 0.40416372323956457
+    },
+    "prost": {
+      "acc": 0.24044619982920581,
+      "acc_stderr": 0.003122204753095043,
+      "acc_norm": 0.3104718189581554,
+      "acc_norm_stderr": 0.0033803389886508614
+    },
+    "piqa": {
+      "acc": 0.6512513601741022,
+      "acc_stderr": 0.011119263056159583,
+      "acc_norm": 0.6485310119695321,
+      "acc_norm_stderr": 0.011139207691931193
+    },
+    "wsc273": {
+      "acc": 0.5860805860805861,
+      "acc_stderr": 0.029864284486117058
+    },
+    "winogrande": {
+      "acc": 0.5248618784530387,
+      "acc_stderr": 0.01403510288362775
+    },
+    "swag": {
+      "acc": 0.4111266620013996,
+      "acc_stderr": 0.0034788002208961154,
+      "acc_norm": 0.5425872238328502,
+      "acc_norm_stderr": 0.003522245713261821
+    },
+    "boolq": {
+      "acc": 0.5333333333333333,
+      "acc_stderr": 0.008725599880049204
+    },
+    "hellaswag": {
+      "acc": 0.30780720971917946,
+      "acc_stderr": 0.004606429684604547,
+      "acc_norm": 0.3520215096594304,
+      "acc_norm_stderr": 0.004766245539606642
+    },
+    "arc_challenge": {
+      "acc": 0.19965870307167236,
+      "acc_stderr": 0.011681625756888688,
+      "acc_norm": 0.24232081911262798,
+      "acc_norm_stderr": 0.012521593295800113
+    },
+    "arc_easy": {
+      "acc": 0.45707070707070707,
+      "acc_stderr": 0.010221897564256037,
+      "acc_norm": 0.41203703703703703,
+      "acc_norm_stderr": 0.010099765857562767
+    },
+    "openbookqa": {
+      "acc": 0.15,
+      "acc_stderr": 0.015984712135164926,
+      "acc_norm": 0.288,
+      "acc_norm_stderr": 0.020271503835075214
+    }
+  },
+  "versions": {
+    "copa": 0,
+    "mc_taco": 0,
+    "prost": 0,
+    "piqa": 0,
+    "wsc273": 0,
+    "winogrande": 0,
+    "swag": 0,
+    "boolq": 1,
+    "hellaswag": 0,
+    "arc_challenge": 0,
+    "arc_easy": 0,
+    "openbookqa": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=facebook/xglm-564M,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda:0",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/xglm/xglm-564M/xglm-564M_gsm8k_8-shot.json

+{
+  "results": {
+    "gsm8k": {
+      "acc": 0.012130401819560273,
+      "acc_stderr": 0.003015294242890939
+    }
+  },
+  "versions": {
+    "gsm8k": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=facebook/xglm-564M,use_accelerate=True",
+    "num_fewshot": 8,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/xglm/xglm-564M/xglm-564M_lambada_openai_0-shot.json

+{
+  "results": {
+    "lambada_openai": {
+      "ppl": 28.56712020387803,
+      "ppl_stderr": 1.029833227972257,
+      "acc": 0.35940228992819717,
+      "acc_stderr": 0.006684904136033
+    },
+    "lambada_openai_cloze": {
+      "ppl": 6898.435163550819,
+      "ppl_stderr": 322.9340159515305,
+      "acc": 0.00038812342324859306,
+      "acc_stderr": 0.00027441806845050347
+    }
+  },
+  "versions": {
+    "lambada_openai": 0,
+    "lambada_openai_cloze": 0
+  },
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=facebook/xglm-564M",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/xglm/xglm-564M/xglm-564M_mathematical_reasoning_few_shot_5-shot.json

+{
+  "results": {
+    "gsm8k": {
+      "acc": 0.008339651250947688,
+      "acc_stderr": 0.0025049422268605386
+    },
+    "math_prealgebra": {
+      "acc": 0.002296211251435132,
+      "acc_stderr": 0.001622733136934626
+    },
+    "math_precalc": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "math_geometry": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "math_num_theory": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "drop": {
+      "em": 0.0037751677852348995,
+      "em_stderr": 0.0006280387809484274,
+      "f1": 0.03062604865771817,
+      "f1_stderr": 0.0011423873658349746
+    },
+    "math_algebra": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "math_intermediate_algebra": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "math_counting_and_prob": {
+      "acc": 0.002109704641350211,
+      "acc_stderr": 0.002109704641350211
+    },
+    "mathqa": {
+      "acc": 0.21105527638190955,
+      "acc_stderr": 0.007470023801451704,
+      "acc_norm": 0.2117252931323283,
+      "acc_norm_stderr": 0.007478693896187993
+    }
+  },
+  "versions": {
+    "gsm8k": 0,
+    "math_prealgebra": 1,
+    "math_precalc": 1,
+    "math_geometry": 1,
+    "math_num_theory": 1,
+    "drop": 1,
+    "math_algebra": 1,
+    "math_intermediate_algebra": 1,
+    "mathqa": 0,
+    "math_counting_and_prob": 1
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=facebook/xglm-564M,use_accelerate=True",
+    "num_fewshot": 5,
+    "batch_size": "auto",
+    "device": "cuda:0",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/xglm/xglm-564M/xglm-564M_pawsx_0-shot.json

+{
+  "results": {
+    "pawsx_de": {
+      "acc": 0.491,
+      "acc_stderr": 0.01118132420626029
+    },
+    "pawsx_ko": {
+      "acc": 0.4625,
+      "acc_stderr": 0.011151639095992287
+    },
+    "pawsx_en": {
+      "acc": 0.5065,
+      "acc_stderr": 0.0111821910061423
+    },
+    "pawsx_es": {
+      "acc": 0.5255,
+      "acc_stderr": 0.011168582883330072
+    },
+    "pawsx_fr": {
+      "acc": 0.508,
+      "acc_stderr": 0.011181704488030002
+    },
+    "pawsx_ja": {
+      "acc": 0.441,
+      "acc_stderr": 0.011105006104468738
+    },
+    "pawsx_zh": {
+      "acc": 0.478,
+      "acc_stderr": 0.011172305500884872
+    }
+  },
+  "versions": {
+    "pawsx_de": 0,
+    "pawsx_ko": 0,
+    "pawsx_en": 0,
+    "pawsx_es": 0,
+    "pawsx_fr": 0,
+    "pawsx_ja": 0,
+    "pawsx_zh": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=facebook/xglm-564M,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/xglm/xglm-564M/xglm-564M_xcopa_0-shot.json

+{
+  "results": {
+    "xcopa_id": {
+      "acc": 0.572,
+      "acc_stderr": 0.02214979066386193
+    },
+    "xcopa_th": {
+      "acc": 0.552,
+      "acc_stderr": 0.02226169729227013
+    },
+    "xcopa_it": {
+      "acc": 0.538,
+      "acc_stderr": 0.022318338119870534
+    },
+    "xcopa_ht": {
+      "acc": 0.55,
+      "acc_stderr": 0.022270877485360437
+    },
+    "xcopa_tr": {
+      "acc": 0.544,
+      "acc_stderr": 0.02229623834840705
+    },
+    "xcopa_zh": {
+      "acc": 0.556,
+      "acc_stderr": 0.02224224437573102
+    },
+    "xcopa_vi": {
+      "acc": 0.584,
+      "acc_stderr": 0.02206494331392886
+    },
+    "xcopa_sw": {
+      "acc": 0.532,
+      "acc_stderr": 0.022337186479044296
+    },
+    "xcopa_ta": {
+      "acc": 0.562,
+      "acc_stderr": 0.022210326363977413
+    },
+    "xcopa_qu": {
+      "acc": 0.492,
+      "acc_stderr": 0.022380208834928028
+    },
+    "xcopa_et": {
+      "acc": 0.556,
+      "acc_stderr": 0.02224224437573102
+    }
+  },
+  "versions": {
+    "xcopa_id": 0,
+    "xcopa_th": 0,
+    "xcopa_it": 0,
+    "xcopa_ht": 0,
+    "xcopa_tr": 0,
+    "xcopa_zh": 0,
+    "xcopa_vi": 0,
+    "xcopa_sw": 0,
+    "xcopa_ta": 0,
+    "xcopa_qu": 0,
+    "xcopa_et": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=facebook/xglm-564M,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/xglm/xglm-564M/xglm-564M_xnli_0-shot.json

+{
+  "results": {
+    "xnli_de": {
+      "acc": 0.44491017964071855,
+      "acc_stderr": 0.007021700267328247
+    },
+    "xnli_sw": {
+      "acc": 0.36067864271457084,
+      "acc_stderr": 0.006784913322438225
+    },
+    "xnli_ar": {
+      "acc": 0.3341317365269461,
+      "acc_stderr": 0.006664652441694282
+    },
+    "xnli_ru": {
+      "acc": 0.44630738522954094,
+      "acc_stderr": 0.007023860641475025
+    },
+    "xnli_fr": {
+      "acc": 0.4548902195608782,
+      "acc_stderr": 0.007035901825327945
+    },
+    "xnli_zh": {
+      "acc": 0.33512974051896205,
+      "acc_stderr": 0.0066695943825036295
+    },
+    "xnli_th": {
+      "acc": 0.38782435129740517,
+      "acc_stderr": 0.006884621148080227
+    },
+    "xnli_el": {
+      "acc": 0.39560878243512976,
+      "acc_stderr": 0.0069090210962589015
+    },
+    "xnli_ur": {
+      "acc": 0.3447105788423154,
+      "acc_stderr": 0.006715345603576113
+    },
+    "xnli_hi": {
+      "acc": 0.38682634730538923,
+      "acc_stderr": 0.00688135955102393
+    },
+    "xnli_es": {
+      "acc": 0.42035928143712575,
+      "acc_stderr": 0.006974518775334314
+    },
+    "xnli_tr": {
+      "acc": 0.40199600798403196,
+      "acc_stderr": 0.006927673254973623
+    },
+    "xnli_bg": {
+      "acc": 0.41297405189620756,
+      "acc_stderr": 0.006956880058406987
+    },
+    "xnli_vi": {
+      "acc": 0.3848303393213573,
+      "acc_stderr": 0.0068747449435859675
+    },
+    "xnli_en": {
+      "acc": 0.48283433133732534,
+      "acc_stderr": 0.0070605478229289675
+    }
+  },
+  "versions": {
+    "xnli_de": 0,
+    "xnli_sw": 0,
+    "xnli_ar": 0,
+    "xnli_ru": 0,
+    "xnli_fr": 0,
+    "xnli_zh": 0,
+    "xnli_th": 0,
+    "xnli_el": 0,
+    "xnli_ur": 0,
+    "xnli_hi": 0,
+    "xnli_es": 0,
+    "xnli_tr": 0,
+    "xnli_bg": 0,
+    "xnli_vi": 0,
+    "xnli_en": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=facebook/xglm-564M,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/xglm/xglm-564M/xglm-564M_xstory_cloze_0-shot.json

+{
+  "results": {
+    "xstory_cloze_ar": {
+      "acc": 0.500992720052945,
+      "acc_stderr": 0.012867099955422925
+    },
+    "xstory_cloze_id": {
+      "acc": 0.5400397088021178,
+      "acc_stderr": 0.012825802370083988
+    },
+    "xstory_cloze_sw": {
+      "acc": 0.5307743216412971,
+      "acc_stderr": 0.01284273034058578
+    },
+    "xstory_cloze_en": {
+      "acc": 0.6055592322964924,
+      "acc_stderr": 0.012577106513936133
+    },
+    "xstory_cloze_te": {
+      "acc": 0.5585704831237591,
+      "acc_stderr": 0.012778538985880637
+    },
+    "xstory_cloze_zh": {
+      "acc": 0.5327597617471873,
+      "acc_stderr": 0.012839477563855915
+    },
+    "xstory_cloze_my": {
+      "acc": 0.514890800794176,
+      "acc_stderr": 0.012861417842074004
+    },
+    "xstory_cloze_hi": {
+      "acc": 0.5228325612177366,
+      "acc_stderr": 0.01285370238487085
+    },
+    "xstory_cloze_ru": {
+      "acc": 0.5618795499669094,
+      "acc_stderr": 0.01276820661627776
+    },
+    "xstory_cloze_es": {
+      "acc": 0.5506287227001986,
+      "acc_stderr": 0.012800991591293383
+    },
+    "xstory_cloze_eu": {
+      "acc": 0.5314361350099271,
+      "acc_stderr": 0.012841668760976905
+    }
+  },
+  "versions": {
+    "xstory_cloze_ar": 0,
+    "xstory_cloze_id": 0,
+    "xstory_cloze_sw": 0,
+    "xstory_cloze_en": 0,
+    "xstory_cloze_te": 0,
+    "xstory_cloze_zh": 0,
+    "xstory_cloze_my": 0,
+    "xstory_cloze_hi": 0,
+    "xstory_cloze_ru": 0,
+    "xstory_cloze_es": 0,
+    "xstory_cloze_eu": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=facebook/xglm-564M,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/xglm/xglm-564M/xglm-564M_xwinograd_0-shot.json

+{
+  "results": {
+    "xwinograd_pt": {
+      "acc": 0.5855513307984791,
+      "acc_stderr": 0.030434573161228055
+    },
+    "xwinograd_zh": {
+      "acc": 0.6567460317460317,
+      "acc_stderr": 0.0211700809891982
+    },
+    "xwinograd_ru": {
+      "acc": 0.5904761904761905,
+      "acc_stderr": 0.02775082824017435
+    },
+    "xwinograd_fr": {
+      "acc": 0.5783132530120482,
+      "acc_stderr": 0.054534284852951115
+    },
+    "xwinograd_en": {
+      "acc": 0.6262365591397849,
+      "acc_stderr": 0.01003574358830904
+    },
+    "xwinograd_jp": {
+      "acc": 0.5453597497393118,
+      "acc_stderr": 0.01608765437474968
+    }
+  },
+  "versions": {
+    "xwinograd_pt": 0,
+    "xwinograd_zh": 0,
+    "xwinograd_ru": 0,
+    "xwinograd_fr": 0,
+    "xwinograd_en": 0,
+    "xwinograd_jp": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=facebook/xglm-564M,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/xglm/xglm-7.5B/xglm-7.5B_common_sense_reasoning_0-shot.json

+{
+  "results": {
+    "openbookqa": {
+      "acc": 0.254,
+      "acc_stderr": 0.019486596801643385,
+      "acc_norm": 0.358,
+      "acc_norm_stderr": 0.02146143486285912
+    },
+    "winogrande": {
+      "acc": 0.5785319652722968,
+      "acc_stderr": 0.0138780723774976
+    },
+    "arc_easy": {
+      "acc": 0.6237373737373737,
+      "acc_stderr": 0.009940646221513789,
+      "acc_norm": 0.5862794612794613,
+      "acc_norm_stderr": 0.010105878530238135
+    },
+    "copa": {
+      "acc": 0.79,
+      "acc_stderr": 0.040936018074033256
+    },
+    "mc_taco": {
+      "em": 0.13813813813813813,
+      "f1": 0.479152974631639
+    },
+    "wsc273": {
+      "acc": 0.7582417582417582,
+      "acc_stderr": 0.02596031999685269
+    },
+    "hellaswag": {
+      "acc": 0.45688109938259314,
+      "acc_stderr": 0.004971192387202445,
+      "acc_norm": 0.6123282214698267,
+      "acc_norm_stderr": 0.004862232790041574
+    },
+    "boolq": {
+      "acc": 0.6018348623853211,
+      "acc_stderr": 0.008561755594317445
+    },
+    "swag": {
+      "acc": 0.505148455463361,
+      "acc_stderr": 0.003534904635576977,
+      "acc_norm": 0.692292312306308,
+      "acc_norm_stderr": 0.003263207195550976
+    },
+    "piqa": {
+      "acc": 0.7393906420021763,
+      "acc_stderr": 0.010241826155811627,
+      "acc_norm": 0.749183895538629,
+      "acc_norm_stderr": 0.010113869547069046
+    },
+    "prost": {
+      "acc": 0.2588599487617421,
+      "acc_stderr": 0.0032000423309913543,
+      "acc_norm": 0.26361016225448336,
+      "acc_norm_stderr": 0.0032189046983713983
+    },
+    "arc_challenge": {
+      "acc": 0.28754266211604096,
+      "acc_stderr": 0.013226719056266129,
+      "acc_norm": 0.3191126279863481,
+      "acc_norm_stderr": 0.013621696119173304
+    }
+  },
+  "versions": {
+    "openbookqa": 0,
+    "winogrande": 0,
+    "arc_easy": 0,
+    "copa": 0,
+    "mc_taco": 0,
+    "wsc273": 0,
+    "hellaswag": 0,
+    "boolq": 1,
+    "swag": 0,
+    "piqa": 0,
+    "prost": 0,
+    "arc_challenge": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=facebook/xglm-7.5B,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda:0",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/xglm/xglm-7.5B/xglm-7.5B_gsm8k_8-shot.json

+{
+  "results": {
+    "gsm8k": {
+      "acc": 0.001516300227445034,
+      "acc_stderr": 0.0010717793485492655
+    }
+  },
+  "versions": {
+    "gsm8k": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=facebook/xglm-7.5B,use_accelerate=True",
+    "num_fewshot": 8,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/xglm/xglm-7.5B/xglm-7.5B_mathematical_reasoning_few_shot_5-shot.json

+{
+  "results": {
+    "math_num_theory": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "gsm8k": {
+      "acc": 0.002274450341167551,
+      "acc_stderr": 0.0013121578148674316
+    },
+    "math_geometry": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "drop": {
+      "em": 0.05421560402684564,
+      "em_stderr": 0.002318984649948223,
+      "f1": 0.08962458053691245,
+      "f1_stderr": 0.0026401926224488034
+    },
+    "math_prealgebra": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "math_counting_and_prob": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "math_precalc": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "math_intermediate_algebra": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "math_algebra": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "mathqa": {
+      "acc": 0.23986599664991626,
+      "acc_stderr": 0.007816818250028128,
+      "acc_norm": 0.23517587939698492,
+      "acc_norm_stderr": 0.0077638612776946255
+    }
+  },
+  "versions": {
+    "math_num_theory": 1,
+    "gsm8k": 0,
+    "math_geometry": 1,
+    "drop": 1,
+    "math_prealgebra": 1,
+    "math_counting_and_prob": 1,
+    "math_precalc": 1,
+    "math_intermediate_algebra": 1,
+    "math_algebra": 1,
+    "mathqa": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=facebook/xglm-7.5B,use_accelerate=True",
+    "num_fewshot": 5,
+    "batch_size": "auto",
+    "device": "cuda:0",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/xglm/xglm-7.5B/xglm-7.5B_pawsx_0-shot.json

+{
+  "results": {
+    "pawsx_en": {
+      "acc": 0.5885,
+      "acc_stderr": 0.011006563824537298
+    },
+    "pawsx_es": {
+      "acc": 0.528,
+      "acc_stderr": 0.011165587094621537
+    },
+    "pawsx_fr": {
+      "acc": 0.518,
+      "acc_stderr": 0.011175886999478619
+    },
+    "pawsx_zh": {
+      "acc": 0.513,
+      "acc_stderr": 0.01117935548207038
+    },
+    "pawsx_ja": {
+      "acc": 0.52,
+      "acc_stderr": 0.011174185930778312
+    },
+    "pawsx_de": {
+      "acc": 0.559,
+      "acc_stderr": 0.011105006104468736
+    },
+    "pawsx_ko": {
+      "acc": 0.4595,
+      "acc_stderr": 0.011146389370464362
+    }
+  },
+  "versions": {
+    "pawsx_en": 0,
+    "pawsx_es": 0,
+    "pawsx_fr": 0,
+    "pawsx_zh": 0,
+    "pawsx_ja": 0,
+    "pawsx_de": 0,
+    "pawsx_ko": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=facebook/xglm-7.5B,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/xglm/xglm-7.5B/xglm-7.5B_xcopa_0-shot.json

+{
+  "results": {
+    "xcopa_et": {
+      "acc": 0.612,
+      "acc_stderr": 0.021814300984787635
+    },
+    "xcopa_th": {
+      "acc": 0.594,
+      "acc_stderr": 0.02198396209008634
+    },
+    "xcopa_qu": {
+      "acc": 0.488,
+      "acc_stderr": 0.02237662679792717
+    },
+    "xcopa_ta": {
+      "acc": 0.544,
+      "acc_stderr": 0.02229623834840705
+    },
+    "xcopa_zh": {
+      "acc": 0.638,
+      "acc_stderr": 0.0215136625275824
+    },
+    "xcopa_vi": {
+      "acc": 0.702,
+      "acc_stderr": 0.02047511809298897
+    },
+    "xcopa_sw": {
+      "acc": 0.6,
+      "acc_stderr": 0.021930844120728505
+    },
+    "xcopa_it": {
+      "acc": 0.636,
+      "acc_stderr": 0.021539170637317685
+    },
+    "xcopa_tr": {
+      "acc": 0.584,
+      "acc_stderr": 0.022064943313928848
+    },
+    "xcopa_id": {
+      "acc": 0.694,
+      "acc_stderr": 0.0206295699983454
+    },
+    "xcopa_ht": {
+      "acc": 0.574,
+      "acc_stderr": 0.022136577335085637
+    }
+  },
+  "versions": {
+    "xcopa_et": 0,
+    "xcopa_th": 0,
+    "xcopa_qu": 0,
+    "xcopa_ta": 0,
+    "xcopa_zh": 0,
+    "xcopa_vi": 0,
+    "xcopa_sw": 0,
+    "xcopa_it": 0,
+    "xcopa_tr": 0,
+    "xcopa_id": 0,
+    "xcopa_ht": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=facebook/xglm-7.5B,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/xglm/xglm-7.5B/xglm-7.5B_xnli_0-shot.json

+{
+  "results": {
+    "xnli_ar": {
+      "acc": 0.3337325349301397,
+      "acc_stderr": 0.00666266628252267
+    },
+    "xnli_bg": {
+      "acc": 0.4489021956087824,
+      "acc_stderr": 0.007027723874210379
+    },
+    "xnli_de": {
+      "acc": 0.48982035928143713,
+      "acc_stderr": 0.0070632481147059134
+    },
+    "xnli_el": {
+      "acc": 0.40658682634730536,
+      "acc_stderr": 0.006940323712177368
+    },
+    "xnli_en": {
+      "acc": 0.5385229540918164,
+      "acc_stderr": 0.0070437128985425335
+    },
+    "xnli_es": {
+      "acc": 0.47704590818363274,
+      "acc_stderr": 0.007057263845316342
+    },
+    "xnli_fr": {
+      "acc": 0.4694610778443114,
+      "acc_stderr": 0.007051522651006734
+    },
+    "xnli_hi": {
+      "acc": 0.4720558882235529,
+      "acc_stderr": 0.007053670508441103
+    },
+    "xnli_ru": {
+      "acc": 0.46327345309381235,
+      "acc_stderr": 0.007045628330322907
+    },
+    "xnli_sw": {
+      "acc": 0.45828343313373254,
+      "acc_stderr": 0.007040080446339805
+    },
+    "xnli_th": {
+      "acc": 0.437125748502994,
+      "acc_stderr": 0.007008633817895695
+    },
+    "xnli_tr": {
+      "acc": 0.4626746506986028,
+      "acc_stderr": 0.007045000071900887
+    },
+    "xnli_ur": {
+      "acc": 0.42095808383233535,
+      "acc_stderr": 0.006975878576227385
+    },
+    "xnli_vi": {
+      "acc": 0.46327345309381235,
+      "acc_stderr": 0.007045628330322896
+    },
+    "xnli_zh": {
+      "acc": 0.3536926147704591,
+      "acc_stderr": 0.006755492859492898
+    }
+  },
+  "versions": {
+    "xnli_ar": 0,
+    "xnli_bg": 0,
+    "xnli_de": 0,
+    "xnli_el": 0,
+    "xnli_en": 0,
+    "xnli_es": 0,
+    "xnli_fr": 0,
+    "xnli_hi": 0,
+    "xnli_ru": 0,
+    "xnli_sw": 0,
+    "xnli_th": 0,
+    "xnli_tr": 0,
+    "xnli_ur": 0,
+    "xnli_vi": 0,
+    "xnli_zh": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=facebook/xglm-7.5B,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/xglm/xglm-7.5B/xglm-7.5B_xstory_cloze_0-shot.json

+{
+  "results": {
+    "xstory_cloze_es": {
+      "acc": 0.6406353408338848,
+      "acc_stderr": 0.012347659802101675
+    },
+    "xstory_cloze_zh": {
+      "acc": 0.5890138980807412,
+      "acc_stderr": 0.012661578894368948
+    },
+    "xstory_cloze_sw": {
+      "acc": 0.5929847782925215,
+      "acc_stderr": 0.012642664836816926
+    },
+    "xstory_cloze_en": {
+      "acc": 0.6982131039046989,
+      "acc_stderr": 0.011812877848905303
+    },
+    "xstory_cloze_hi": {
+      "acc": 0.5876902713434812,
+      "acc_stderr": 0.012667694122397068
+    },
+    "xstory_cloze_ar": {
+      "acc": 0.5618795499669094,
+      "acc_stderr": 0.012768206616277757
+    },
+    "xstory_cloze_eu": {
+      "acc": 0.5771012574454004,
+      "acc_stderr": 0.0127132250091262
+    },
+    "xstory_cloze_id": {
+      "acc": 0.6293845135671741,
+      "acc_stderr": 0.012428861084065903
+    },
+    "xstory_cloze_ru": {
+      "acc": 0.6353408338848445,
+      "acc_stderr": 0.012386781532906161
+    },
+    "xstory_cloze_te": {
+      "acc": 0.6022501654533422,
+      "acc_stderr": 0.012595197856703525
+    },
+    "xstory_cloze_my": {
+      "acc": 0.57114493712773,
+      "acc_stderr": 0.01273620271314778
+    }
+  },
+  "versions": {
+    "xstory_cloze_es": 0,
+    "xstory_cloze_zh": 0,
+    "xstory_cloze_sw": 0,
+    "xstory_cloze_en": 0,
+    "xstory_cloze_hi": 0,
+    "xstory_cloze_ar": 0,
+    "xstory_cloze_eu": 0,
+    "xstory_cloze_id": 0,
+    "xstory_cloze_ru": 0,
+    "xstory_cloze_te": 0,
+    "xstory_cloze_my": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=facebook/xglm-7.5B,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}