add bloom, xglm and llama results

results/bloom/bloom-1b7/bloom-1b7_xwinograd_0-shot.json

+{
+  "results": {
+    "xwinograd_zh": {
+      "acc": 0.6964285714285714,
+      "acc_stderr": 0.02050145785082139
+    },
+    "xwinograd_jp": {
+      "acc": 0.5411887382690302,
+      "acc_stderr": 0.016099361618063936
+    },
+    "xwinograd_pt": {
+      "acc": 0.6349809885931559,
+      "acc_stderr": 0.02974318401093692
+    },
+    "xwinograd_fr": {
+      "acc": 0.6867469879518072,
+      "acc_stderr": 0.051219942106581456
+    },
+    "xwinograd_ru": {
+      "acc": 0.5238095238095238,
+      "acc_stderr": 0.02818462259599846
+    },
+    "xwinograd_en": {
+      "acc": 0.7470967741935484,
+      "acc_stderr": 0.009016698607546121
+    }
+  },
+  "versions": {
+    "xwinograd_zh": 0,
+    "xwinograd_jp": 0,
+    "xwinograd_pt": 0,
+    "xwinograd_fr": 0,
+    "xwinograd_ru": 0,
+    "xwinograd_en": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=bigscience/bloom-1b7",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/bloom/bloom-3b/bloom-3b_common_sense_reasoning_0-shot.json

+{
+  "results": {
+    "openbookqa": {
+      "acc": 0.216,
+      "acc_stderr": 0.01842190906141194,
+      "acc_norm": 0.322,
+      "acc_norm_stderr": 0.020916668330019882
+    },
+    "wsc273": {
+      "acc": 0.7692307692307693,
+      "acc_stderr": 0.02554658323673352
+    },
+    "swag": {
+      "acc": 0.47355793262021395,
+      "acc_stderr": 0.0035301452338806077,
+      "acc_norm": 0.6459062281315605,
+      "acc_norm_stderr": 0.003381229133319312
+    },
+    "mc_taco": {
+      "em": 0.11936936936936937,
+      "f1": 0.4957122298258418
+    },
+    "winogrande": {
+      "acc": 0.5872138910812944,
+      "acc_stderr": 0.0138370606486821
+    },
+    "prost": {
+      "acc": 0.22688941076003416,
+      "acc_stderr": 0.003059856614352936,
+      "acc_norm": 0.26361016225448336,
+      "acc_norm_stderr": 0.0032189046983713957
+    },
+    "arc_challenge": {
+      "acc": 0.27986348122866894,
+      "acc_stderr": 0.013119040897725922,
+      "acc_norm": 0.3054607508532423,
+      "acc_norm_stderr": 0.013460080478002498
+    },
+    "arc_easy": {
+      "acc": 0.5946969696969697,
+      "acc_stderr": 0.010074093589739182,
+      "acc_norm": 0.5324074074074074,
+      "acc_norm_stderr": 0.010238210368801902
+    },
+    "piqa": {
+      "acc": 0.7083786724700761,
+      "acc_stderr": 0.010604441527428793,
+      "acc_norm": 0.705114254624592,
+      "acc_norm_stderr": 0.010639030620156982
+    },
+    "copa": {
+      "acc": 0.74,
+      "acc_stderr": 0.04408440022768078
+    },
+    "boolq": {
+      "acc": 0.6162079510703364,
+      "acc_stderr": 0.008505584729104964
+    },
+    "hellaswag": {
+      "acc": 0.41256721768571997,
+      "acc_stderr": 0.004912900450370833,
+      "acc_norm": 0.527185819557857,
+      "acc_norm_stderr": 0.0049824003689396615
+    }
+  },
+  "versions": {
+    "openbookqa": 0,
+    "wsc273": 0,
+    "swag": 0,
+    "mc_taco": 0,
+    "winogrande": 0,
+    "prost": 0,
+    "arc_challenge": 0,
+    "arc_easy": 0,
+    "piqa": 0,
+    "copa": 0,
+    "boolq": 1,
+    "hellaswag": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=bigscience/bloom-3b,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda:0",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/bloom/bloom-3b/bloom-3b_gsm8k_8-shot.json

+{
+  "results": {
+    "gsm8k": {
+      "acc": 0.012130401819560273,
+      "acc_stderr": 0.0030152942428909486
+    }
+  },
+  "versions": {
+    "gsm8k": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=bigscience/bloom-3b,use_accelerate=True",
+    "num_fewshot": 8,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/bloom/bloom-3b/bloom-3b_mathematical_reasoning_few_shot_5-shot.json

+{
+  "results": {
+    "math_counting_and_prob": {
+      "acc": 0.002109704641350211,
+      "acc_stderr": 0.00210970464135021
+    },
+    "math_algebra": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "math_precalc": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "gsm8k": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "math_prealgebra": {
+      "acc": 0.001148105625717566,
+      "acc_stderr": 0.0011481056257175708
+    },
+    "math_geometry": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "drop": {
+      "em": 0.02097315436241611,
+      "em_stderr": 0.001467468637213982,
+      "f1": 0.04631921140939603,
+      "f1_stderr": 0.001664167972365937
+    },
+    "math_num_theory": {
+      "acc": 0.001851851851851852,
+      "acc_stderr": 0.0018518518518518448
+    },
+    "math_intermediate_algebra": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "mathqa": {
+      "acc": 0.2525963149078727,
+      "acc_stderr": 0.007954112207299597,
+      "acc_norm": 0.25058626465661643,
+      "acc_norm_stderr": 0.00793304734353984
+    }
+  },
+  "versions": {
+    "math_counting_and_prob": 1,
+    "math_algebra": 1,
+    "math_precalc": 1,
+    "mathqa": 0,
+    "gsm8k": 0,
+    "math_prealgebra": 1,
+    "math_geometry": 1,
+    "drop": 1,
+    "math_num_theory": 1,
+    "math_intermediate_algebra": 1
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=bigscience/bloom-3b,use_accelerate=True",
+    "num_fewshot": 5,
+    "batch_size": "auto",
+    "device": "cuda:0",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/bloom/bloom-3b/bloom-3b_pawsx_0-shot.json

+{
+  "results": {
+    "pawsx_es": {
+      "acc": 0.564,
+      "acc_stderr": 0.011091145421162657
+    },
+    "pawsx_ja": {
+      "acc": 0.446,
+      "acc_stderr": 0.011117724672834362
+    },
+    "pawsx_ko": {
+      "acc": 0.463,
+      "acc_stderr": 0.011152474561478177
+    },
+    "pawsx_zh": {
+      "acc": 0.471,
+      "acc_stderr": 0.011164310140373722
+    },
+    "pawsx_en": {
+      "acc": 0.568,
+      "acc_stderr": 0.011079231683079107
+    },
+    "pawsx_de": {
+      "acc": 0.546,
+      "acc_stderr": 0.0111357084193598
+    },
+    "pawsx_fr": {
+      "acc": 0.476,
+      "acc_stderr": 0.011170245619215438
+    }
+  },
+  "versions": {
+    "pawsx_es": 0,
+    "pawsx_ja": 0,
+    "pawsx_ko": 0,
+    "pawsx_zh": 0,
+    "pawsx_en": 0,
+    "pawsx_de": 0,
+    "pawsx_fr": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=bigscience/bloom-3b",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/bloom/bloom-3b/bloom-3b_question_answering_0-shot.json

+{
+  "results": {
+    "squad2": {
+      "exact": 6.914848816642803,
+      "f1": 11.511512971067512,
+      "HasAns_exact": 11.099865047233468,
+      "HasAns_f1": 20.306375422652543,
+      "NoAns_exact": 2.7417998317914214,
+      "NoAns_f1": 2.7417998317914214,
+      "best_exact": 50.07159100480081,
+      "best_f1": 50.08024690773861
+    },
+    "logiqa": {
+      "acc": 0.2073732718894009,
+      "acc_stderr": 0.015902084913876333,
+      "acc_norm": 0.29185867895545314,
+      "acc_norm_stderr": 0.017831570553971925
+    },
+    "headqa_en": {
+      "acc": 0.2840991976659373,
+      "acc_stderr": 0.008614040521644994,
+      "acc_norm": 0.3336980306345733,
+      "acc_norm_stderr": 0.009006537310888562
+    },
+    "truthfulqa_mc": {
+      "mc1": 0.23255813953488372,
+      "mc1_stderr": 0.014789157531080503,
+      "mc2": 0.40572206357204965,
+      "mc2_stderr": 0.014390512893375817
+    },
+    "webqs": {
+      "acc": 0.01673228346456693,
+      "acc_stderr": 0.0028461549169432184
+    },
+    "triviaqa": {
+      "acc": 0.04154512507734465,
+      "acc_stderr": 0.0018761872163031025
+    },
+    "headqa_es": {
+      "acc": 0.26440554339897887,
+      "acc_stderr": 0.008423643607316284,
+      "acc_norm": 0.3099927060539752,
+      "acc_norm_stderr": 0.008833810133604958
+    }
+  },
+  "versions": {
+    "squad2": 1,
+    "logiqa": 0,
+    "headqa_en": 0,
+    "truthfulqa_mc": 1,
+    "webqs": 0,
+    "triviaqa": 1,
+    "headqa_es": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=bigscience/bloom-3b,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda:0",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/bloom/bloom-3b/bloom-3b_reading_comprehension_0-shot.json

+{
+  "results": {
+    "race": {
+      "acc": 0.3521531100478469,
+      "acc_stderr": 0.014782629897202266
+    },
+    "coqa": {
+      "f1": 0.6149953283852732,
+      "f1_stderr": 0.017671101287646335,
+      "em": 0.4606666666666665,
+      "em_stderr": 0.020185214533701167
+    },
+    "drop": {
+      "em": 0.0194001677852349,
+      "em_stderr": 0.0014124994962717904,
+      "f1": 0.08879089765100667,
+      "f1_stderr": 0.002002230574295575
+    }
+  },
+  "versions": {
+    "race": 1,
+    "coqa": 1,
+    "drop": 1
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=bigscience/bloom-3b,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda:0",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/bloom/bloom-3b/bloom-3b_xcopa_0-shot.json

+{
+  "results": {
+    "xcopa_qu": {
+      "acc": 0.506,
+      "acc_stderr": 0.022381462412439324
+    },
+    "xcopa_ta": {
+      "acc": 0.58,
+      "acc_stderr": 0.02209471322976178
+    },
+    "xcopa_it": {
+      "acc": 0.516,
+      "acc_stderr": 0.0223716109825804
+    },
+    "xcopa_vi": {
+      "acc": 0.688,
+      "acc_stderr": 0.02074059653648807
+    },
+    "xcopa_id": {
+      "acc": 0.692,
+      "acc_stderr": 0.0206670329874661
+    },
+    "xcopa_sw": {
+      "acc": 0.514,
+      "acc_stderr": 0.02237429816635319
+    },
+    "xcopa_et": {
+      "acc": 0.492,
+      "acc_stderr": 0.022380208834928028
+    },
+    "xcopa_tr": {
+      "acc": 0.534,
+      "acc_stderr": 0.02233126442325838
+    },
+    "xcopa_th": {
+      "acc": 0.526,
+      "acc_stderr": 0.02235279165091416
+    },
+    "xcopa_ht": {
+      "acc": 0.502,
+      "acc_stderr": 0.022382894986483524
+    },
+    "xcopa_zh": {
+      "acc": 0.62,
+      "acc_stderr": 0.021728881438701702
+    }
+  },
+  "versions": {
+    "xcopa_qu": 0,
+    "xcopa_ta": 0,
+    "xcopa_it": 0,
+    "xcopa_vi": 0,
+    "xcopa_id": 0,
+    "xcopa_sw": 0,
+    "xcopa_et": 0,
+    "xcopa_tr": 0,
+    "xcopa_th": 0,
+    "xcopa_ht": 0,
+    "xcopa_zh": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=bigscience/bloom-3b",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/bloom/bloom-3b/bloom-3b_xnli_0-shot.json

+{
+  "results": {
+    "xnli_ar": {
+      "acc": 0.3343313373253493,
+      "acc_stderr": 0.006665643509474758
+    },
+    "xnli_bg": {
+      "acc": 0.3790419161676647,
+      "acc_stderr": 0.006854870366766951
+    },
+    "xnli_de": {
+      "acc": 0.4039920159680639,
+      "acc_stderr": 0.00693325085793088
+    },
+    "xnli_el": {
+      "acc": 0.3321357285429142,
+      "acc_stderr": 0.006654667909419873
+    },
+    "xnli_en": {
+      "acc": 0.5341317365269461,
+      "acc_stderr": 0.007048232775587604
+    },
+    "xnli_es": {
+      "acc": 0.4908183632734531,
+      "acc_stderr": 0.007063521183707481
+    },
+    "xnli_fr": {
+      "acc": 0.49181636726546907,
+      "acc_stderr": 0.007063766092285604
+    },
+    "xnli_hi": {
+      "acc": 0.4554890219560878,
+      "acc_stderr": 0.007036663146398998
+    },
+    "xnli_ru": {
+      "acc": 0.41397205588822356,
+      "acc_stderr": 0.006959357713092714
+    },
+    "xnli_sw": {
+      "acc": 0.3582834331337325,
+      "acc_stderr": 0.006775002711732832
+    },
+    "xnli_th": {
+      "acc": 0.3339321357285429,
+      "acc_stderr": 0.006663660032909966
+    },
+    "xnli_tr": {
+      "acc": 0.33812375249501,
+      "acc_stderr": 0.006684219795088851
+    },
+    "xnli_ur": {
+      "acc": 0.4,
+      "acc_stderr": 0.006921976252528393
+    },
+    "xnli_vi": {
+      "acc": 0.46506986027944114,
+      "acc_stderr": 0.0070474518252208835
+    },
+    "xnli_zh": {
+      "acc": 0.37425149700598803,
+      "acc_stderr": 0.006837638981887555
+    }
+  },
+  "versions": {
+    "xnli_ar": 0,
+    "xnli_bg": 0,
+    "xnli_de": 0,
+    "xnli_el": 0,
+    "xnli_en": 0,
+    "xnli_es": 0,
+    "xnli_fr": 0,
+    "xnli_hi": 0,
+    "xnli_ru": 0,
+    "xnli_sw": 0,
+    "xnli_th": 0,
+    "xnli_tr": 0,
+    "xnli_ur": 0,
+    "xnli_vi": 0,
+    "xnli_zh": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=bigscience/bloom-3b",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/bloom/bloom-3b/bloom-3b_xstory_cloze_0-shot.json

+{
+  "results": {
+    "xstory_cloze_eu": {
+      "acc": 0.5565850430178689,
+      "acc_stderr": 0.012784462136657198
+    },
+    "xstory_cloze_en": {
+      "acc": 0.6677696889477167,
+      "acc_stderr": 0.01212116892354459
+    },
+    "xstory_cloze_hi": {
+      "acc": 0.5757776307081403,
+      "acc_stderr": 0.012718494399531051
+    },
+    "xstory_cloze_sw": {
+      "acc": 0.5301125082726671,
+      "acc_stderr": 0.012843769248432169
+    },
+    "xstory_cloze_id": {
+      "acc": 0.6082064857710126,
+      "acc_stderr": 0.012562199063960642
+    },
+    "xstory_cloze_my": {
+      "acc": 0.46591661151555264,
+      "acc_stderr": 0.012837195610619434
+    },
+    "xstory_cloze_ru": {
+      "acc": 0.5069490403706155,
+      "acc_stderr": 0.01286588257096072
+    },
+    "xstory_cloze_te": {
+      "acc": 0.5817339510258107,
+      "acc_stderr": 0.012694045150564688
+    },
+    "xstory_cloze_zh": {
+      "acc": 0.6088682991396426,
+      "acc_stderr": 0.012558411693622684
+    },
+    "xstory_cloze_ar": {
+      "acc": 0.5658504301786896,
+      "acc_stderr": 0.01275504628991221
+    },
+    "xstory_cloze_es": {
+      "acc": 0.6412971542025149,
+      "acc_stderr": 0.012342655113112371
+    }
+  },
+  "versions": {
+    "xstory_cloze_eu": 0,
+    "xstory_cloze_en": 0,
+    "xstory_cloze_hi": 0,
+    "xstory_cloze_sw": 0,
+    "xstory_cloze_id": 0,
+    "xstory_cloze_my": 0,
+    "xstory_cloze_ru": 0,
+    "xstory_cloze_te": 0,
+    "xstory_cloze_zh": 0,
+    "xstory_cloze_ar": 0,
+    "xstory_cloze_es": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=bigscience/bloom-3b",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/bloom/bloom-3b/bloom-3b_xwinograd_0-shot.json

+{
+  "results": {
+    "xwinograd_en": {
+      "acc": 0.7909677419354839,
+      "acc_stderr": 0.008434667753827384
+    },
+    "xwinograd_pt": {
+      "acc": 0.7034220532319392,
+      "acc_stderr": 0.028218035233491295
+    },
+    "xwinograd_jp": {
+      "acc": 0.5662148070907195,
+      "acc_stderr": 0.016011986615263228
+    },
+    "xwinograd_fr": {
+      "acc": 0.7108433734939759,
+      "acc_stderr": 0.050066428050419186
+    },
+    "xwinograd_zh": {
+      "acc": 0.7361111111111112,
+      "acc_stderr": 0.019651614961528674
+    },
+    "xwinograd_ru": {
+      "acc": 0.5365079365079365,
+      "acc_stderr": 0.028141315964997568
+    }
+  },
+  "versions": {
+    "xwinograd_en": 0,
+    "xwinograd_pt": 0,
+    "xwinograd_jp": 0,
+    "xwinograd_fr": 0,
+    "xwinograd_zh": 0,
+    "xwinograd_ru": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=bigscience/bloom-3b",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/bloom/bloom-560m/bloom-560m_common_sense_reasoning_0-shot.json

+{
+  "results": {
+    "hellaswag": {
+      "acc": 0.31557458673571004,
+      "acc_stderr": 0.004637944965914592,
+      "acc_norm": 0.3655646285600478,
+      "acc_norm_stderr": 0.00480603903900897
+    },
+    "prost": {
+      "acc": 0.22080486763450044,
+      "acc_stderr": 0.0030304044027250577,
+      "acc_norm": 0.3207728437233134,
+      "acc_norm_stderr": 0.003410197007857463
+    },
+    "piqa": {
+      "acc": 0.6409140369967355,
+      "acc_stderr": 0.011192949073844103,
+      "acc_norm": 0.6512513601741022,
+      "acc_norm_stderr": 0.011119263056159595
+    },
+    "arc_easy": {
+      "acc": 0.4734848484848485,
+      "acc_stderr": 0.010245347015573713,
+      "acc_norm": 0.4166666666666667,
+      "acc_norm_stderr": 0.01011628297778124
+    },
+    "winogrande": {
+      "acc": 0.5280189423835833,
+      "acc_stderr": 0.014030404213405784
+    },
+    "mc_taco": {
+      "em": 0.17417417417417416,
+      "f1": 0.31427590778450365
+    },
+    "openbookqa": {
+      "acc": 0.172,
+      "acc_stderr": 0.01689386887634748,
+      "acc_norm": 0.282,
+      "acc_norm_stderr": 0.020143572847290795
+    },
+    "copa": {
+      "acc": 0.61,
+      "acc_stderr": 0.04902071300001975
+    },
+    "boolq": {
+      "acc": 0.5513761467889908,
+      "acc_stderr": 0.008698767182005265
+    },
+    "swag": {
+      "acc": 0.40347895631310604,
+      "acc_stderr": 0.003468598652499914,
+      "acc_norm": 0.5296411076676997,
+      "acc_norm_stderr": 0.003528874749486556
+    },
+    "arc_challenge": {
+      "acc": 0.22440273037542663,
+      "acc_stderr": 0.012191404938603838,
+      "acc_norm": 0.23976109215017063,
+      "acc_norm_stderr": 0.012476304127453947
+    },
+    "wsc273": {
+      "acc": 0.6666666666666666,
+      "acc_stderr": 0.028583097523751506
+    }
+  },
+  "versions": {
+    "hellaswag": 0,
+    "prost": 0,
+    "piqa": 0,
+    "arc_easy": 0,
+    "winogrande": 0,
+    "mc_taco": 0,
+    "openbookqa": 0,
+    "copa": 0,
+    "boolq": 1,
+    "swag": 0,
+    "arc_challenge": 0,
+    "wsc273": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=bigscience/bloom-560m,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda:0",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/bloom/bloom-560m/bloom-560m_gsm8k_8-shot.json

+{
+  "results": {
+    "gsm8k": {
+      "acc": 0.00530705079605762,
+      "acc_stderr": 0.002001305720948044
+    }
+  },
+  "versions": {
+    "gsm8k": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=bigscience/bloom-560m,use_accelerate=True",
+    "num_fewshot": 8,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/bloom/bloom-560m/bloom-560m_lambada_openai_0-shot.json

+{
+  "results": {
+    "lambada_openai_cloze": {
+      "ppl": 6212.811566448059,
+      "ppl_stderr": 267.17480839849884,
+      "acc": 0.00446341936735882,
+      "acc_stderr": 0.0009286980441682362
+    },
+    "lambada_openai": {
+      "ppl": 28.67792043529687,
+      "ppl_stderr": 1.0838905590384336,
+      "acc": 0.35396856200271687,
+      "acc_stderr": 0.006662254900290991
+    }
+  },
+  "versions": {
+    "lambada_openai_cloze": 0,
+    "lambada_openai": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=bigscience/bloom-560m,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda:0",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/bloom/bloom-560m/bloom-560m_mathematical_reasoning_few_shot_5-shot.json

+{
+  "results": {
+    "math_intermediate_algebra": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "math_geometry": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "math_prealgebra": {
+      "acc": 0.002296211251435132,
+      "acc_stderr": 0.001622733136934621
+    },
+    "drop": {
+      "em": 0.012583892617449664,
+      "em_stderr": 0.0011415560941551478,
+      "f1": 0.034977978187919485,
+      "f1_stderr": 0.0013870691301022255
+    },
+    "math_counting_and_prob": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "gsm8k": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "math_num_theory": {
+      "acc": 0.001851851851851852,
+      "acc_stderr": 0.0018518518518518465
+    },
+    "math_precalc": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "math_algebra": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "mathqa": {
+      "acc": 0.22512562814070353,
+      "acc_stderr": 0.00764590166234271,
+      "acc_norm": 0.2234505862646566,
+      "acc_norm_stderr": 0.00762563278617748
+    }
+  },
+  "versions": {
+    "math_intermediate_algebra": 1,
+    "math_geometry": 1,
+    "math_prealgebra": 1,
+    "drop": 1,
+    "mathqa": 0,
+    "math_counting_and_prob": 1,
+    "gsm8k": 0,
+    "math_num_theory": 1,
+    "math_precalc": 1,
+    "math_algebra": 1
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=bigscience/bloom-560m,use_accelerate=True",
+    "num_fewshot": 5,
+    "batch_size": "auto",
+    "device": "cuda:0",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/bloom/bloom-560m/bloom-560m_pawsx_0-shot.json

+{
+  "results": {
+    "pawsx_zh": {
+      "acc": 0.452,
+      "acc_stderr": 0.011131484850525779
+    },
+    "pawsx_de": {
+      "acc": 0.528,
+      "acc_stderr": 0.01116558709462154
+    },
+    "pawsx_ja": {
+      "acc": 0.449,
+      "acc_stderr": 0.011124809242874427
+    },
+    "pawsx_en": {
+      "acc": 0.52,
+      "acc_stderr": 0.011174185930778313
+    },
+    "pawsx_fr": {
+      "acc": 0.4795,
+      "acc_stderr": 0.011173732641806813
+    },
+    "pawsx_es": {
+      "acc": 0.5325,
+      "acc_stderr": 0.011159486640120933
+    },
+    "pawsx_ko": {
+      "acc": 0.519,
+      "acc_stderr": 0.011175058879956058
+    }
+  },
+  "versions": {
+    "pawsx_zh": 0,
+    "pawsx_de": 0,
+    "pawsx_ja": 0,
+    "pawsx_en": 0,
+    "pawsx_fr": 0,
+    "pawsx_es": 0,
+    "pawsx_ko": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=bigscience/bloom-560m",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/bloom/bloom-560m/bloom-560m_question_answering_0-shot.json

+{
+  "results": {
+    "logiqa": {
+      "acc": 0.22580645161290322,
+      "acc_stderr": 0.01639971378844507,
+      "acc_norm": 0.271889400921659,
+      "acc_norm_stderr": 0.01745171600943683
+    },
+    "webqs": {
+      "acc": 0.008366141732283465,
+      "acc_stderr": 0.002021079144496907
+    },
+    "triviaqa": {
+      "acc": 0.014408202952355696,
+      "acc_stderr": 0.001120427476389321
+    },
+    "truthfulqa_mc": {
+      "mc1": 0.24479804161566707,
+      "mc1_stderr": 0.01505186948671501,
+      "mc2": 0.42431454034377986,
+      "mc2_stderr": 0.015119647586282134
+    },
+    "headqa_en": {
+      "acc": 0.2567469000729395,
+      "acc_stderr": 0.008343849291869198,
+      "acc_norm": 0.29576951130561635,
+      "acc_norm_stderr": 0.008717251898361426
+    },
+    "headqa_es": {
+      "acc": 0.23960612691466082,
+      "acc_stderr": 0.008152930613263032,
+      "acc_norm": 0.27169948942377825,
+      "acc_norm_stderr": 0.00849660053640109
+    },
+    "squad2": {
+      "exact": 0.4295460288048513,
+      "f1": 1.8591225589833205,
+      "HasAns_exact": 0.7591093117408907,
+      "HasAns_f1": 3.622362034886802,
+      "NoAns_exact": 0.10092514718250631,
+      "NoAns_f1": 0.10092514718250631,
+      "best_exact": 50.07159100480081,
+      "best_f1": 50.07159100480081
+    }
+  },
+  "versions": {
+    "logiqa": 0,
+    "webqs": 0,
+    "triviaqa": 1,
+    "truthfulqa_mc": 1,
+    "headqa_en": 0,
+    "headqa_es": 0,
+    "squad2": 1
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=bigscience/bloom-560m,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda:0",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/bloom/bloom-560m/bloom-560m_reading_comprehension_0-shot.json

+{
+  "results": {
+    "drop": {
+      "em": 0.014995805369127516,
+      "em_stderr": 0.0012446395261788805,
+      "f1": 0.0621245805369127,
+      "f1_stderr": 0.001730980795797461
+    },
+    "coqa": {
+      "f1": 0.22712736568843772,
+      "f1_stderr": 0.01673094848597647,
+      "em": 0.174,
+      "em_stderr": 0.016190705499013296
+    },
+    "race": {
+      "acc": 0.30239234449760766,
+      "acc_stderr": 0.014214800395178313
+    }
+  },
+  "versions": {
+    "drop": 1,
+    "race": 1,
+    "coqa": 1
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=bigscience/bloom-560m,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda:0",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/bloom/bloom-560m/bloom-560m_xcopa_0-shot.json

+{
+  "results": {
+    "xcopa_id": {
+      "acc": 0.592,
+      "acc_stderr": 0.02200091089387719
+    },
+    "xcopa_ht": {
+      "acc": 0.502,
+      "acc_stderr": 0.022382894986483524
+    },
+    "xcopa_qu": {
+      "acc": 0.502,
+      "acc_stderr": 0.02238289498648353
+    },
+    "xcopa_et": {
+      "acc": 0.49,
+      "acc_stderr": 0.022378596989230785
+    },
+    "xcopa_th": {
+      "acc": 0.544,
+      "acc_stderr": 0.022296238348407056
+    },
+    "xcopa_tr": {
+      "acc": 0.53,
+      "acc_stderr": 0.02234274819250285
+    },
+    "xcopa_it": {
+      "acc": 0.508,
+      "acc_stderr": 0.02238020883492804
+    },
+    "xcopa_ta": {
+      "acc": 0.558,
+      "acc_stderr": 0.02223197069632112
+    },
+    "xcopa_sw": {
+      "acc": 0.516,
+      "acc_stderr": 0.022371610982580396
+    },
+    "xcopa_vi": {
+      "acc": 0.61,
+      "acc_stderr": 0.021834685869369208
+    },
+    "xcopa_zh": {
+      "acc": 0.586,
+      "acc_stderr": 0.022049497969827865
+    }
+  },
+  "versions": {
+    "xcopa_id": 0,
+    "xcopa_ht": 0,
+    "xcopa_qu": 0,
+    "xcopa_et": 0,
+    "xcopa_th": 0,
+    "xcopa_tr": 0,
+    "xcopa_it": 0,
+    "xcopa_ta": 0,
+    "xcopa_sw": 0,
+    "xcopa_vi": 0,
+    "xcopa_zh": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=bigscience/bloom-560m",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/bloom/bloom-560m/bloom-560m_xnli_0-shot.json

+{
+  "results": {
+    "xnli_sw": {
+      "acc": 0.3317365269461078,
+      "acc_stderr": 0.006652654857813421
+    },
+    "xnli_th": {
+      "acc": 0.33572854291417165,
+      "acc_stderr": 0.0066725434859242665
+    },
+    "xnli_ur": {
+      "acc": 0.3712574850299401,
+      "acc_stderr": 0.0068265064875345964
+    },
+    "xnli_bg": {
+      "acc": 0.3339321357285429,
+      "acc_stderr": 0.00666366003290998
+    },
+    "xnli_tr": {
+      "acc": 0.3343313373253493,
+      "acc_stderr": 0.006665643509474755
+    },
+    "xnli_zh": {
+      "acc": 0.3395209580838323,
+      "acc_stderr": 0.006690942515072474
+    },
+    "xnli_ar": {
+      "acc": 0.3335329341317365,
+      "acc_stderr": 0.006661671189931638
+    },
+    "xnli_el": {
+      "acc": 0.3333333333333333,
+      "acc_stderr": 0.006660674754535592
+    },
+    "xnli_hi": {
+      "acc": 0.40838323353293415,
+      "acc_stderr": 0.006945102706766183
+    },
+    "xnli_fr": {
+      "acc": 0.4528942115768463,
+      "acc_stderr": 0.007033289986695001
+    },
+    "xnli_es": {
+      "acc": 0.45229540918163674,
+      "acc_stderr": 0.007032484191375647
+    },
+    "xnli_vi": {
+      "acc": 0.405189620758483,
+      "acc_stderr": 0.006936540228025353
+    },
+    "xnli_en": {
+      "acc": 0.49500998003992014,
+      "acc_stderr": 0.007064360593648105
+    },
+    "xnli_de": {
+      "acc": 0.34790419161676644,
+      "acc_stderr": 0.006729921818907755
+    },
+    "xnli_ru": {
+      "acc": 0.3401197604790419,
+      "acc_stderr": 0.006693803790492355
+    }
+  },
+  "versions": {
+    "xnli_sw": 0,
+    "xnli_th": 0,
+    "xnli_ur": 0,
+    "xnli_bg": 0,
+    "xnli_tr": 0,
+    "xnli_zh": 0,
+    "xnli_ar": 0,
+    "xnli_el": 0,
+    "xnli_hi": 0,
+    "xnli_fr": 0,
+    "xnli_es": 0,
+    "xnli_vi": 0,
+    "xnli_en": 0,
+    "xnli_de": 0,
+    "xnli_ru": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=bigscience/bloom-560m",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}