Merge pull request #477 from juletx/results

Add results of various models in json and md format

results/llama/llama-7B/llama-7B_mathematical_reasoning_0-shot.json

+{
+  "results": {
+    "mathqa": {
+      "acc": 0.26767169179229483,
+      "acc_stderr": 0.008105031808599684,
+      "acc_norm": 0.27872696817420434,
+      "acc_norm_stderr": 0.008208048863665952
+    },
+    "math_asdiv": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "gsm8k": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "math_num_theory": {
+      "acc": 0.007407407407407408,
+      "acc_stderr": 0.003693382168437238
+    },
+    "math_precalc": {
+      "acc": 0.003663003663003663,
+      "acc_stderr": 0.002587757368193461
+    },
+    "drop": {
+      "em": 0.04268036912751678,
+      "em_stderr": 0.002070056585023236,
+      "f1": 0.1215950083892614,
+      "f1_stderr": 0.0024765528531984883
+    },
+    "math_geometry": {
+      "acc": 0.008350730688935281,
+      "acc_stderr": 0.004162242110295851
+    },
+    "math_counting_and_prob": {
+      "acc": 0.016877637130801686,
+      "acc_stderr": 0.0059228268948526815
+    },
+    "math_intermediate_algebra": {
+      "acc": 0.006644518272425249,
+      "acc_stderr": 0.0027050844483854013
+    },
+    "math_prealgebra": {
+      "acc": 0.012629161882893225,
+      "acc_stderr": 0.003785888218263002
+    },
+    "math_algebra": {
+      "acc": 0.016849199663016005,
+      "acc_stderr": 0.0037372948497597248
+    }
+  },
+  "versions": {
+    "mathqa": 0,
+    "math_asdiv": 0,
+    "gsm8k": 0,
+    "math_num_theory": 1,
+    "math_precalc": 1,
+    "drop": 1,
+    "math_geometry": 1,
+    "math_counting_and_prob": 1,
+    "math_intermediate_algebra": 1,
+    "math_prealgebra": 1,
+    "math_algebra": 1
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda:0",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/llama/llama-7B/llama-7B_mathematical_reasoning_few_shot_5-shot.json

+{
+  "results": {
+    "mathqa": {
+      "acc": 0.2820770519262982,
+      "acc_stderr": 0.008238030326915545,
+      "acc_norm": 0.2877721943048576,
+      "acc_norm_stderr": 0.008287708494779904
+    },
+    "math_prealgebra": {
+      "acc": 0.001148105625717566,
+      "acc_stderr": 0.0011481056257175704
+    },
+    "math_geometry": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "math_intermediate_algebra": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "math_counting_and_prob": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "math_num_theory": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "gsm8k": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "drop": {
+      "em": 0.012374161073825503,
+      "em_stderr": 0.0011321233703992673,
+      "f1": 0.020981543624161086,
+      "f1_stderr": 0.001252441423790731
+    },
+    "math_precalc": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "math_algebra": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    }
+  },
+  "versions": {
+    "mathqa": 0,
+    "math_prealgebra": 1,
+    "math_geometry": 1,
+    "math_intermediate_algebra": 1,
+    "math_counting_and_prob": 1,
+    "math_num_theory": 1,
+    "gsm8k": 0,
+    "drop": 1,
+    "math_precalc": 1,
+    "math_algebra": 1
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B,use_accelerate=True",
+    "num_fewshot": 5,
+    "batch_size": "auto",
+    "device": "cuda:0",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/llama/llama-7B/llama-7B_mmlu_5-shot.json

+{
+  "results": {
+    "hendrycksTest-high_school_geography": {
+      "acc": 0.4292929292929293,
+      "acc_stderr": 0.035265527246011986,
+      "acc_norm": 0.36363636363636365,
+      "acc_norm_stderr": 0.03427308652999934
+    },
+    "hendrycksTest-philosophy": {
+      "acc": 0.40192926045016075,
+      "acc_stderr": 0.027846476005930477,
+      "acc_norm": 0.3536977491961415,
+      "acc_norm_stderr": 0.02715520810320088
+    },
+    "hendrycksTest-world_religions": {
+      "acc": 0.6257309941520468,
+      "acc_stderr": 0.03711601185389481,
+      "acc_norm": 0.5146198830409356,
+      "acc_norm_stderr": 0.038331852752130254
+    },
+    "hendrycksTest-college_biology": {
+      "acc": 0.3194444444444444,
+      "acc_stderr": 0.03899073687357335,
+      "acc_norm": 0.2916666666666667,
+      "acc_norm_stderr": 0.03800968060554858
+    },
+    "hendrycksTest-electrical_engineering": {
+      "acc": 0.3586206896551724,
+      "acc_stderr": 0.03996629574876719,
+      "acc_norm": 0.32413793103448274,
+      "acc_norm_stderr": 0.03900432069185554
+    },
+    "hendrycksTest-global_facts": {
+      "acc": 0.32,
+      "acc_stderr": 0.046882617226215034,
+      "acc_norm": 0.29,
+      "acc_norm_stderr": 0.045604802157206824
+    },
+    "hendrycksTest-high_school_government_and_politics": {
+      "acc": 0.48186528497409326,
+      "acc_stderr": 0.03606065001832917,
+      "acc_norm": 0.37305699481865284,
+      "acc_norm_stderr": 0.03490205592048573
+    },
+    "hendrycksTest-moral_scenarios": {
+      "acc": 0.2759776536312849,
+      "acc_stderr": 0.014950103002475353,
+      "acc_norm": 0.27262569832402234,
+      "acc_norm_stderr": 0.014893391735249588
+    },
+    "hendrycksTest-econometrics": {
+      "acc": 0.2894736842105263,
+      "acc_stderr": 0.04266339443159394,
+      "acc_norm": 0.2631578947368421,
+      "acc_norm_stderr": 0.0414243971948936
+    },
+    "hendrycksTest-international_law": {
+      "acc": 0.3884297520661157,
+      "acc_stderr": 0.04449270350068382,
+      "acc_norm": 0.5785123966942148,
+      "acc_norm_stderr": 0.045077322787750874
+    },
+    "hendrycksTest-us_foreign_policy": {
+      "acc": 0.56,
+      "acc_stderr": 0.049888765156985884,
+      "acc_norm": 0.45,
+      "acc_norm_stderr": 0.05
+    },
+    "hendrycksTest-high_school_macroeconomics": {
+      "acc": 0.31794871794871793,
+      "acc_stderr": 0.02361088430892786,
+      "acc_norm": 0.30256410256410254,
+      "acc_norm_stderr": 0.023290888053772742
+    },
+    "hendrycksTest-virology": {
+      "acc": 0.39759036144578314,
+      "acc_stderr": 0.038099730845402184,
+      "acc_norm": 0.2891566265060241,
+      "acc_norm_stderr": 0.035294868015111155
+    },
+    "hendrycksTest-high_school_mathematics": {
+      "acc": 0.22592592592592592,
+      "acc_stderr": 0.025497532639609542,
+      "acc_norm": 0.3074074074074074,
+      "acc_norm_stderr": 0.02813325257881564
+    },
+    "hendrycksTest-clinical_knowledge": {
+      "acc": 0.3886792452830189,
+      "acc_stderr": 0.03000048544867599,
+      "acc_norm": 0.38113207547169814,
+      "acc_norm_stderr": 0.029890609686286627
+    },
+    "hendrycksTest-professional_psychology": {
+      "acc": 0.3839869281045752,
+      "acc_stderr": 0.01967580813528152,
+      "acc_norm": 0.29901960784313725,
+      "acc_norm_stderr": 0.01852175621542302
+    },
+    "hendrycksTest-formal_logic": {
+      "acc": 0.30952380952380953,
+      "acc_stderr": 0.04134913018303316,
+      "acc_norm": 0.3492063492063492,
+      "acc_norm_stderr": 0.042639068927951315
+    },
+    "hendrycksTest-management": {
+      "acc": 0.4854368932038835,
+      "acc_stderr": 0.04948637324026637,
+      "acc_norm": 0.36893203883495146,
+      "acc_norm_stderr": 0.0477761518115674
+    },
+    "hendrycksTest-human_sexuality": {
+      "acc": 0.5114503816793893,
+      "acc_stderr": 0.043841400240780176,
+      "acc_norm": 0.366412213740458,
+      "acc_norm_stderr": 0.042258754519696386
+    },
+    "hendrycksTest-high_school_world_history": {
+      "acc": 0.3924050632911392,
+      "acc_stderr": 0.03178471874564729,
+      "acc_norm": 0.33755274261603374,
+      "acc_norm_stderr": 0.030781549102026216
+    },
+    "hendrycksTest-medical_genetics": {
+      "acc": 0.44,
+      "acc_stderr": 0.04988876515698589,
+      "acc_norm": 0.4,
+      "acc_norm_stderr": 0.04923659639173309
+    },
+    "hendrycksTest-computer_security": {
+      "acc": 0.37,
+      "acc_stderr": 0.048523658709391,
+      "acc_norm": 0.44,
+      "acc_norm_stderr": 0.04988876515698589
+    },
+    "hendrycksTest-miscellaneous": {
+      "acc": 0.5836526181353767,
+      "acc_stderr": 0.0176279480304303,
+      "acc_norm": 0.3895274584929757,
+      "acc_norm_stderr": 0.017438082556264597
+    },
+    "hendrycksTest-public_relations": {
+      "acc": 0.39090909090909093,
+      "acc_stderr": 0.046737523336702384,
+      "acc_norm": 0.22727272727272727,
+      "acc_norm_stderr": 0.040139645540727735
+    },
+    "hendrycksTest-college_physics": {
+      "acc": 0.23529411764705882,
+      "acc_stderr": 0.04220773659171453,
+      "acc_norm": 0.3235294117647059,
+      "acc_norm_stderr": 0.046550104113196177
+    },
+    "hendrycksTest-professional_accounting": {
+      "acc": 0.30141843971631205,
+      "acc_stderr": 0.02737412888263115,
+      "acc_norm": 0.29432624113475175,
+      "acc_norm_stderr": 0.027187127011503793
+    },
+    "hendrycksTest-logical_fallacies": {
+      "acc": 0.3803680981595092,
+      "acc_stderr": 0.03814269893261837,
+      "acc_norm": 0.3496932515337423,
+      "acc_norm_stderr": 0.037466683254700206
+    },
+    "hendrycksTest-business_ethics": {
+      "acc": 0.53,
+      "acc_stderr": 0.050161355804659205,
+      "acc_norm": 0.46,
+      "acc_norm_stderr": 0.05009082659620332
+    },
+    "hendrycksTest-high_school_chemistry": {
+      "acc": 0.2512315270935961,
+      "acc_stderr": 0.030516530732694436,
+      "acc_norm": 0.2955665024630542,
+      "acc_norm_stderr": 0.03210494433751458
+    },
+    "hendrycksTest-astronomy": {
+      "acc": 0.45394736842105265,
+      "acc_stderr": 0.04051646342874143,
+      "acc_norm": 0.4605263157894737,
+      "acc_norm_stderr": 0.04056242252249033
+    },
+    "hendrycksTest-high_school_us_history": {
+      "acc": 0.4264705882352941,
+      "acc_stderr": 0.03471157907953424,
+      "acc_norm": 0.3137254901960784,
+      "acc_norm_stderr": 0.032566854844603886
+    },
+    "hendrycksTest-college_chemistry": {
+      "acc": 0.33,
+      "acc_stderr": 0.047258156262526045,
+      "acc_norm": 0.3,
+      "acc_norm_stderr": 0.046056618647183814
+    },
+    "hendrycksTest-abstract_algebra": {
+      "acc": 0.23,
+      "acc_stderr": 0.042295258468165065,
+      "acc_norm": 0.26,
+      "acc_norm_stderr": 0.0440844002276808
+    },
+    "hendrycksTest-moral_disputes": {
+      "acc": 0.36416184971098264,
+      "acc_stderr": 0.025906632631016117,
+      "acc_norm": 0.33236994219653176,
+      "acc_norm_stderr": 0.02536116874968821
+    },
+    "hendrycksTest-college_computer_science": {
+      "acc": 0.33,
+      "acc_stderr": 0.04725815626252605,
+      "acc_norm": 0.28,
+      "acc_norm_stderr": 0.04512608598542128
+    },
+    "hendrycksTest-professional_law": {
+      "acc": 0.2966101694915254,
+      "acc_stderr": 0.011665946586082849,
+      "acc_norm": 0.28552803129074317,
+      "acc_norm_stderr": 0.011535751586665664
+    },
+    "hendrycksTest-college_mathematics": {
+      "acc": 0.32,
+      "acc_stderr": 0.046882617226215034,
+      "acc_norm": 0.32,
+      "acc_norm_stderr": 0.04688261722621505
+    },
+    "hendrycksTest-high_school_microeconomics": {
+      "acc": 0.3865546218487395,
+      "acc_stderr": 0.0316314580755238,
+      "acc_norm": 0.36554621848739494,
+      "acc_norm_stderr": 0.0312821770636846
+    },
+    "hendrycksTest-high_school_european_history": {
+      "acc": 0.40606060606060607,
+      "acc_stderr": 0.03834816355401181,
+      "acc_norm": 0.3696969696969697,
+      "acc_norm_stderr": 0.03769430314512568
+    },
+    "hendrycksTest-high_school_biology": {
+      "acc": 0.3580645161290323,
+      "acc_stderr": 0.027273890594300642,
+      "acc_norm": 0.3580645161290323,
+      "acc_norm_stderr": 0.02727389059430063
+    },
+    "hendrycksTest-security_studies": {
+      "acc": 0.40816326530612246,
+      "acc_stderr": 0.03146465712827424,
+      "acc_norm": 0.31020408163265306,
+      "acc_norm_stderr": 0.029613459872484375
+    },
+    "hendrycksTest-high_school_psychology": {
+      "acc": 0.46605504587155966,
+      "acc_stderr": 0.02138786335035399,
+      "acc_norm": 0.30825688073394497,
+      "acc_norm_stderr": 0.01979836669836726
+    },
+    "hendrycksTest-conceptual_physics": {
+      "acc": 0.3276595744680851,
+      "acc_stderr": 0.030683020843231004,
+      "acc_norm": 0.2170212765957447,
+      "acc_norm_stderr": 0.026947483121496228
+    },
+    "hendrycksTest-human_aging": {
+      "acc": 0.3721973094170404,
+      "acc_stderr": 0.03244305283008731,
+      "acc_norm": 0.25112107623318386,
+      "acc_norm_stderr": 0.02910522083322462
+    },
+    "hendrycksTest-prehistory": {
+      "acc": 0.4012345679012346,
+      "acc_stderr": 0.0272725828498398,
+      "acc_norm": 0.2777777777777778,
+      "acc_norm_stderr": 0.02492200116888633
+    },
+    "hendrycksTest-sociology": {
+      "acc": 0.47761194029850745,
+      "acc_stderr": 0.035319879302087305,
+      "acc_norm": 0.42786069651741293,
+      "acc_norm_stderr": 0.03498541988407795
+    },
+    "hendrycksTest-marketing": {
+      "acc": 0.6111111111111112,
+      "acc_stderr": 0.031937057262002924,
+      "acc_norm": 0.5042735042735043,
+      "acc_norm_stderr": 0.03275489264382132
+    },
+    "hendrycksTest-high_school_computer_science": {
+      "acc": 0.41,
+      "acc_stderr": 0.049431107042371025,
+      "acc_norm": 0.34,
+      "acc_norm_stderr": 0.047609522856952365
+    },
+    "hendrycksTest-machine_learning": {
+      "acc": 0.30357142857142855,
+      "acc_stderr": 0.04364226155841044,
+      "acc_norm": 0.26785714285714285,
+      "acc_norm_stderr": 0.04203277291467762
+    },
+    "hendrycksTest-elementary_mathematics": {
+      "acc": 0.3201058201058201,
+      "acc_stderr": 0.024026846392873506,
+      "acc_norm": 0.291005291005291,
+      "acc_norm_stderr": 0.023393826500484865
+    },
+    "hendrycksTest-nutrition": {
+      "acc": 0.3954248366013072,
+      "acc_stderr": 0.027996723180631435,
+      "acc_norm": 0.43790849673202614,
+      "acc_norm_stderr": 0.028408302020332694
+    },
+    "hendrycksTest-anatomy": {
+      "acc": 0.3851851851851852,
+      "acc_stderr": 0.042039210401562783,
+      "acc_norm": 0.2814814814814815,
+      "acc_norm_stderr": 0.03885004245800254
+    },
+    "hendrycksTest-jurisprudence": {
+      "acc": 0.4351851851851852,
+      "acc_stderr": 0.04792898170907062,
+      "acc_norm": 0.5,
+      "acc_norm_stderr": 0.04833682445228318
+    },
+    "hendrycksTest-college_medicine": {
+      "acc": 0.37572254335260113,
+      "acc_stderr": 0.036928207672648664,
+      "acc_norm": 0.3063583815028902,
+      "acc_norm_stderr": 0.03514942551267439
+    },
+    "hendrycksTest-high_school_statistics": {
+      "acc": 0.3425925925925926,
+      "acc_stderr": 0.03236585252602156,
+      "acc_norm": 0.3425925925925926,
+      "acc_norm_stderr": 0.03236585252602156
+    },
+    "hendrycksTest-high_school_physics": {
+      "acc": 0.2052980132450331,
+      "acc_stderr": 0.03297986648473834,
+      "acc_norm": 0.271523178807947,
+      "acc_norm_stderr": 0.036313298039696525
+    },
+    "hendrycksTest-professional_medicine": {
+      "acc": 0.3382352941176471,
+      "acc_stderr": 0.028739328513983576,
+      "acc_norm": 0.27941176470588236,
+      "acc_norm_stderr": 0.027257202606114948
+    }
+  },
+  "versions": {
+    "hendrycksTest-high_school_geography": 0,
+    "hendrycksTest-philosophy": 0,
+    "hendrycksTest-world_religions": 0,
+    "hendrycksTest-college_biology": 0,
+    "hendrycksTest-electrical_engineering": 0,
+    "hendrycksTest-global_facts": 0,
+    "hendrycksTest-high_school_government_and_politics": 0,
+    "hendrycksTest-moral_scenarios": 0,
+    "hendrycksTest-econometrics": 0,
+    "hendrycksTest-international_law": 0,
+    "hendrycksTest-us_foreign_policy": 0,
+    "hendrycksTest-high_school_macroeconomics": 0,
+    "hendrycksTest-virology": 0,
+    "hendrycksTest-high_school_mathematics": 0,
+    "hendrycksTest-clinical_knowledge": 0,
+    "hendrycksTest-professional_psychology": 0,
+    "hendrycksTest-formal_logic": 0,
+    "hendrycksTest-management": 0,
+    "hendrycksTest-human_sexuality": 0,
+    "hendrycksTest-high_school_world_history": 0,
+    "hendrycksTest-medical_genetics": 0,
+    "hendrycksTest-computer_security": 0,
+    "hendrycksTest-miscellaneous": 0,
+    "hendrycksTest-public_relations": 0,
+    "hendrycksTest-college_physics": 0,
+    "hendrycksTest-professional_accounting": 0,
+    "hendrycksTest-logical_fallacies": 0,
+    "hendrycksTest-business_ethics": 0,
+    "hendrycksTest-high_school_chemistry": 0,
+    "hendrycksTest-astronomy": 0,
+    "hendrycksTest-high_school_us_history": 0,
+    "hendrycksTest-college_chemistry": 0,
+    "hendrycksTest-abstract_algebra": 0,
+    "hendrycksTest-moral_disputes": 0,
+    "hendrycksTest-college_computer_science": 0,
+    "hendrycksTest-professional_law": 0,
+    "hendrycksTest-college_mathematics": 0,
+    "hendrycksTest-high_school_microeconomics": 0,
+    "hendrycksTest-high_school_european_history": 0,
+    "hendrycksTest-high_school_biology": 0,
+    "hendrycksTest-security_studies": 0,
+    "hendrycksTest-high_school_psychology": 0,
+    "hendrycksTest-conceptual_physics": 0,
+    "hendrycksTest-human_aging": 0,
+    "hendrycksTest-prehistory": 0,
+    "hendrycksTest-sociology": 0,
+    "hendrycksTest-marketing": 0,
+    "hendrycksTest-high_school_computer_science": 0,
+    "hendrycksTest-machine_learning": 0,
+    "hendrycksTest-elementary_mathematics": 0,
+    "hendrycksTest-nutrition": 0,
+    "hendrycksTest-anatomy": 0,
+    "hendrycksTest-jurisprudence": 0,
+    "hendrycksTest-college_medicine": 0,
+    "hendrycksTest-high_school_statistics": 0,
+    "hendrycksTest-high_school_physics": 0,
+    "hendrycksTest-professional_medicine": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B,use_accelerate=True",
+    "num_fewshot": 5,
+    "batch_size": "auto",
+    "device": "cuda:0",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/llama/llama-7B/llama-7B_pawsx_0-shot.json

+{
+  "results": {
+    "pawsx_en": {
+      "acc": 0.6185,
+      "acc_stderr": 0.010864524561478635
+    },
+    "pawsx_es": {
+      "acc": 0.561,
+      "acc_stderr": 0.011099599116647334
+    },
+    "pawsx_zh": {
+      "acc": 0.491,
+      "acc_stderr": 0.01118132420626029
+    },
+    "pawsx_fr": {
+      "acc": 0.5295,
+      "acc_stderr": 0.01116365480451166
+    },
+    "pawsx_de": {
+      "acc": 0.5465,
+      "acc_stderr": 0.011134669525078668
+    },
+    "pawsx_ko": {
+      "acc": 0.497,
+      "acc_stderr": 0.011182934722804558
+    },
+    "pawsx_ja": {
+      "acc": 0.567,
+      "acc_stderr": 0.011082279027990133
+    }
+  },
+  "versions": {
+    "pawsx_en": 0,
+    "pawsx_es": 0,
+    "pawsx_zh": 0,
+    "pawsx_fr": 0,
+    "pawsx_de": 0,
+    "pawsx_ko": 0,
+    "pawsx_ja": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/llama/llama-7B/llama-7B_question_answering_0-shot.json

+{
+  "results": {
+    "webqs": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "truthfulqa_mc": {
+      "mc1": 0.21052631578947367,
+      "mc1_stderr": 0.014271740645964186,
+      "mc2": 0.3413779054949544,
+      "mc2_stderr": 0.013147086422190785
+    },
+    "headqa_en": {
+      "acc": 0.3242159008023341,
+      "acc_stderr": 0.008940599111132593,
+      "acc_norm": 0.35922684172137126,
+      "acc_norm_stderr": 0.009163935584608705
+    },
+    "triviaqa": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "headqa_es": {
+      "acc": 0.2826404084609774,
+      "acc_stderr": 0.00860064580960105,
+      "acc_norm": 0.3242159008023341,
+      "acc_norm_stderr": 0.008940599111132597
+    },
+    "logiqa": {
+      "acc": 0.21812596006144394,
+      "acc_stderr": 0.01619814925841932,
+      "acc_norm": 0.30261136712749614,
+      "acc_norm_stderr": 0.018018696598158843
+    },
+    "squad2": {
+      "exact": 9.416322749094585,
+      "f1": 19.449005859216243,
+      "HasAns_exact": 18.488529014844804,
+      "HasAns_f1": 38.58266642484388,
+      "NoAns_exact": 0.3700588730025231,
+      "NoAns_f1": 0.3700588730025231,
+      "best_exact": 50.07159100480081,
+      "best_f1": 50.0801228586961
+    }
+  },
+  "versions": {
+    "webqs": 0,
+    "truthfulqa_mc": 1,
+    "headqa_en": 0,
+    "triviaqa": 1,
+    "headqa_es": 0,
+    "logiqa": 0,
+    "squad2": 1
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda:0",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/llama/llama-7B/llama-7B_reading_comprehension_0-shot.json

+{
+  "results": {
+    "coqa": {
+      "f1": 0.752079907334399,
+      "f1_stderr": 0.01529202913294504,
+      "em": 0.6266666666666667,
+      "em_stderr": 0.01881953816383566
+    },
+    "drop": {
+      "em": 0.035864093959731544,
+      "em_stderr": 0.001904314663911949,
+      "f1": 0.11349412751677862,
+      "f1_stderr": 0.002340148712955784
+    },
+    "race": {
+      "acc": 0.3990430622009569,
+      "acc_stderr": 0.015155885289809327
+    }
+  },
+  "versions": {
+    "coqa": 1,
+    "drop": 1,
+    "race": 1
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda:0",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/llama/llama-7B/llama-7B_unscramble_0-shot.json

+{
+  "results": {
+    "anagrams1": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "anagrams2": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "cycle_letters": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "random_insertion": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "reversed_words": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    }
+  },
+  "versions": {
+    "anagrams1": 0,
+    "anagrams2": 0,
+    "cycle_letters": 0,
+    "random_insertion": 0,
+    "reversed_words": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/llama/llama-7B/llama-7B_xcopa_0-shot.json

+{
+  "results": {
+    "xcopa_sw": {
+      "acc": 0.508,
+      "acc_stderr": 0.022380208834928025
+    },
+    "xcopa_zh": {
+      "acc": 0.562,
+      "acc_stderr": 0.022210326363977417
+    },
+    "xcopa_et": {
+      "acc": 0.488,
+      "acc_stderr": 0.02237662679792717
+    },
+    "xcopa_th": {
+      "acc": 0.558,
+      "acc_stderr": 0.022231970696321122
+    },
+    "xcopa_tr": {
+      "acc": 0.556,
+      "acc_stderr": 0.022242244375731017
+    },
+    "xcopa_qu": {
+      "acc": 0.514,
+      "acc_stderr": 0.022374298166353185
+    },
+    "xcopa_id": {
+      "acc": 0.546,
+      "acc_stderr": 0.02228814759117695
+    },
+    "xcopa_ta": {
+      "acc": 0.552,
+      "acc_stderr": 0.02226169729227013
+    },
+    "xcopa_ht": {
+      "acc": 0.51,
+      "acc_stderr": 0.02237859698923078
+    },
+    "xcopa_vi": {
+      "acc": 0.516,
+      "acc_stderr": 0.0223716109825804
+    },
+    "xcopa_it": {
+      "acc": 0.62,
+      "acc_stderr": 0.021728881438701705
+    }
+  },
+  "versions": {
+    "xcopa_sw": 0,
+    "xcopa_zh": 0,
+    "xcopa_et": 0,
+    "xcopa_th": 0,
+    "xcopa_tr": 0,
+    "xcopa_qu": 0,
+    "xcopa_id": 0,
+    "xcopa_ta": 0,
+    "xcopa_ht": 0,
+    "xcopa_vi": 0,
+    "xcopa_it": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/llama/llama-7B/llama-7B_xnli_0-shot.json

+{
+  "results": {
+    "xnli_hi": {
+      "acc": 0.36107784431137724,
+      "acc_stderr": 0.006786547275949383
+    },
+    "xnli_vi": {
+      "acc": 0.35588822355289423,
+      "acc_stderr": 0.006764908277770048
+    },
+    "xnli_fr": {
+      "acc": 0.4375249500998004,
+      "acc_stderr": 0.007009346470228512
+    },
+    "xnli_tr": {
+      "acc": 0.35588822355289423,
+      "acc_stderr": 0.006764908277770052
+    },
+    "xnli_el": {
+      "acc": 0.34930139720558884,
+      "acc_stderr": 0.00673619394518908
+    },
+    "xnli_zh": {
+      "acc": 0.36227544910179643,
+      "acc_stderr": 0.00679141867023231
+    },
+    "xnli_es": {
+      "acc": 0.406187624750499,
+      "acc_stderr": 0.006939248658213527
+    },
+    "xnli_sw": {
+      "acc": 0.33712574850299404,
+      "acc_stderr": 0.006679377985812511
+    },
+    "xnli_th": {
+      "acc": 0.34510978043912177,
+      "acc_stderr": 0.0067171859376095565
+    },
+    "xnli_bg": {
+      "acc": 0.36986027944111777,
+      "acc_stderr": 0.006821215321725033
+    },
+    "xnli_ru": {
+      "acc": 0.3936127744510978,
+      "acc_stderr": 0.006902939997053364
+    },
+    "xnli_ur": {
+      "acc": 0.3339321357285429,
+      "acc_stderr": 0.006663660032909966
+    },
+    "xnli_ar": {
+      "acc": 0.33572854291417165,
+      "acc_stderr": 0.006672543485924257
+    },
+    "xnli_en": {
+      "acc": 0.5105788423153692,
+      "acc_stderr": 0.007063131001466571
+    },
+    "xnli_de": {
+      "acc": 0.4477045908183633,
+      "acc_stderr": 0.007025964880868372
+    }
+  },
+  "versions": {
+    "xnli_hi": 0,
+    "xnli_vi": 0,
+    "xnli_fr": 0,
+    "xnli_tr": 0,
+    "xnli_el": 0,
+    "xnli_zh": 0,
+    "xnli_es": 0,
+    "xnli_sw": 0,
+    "xnli_th": 0,
+    "xnli_bg": 0,
+    "xnli_ru": 0,
+    "xnli_ur": 0,
+    "xnli_ar": 0,
+    "xnli_en": 0,
+    "xnli_de": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/llama/llama-7B/llama-7B_xstory_cloze_0-shot.json

+{
+  "results": {
+    "xstory_cloze_id": {
+      "acc": 0.5208471211118465,
+      "acc_stderr": 0.012855936282881265
+    },
+    "xstory_cloze_en": {
+      "acc": 0.7478491065519524,
+      "acc_stderr": 0.011175031901561881
+    },
+    "xstory_cloze_ar": {
+      "acc": 0.4831237590999338,
+      "acc_stderr": 0.012859793919977602
+    },
+    "xstory_cloze_sw": {
+      "acc": 0.5036399735274653,
+      "acc_stderr": 0.012866784348289232
+    },
+    "xstory_cloze_my": {
+      "acc": 0.4870946393117141,
+      "acc_stderr": 0.012862838605728472
+    },
+    "xstory_cloze_hi": {
+      "acc": 0.5268034414295168,
+      "acc_stderr": 0.012848623899505763
+    },
+    "xstory_cloze_te": {
+      "acc": 0.528788881535407,
+      "acc_stderr": 0.012845779070719484
+    },
+    "xstory_cloze_es": {
+      "acc": 0.6512243547319656,
+      "acc_stderr": 0.012264502012981189
+    },
+    "xstory_cloze_zh": {
+      "acc": 0.5433487756452681,
+      "acc_stderr": 0.012818676452481954
+    },
+    "xstory_cloze_eu": {
+      "acc": 0.500992720052945,
+      "acc_stderr": 0.012867099955422926
+    },
+    "xstory_cloze_ru": {
+      "acc": 0.613500992720053,
+      "acc_stderr": 0.012531219943771484
+    }
+  },
+  "versions": {
+    "xstory_cloze_id": 0,
+    "xstory_cloze_en": 0,
+    "xstory_cloze_ar": 0,
+    "xstory_cloze_sw": 0,
+    "xstory_cloze_my": 0,
+    "xstory_cloze_hi": 0,
+    "xstory_cloze_te": 0,
+    "xstory_cloze_es": 0,
+    "xstory_cloze_zh": 0,
+    "xstory_cloze_eu": 0,
+    "xstory_cloze_ru": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/llama/llama-7B/llama-7B_xwinograd_0-shot.json

+{
+  "results": {
+    "xwinograd_en": {
+      "acc": 0.8494623655913979,
+      "acc_stderr": 0.007417824483418801
+    },
+    "xwinograd_fr": {
+      "acc": 0.7228915662650602,
+      "acc_stderr": 0.049425892997830914
+    },
+    "xwinograd_jp": {
+      "acc": 0.5891553701772679,
+      "acc_stderr": 0.01589538213585439
+    },
+    "xwinograd_zh": {
+      "acc": 0.6369047619047619,
+      "acc_stderr": 0.02144191312603829
+    },
+    "xwinograd_ru": {
+      "acc": 0.6444444444444445,
+      "acc_stderr": 0.02701354947053879
+    },
+    "xwinograd_pt": {
+      "acc": 0.7072243346007605,
+      "acc_stderr": 0.02811223926963042
+    }
+  },
+  "versions": {
+    "xwinograd_en": 0,
+    "xwinograd_fr": 0,
+    "xwinograd_jp": 0,
+    "xwinograd_zh": 0,
+    "xwinograd_ru": 0,
+    "xwinograd_pt": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/mpt/mpt-7b/README.md

+# mpt-7b
+
+## mpt-7b_anli_0-shot.json
+| Task  |Version|Metric|Value|   |Stderr|
+|-------|------:|------|----:|---|-----:|
+|anli_r1|      0|acc   | 33.2|±  |  1.49|
+|anli_r2|      0|acc   | 33.6|±  |  1.49|
+|anli_r3|      0|acc   | 34.5|±  |  1.37|
+
+## mpt-7b_arithmetic_5-shot.json
+|     Task     |Version|Metric|Value|   |Stderr|
+|--------------|------:|------|----:|---|-----:|
+|arithmetic_1dc|      0|acc   | 8.10|±  |  0.61|
+|arithmetic_2da|      0|acc   |91.80|±  |  0.61|
+|arithmetic_2dm|      0|acc   |25.60|±  |  0.98|
+|arithmetic_2ds|      0|acc   |78.75|±  |  0.91|
+|arithmetic_3da|      0|acc   |29.15|±  |  1.02|
+|arithmetic_3ds|      0|acc   |42.80|±  |  1.11|
+|arithmetic_4da|      0|acc   | 2.60|±  |  0.36|
+|arithmetic_4ds|      0|acc   | 2.60|±  |  0.36|
+|arithmetic_5da|      0|acc   | 0.45|±  |  0.15|
+|arithmetic_5ds|      0|acc   | 0.20|±  |  0.10|
+
+## mpt-7b_bbh_3-shot.json
+|                      Task                      |Version|       Metric        |Value|   |Stderr|
+|------------------------------------------------|------:|---------------------|----:|---|-----:|
+|bigbench_causal_judgement                       |      0|multiple_choice_grade|56.32|±  |  3.61|
+|bigbench_date_understanding                     |      0|multiple_choice_grade|58.27|±  |  2.57|
+|bigbench_disambiguation_qa                      |      0|multiple_choice_grade|36.43|±  |  3.00|
+|bigbench_dyck_languages                         |      0|multiple_choice_grade|12.30|±  |  1.04|
+|bigbench_formal_fallacies_syllogisms_negation   |      0|multiple_choice_grade|49.92|±  |  0.42|
+|bigbench_geometric_shapes                       |      0|multiple_choice_grade|20.33|±  |  2.13|
+|                                                |       |exact_str_match      |12.26|±  |  1.73|
+|bigbench_hyperbaton                             |      0|multiple_choice_grade|49.36|±  |  0.22|
+|bigbench_logical_deduction_five_objects         |      0|multiple_choice_grade|24.00|±  |  1.91|
+|bigbench_logical_deduction_seven_objects        |      0|multiple_choice_grade|16.57|±  |  1.41|
+|bigbench_logical_deduction_three_objects        |      0|multiple_choice_grade|38.67|±  |  2.82|
+|bigbench_movie_recommendation                   |      0|multiple_choice_grade|43.80|±  |  2.22|
+|bigbench_navigate                               |      0|multiple_choice_grade|48.60|±  |  1.58|
+|bigbench_reasoning_about_colored_objects        |      0|multiple_choice_grade|29.85|±  |  1.02|
+|bigbench_ruin_names                             |      0|multiple_choice_grade|29.69|±  |  2.16|
+|bigbench_salient_translation_error_detection    |      0|multiple_choice_grade|17.94|±  |  1.22|
+|bigbench_snarks                                 |      0|multiple_choice_grade|53.04|±  |  3.72|
+|bigbench_sports_understanding                   |      0|multiple_choice_grade|49.49|±  |  1.59|
+|bigbench_temporal_sequences                     |      0|multiple_choice_grade|29.60|±  |  1.44|
+|bigbench_tracking_shuffled_objects_five_objects |      0|multiple_choice_grade|19.44|±  |  1.12|
+|bigbench_tracking_shuffled_objects_seven_objects|      0|multiple_choice_grade|13.43|±  |  0.82|
+|bigbench_tracking_shuffled_objects_three_objects|      0|multiple_choice_grade|38.67|±  |  2.82|
+
+## mpt-7b_blimp_0-shot.json
+|                          Task                           |Version|Metric|Value|   |Stderr|
+|---------------------------------------------------------|------:|------|----:|---|-----:|
+|blimp_adjunct_island                                     |      0|acc   | 87.8|±  |  1.04|
+|blimp_anaphor_gender_agreement                           |      0|acc   | 99.5|±  |  0.22|
+|blimp_anaphor_number_agreement                           |      0|acc   | 99.5|±  |  0.22|
+|blimp_animate_subject_passive                            |      0|acc   | 77.5|±  |  1.32|
+|blimp_animate_subject_trans                              |      0|acc   | 88.4|±  |  1.01|
+|blimp_causative                                          |      0|acc   | 74.7|±  |  1.38|
+|blimp_complex_NP_island                                  |      0|acc   | 52.8|±  |  1.58|
+|blimp_coordinate_structure_constraint_complex_left_branch|      0|acc   | 77.9|±  |  1.31|
+|blimp_coordinate_structure_constraint_object_extraction  |      0|acc   | 84.8|±  |  1.14|
+|blimp_determiner_noun_agreement_1                        |      0|acc   | 99.2|±  |  0.28|
+|blimp_determiner_noun_agreement_2                        |      0|acc   | 97.4|±  |  0.50|
+|blimp_determiner_noun_agreement_irregular_1              |      0|acc   | 93.6|±  |  0.77|
+|blimp_determiner_noun_agreement_irregular_2              |      0|acc   | 92.8|±  |  0.82|
+|blimp_determiner_noun_agreement_with_adj_2               |      0|acc   | 93.5|±  |  0.78|
+|blimp_determiner_noun_agreement_with_adj_irregular_1     |      0|acc   | 88.3|±  |  1.02|
+|blimp_determiner_noun_agreement_with_adj_irregular_2     |      0|acc   | 91.9|±  |  0.86|
+|blimp_determiner_noun_agreement_with_adjective_1         |      0|acc   | 97.2|±  |  0.52|
+|blimp_distractor_agreement_relational_noun               |      0|acc   | 88.9|±  |  0.99|
+|blimp_distractor_agreement_relative_clause               |      0|acc   | 74.3|±  |  1.38|
+|blimp_drop_argument                                      |      0|acc   | 78.7|±  |  1.30|
+|blimp_ellipsis_n_bar_1                                   |      0|acc   | 79.0|±  |  1.29|
+|blimp_ellipsis_n_bar_2                                   |      0|acc   | 92.8|±  |  0.82|
+|blimp_existential_there_object_raising                   |      0|acc   | 83.4|±  |  1.18|
+|blimp_existential_there_quantifiers_1                    |      0|acc   | 98.8|±  |  0.34|
+|blimp_existential_there_quantifiers_2                    |      0|acc   | 27.0|±  |  1.40|
+|blimp_existential_there_subject_raising                  |      0|acc   | 88.8|±  |  1.00|
+|blimp_expletive_it_object_raising                        |      0|acc   | 80.0|±  |  1.27|
+|blimp_inchoative                                         |      0|acc   | 67.3|±  |  1.48|
+|blimp_intransitive                                       |      0|acc   | 83.2|±  |  1.18|
+|blimp_irregular_past_participle_adjectives               |      0|acc   | 97.2|±  |  0.52|
+|blimp_irregular_past_participle_verbs                    |      0|acc   | 88.5|±  |  1.01|
+|blimp_irregular_plural_subject_verb_agreement_1          |      0|acc   | 92.0|±  |  0.86|
+|blimp_irregular_plural_subject_verb_agreement_2          |      0|acc   | 90.8|±  |  0.91|
+|blimp_left_branch_island_echo_question                   |      0|acc   | 43.2|±  |  1.57|
+|blimp_left_branch_island_simple_question                 |      0|acc   | 89.7|±  |  0.96|
+|blimp_matrix_question_npi_licensor_present               |      0|acc   | 70.5|±  |  1.44|
+|blimp_npi_present_1                                      |      0|acc   | 57.9|±  |  1.56|
+|blimp_npi_present_2                                      |      0|acc   | 68.8|±  |  1.47|
+|blimp_only_npi_licensor_present                          |      0|acc   | 91.6|±  |  0.88|
+|blimp_only_npi_scope                                     |      0|acc   | 73.2|±  |  1.40|
+|blimp_passive_1                                          |      0|acc   | 88.7|±  |  1.00|
+|blimp_passive_2                                          |      0|acc   | 89.5|±  |  0.97|
+|blimp_principle_A_c_command                              |      0|acc   | 75.0|±  |  1.37|
+|blimp_principle_A_case_1                                 |      0|acc   |100.0|±  |  0.00|
+|blimp_principle_A_case_2                                 |      0|acc   | 94.0|±  |  0.75|
+|blimp_principle_A_domain_1                               |      0|acc   | 99.7|±  |  0.17|
+|blimp_principle_A_domain_2                               |      0|acc   | 82.8|±  |  1.19|
+|blimp_principle_A_domain_3                               |      0|acc   | 76.1|±  |  1.35|
+|blimp_principle_A_reconstruction                         |      0|acc   | 41.0|±  |  1.56|
+|blimp_regular_plural_subject_verb_agreement_1            |      0|acc   | 97.1|±  |  0.53|
+|blimp_regular_plural_subject_verb_agreement_2            |      0|acc   | 90.7|±  |  0.92|
+|blimp_sentential_negation_npi_licensor_present           |      0|acc   | 98.9|±  |  0.33|
+|blimp_sentential_negation_npi_scope                      |      0|acc   | 73.3|±  |  1.40|
+|blimp_sentential_subject_island                          |      0|acc   | 39.9|±  |  1.55|
+|blimp_superlative_quantifiers_1                          |      0|acc   | 82.2|±  |  1.21|
+|blimp_superlative_quantifiers_2                          |      0|acc   | 89.7|±  |  0.96|
+|blimp_tough_vs_raising_1                                 |      0|acc   | 69.0|±  |  1.46|
+|blimp_tough_vs_raising_2                                 |      0|acc   | 82.9|±  |  1.19|
+|blimp_transitive                                         |      0|acc   | 87.2|±  |  1.06|
+|blimp_wh_island                                          |      0|acc   | 81.3|±  |  1.23|
+|blimp_wh_questions_object_gap                            |      0|acc   | 76.3|±  |  1.35|
+|blimp_wh_questions_subject_gap                           |      0|acc   | 89.0|±  |  0.99|
+|blimp_wh_questions_subject_gap_long_distance             |      0|acc   | 89.3|±  |  0.98|
+|blimp_wh_vs_that_no_gap                                  |      0|acc   | 94.6|±  |  0.72|
+|blimp_wh_vs_that_no_gap_long_distance                    |      0|acc   | 95.1|±  |  0.68|
+|blimp_wh_vs_that_with_gap                                |      0|acc   | 32.1|±  |  1.48|
+|blimp_wh_vs_that_with_gap_long_distance                  |      0|acc   | 29.2|±  |  1.44|
+
+## mpt-7b_common_sense_reasoning_0-shot.json
+|    Task     |Version| Metric |Value|   |Stderr|
+|-------------|------:|--------|----:|---|-----:|
+|arc_challenge|      0|acc     |40.61|±  |  1.44|
+|             |       |acc_norm|41.81|±  |  1.44|
+|arc_easy     |      0|acc     |74.87|±  |  0.89|
+|             |       |acc_norm|70.29|±  |  0.94|
+|boolq        |      1|acc     |73.52|±  |  0.77|
+|copa         |      0|acc     |85.00|±  |  3.59|
+|hellaswag    |      0|acc     |57.24|±  |  0.49|
+|             |       |acc_norm|76.12|±  |  0.43|
+|mc_taco      |      0|em      |13.51|   |      |
+|             |       |f1      |45.48|   |      |
+|openbookqa   |      0|acc     |32.00|±  |  2.09|
+|             |       |acc_norm|42.60|±  |  2.21|
+|piqa         |      0|acc     |79.16|±  |  0.95|
+|             |       |acc_norm|80.41|±  |  0.93|
+|prost        |      0|acc     |25.73|±  |  0.32|
+|             |       |acc_norm|30.12|±  |  0.34|
+|swag         |      0|acc     |56.17|±  |  0.35|
+|             |       |acc_norm|75.80|±  |  0.30|
+|winogrande   |      0|acc     |68.67|±  |  1.30|
+|wsc273       |      0|acc     |85.71|±  |  2.12|
+
+## mpt-7b_glue_0-shot.json
+|     Task      |Version|Metric|Value|   |Stderr|
+|---------------|------:|------|----:|---|-----:|
+|cola           |      0|mcc   |-4.41|±  |  3.12|
+|mnli           |      0|acc   |37.83|±  |  0.49|
+|mnli_mismatched|      0|acc   |37.49|±  |  0.49|
+|mrpc           |      0|acc   |62.99|±  |  2.39|
+|               |       |f1    |75.61|±  |  1.93|
+|qnli           |      0|acc   |51.35|±  |  0.68|
+|qqp            |      0|acc   |50.36|±  |  0.25|
+|               |       |f1    |54.14|±  |  0.29|
+|rte            |      0|acc   |63.90|±  |  2.89|
+|sst            |      0|acc   |76.83|±  |  1.43|
+|wnli           |      1|acc   |47.89|±  |  5.97|
+
+## mpt-7b_human_alignment_0-shot.json
+|                 Task                  |Version|       Metric        |Value |   |Stderr|
+|---------------------------------------|------:|---------------------|-----:|---|-----:|
+|crows_pairs_english_age                |      0|likelihood_difference|415.11|±  | 38.32|
+|                                       |       |pct_stereotype       | 73.63|±  |  4.64|
+|crows_pairs_english_autre              |      0|likelihood_difference|505.68|±  |177.03|
+|                                       |       |pct_stereotype       | 72.73|±  | 14.08|
+|crows_pairs_english_disability         |      0|likelihood_difference|601.92|±  | 63.31|
+|                                       |       |pct_stereotype       | 76.92|±  |  5.27|
+|crows_pairs_english_gender             |      0|likelihood_difference|268.24|±  | 17.01|
+|                                       |       |pct_stereotype       | 63.75|±  |  2.69|
+|crows_pairs_english_nationality        |      0|likelihood_difference|349.83|±  | 21.51|
+|                                       |       |pct_stereotype       | 61.57|±  |  3.32|
+|crows_pairs_english_physical_appearance|      0|likelihood_difference|373.78|±  | 33.85|
+|                                       |       |pct_stereotype       | 72.22|±  |  5.32|
+|crows_pairs_english_race_color         |      0|likelihood_difference|336.20|±  | 14.10|
+|                                       |       |pct_stereotype       | 57.28|±  |  2.20|
+|crows_pairs_english_religion           |      0|likelihood_difference|366.44|±  | 33.86|
+|                                       |       |pct_stereotype       | 72.97|±  |  4.23|
+|crows_pairs_english_sexual_orientation |      0|likelihood_difference|463.04|±  | 45.75|
+|                                       |       |pct_stereotype       | 82.80|±  |  3.93|
+|crows_pairs_english_socioeconomic      |      0|likelihood_difference|406.51|±  | 23.52|
+|                                       |       |pct_stereotype       | 67.89|±  |  3.40|
+|crows_pairs_french_age                 |      0|likelihood_difference|360.97|±  | 36.15|
+|                                       |       |pct_stereotype       | 42.22|±  |  5.24|
+|crows_pairs_french_autre               |      0|likelihood_difference|269.23|±  | 92.30|
+|                                       |       |pct_stereotype       | 61.54|±  | 14.04|
+|crows_pairs_french_disability          |      0|likelihood_difference|495.83|±  | 42.69|
+|                                       |       |pct_stereotype       | 63.64|±  |  5.97|
+|crows_pairs_french_gender              |      0|likelihood_difference|321.38|±  | 17.59|
+|                                       |       |pct_stereotype       | 51.09|±  |  2.79|
+|crows_pairs_french_nationality         |      0|likelihood_difference|388.34|±  | 21.84|
+|                                       |       |pct_stereotype       | 34.39|±  |  2.99|
+|crows_pairs_french_physical_appearance |      0|likelihood_difference|322.74|±  | 43.29|
+|                                       |       |pct_stereotype       | 59.72|±  |  5.82|
+|crows_pairs_french_race_color          |      0|likelihood_difference|316.14|±  | 16.56|
+|                                       |       |pct_stereotype       | 43.70|±  |  2.32|
+|crows_pairs_french_religion            |      0|likelihood_difference|356.74|±  | 33.68|
+|                                       |       |pct_stereotype       | 62.61|±  |  4.53|
+|crows_pairs_french_sexual_orientation  |      0|likelihood_difference|479.12|±  | 40.10|
+|                                       |       |pct_stereotype       | 78.02|±  |  4.36|
+|crows_pairs_french_socioeconomic       |      0|likelihood_difference|399.39|±  | 26.31|
+|                                       |       |pct_stereotype       | 65.82|±  |  3.40|
+|ethics_cm                              |      0|acc                  | 54.59|±  |  0.80|
+|ethics_deontology                      |      0|acc                  | 50.25|±  |  0.83|
+|                                       |       |em                   |  0.44|   |      |
+|ethics_justice                         |      0|acc                  | 51.96|±  |  0.96|
+|                                       |       |em                   |  1.18|   |      |
+|ethics_utilitarianism                  |      0|acc                  | 57.49|±  |  0.71|
+|ethics_utilitarianism_original         |      0|acc                  | 99.56|±  |  0.10|
+|ethics_virtue                          |      0|acc                  | 80.40|±  |  0.56|
+|                                       |       |em                   | 12.56|   |      |
+|toxigen                                |      0|acc                  | 43.19|±  |  1.62|
+|                                       |       |acc_norm             | 43.19|±  |  1.62|
+
+## mpt-7b_lambada_0-shot.json
+|         Task         |Version|Metric|Value |   |Stderr|
+|----------------------|------:|------|-----:|---|-----:|
+|lambada_openai        |      0|ppl   |  3.87|±  |  0.08|
+|                      |       |acc   | 68.35|±  |  0.65|
+|lambada_openai_cloze  |      0|ppl   | 26.56|±  |  0.70|
+|                      |       |acc   | 39.65|±  |  0.68|
+|lambada_openai_mt_de  |      0|ppl   | 70.12|±  |  4.04|
+|                      |       |acc   | 33.77|±  |  0.66|
+|lambada_openai_mt_en  |      0|ppl   |  3.87|±  |  0.08|
+|                      |       |acc   | 68.35|±  |  0.65|
+|lambada_openai_mt_es  |      0|ppl   | 67.23|±  |  3.69|
+|                      |       |acc   | 36.95|±  |  0.67|
+|lambada_openai_mt_fr  |      0|ppl   | 42.93|±  |  2.37|
+|                      |       |acc   | 43.02|±  |  0.69|
+|lambada_openai_mt_it  |      0|ppl   | 65.76|±  |  3.87|
+|                      |       |acc   | 39.20|±  |  0.68|
+|lambada_standard      |      0|ppl   |  4.92|±  |  0.11|
+|                      |       |acc   | 61.91|±  |  0.68|
+|lambada_standard_cloze|      0|ppl   |109.10|±  |  3.04|
+|                      |       |acc   | 16.75|±  |  0.52|
+
+## mpt-7b_mmlu_5-shot.json
+|                      Task                       |Version| Metric |Value|   |Stderr|
+|-------------------------------------------------|------:|--------|----:|---|-----:|
+|hendrycksTest-abstract_algebra                   |      0|acc     |18.00|±  |  3.86|
+|                                                 |       |acc_norm|21.00|±  |  4.09|
+|hendrycksTest-anatomy                            |      0|acc     |38.52|±  |  4.20|
+|                                                 |       |acc_norm|37.78|±  |  4.19|
+|hendrycksTest-astronomy                          |      0|acc     |39.47|±  |  3.98|
+|                                                 |       |acc_norm|42.11|±  |  4.02|
+|hendrycksTest-business_ethics                    |      0|acc     |49.00|±  |  5.02|
+|                                                 |       |acc_norm|48.00|±  |  5.02|
+|hendrycksTest-clinical_knowledge                 |      0|acc     |33.21|±  |  2.90|
+|                                                 |       |acc_norm|37.74|±  |  2.98|
+|hendrycksTest-college_biology                    |      0|acc     |38.19|±  |  4.06|
+|                                                 |       |acc_norm|35.42|±  |  4.00|
+|hendrycksTest-college_chemistry                  |      0|acc     |39.00|±  |  4.90|
+|                                                 |       |acc_norm|41.00|±  |  4.94|
+|hendrycksTest-college_computer_science           |      0|acc     |34.00|±  |  4.76|
+|                                                 |       |acc_norm|32.00|±  |  4.69|
+|hendrycksTest-college_mathematics                |      0|acc     |27.00|±  |  4.46|
+|                                                 |       |acc_norm|33.00|±  |  4.73|
+|hendrycksTest-college_medicine                   |      0|acc     |36.42|±  |  3.67|
+|                                                 |       |acc_norm|34.68|±  |  3.63|
+|hendrycksTest-college_physics                    |      0|acc     |30.39|±  |  4.58|
+|                                                 |       |acc_norm|33.33|±  |  4.69|
+|hendrycksTest-computer_security                  |      0|acc     |41.00|±  |  4.94|
+|                                                 |       |acc_norm|41.00|±  |  4.94|
+|hendrycksTest-conceptual_physics                 |      0|acc     |32.77|±  |  3.07|
+|                                                 |       |acc_norm|25.53|±  |  2.85|
+|hendrycksTest-econometrics                       |      0|acc     |27.19|±  |  4.19|
+|                                                 |       |acc_norm|23.68|±  |  4.00|
+|hendrycksTest-electrical_engineering             |      0|acc     |36.55|±  |  4.01|
+|                                                 |       |acc_norm|33.79|±  |  3.94|
+|hendrycksTest-elementary_mathematics             |      0|acc     |29.89|±  |  2.36|
+|                                                 |       |acc_norm|28.84|±  |  2.33|
+|hendrycksTest-formal_logic                       |      0|acc     |30.95|±  |  4.13|
+|                                                 |       |acc_norm|28.57|±  |  4.04|
+|hendrycksTest-global_facts                       |      0|acc     |35.00|±  |  4.79|
+|                                                 |       |acc_norm|33.00|±  |  4.73|
+|hendrycksTest-high_school_biology                |      0|acc     |36.45|±  |  2.74|
+|                                                 |       |acc_norm|39.03|±  |  2.78|
+|hendrycksTest-high_school_chemistry              |      0|acc     |21.18|±  |  2.87|
+|                                                 |       |acc_norm|21.67|±  |  2.90|
+|hendrycksTest-high_school_computer_science       |      0|acc     |43.00|±  |  4.98|
+|                                                 |       |acc_norm|41.00|±  |  4.94|
+|hendrycksTest-high_school_european_history       |      0|acc     |38.18|±  |  3.79|
+|                                                 |       |acc_norm|37.58|±  |  3.78|
+|hendrycksTest-high_school_geography              |      0|acc     |38.38|±  |  3.46|
+|                                                 |       |acc_norm|40.40|±  |  3.50|
+|hendrycksTest-high_school_government_and_politics|      0|acc     |41.45|±  |  3.56|
+|                                                 |       |acc_norm|41.45|±  |  3.56|
+|hendrycksTest-high_school_macroeconomics         |      0|acc     |34.87|±  |  2.42|
+|                                                 |       |acc_norm|29.74|±  |  2.32|
+|hendrycksTest-high_school_mathematics            |      0|acc     |29.26|±  |  2.77|
+|                                                 |       |acc_norm|30.37|±  |  2.80|
+|hendrycksTest-high_school_microeconomics         |      0|acc     |33.61|±  |  3.07|
+|                                                 |       |acc_norm|36.97|±  |  3.14|
+|hendrycksTest-high_school_physics                |      0|acc     |27.81|±  |  3.66|
+|                                                 |       |acc_norm|27.81|±  |  3.66|
+|hendrycksTest-high_school_psychology             |      0|acc     |46.97|±  |  2.14|
+|                                                 |       |acc_norm|44.59|±  |  2.13|
+|hendrycksTest-high_school_statistics             |      0|acc     |32.87|±  |  3.20|
+|                                                 |       |acc_norm|32.41|±  |  3.19|
+|hendrycksTest-high_school_us_history             |      0|acc     |34.31|±  |  3.33|
+|                                                 |       |acc_norm|31.37|±  |  3.26|
+|hendrycksTest-high_school_world_history          |      0|acc     |29.54|±  |  2.97|
+|                                                 |       |acc_norm|28.69|±  |  2.94|
+|hendrycksTest-human_aging                        |      0|acc     |33.63|±  |  3.17|
+|                                                 |       |acc_norm|32.74|±  |  3.15|
+|hendrycksTest-human_sexuality                    |      0|acc     |27.48|±  |  3.92|
+|                                                 |       |acc_norm|32.82|±  |  4.12|
+|hendrycksTest-international_law                  |      0|acc     |37.19|±  |  4.41|
+|                                                 |       |acc_norm|49.59|±  |  4.56|
+|hendrycksTest-jurisprudence                      |      0|acc     |34.26|±  |  4.59|
+|                                                 |       |acc_norm|39.81|±  |  4.73|
+|hendrycksTest-logical_fallacies                  |      0|acc     |38.04|±  |  3.81|
+|                                                 |       |acc_norm|36.81|±  |  3.79|
+|hendrycksTest-machine_learning                   |      0|acc     |26.79|±  |  4.20|
+|                                                 |       |acc_norm|24.11|±  |  4.06|
+|hendrycksTest-management                         |      0|acc     |42.72|±  |  4.90|
+|                                                 |       |acc_norm|39.81|±  |  4.85|
+|hendrycksTest-marketing                          |      0|acc     |55.13|±  |  3.26|
+|                                                 |       |acc_norm|55.13|±  |  3.26|
+|hendrycksTest-medical_genetics                   |      0|acc     |39.00|±  |  4.90|
+|                                                 |       |acc_norm|38.00|±  |  4.88|
+|hendrycksTest-miscellaneous                      |      0|acc     |55.56|±  |  1.78|
+|                                                 |       |acc_norm|55.68|±  |  1.78|
+|hendrycksTest-moral_disputes                     |      0|acc     |32.08|±  |  2.51|
+|                                                 |       |acc_norm|30.06|±  |  2.47|
+|hendrycksTest-moral_scenarios                    |      0|acc     |26.03|±  |  1.47|
+|                                                 |       |acc_norm|27.26|±  |  1.49|
+|hendrycksTest-nutrition                          |      0|acc     |34.31|±  |  2.72|
+|                                                 |       |acc_norm|40.20|±  |  2.81|
+|hendrycksTest-philosophy                         |      0|acc     |37.62|±  |  2.75|
+|                                                 |       |acc_norm|36.98|±  |  2.74|
+|hendrycksTest-prehistory                         |      0|acc     |33.64|±  |  2.63|
+|                                                 |       |acc_norm|30.56|±  |  2.56|
+|hendrycksTest-professional_accounting            |      0|acc     |30.50|±  |  2.75|
+|                                                 |       |acc_norm|29.08|±  |  2.71|
+|hendrycksTest-professional_law                   |      0|acc     |25.95|±  |  1.12|
+|                                                 |       |acc_norm|28.42|±  |  1.15|
+|hendrycksTest-professional_medicine              |      0|acc     |29.41|±  |  2.77|
+|                                                 |       |acc_norm|31.62|±  |  2.82|
+|hendrycksTest-professional_psychology            |      0|acc     |31.54|±  |  1.88|
+|                                                 |       |acc_norm|30.23|±  |  1.86|
+|hendrycksTest-public_relations                   |      0|acc     |41.82|±  |  4.72|
+|                                                 |       |acc_norm|42.73|±  |  4.74|
+|hendrycksTest-security_studies                   |      0|acc     |28.16|±  |  2.88|
+|                                                 |       |acc_norm|24.08|±  |  2.74|
+|hendrycksTest-sociology                          |      0|acc     |34.33|±  |  3.36|
+|                                                 |       |acc_norm|36.82|±  |  3.41|
+|hendrycksTest-us_foreign_policy                  |      0|acc     |38.00|±  |  4.88|
+|                                                 |       |acc_norm|39.00|±  |  4.90|
+|hendrycksTest-virology                           |      0|acc     |32.53|±  |  3.65|
+|                                                 |       |acc_norm|32.53|±  |  3.65|
+|hendrycksTest-world_religions                    |      0|acc     |54.39|±  |  3.82|
+|                                                 |       |acc_norm|57.89|±  |  3.79|
+
+## mpt-7b_pawsx_0-shot.json
+|  Task  |Version|Metric|Value|   |Stderr|
+|--------|------:|------|----:|---|-----:|
+|pawsx_de|      0|acc   |61.40|±  |  1.09|
+|pawsx_en|      0|acc   |70.35|±  |  1.02|
+|pawsx_es|      0|acc   |64.95|±  |  1.07|
+|pawsx_fr|      0|acc   |62.85|±  |  1.08|
+|pawsx_ja|      0|acc   |49.30|±  |  1.12|
+|pawsx_ko|      0|acc   |53.65|±  |  1.12|
+|pawsx_zh|      0|acc   |56.25|±  |  1.11|
+
+## mpt-7b_reading_comprehension_0-shot.json
+|Task|Version|Metric|Value|   |Stderr|
+|----|------:|------|----:|---|-----:|
+|coqa|      1|f1    |76.51|±  |  1.48|
+|    |       |em    |63.02|±  |  1.87|
+|drop|      1|em    | 3.43|±  |  0.19|
+|    |       |f1    |13.39|±  |  0.25|
+|race|      1|acc   |38.66|±  |  1.51|
+
+## mpt-7b_superglue_0-shot.json
+| Task  |Version|Metric|Value|   |Stderr|
+|-------|------:|------|----:|---|-----:|
+|boolq  |      1|acc   |73.82|±  |  0.77|
+|cb     |      1|acc   |41.07|±  |  6.63|
+|       |       |f1    |21.27|   |      |
+|copa   |      0|acc   |84.00|±  |  3.68|
+|multirc|      1|acc   | 0.84|±  |  0.30|
+|record |      0|f1    |90.10|±  |  0.29|
+|       |       |em    |89.30|±  |  0.31|
+|wic    |      0|acc   |48.43|±  |  1.98|
+|wsc    |      0|acc   |63.46|±  |  4.74|
+
+## mpt-7b_unscramble_0-shot.json
+|      Task      |Version|Metric|Value|   |Stderr|
+|----------------|------:|------|----:|---|-----:|
+|anagrams1       |      0|acc   | 0.00|±  |  0.00|
+|anagrams2       |      0|acc   | 0.01|±  |  0.01|
+|cycle_letters   |      0|acc   | 0.00|±  |  0.00|
+|random_insertion|      0|acc   | 0.04|±  |  0.02|
+|reversed_words  |      0|acc   | 0.00|±  |  0.00|
+
+## mpt-7b_xcopa_0-shot.json
+|  Task  |Version|Metric|Value|   |Stderr|
+|--------|------:|------|----:|---|-----:|
+|xcopa_et|      0|acc   | 47.4|±  |  2.24|
+|xcopa_ht|      0|acc   | 49.8|±  |  2.24|
+|xcopa_id|      0|acc   | 56.8|±  |  2.22|
+|xcopa_it|      0|acc   | 59.4|±  |  2.20|
+|xcopa_qu|      0|acc   | 48.4|±  |  2.24|
+|xcopa_sw|      0|acc   | 51.6|±  |  2.24|
+|xcopa_ta|      0|acc   | 54.0|±  |  2.23|
+|xcopa_th|      0|acc   | 54.2|±  |  2.23|
+|xcopa_tr|      0|acc   | 51.6|±  |  2.24|
+|xcopa_vi|      0|acc   | 53.6|±  |  2.23|
+|xcopa_zh|      0|acc   | 63.2|±  |  2.16|
+
+## mpt-7b_xnli_0-shot.json
+| Task  |Version|Metric|Value|   |Stderr|
+|-------|------:|------|----:|---|-----:|
+|xnli_ar|      0|acc   |33.31|±  |  0.67|
+|xnli_bg|      0|acc   |36.83|±  |  0.68|
+|xnli_de|      0|acc   |46.45|±  |  0.70|
+|xnli_el|      0|acc   |36.19|±  |  0.68|
+|xnli_en|      0|acc   |54.33|±  |  0.70|
+|xnli_es|      0|acc   |45.65|±  |  0.70|
+|xnli_fr|      0|acc   |48.80|±  |  0.71|
+|xnli_hi|      0|acc   |34.73|±  |  0.67|
+|xnli_ru|      0|acc   |44.43|±  |  0.70|
+|xnli_sw|      0|acc   |33.41|±  |  0.67|
+|xnli_th|      0|acc   |36.13|±  |  0.68|
+|xnli_tr|      0|acc   |37.68|±  |  0.68|
+|xnli_ur|      0|acc   |33.63|±  |  0.67|
+|xnli_vi|      0|acc   |37.33|±  |  0.68|
+|xnli_zh|      0|acc   |35.35|±  |  0.68|
+
+## mpt-7b_xstory_cloze_0-shot.json
+|     Task      |Version|Metric|Value|   |Stderr|
+|---------------|------:|------|----:|---|-----:|
+|xstory_cloze_ar|      0|acc   |48.51|±  |  1.29|
+|xstory_cloze_en|      0|acc   |77.90|±  |  1.07|
+|xstory_cloze_es|      0|acc   |66.05|±  |  1.22|
+|xstory_cloze_eu|      0|acc   |51.09|±  |  1.29|
+|xstory_cloze_hi|      0|acc   |51.69|±  |  1.29|
+|xstory_cloze_id|      0|acc   |55.20|±  |  1.28|
+|xstory_cloze_my|      0|acc   |48.38|±  |  1.29|
+|xstory_cloze_ru|      0|acc   |57.25|±  |  1.27|
+|xstory_cloze_sw|      0|acc   |49.90|±  |  1.29|
+|xstory_cloze_te|      0|acc   |52.95|±  |  1.28|
+|xstory_cloze_zh|      0|acc   |59.56|±  |  1.26|
+
+## mpt-7b_xwinograd_0-shot.json
+|    Task    |Version|Metric|Value|   |Stderr|
+|------------|------:|------|----:|---|-----:|
+|xwinograd_en|      0|acc   |86.67|±  |  0.71|
+|xwinograd_fr|      0|acc   |66.27|±  |  5.22|
+|xwinograd_jp|      0|acc   |60.27|±  |  1.58|
+|xwinograd_pt|      0|acc   |66.92|±  |  2.91|
+|xwinograd_ru|      0|acc   |69.52|±  |  2.60|
+|xwinograd_zh|      0|acc   |71.63|±  |  2.01|

results/mpt/mpt-7b/mpt-7b_anli_0-shot.json

+{
+  "results": {
+    "anli_r1": {
+      "acc": 0.332,
+      "acc_stderr": 0.014899597242811485
+    },
+    "anli_r2": {
+      "acc": 0.336,
+      "acc_stderr": 0.014944140233795027
+    },
+    "anli_r3": {
+      "acc": 0.345,
+      "acc_stderr": 0.013728421539454881
+    }
+  },
+  "versions": {
+    "anli_r1": 0,
+    "anli_r2": 0,
+    "anli_r3": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=mosaicml/mpt-7b,trust_remote_code=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/mpt/mpt-7b/mpt-7b_arithmetic_5-shot.json

+{
+  "results": {
+    "arithmetic_1dc": {
+      "acc": 0.081,
+      "acc_stderr": 0.006102304405675846
+    },
+    "arithmetic_2da": {
+      "acc": 0.918,
+      "acc_stderr": 0.006136515983374211
+    },
+    "arithmetic_2dm": {
+      "acc": 0.256,
+      "acc_stderr": 0.009761129023832868
+    },
+    "arithmetic_2ds": {
+      "acc": 0.7875,
+      "acc_stderr": 0.009149520854445372
+    },
+    "arithmetic_3da": {
+      "acc": 0.2915,
+      "acc_stderr": 0.010164424861564223
+    },
+    "arithmetic_3ds": {
+      "acc": 0.428,
+      "acc_stderr": 0.011066581884995264
+    },
+    "arithmetic_4da": {
+      "acc": 0.026,
+      "acc_stderr": 0.0035592603398856676
+    },
+    "arithmetic_4ds": {
+      "acc": 0.026,
+      "acc_stderr": 0.003559260339885677
+    },
+    "arithmetic_5da": {
+      "acc": 0.0045,
+      "acc_stderr": 0.0014969954902233202
+    },
+    "arithmetic_5ds": {
+      "acc": 0.002,
+      "acc_stderr": 0.0009992493430694993
+    }
+  },
+  "versions": {
+    "arithmetic_1dc": 0,
+    "arithmetic_2da": 0,
+    "arithmetic_2dm": 0,
+    "arithmetic_2ds": 0,
+    "arithmetic_3da": 0,
+    "arithmetic_3ds": 0,
+    "arithmetic_4da": 0,
+    "arithmetic_4ds": 0,
+    "arithmetic_5da": 0,
+    "arithmetic_5ds": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=mosaicml/mpt-7b,trust_remote_code=True",
+    "num_fewshot": 5,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/mpt/mpt-7b/mpt-7b_bbh_3-shot.json

+{
+  "results": {
+    "bigbench_causal_judgement": {
+      "multiple_choice_grade": 0.5631578947368421,
+      "multiple_choice_grade_stderr": 0.036078330444807245
+    },
+    "bigbench_date_understanding": {
+      "multiple_choice_grade": 0.5826558265582655,
+      "multiple_choice_grade_stderr": 0.025705692903559226
+    },
+    "bigbench_disambiguation_qa": {
+      "multiple_choice_grade": 0.3643410852713178,
+      "multiple_choice_grade_stderr": 0.03001923241206336
+    },
+    "bigbench_dyck_languages": {
+      "multiple_choice_grade": 0.123,
+      "multiple_choice_grade_stderr": 0.010391293421849874
+    },
+    "bigbench_formal_fallacies_syllogisms_negation": {
+      "multiple_choice_grade": 0.49915492957746477,
+      "multiple_choice_grade_stderr": 0.0041960485493055645
+    },
+    "bigbench_geometric_shapes": {
+      "multiple_choice_grade": 0.20334261838440112,
+      "multiple_choice_grade_stderr": 0.021272007856536258,
+      "exact_str_match": 0.12256267409470752,
+      "exact_str_match_stderr": 0.017331879192703025
+    },
+    "bigbench_hyperbaton": {
+      "multiple_choice_grade": 0.4936,
+      "multiple_choice_grade_stderr": 0.002235907150490653
+    },
+    "bigbench_logical_deduction_five_objects": {
+      "multiple_choice_grade": 0.24,
+      "multiple_choice_grade_stderr": 0.01911886665375975
+    },
+    "bigbench_logical_deduction_seven_objects": {
+      "multiple_choice_grade": 0.1657142857142857,
+      "multiple_choice_grade_stderr": 0.014063673984033173
+    },
+    "bigbench_logical_deduction_three_objects": {
+      "multiple_choice_grade": 0.38666666666666666,
+      "multiple_choice_grade_stderr": 0.028163138908196852
+    },
+    "bigbench_movie_recommendation": {
+      "multiple_choice_grade": 0.438,
+      "multiple_choice_grade_stderr": 0.022210326363977417
+    },
+    "bigbench_navigate": {
+      "multiple_choice_grade": 0.486,
+      "multiple_choice_grade_stderr": 0.01581309754773099
+    },
+    "bigbench_reasoning_about_colored_objects": {
+      "multiple_choice_grade": 0.2985,
+      "multiple_choice_grade_stderr": 0.010234805842091589
+    },
+    "bigbench_ruin_names": {
+      "multiple_choice_grade": 0.296875,
+      "multiple_choice_grade_stderr": 0.021609729061250887
+    },
+    "bigbench_salient_translation_error_detection": {
+      "multiple_choice_grade": 0.17935871743486975,
+      "multiple_choice_grade_stderr": 0.012150393578288319
+    },
+    "bigbench_snarks": {
+      "multiple_choice_grade": 0.5303867403314917,
+      "multiple_choice_grade_stderr": 0.03719891321680327
+    },
+    "bigbench_sports_understanding": {
+      "multiple_choice_grade": 0.4949290060851927,
+      "multiple_choice_grade_stderr": 0.015930505328489487
+    },
+    "bigbench_temporal_sequences": {
+      "multiple_choice_grade": 0.296,
+      "multiple_choice_grade_stderr": 0.014442734941575016
+    },
+    "bigbench_tracking_shuffled_objects_five_objects": {
+      "multiple_choice_grade": 0.1944,
+      "multiple_choice_grade_stderr": 0.011197643581460408
+    },
+    "bigbench_tracking_shuffled_objects_seven_objects": {
+      "multiple_choice_grade": 0.13428571428571429,
+      "multiple_choice_grade_stderr": 0.008152809490408933
+    },
+    "bigbench_tracking_shuffled_objects_three_objects": {
+      "multiple_choice_grade": 0.38666666666666666,
+      "multiple_choice_grade_stderr": 0.028163138908196852
+    }
+  },
+  "versions": {
+    "bigbench_causal_judgement": 0,
+    "bigbench_date_understanding": 0,
+    "bigbench_disambiguation_qa": 0,
+    "bigbench_dyck_languages": 0,
+    "bigbench_formal_fallacies_syllogisms_negation": 0,
+    "bigbench_geometric_shapes": 0,
+    "bigbench_hyperbaton": 0,
+    "bigbench_logical_deduction_five_objects": 0,
+    "bigbench_logical_deduction_seven_objects": 0,
+    "bigbench_logical_deduction_three_objects": 0,
+    "bigbench_movie_recommendation": 0,
+    "bigbench_navigate": 0,
+    "bigbench_reasoning_about_colored_objects": 0,
+    "bigbench_ruin_names": 0,
+    "bigbench_salient_translation_error_detection": 0,
+    "bigbench_snarks": 0,
+    "bigbench_sports_understanding": 0,
+    "bigbench_temporal_sequences": 0,
+    "bigbench_tracking_shuffled_objects_five_objects": 0,
+    "bigbench_tracking_shuffled_objects_seven_objects": 0,
+    "bigbench_tracking_shuffled_objects_three_objects": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=mosaicml/mpt-7b,trust_remote_code=True",
+    "num_fewshot": 3,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/mpt/mpt-7b/mpt-7b_blimp_0-shot.json

+{
+  "results": {
+    "blimp_adjunct_island": {
+      "acc": 0.878,
+      "acc_stderr": 0.010354864712936724
+    },
+    "blimp_anaphor_gender_agreement": {
+      "acc": 0.995,
+      "acc_stderr": 0.002231586874844882
+    },
+    "blimp_anaphor_number_agreement": {
+      "acc": 0.995,
+      "acc_stderr": 0.0022315868748448817
+    },
+    "blimp_animate_subject_passive": {
+      "acc": 0.775,
+      "acc_stderr": 0.013211720158614753
+    },
+    "blimp_animate_subject_trans": {
+      "acc": 0.884,
+      "acc_stderr": 0.010131468138756997
+    },
+    "blimp_causative": {
+      "acc": 0.747,
+      "acc_stderr": 0.01375427861358708
+    },
+    "blimp_complex_NP_island": {
+      "acc": 0.528,
+      "acc_stderr": 0.01579447578951147
+    },
+    "blimp_coordinate_structure_constraint_complex_left_branch": {
+      "acc": 0.779,
+      "acc_stderr": 0.013127502859696239
+    },
+    "blimp_coordinate_structure_constraint_object_extraction": {
+      "acc": 0.848,
+      "acc_stderr": 0.011358918303475293
+    },
+    "blimp_determiner_noun_agreement_1": {
+      "acc": 0.992,
+      "acc_stderr": 0.002818500300504507
+    },
+    "blimp_determiner_noun_agreement_2": {
+      "acc": 0.974,
+      "acc_stderr": 0.0050348137353182195
+    },
+    "blimp_determiner_noun_agreement_irregular_1": {
+      "acc": 0.936,
+      "acc_stderr": 0.007743640226919308
+    },
+    "blimp_determiner_noun_agreement_irregular_2": {
+      "acc": 0.928,
+      "acc_stderr": 0.008178195576218681
+    },
+    "blimp_determiner_noun_agreement_with_adj_2": {
+      "acc": 0.935,
+      "acc_stderr": 0.007799733061832037
+    },
+    "blimp_determiner_noun_agreement_with_adj_irregular_1": {
+      "acc": 0.883,
+      "acc_stderr": 0.010169287802713327
+    },
+    "blimp_determiner_noun_agreement_with_adj_irregular_2": {
+      "acc": 0.919,
+      "acc_stderr": 0.008632121032139973
+    },
+    "blimp_determiner_noun_agreement_with_adjective_1": {
+      "acc": 0.972,
+      "acc_stderr": 0.005219506034410047
+    },
+    "blimp_distractor_agreement_relational_noun": {
+      "acc": 0.889,
+      "acc_stderr": 0.009938701010583726
+    },
+    "blimp_distractor_agreement_relative_clause": {
+      "acc": 0.743,
+      "acc_stderr": 0.013825416526895042
+    },
+    "blimp_drop_argument": {
+      "acc": 0.787,
+      "acc_stderr": 0.012953717566737234
+    },
+    "blimp_ellipsis_n_bar_1": {
+      "acc": 0.79,
+      "acc_stderr": 0.012886662332274522
+    },
+    "blimp_ellipsis_n_bar_2": {
+      "acc": 0.928,
+      "acc_stderr": 0.008178195576218681
+    },
+    "blimp_existential_there_object_raising": {
+      "acc": 0.834,
+      "acc_stderr": 0.01177211037081219
+    },
+    "blimp_existential_there_quantifiers_1": {
+      "acc": 0.988,
+      "acc_stderr": 0.003444977194099841
+    },
+    "blimp_existential_there_quantifiers_2": {
+      "acc": 0.27,
+      "acc_stderr": 0.014046255632633908
+    },
+    "blimp_existential_there_subject_raising": {
+      "acc": 0.888,
+      "acc_stderr": 0.00997775303139724
+    },
+    "blimp_expletive_it_object_raising": {
+      "acc": 0.8,
+      "acc_stderr": 0.012655439943366646
+    },
+    "blimp_inchoative": {
+      "acc": 0.673,
+      "acc_stderr": 0.01484221315341125
+    },
+    "blimp_intransitive": {
+      "acc": 0.832,
+      "acc_stderr": 0.01182860583145426
+    },
+    "blimp_irregular_past_participle_adjectives": {
+      "acc": 0.972,
+      "acc_stderr": 0.005219506034410053
+    },
+    "blimp_irregular_past_participle_verbs": {
+      "acc": 0.885,
+      "acc_stderr": 0.010093407594904614
+    },
+    "blimp_irregular_plural_subject_verb_agreement_1": {
+      "acc": 0.92,
+      "acc_stderr": 0.008583336977753651
+    },
+    "blimp_irregular_plural_subject_verb_agreement_2": {
+      "acc": 0.908,
+      "acc_stderr": 0.009144376393151112
+    },
+    "blimp_left_branch_island_echo_question": {
+      "acc": 0.432,
+      "acc_stderr": 0.01567232023733621
+    },
+    "blimp_left_branch_island_simple_question": {
+      "acc": 0.897,
+      "acc_stderr": 0.009616833339695798
+    },
+    "blimp_matrix_question_npi_licensor_present": {
+      "acc": 0.705,
+      "acc_stderr": 0.014428554438445514
+    },
+    "blimp_npi_present_1": {
+      "acc": 0.579,
+      "acc_stderr": 0.015620595475301318
+    },
+    "blimp_npi_present_2": {
+      "acc": 0.688,
+      "acc_stderr": 0.01465847437050901
+    },
+    "blimp_only_npi_licensor_present": {
+      "acc": 0.916,
+      "acc_stderr": 0.008776162089491096
+    },
+    "blimp_only_npi_scope": {
+      "acc": 0.732,
+      "acc_stderr": 0.014013292702729491
+    },
+    "blimp_passive_1": {
+      "acc": 0.887,
+      "acc_stderr": 0.010016552866696863
+    },
+    "blimp_passive_2": {
+      "acc": 0.895,
+      "acc_stderr": 0.009698921026024947
+    },
+    "blimp_principle_A_c_command": {
+      "acc": 0.75,
+      "acc_stderr": 0.013699915608779773
+    },
+    "blimp_principle_A_case_1": {
+      "acc": 1.0,
+      "acc_stderr": 0.0
+    },
+    "blimp_principle_A_case_2": {
+      "acc": 0.94,
+      "acc_stderr": 0.007513751157474928
+    },
+    "blimp_principle_A_domain_1": {
+      "acc": 0.997,
+      "acc_stderr": 0.0017303161543469417
+    },
+    "blimp_principle_A_domain_2": {
+      "acc": 0.828,
+      "acc_stderr": 0.011939788882495321
+    },
+    "blimp_principle_A_domain_3": {
+      "acc": 0.761,
+      "acc_stderr": 0.013493000446937591
+    },
+    "blimp_principle_A_reconstruction": {
+      "acc": 0.41,
+      "acc_stderr": 0.015560917136921646
+    },
+    "blimp_regular_plural_subject_verb_agreement_1": {
+      "acc": 0.971,
+      "acc_stderr": 0.005309160685756986
+    },
+    "blimp_regular_plural_subject_verb_agreement_2": {
+      "acc": 0.907,
+      "acc_stderr": 0.009188875634996659
+    },
+    "blimp_sentential_negation_npi_licensor_present": {
+      "acc": 0.989,
+      "acc_stderr": 0.0032999833166078166
+    },
+    "blimp_sentential_negation_npi_scope": {
+      "acc": 0.733,
+      "acc_stderr": 0.013996674851796264
+    },
+    "blimp_sentential_subject_island": {
+      "acc": 0.399,
+      "acc_stderr": 0.015493193313162906
+    },
+    "blimp_superlative_quantifiers_1": {
+      "acc": 0.822,
+      "acc_stderr": 0.012102167676183596
+    },
+    "blimp_superlative_quantifiers_2": {
+      "acc": 0.897,
+      "acc_stderr": 0.009616833339695796
+    },
+    "blimp_tough_vs_raising_1": {
+      "acc": 0.69,
+      "acc_stderr": 0.014632638658632898
+    },
+    "blimp_tough_vs_raising_2": {
+      "acc": 0.829,
+      "acc_stderr": 0.011912216456264606
+    },
+    "blimp_transitive": {
+      "acc": 0.872,
+      "acc_stderr": 0.010570133761108658
+    },
+    "blimp_wh_island": {
+      "acc": 0.813,
+      "acc_stderr": 0.012336254828074126
+    },
+    "blimp_wh_questions_object_gap": {
+      "acc": 0.763,
+      "acc_stderr": 0.01345407046257794
+    },
+    "blimp_wh_questions_subject_gap": {
+      "acc": 0.89,
+      "acc_stderr": 0.009899393819724435
+    },
+    "blimp_wh_questions_subject_gap_long_distance": {
+      "acc": 0.893,
+      "acc_stderr": 0.009779910359847169
+    },
+    "blimp_wh_vs_that_no_gap": {
+      "acc": 0.946,
+      "acc_stderr": 0.007150883521295428
+    },
+    "blimp_wh_vs_that_no_gap_long_distance": {
+      "acc": 0.951,
+      "acc_stderr": 0.006829761756140921
+    },
+    "blimp_wh_vs_that_with_gap": {
+      "acc": 0.321,
+      "acc_stderr": 0.01477082181793466
+    },
+    "blimp_wh_vs_that_with_gap_long_distance": {
+      "acc": 0.292,
+      "acc_stderr": 0.014385511563477347
+    }
+  },
+  "versions": {
+    "blimp_adjunct_island": 0,
+    "blimp_anaphor_gender_agreement": 0,
+    "blimp_anaphor_number_agreement": 0,
+    "blimp_animate_subject_passive": 0,
+    "blimp_animate_subject_trans": 0,
+    "blimp_causative": 0,
+    "blimp_complex_NP_island": 0,
+    "blimp_coordinate_structure_constraint_complex_left_branch": 0,
+    "blimp_coordinate_structure_constraint_object_extraction": 0,
+    "blimp_determiner_noun_agreement_1": 0,
+    "blimp_determiner_noun_agreement_2": 0,
+    "blimp_determiner_noun_agreement_irregular_1": 0,
+    "blimp_determiner_noun_agreement_irregular_2": 0,
+    "blimp_determiner_noun_agreement_with_adj_2": 0,
+    "blimp_determiner_noun_agreement_with_adj_irregular_1": 0,
+    "blimp_determiner_noun_agreement_with_adj_irregular_2": 0,
+    "blimp_determiner_noun_agreement_with_adjective_1": 0,
+    "blimp_distractor_agreement_relational_noun": 0,
+    "blimp_distractor_agreement_relative_clause": 0,
+    "blimp_drop_argument": 0,
+    "blimp_ellipsis_n_bar_1": 0,
+    "blimp_ellipsis_n_bar_2": 0,
+    "blimp_existential_there_object_raising": 0,
+    "blimp_existential_there_quantifiers_1": 0,
+    "blimp_existential_there_quantifiers_2": 0,
+    "blimp_existential_there_subject_raising": 0,
+    "blimp_expletive_it_object_raising": 0,
+    "blimp_inchoative": 0,
+    "blimp_intransitive": 0,
+    "blimp_irregular_past_participle_adjectives": 0,
+    "blimp_irregular_past_participle_verbs": 0,
+    "blimp_irregular_plural_subject_verb_agreement_1": 0,
+    "blimp_irregular_plural_subject_verb_agreement_2": 0,
+    "blimp_left_branch_island_echo_question": 0,
+    "blimp_left_branch_island_simple_question": 0,
+    "blimp_matrix_question_npi_licensor_present": 0,
+    "blimp_npi_present_1": 0,
+    "blimp_npi_present_2": 0,
+    "blimp_only_npi_licensor_present": 0,
+    "blimp_only_npi_scope": 0,
+    "blimp_passive_1": 0,
+    "blimp_passive_2": 0,
+    "blimp_principle_A_c_command": 0,
+    "blimp_principle_A_case_1": 0,
+    "blimp_principle_A_case_2": 0,
+    "blimp_principle_A_domain_1": 0,
+    "blimp_principle_A_domain_2": 0,
+    "blimp_principle_A_domain_3": 0,
+    "blimp_principle_A_reconstruction": 0,
+    "blimp_regular_plural_subject_verb_agreement_1": 0,
+    "blimp_regular_plural_subject_verb_agreement_2": 0,
+    "blimp_sentential_negation_npi_licensor_present": 0,
+    "blimp_sentential_negation_npi_scope": 0,
+    "blimp_sentential_subject_island": 0,
+    "blimp_superlative_quantifiers_1": 0,
+    "blimp_superlative_quantifiers_2": 0,
+    "blimp_tough_vs_raising_1": 0,
+    "blimp_tough_vs_raising_2": 0,
+    "blimp_transitive": 0,
+    "blimp_wh_island": 0,
+    "blimp_wh_questions_object_gap": 0,
+    "blimp_wh_questions_subject_gap": 0,
+    "blimp_wh_questions_subject_gap_long_distance": 0,
+    "blimp_wh_vs_that_no_gap": 0,
+    "blimp_wh_vs_that_no_gap_long_distance": 0,
+    "blimp_wh_vs_that_with_gap": 0,
+    "blimp_wh_vs_that_with_gap_long_distance": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=mosaicml/mpt-7b,trust_remote_code=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/mpt/mpt-7b/mpt-7b_common_sense_reasoning_0-shot.json

+{
+  "results": {
+    "arc_challenge": {
+      "acc": 0.4061433447098976,
+      "acc_stderr": 0.014351656690097862,
+      "acc_norm": 0.4180887372013652,
+      "acc_norm_stderr": 0.01441398839699608
+    },
+    "arc_easy": {
+      "acc": 0.7487373737373737,
+      "acc_stderr": 0.008900141191221648,
+      "acc_norm": 0.7028619528619529,
+      "acc_norm_stderr": 0.009377397867796849
+    },
+    "boolq": {
+      "acc": 0.7351681957186544,
+      "acc_stderr": 0.007717399182659714
+    },
+    "copa": {
+      "acc": 0.85,
+      "acc_stderr": 0.03588702812826373
+    },
+    "hellaswag": {
+      "acc": 0.5723959370643298,
+      "acc_stderr": 0.004937199759947679,
+      "acc_norm": 0.761202947619996,
+      "acc_norm_stderr": 0.004254771367531344
+    },
+    "mc_taco": {
+      "em": 0.13513513513513514,
+      "f1": 0.45480193909643063
+    },
+    "openbookqa": {
+      "acc": 0.32,
+      "acc_stderr": 0.020882340488761805,
+      "acc_norm": 0.426,
+      "acc_norm_stderr": 0.022136577335085637
+    },
+    "piqa": {
+      "acc": 0.7916213275299239,
+      "acc_stderr": 0.009476125383049457,
+      "acc_norm": 0.8041349292709467,
+      "acc_norm_stderr": 0.009259518041395765
+    },
+    "prost": {
+      "acc": 0.25731212638770284,
+      "acc_stderr": 0.0031937906462958443,
+      "acc_norm": 0.30118488471391974,
+      "acc_norm_stderr": 0.003351744324251047
+    },
+    "swag": {
+      "acc": 0.5616814955513346,
+      "acc_stderr": 0.0035080896485241934,
+      "acc_norm": 0.7579726082175348,
+      "acc_norm_stderr": 0.003028236139561354
+    },
+    "winogrande": {
+      "acc": 0.6866614048934491,
+      "acc_stderr": 0.013036512096747986
+    },
+    "wsc273": {
+      "acc": 0.8571428571428571,
+      "acc_stderr": 0.021217447349500165
+    }
+  },
+  "versions": {
+    "arc_challenge": 0,
+    "arc_easy": 0,
+    "boolq": 1,
+    "copa": 0,
+    "hellaswag": 0,
+    "mc_taco": 0,
+    "openbookqa": 0,
+    "piqa": 0,
+    "prost": 0,
+    "swag": 0,
+    "winogrande": 0,
+    "wsc273": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=mosaicml/mpt-7b,trust_remote_code=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/mpt/mpt-7b/mpt-7b_glue_0-shot.json

+{
+  "results": {
+    "cola": {
+      "mcc": -0.044109532013985,
+      "mcc_stderr": 0.031207420387242075
+    },
+    "mnli": {
+      "acc": 0.3782985226693836,
+      "acc_stderr": 0.0048953668826959605
+    },
+    "mnli_mismatched": {
+      "acc": 0.37489829129373475,
+      "acc_stderr": 0.004882398521925481
+    },
+    "mrpc": {
+      "acc": 0.6299019607843137,
+      "acc_stderr": 0.023933029030729185,
+      "f1": 0.7560581583198709,
+      "f1_stderr": 0.019295620717095126
+    },
+    "qnli": {
+      "acc": 0.513454146073586,
+      "acc_stderr": 0.006762960839582666
+    },
+    "qqp": {
+      "acc": 0.5035864457086322,
+      "acc_stderr": 0.002486636660034357,
+      "f1": 0.5414038936111873,
+      "f1_stderr": 0.0028706161186533327
+    },
+    "rte": {
+      "acc": 0.6389891696750902,
+      "acc_stderr": 0.028910281676964168
+    },
+    "sst": {
+      "acc": 0.768348623853211,
+      "acc_stderr": 0.014295110726150017
+    },
+    "wnli": {
+      "acc": 0.4788732394366197,
+      "acc_stderr": 0.05970805879899505
+    }
+  },
+  "versions": {
+    "cola": 0,
+    "mnli": 0,
+    "mnli_mismatched": 0,
+    "mrpc": 0,
+    "qnli": 0,
+    "qqp": 0,
+    "rte": 0,
+    "sst": 0,
+    "wnli": 1
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=mosaicml/mpt-7b,trust_remote_code=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/mpt/mpt-7b/mpt-7b_human_alignment_0-shot.json

+{
+  "results": {
+    "crows_pairs_english_age": {
+      "likelihood_difference": 4.1510989010989015,
+      "likelihood_difference_stderr": 0.38322042294913006,
+      "pct_stereotype": 0.7362637362637363,
+      "pct_stereotype_stderr": 0.04644942852497395
+    },
+    "crows_pairs_english_autre": {
+      "likelihood_difference": 5.056818181818182,
+      "likelihood_difference_stderr": 1.7702853227564594,
+      "pct_stereotype": 0.7272727272727273,
+      "pct_stereotype_stderr": 0.14083575804390605
+    },
+    "crows_pairs_english_disability": {
+      "likelihood_difference": 6.019230769230769,
+      "likelihood_difference_stderr": 0.6330801651940536,
+      "pct_stereotype": 0.7692307692307693,
+      "pct_stereotype_stderr": 0.05266563052934291
+    },
+    "crows_pairs_english_gender": {
+      "likelihood_difference": 2.682421875,
+      "likelihood_difference_stderr": 0.17011984636779479,
+      "pct_stereotype": 0.6375,
+      "pct_stereotype_stderr": 0.02691527109619775
+    },
+    "crows_pairs_english_nationality": {
+      "likelihood_difference": 3.498263888888889,
+      "likelihood_difference_stderr": 0.21512426671182808,
+      "pct_stereotype": 0.6157407407407407,
+      "pct_stereotype_stderr": 0.03317354514310742
+    },
+    "crows_pairs_english_physical_appearance": {
+      "likelihood_difference": 3.7378472222222223,
+      "likelihood_difference_stderr": 0.3384578916047944,
+      "pct_stereotype": 0.7222222222222222,
+      "pct_stereotype_stderr": 0.053156331218399945
+    },
+    "crows_pairs_english_race_color": {
+      "likelihood_difference": 3.3619586614173227,
+      "likelihood_difference_stderr": 0.14103384370541044,
+      "pct_stereotype": 0.5728346456692913,
+      "pct_stereotype_stderr": 0.021968918082519016
+    },
+    "crows_pairs_english_religion": {
+      "likelihood_difference": 3.6644144144144146,
+      "likelihood_difference_stderr": 0.33863382204528913,
+      "pct_stereotype": 0.7297297297297297,
+      "pct_stereotype_stderr": 0.04234321361084538
+    },
+    "crows_pairs_english_sexual_orientation": {
+      "likelihood_difference": 4.630376344086022,
+      "likelihood_difference_stderr": 0.4574502494208299,
+      "pct_stereotype": 0.8279569892473119,
+      "pct_stereotype_stderr": 0.03934852812061865
+    },
+    "crows_pairs_english_socioeconomic": {
+      "likelihood_difference": 4.065131578947368,
+      "likelihood_difference_stderr": 0.23523235884669547,
+      "pct_stereotype": 0.6789473684210526,
+      "pct_stereotype_stderr": 0.03396059335824887
+    },
+    "crows_pairs_french_age": {
+      "likelihood_difference": 3.609722222222222,
+      "likelihood_difference_stderr": 0.3615396777446631,
+      "pct_stereotype": 0.4222222222222222,
+      "pct_stereotype_stderr": 0.05235473399540658
+    },
+    "crows_pairs_french_autre": {
+      "likelihood_difference": 2.6923076923076925,
+      "likelihood_difference_stderr": 0.9229767573858816,
+      "pct_stereotype": 0.6153846153846154,
+      "pct_stereotype_stderr": 0.14044168141158106
+    },
+    "crows_pairs_french_disability": {
+      "likelihood_difference": 4.958333333333333,
+      "likelihood_difference_stderr": 0.42689940749326905,
+      "pct_stereotype": 0.6363636363636364,
+      "pct_stereotype_stderr": 0.05966637484671758
+    },
+    "crows_pairs_french_gender": {
+      "likelihood_difference": 3.213785046728972,
+      "likelihood_difference_stderr": 0.17588386956758606,
+      "pct_stereotype": 0.5109034267912772,
+      "pct_stereotype_stderr": 0.027944203070818633
+    },
+    "crows_pairs_french_nationality": {
+      "likelihood_difference": 3.883399209486166,
+      "likelihood_difference_stderr": 0.21844567559439967,
+      "pct_stereotype": 0.3438735177865613,
+      "pct_stereotype_stderr": 0.029922155720849428
+    },
+    "crows_pairs_french_physical_appearance": {
+      "likelihood_difference": 3.2274305555555554,
+      "likelihood_difference_stderr": 0.4328667471914375,
+      "pct_stereotype": 0.5972222222222222,
+      "pct_stereotype_stderr": 0.05820650942569533
+    },
+    "crows_pairs_french_race_color": {
+      "likelihood_difference": 3.161413043478261,
+      "likelihood_difference_stderr": 0.16557903974411925,
+      "pct_stereotype": 0.4369565217391304,
+      "pct_stereotype_stderr": 0.023151745316873387
+    },
+    "crows_pairs_french_religion": {
+      "likelihood_difference": 3.5673913043478263,
+      "likelihood_difference_stderr": 0.3368331015818195,
+      "pct_stereotype": 0.6260869565217392,
+      "pct_stereotype_stderr": 0.045315858286449635
+    },
+    "crows_pairs_french_sexual_orientation": {
+      "likelihood_difference": 4.791208791208791,
+      "likelihood_difference_stderr": 0.4009539855629619,
+      "pct_stereotype": 0.7802197802197802,
+      "pct_stereotype_stderr": 0.043649726328985346
+    },
+    "crows_pairs_french_socioeconomic": {
+      "likelihood_difference": 3.9939413265306123,
+      "likelihood_difference_stderr": 0.263093158126228,
+      "pct_stereotype": 0.6581632653061225,
+      "pct_stereotype_stderr": 0.033967132039868675
+    },
+    "ethics_cm": {
+      "acc": 0.5459459459459459,
+      "acc_stderr": 0.007988936899457039
+    },
+    "ethics_deontology": {
+      "acc": 0.5025027808676307,
+      "acc_stderr": 0.008339021933755771,
+      "em": 0.004449388209121246
+    },
+    "ethics_justice": {
+      "acc": 0.5196005917159763,
+      "acc_stderr": 0.009609770755397009,
+      "em": 0.011834319526627219
+    },
+    "ethics_utilitarianism": {
+      "acc": 0.5748752079866889,
+      "acc_stderr": 0.007130302336230959
+    },
+    "ethics_utilitarianism_original": {
+      "acc": 0.9956322795341098,
+      "acc_stderr": 0.000951129914345755
+    },
+    "ethics_virtue": {
+      "acc": 0.8040201005025126,
+      "acc_stderr": 0.005628417801676332,
+      "em": 0.12562814070351758
+    },
+    "toxigen": {
+      "acc": 0.4319148936170213,
+      "acc_stderr": 0.016164899004911828,
+      "acc_norm": 0.4319148936170213,
+      "acc_norm_stderr": 0.016164899004911828
+    }
+  },
+  "versions": {
+    "crows_pairs_english_age": 0,
+    "crows_pairs_english_autre": 0,
+    "crows_pairs_english_disability": 0,
+    "crows_pairs_english_gender": 0,
+    "crows_pairs_english_nationality": 0,
+    "crows_pairs_english_physical_appearance": 0,
+    "crows_pairs_english_race_color": 0,
+    "crows_pairs_english_religion": 0,
+    "crows_pairs_english_sexual_orientation": 0,
+    "crows_pairs_english_socioeconomic": 0,
+    "crows_pairs_french_age": 0,
+    "crows_pairs_french_autre": 0,
+    "crows_pairs_french_disability": 0,
+    "crows_pairs_french_gender": 0,
+    "crows_pairs_french_nationality": 0,
+    "crows_pairs_french_physical_appearance": 0,
+    "crows_pairs_french_race_color": 0,
+    "crows_pairs_french_religion": 0,
+    "crows_pairs_french_sexual_orientation": 0,
+    "crows_pairs_french_socioeconomic": 0,
+    "ethics_cm": 0,
+    "ethics_deontology": 0,
+    "ethics_justice": 0,
+    "ethics_utilitarianism": 0,
+    "ethics_utilitarianism_original": 0,
+    "ethics_virtue": 0,
+    "toxigen": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=mosaicml/mpt-7b,trust_remote_code=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/mpt/mpt-7b/mpt-7b_lambada_0-shot.json

+{
+  "results": {
+    "lambada_openai": {
+      "ppl": 3.8685393479651173,
+      "ppl_stderr": 0.08079962928213236,
+      "acc": 0.6834853483407723,
+      "acc_stderr": 0.006479978824925181
+    },
+    "lambada_openai_cloze": {
+      "ppl": 26.557641771948866,
+      "ppl_stderr": 0.7040059319391023,
+      "acc": 0.3964680768484378,
+      "acc_stderr": 0.006815007030417622
+    },
+    "lambada_openai_mt_de": {
+      "ppl": 70.11746381055114,
+      "ppl_stderr": 4.044349022196945,
+      "acc": 0.33766737822627596,
+      "acc_stderr": 0.006588623616680426
+    },
+    "lambada_openai_mt_en": {
+      "ppl": 3.8685393479651173,
+      "ppl_stderr": 0.08079962928213236,
+      "acc": 0.6834853483407723,
+      "acc_stderr": 0.006479978824925181
+    },
+    "lambada_openai_mt_es": {
+      "ppl": 67.22873557270539,
+      "ppl_stderr": 3.6853291413042055,
+      "acc": 0.3694934989326606,
+      "acc_stderr": 0.006724504027913079
+    },
+    "lambada_openai_mt_fr": {
+      "ppl": 42.93424455960906,
+      "ppl_stderr": 2.3689294613751506,
+      "acc": 0.4302348146710654,
+      "acc_stderr": 0.006897835015074962
+    },
+    "lambada_openai_mt_it": {
+      "ppl": 65.76277239773485,
+      "ppl_stderr": 3.8678074045404967,
+      "acc": 0.392004657481079,
+      "acc_stderr": 0.006801548708056975
+    },
+    "lambada_standard": {
+      "ppl": 4.922814417244969,
+      "ppl_stderr": 0.10844800336124351,
+      "acc": 0.619056860081506,
+      "acc_stderr": 0.006765617200231529
+    },
+    "lambada_standard_cloze": {
+      "ppl": 109.09886744922697,
+      "ppl_stderr": 3.0351063735026287,
+      "acc": 0.1674752571317679,
+      "acc_stderr": 0.005202187177767864
+    }
+  },
+  "versions": {
+    "lambada_openai": 0,
+    "lambada_openai_cloze": 0,
+    "lambada_openai_mt_de": 0,
+    "lambada_openai_mt_en": 0,
+    "lambada_openai_mt_es": 0,
+    "lambada_openai_mt_fr": 0,
+    "lambada_openai_mt_it": 0,
+    "lambada_standard": 0,
+    "lambada_standard_cloze": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=mosaicml/mpt-7b,trust_remote_code=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}