add bloom, xglm and llama results

results/llama/llama-13B/llama-13B_lambada_0-shot.json

+{
+  "results": {
+    "lambada_openai": {
+      "ppl": 1279051.053451683,
+      "ppl_stderr": 60995.62964377304,
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "lambada_openai_mt_de": {
+      "ppl": 1310285.4433720284,
+      "ppl_stderr": 71395.90633942866,
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "lambada_openai_mt_it": {
+      "ppl": 4091504.352954044,
+      "ppl_stderr": 218020.965277226,
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "lambada_standard": {
+      "ppl": 1409047.9981006894,
+      "ppl_stderr": 47832.883755899915,
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "lambada_standard_cloze": {
+      "ppl": 4235345.031433833,
+      "ppl_stderr": 132892.5654001927,
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "lambada_openai_mt_fr": {
+      "ppl": 2461448.491005768,
+      "ppl_stderr": 128013.98724687536,
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "lambada_openai_cloze": {
+      "ppl": 204515.38946166556,
+      "ppl_stderr": 9705.341358126625,
+      "acc": 0.00019406171162429653,
+      "acc_stderr": 0.00019406171162430135
+    },
+    "lambada_openai_mt_en": {
+      "ppl": 1279051.053451683,
+      "ppl_stderr": 60995.62964377304,
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "lambada_openai_mt_es": {
+      "ppl": 1980241.7718905837,
+      "ppl_stderr": 101614.2034914904,
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    }
+  },
+  "versions": {
+    "lambada_openai": 0,
+    "lambada_openai_mt_de": 0,
+    "lambada_openai_mt_it": 0,
+    "lambada_standard": 0,
+    "lambada_standard_cloze": 0,
+    "lambada_openai_mt_fr": 0,
+    "lambada_openai_cloze": 0,
+    "lambada_openai_mt_en": 0,
+    "lambada_openai_mt_es": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/13B",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda:0",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/llama/llama-13B/llama-13B_mathematical_reasoning_0-shot.json

+{
+  "results": {
+    "math_prealgebra": {
+      "acc": 0.02870264064293915,
+      "acc_stderr": 0.0056607946989983855
+    },
+    "math_num_theory": {
+      "acc": 0.014814814814814815,
+      "acc_stderr": 0.005203704987512651
+    },
+    "drop": {
+      "em": 0.0388003355704698,
+      "em_stderr": 0.0019777172311177993,
+      "f1": 0.13990771812080444,
+      "f1_stderr": 0.002512880034517493
+    },
+    "gsm8k": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "math_intermediate_algebra": {
+      "acc": 0.012181616832779624,
+      "acc_stderr": 0.0036524791938863576
+    },
+    "math_algebra": {
+      "acc": 0.018534119629317607,
+      "acc_stderr": 0.003916347676363957
+    },
+    "math_counting_and_prob": {
+      "acc": 0.014767932489451477,
+      "acc_stderr": 0.0055462385896684775
+    },
+    "math_geometry": {
+      "acc": 0.012526096033402923,
+      "acc_stderr": 0.005086941389677977
+    },
+    "math_precalc": {
+      "acc": 0.01098901098901099,
+      "acc_stderr": 0.004465618427331416
+    },
+    "mathqa": {
+      "acc": 0.28442211055276384,
+      "acc_stderr": 0.008258681628795297,
+      "acc_norm": 0.28676716917922945,
+      "acc_norm_stderr": 0.00827905882129993
+    },
+    "math_asdiv": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    }
+  },
+  "versions": {
+    "math_prealgebra": 1,
+    "math_num_theory": 1,
+    "drop": 1,
+    "mathqa": 0,
+    "gsm8k": 0,
+    "math_intermediate_algebra": 1,
+    "math_algebra": 1,
+    "math_counting_and_prob": 1,
+    "math_geometry": 1,
+    "math_precalc": 1,
+    "math_asdiv": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/13B,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda:0",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/llama/llama-13B/llama-13B_mathematical_reasoning_few_shot_5-shot.json

+{
+  "results": {
+    "math_prealgebra": {
+      "acc": 0.001148105625717566,
+      "acc_stderr": 0.0011481056257175704
+    },
+    "drop": {
+      "em": 0.01709312080536913,
+      "em_stderr": 0.001327414384722433,
+      "f1": 0.024450503355704672,
+      "f1_stderr": 0.001413124400630544
+    },
+    "math_intermediate_algebra": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "math_counting_and_prob": {
+      "acc": 0.002109704641350211,
+      "acc_stderr": 0.0021097046413502104
+    },
+    "math_num_theory": {
+      "acc": 0.001851851851851852,
+      "acc_stderr": 0.0018518518518518502
+    },
+    "gsm8k": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "math_geometry": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "math_algebra": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "math_precalc": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "mathqa": {
+      "acc": 0.2998324958123953,
+      "acc_stderr": 0.008387661895516162,
+      "acc_norm": 0.3035175879396985,
+      "acc_norm_stderr": 0.008416811454701563
+    }
+  },
+  "versions": {
+    "math_prealgebra": 1,
+    "drop": 1,
+    "mathqa": 0,
+    "math_intermediate_algebra": 1,
+    "math_counting_and_prob": 1,
+    "math_num_theory": 1,
+    "gsm8k": 0,
+    "math_geometry": 1,
+    "math_algebra": 1,
+    "math_precalc": 1
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/13B,use_accelerate=True",
+    "num_fewshot": 5,
+    "batch_size": "auto",
+    "device": "cuda:0",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/llama/llama-13B/llama-13B_mmlu_5-shot.json

+{
+  "results": {
+    "hendrycksTest-college_biology": {
+      "acc": 0.4583333333333333,
+      "acc_stderr": 0.04166666666666665,
+      "acc_norm": 0.3263888888888889,
+      "acc_norm_stderr": 0.03921067198982266
+    },
+    "hendrycksTest-clinical_knowledge": {
+      "acc": 0.46037735849056605,
+      "acc_stderr": 0.030676096599389188,
+      "acc_norm": 0.3849056603773585,
+      "acc_norm_stderr": 0.029946498567699948
+    },
+    "hendrycksTest-high_school_european_history": {
+      "acc": 0.5272727272727272,
+      "acc_stderr": 0.03898531605579418,
+      "acc_norm": 0.49696969696969695,
+      "acc_norm_stderr": 0.03904272341431855
+    },
+    "hendrycksTest-high_school_psychology": {
+      "acc": 0.6073394495412844,
+      "acc_stderr": 0.02093750516120109,
+      "acc_norm": 0.3688073394495413,
+      "acc_norm_stderr": 0.020686227560729537
+    },
+    "hendrycksTest-business_ethics": {
+      "acc": 0.53,
+      "acc_stderr": 0.05016135580465919,
+      "acc_norm": 0.44,
+      "acc_norm_stderr": 0.04988876515698589
+    },
+    "hendrycksTest-high_school_government_and_politics": {
+      "acc": 0.5854922279792746,
+      "acc_stderr": 0.035553003195576686,
+      "acc_norm": 0.38860103626943004,
+      "acc_norm_stderr": 0.03517739796373132
+    },
+    "hendrycksTest-security_studies": {
+      "acc": 0.45714285714285713,
+      "acc_stderr": 0.03189141832421396,
+      "acc_norm": 0.37551020408163266,
+      "acc_norm_stderr": 0.03100120903989484
+    },
+    "hendrycksTest-high_school_macroeconomics": {
+      "acc": 0.3769230769230769,
+      "acc_stderr": 0.024570975364225995,
+      "acc_norm": 0.31794871794871793,
+      "acc_norm_stderr": 0.02361088430892786
+    },
+    "hendrycksTest-sociology": {
+      "acc": 0.582089552238806,
+      "acc_stderr": 0.034875586404620636,
+      "acc_norm": 0.4577114427860697,
+      "acc_norm_stderr": 0.035228658640995975
+    },
+    "hendrycksTest-college_mathematics": {
+      "acc": 0.29,
+      "acc_stderr": 0.04560480215720683,
+      "acc_norm": 0.34,
+      "acc_norm_stderr": 0.04760952285695235
+    },
+    "hendrycksTest-professional_accounting": {
+      "acc": 0.2978723404255319,
+      "acc_stderr": 0.02728160834446941,
+      "acc_norm": 0.2801418439716312,
+      "acc_norm_stderr": 0.02678917235114023
+    },
+    "hendrycksTest-anatomy": {
+      "acc": 0.42962962962962964,
+      "acc_stderr": 0.04276349494376599,
+      "acc_norm": 0.2962962962962963,
+      "acc_norm_stderr": 0.03944624162501116
+    },
+    "hendrycksTest-professional_psychology": {
+      "acc": 0.42320261437908496,
+      "acc_stderr": 0.019987809769482067,
+      "acc_norm": 0.3300653594771242,
+      "acc_norm_stderr": 0.01902372616072456
+    },
+    "hendrycksTest-moral_scenarios": {
+      "acc": 0.28268156424581004,
+      "acc_stderr": 0.015060381730018082,
+      "acc_norm": 0.27262569832402234,
+      "acc_norm_stderr": 0.014893391735249588
+    },
+    "hendrycksTest-conceptual_physics": {
+      "acc": 0.42127659574468085,
+      "acc_stderr": 0.03227834510146268,
+      "acc_norm": 0.2425531914893617,
+      "acc_norm_stderr": 0.028020226271200217
+    },
+    "hendrycksTest-virology": {
+      "acc": 0.40963855421686746,
+      "acc_stderr": 0.03828401115079021,
+      "acc_norm": 0.30120481927710846,
+      "acc_norm_stderr": 0.035716092300534796
+    },
+    "hendrycksTest-world_religions": {
+      "acc": 0.7426900584795322,
+      "acc_stderr": 0.03352799844161865,
+      "acc_norm": 0.6491228070175439,
+      "acc_norm_stderr": 0.03660298834049162
+    },
+    "hendrycksTest-high_school_computer_science": {
+      "acc": 0.49,
+      "acc_stderr": 0.05024183937956911,
+      "acc_norm": 0.41,
+      "acc_norm_stderr": 0.049431107042371025
+    },
+    "hendrycksTest-abstract_algebra": {
+      "acc": 0.32,
+      "acc_stderr": 0.046882617226215034,
+      "acc_norm": 0.3,
+      "acc_norm_stderr": 0.046056618647183814
+    },
+    "hendrycksTest-medical_genetics": {
+      "acc": 0.49,
+      "acc_stderr": 0.05024183937956911,
+      "acc_norm": 0.48,
+      "acc_norm_stderr": 0.050211673156867795
+    },
+    "hendrycksTest-nutrition": {
+      "acc": 0.45098039215686275,
+      "acc_stderr": 0.02849199358617156,
+      "acc_norm": 0.4673202614379085,
+      "acc_norm_stderr": 0.02856869975222588
+    },
+    "hendrycksTest-elementary_mathematics": {
+      "acc": 0.36772486772486773,
+      "acc_stderr": 0.024833839825562424,
+      "acc_norm": 0.328042328042328,
+      "acc_norm_stderr": 0.024180497164376907
+    },
+    "hendrycksTest-philosophy": {
+      "acc": 0.45980707395498394,
+      "acc_stderr": 0.028306190403305696,
+      "acc_norm": 0.3858520900321543,
+      "acc_norm_stderr": 0.02764814959975146
+    },
+    "hendrycksTest-high_school_microeconomics": {
+      "acc": 0.42016806722689076,
+      "acc_stderr": 0.03206183783236152,
+      "acc_norm": 0.40756302521008403,
+      "acc_norm_stderr": 0.031918633744784645
+    },
+    "hendrycksTest-management": {
+      "acc": 0.6407766990291263,
+      "acc_stderr": 0.04750458399041696,
+      "acc_norm": 0.4174757281553398,
+      "acc_norm_stderr": 0.048828405482122375
+    },
+    "hendrycksTest-us_foreign_policy": {
+      "acc": 0.68,
+      "acc_stderr": 0.046882617226215034,
+      "acc_norm": 0.52,
+      "acc_norm_stderr": 0.050211673156867795
+    },
+    "hendrycksTest-international_law": {
+      "acc": 0.5619834710743802,
+      "acc_stderr": 0.04529146804435792,
+      "acc_norm": 0.6033057851239669,
+      "acc_norm_stderr": 0.044658697805310094
+    },
+    "hendrycksTest-college_chemistry": {
+      "acc": 0.31,
+      "acc_stderr": 0.04648231987117316,
+      "acc_norm": 0.3,
+      "acc_norm_stderr": 0.046056618647183814
+    },
+    "hendrycksTest-high_school_mathematics": {
+      "acc": 0.26666666666666666,
+      "acc_stderr": 0.026962424325073817,
+      "acc_norm": 0.31851851851851853,
+      "acc_norm_stderr": 0.028406533090608463
+    },
+    "hendrycksTest-high_school_world_history": {
+      "acc": 0.4978902953586498,
+      "acc_stderr": 0.032546938018020076,
+      "acc_norm": 0.42616033755274263,
+      "acc_norm_stderr": 0.03219035703131774
+    },
+    "hendrycksTest-human_sexuality": {
+      "acc": 0.549618320610687,
+      "acc_stderr": 0.04363643698524779,
+      "acc_norm": 0.3969465648854962,
+      "acc_norm_stderr": 0.04291135671009224
+    },
+    "hendrycksTest-college_computer_science": {
+      "acc": 0.33,
+      "acc_stderr": 0.047258156262526045,
+      "acc_norm": 0.28,
+      "acc_norm_stderr": 0.045126085985421276
+    },
+    "hendrycksTest-college_medicine": {
+      "acc": 0.4277456647398844,
+      "acc_stderr": 0.037724468575180255,
+      "acc_norm": 0.30057803468208094,
+      "acc_norm_stderr": 0.0349610148119118
+    },
+    "hendrycksTest-formal_logic": {
+      "acc": 0.3253968253968254,
+      "acc_stderr": 0.041905964388711366,
+      "acc_norm": 0.3412698412698413,
+      "acc_norm_stderr": 0.04240799327574925
+    },
+    "hendrycksTest-high_school_physics": {
+      "acc": 0.271523178807947,
+      "acc_stderr": 0.03631329803969653,
+      "acc_norm": 0.25165562913907286,
+      "acc_norm_stderr": 0.035433042343899844
+    },
+    "hendrycksTest-marketing": {
+      "acc": 0.7264957264957265,
+      "acc_stderr": 0.029202540153431173,
+      "acc_norm": 0.6153846153846154,
+      "acc_norm_stderr": 0.03187195347942466
+    },
+    "hendrycksTest-jurisprudence": {
+      "acc": 0.48148148148148145,
+      "acc_stderr": 0.04830366024635331,
+      "acc_norm": 0.5,
+      "acc_norm_stderr": 0.04833682445228318
+    },
+    "hendrycksTest-computer_security": {
+      "acc": 0.57,
+      "acc_stderr": 0.049756985195624284,
+      "acc_norm": 0.44,
+      "acc_norm_stderr": 0.04988876515698589
+    },
+    "hendrycksTest-high_school_chemistry": {
+      "acc": 0.3103448275862069,
+      "acc_stderr": 0.03255086769970103,
+      "acc_norm": 0.32019704433497537,
+      "acc_norm_stderr": 0.032826493853041504
+    },
+    "hendrycksTest-prehistory": {
+      "acc": 0.49691358024691357,
+      "acc_stderr": 0.02782021415859437,
+      "acc_norm": 0.345679012345679,
+      "acc_norm_stderr": 0.026462487777001876
+    },
+    "hendrycksTest-machine_learning": {
+      "acc": 0.2857142857142857,
+      "acc_stderr": 0.04287858751340455,
+      "acc_norm": 0.29464285714285715,
+      "acc_norm_stderr": 0.043270409325787296
+    },
+    "hendrycksTest-professional_medicine": {
+      "acc": 0.39338235294117646,
+      "acc_stderr": 0.02967428828131118,
+      "acc_norm": 0.33088235294117646,
+      "acc_norm_stderr": 0.028582709753898452
+    },
+    "hendrycksTest-global_facts": {
+      "acc": 0.34,
+      "acc_stderr": 0.04760952285695235,
+      "acc_norm": 0.29,
+      "acc_norm_stderr": 0.04560480215720684
+    },
+    "hendrycksTest-high_school_us_history": {
+      "acc": 0.5245098039215687,
+      "acc_stderr": 0.03505093194348798,
+      "acc_norm": 0.37254901960784315,
+      "acc_norm_stderr": 0.033933885849584046
+    },
+    "hendrycksTest-high_school_geography": {
+      "acc": 0.5757575757575758,
+      "acc_stderr": 0.03521224908841586,
+      "acc_norm": 0.42424242424242425,
+      "acc_norm_stderr": 0.03521224908841583
+    },
+    "hendrycksTest-human_aging": {
+      "acc": 0.5739910313901345,
+      "acc_stderr": 0.033188332862172806,
+      "acc_norm": 0.336322869955157,
+      "acc_norm_stderr": 0.03170882426845501
+    },
+    "hendrycksTest-high_school_biology": {
+      "acc": 0.4967741935483871,
+      "acc_stderr": 0.028443414226438316,
+      "acc_norm": 0.36129032258064514,
+      "acc_norm_stderr": 0.027327548447957553
+    },
+    "hendrycksTest-public_relations": {
+      "acc": 0.5454545454545454,
+      "acc_stderr": 0.04769300568972744,
+      "acc_norm": 0.2909090909090909,
+      "acc_norm_stderr": 0.04350271442923243
+    },
+    "hendrycksTest-professional_law": {
+      "acc": 0.30378096479791394,
+      "acc_stderr": 0.011745787720472483,
+      "acc_norm": 0.3089960886571056,
+      "acc_norm_stderr": 0.011801729777239246
+    },
+    "hendrycksTest-electrical_engineering": {
+      "acc": 0.41379310344827586,
+      "acc_stderr": 0.041042692118062316,
+      "acc_norm": 0.3448275862068966,
+      "acc_norm_stderr": 0.039609335494512087
+    },
+    "hendrycksTest-logical_fallacies": {
+      "acc": 0.4539877300613497,
+      "acc_stderr": 0.0391170190467718,
+      "acc_norm": 0.36809815950920244,
+      "acc_norm_stderr": 0.03789213935838396
+    },
+    "hendrycksTest-moral_disputes": {
+      "acc": 0.4479768786127168,
+      "acc_stderr": 0.026772990653361816,
+      "acc_norm": 0.3815028901734104,
+      "acc_norm_stderr": 0.0261521986197268
+    },
+    "hendrycksTest-high_school_statistics": {
+      "acc": 0.38425925925925924,
+      "acc_stderr": 0.03317354514310742,
+      "acc_norm": 0.375,
+      "acc_norm_stderr": 0.033016908987210894
+    },
+    "hendrycksTest-college_physics": {
+      "acc": 0.28431372549019607,
+      "acc_stderr": 0.04488482852329017,
+      "acc_norm": 0.35294117647058826,
+      "acc_norm_stderr": 0.04755129616062947
+    },
+    "hendrycksTest-econometrics": {
+      "acc": 0.2719298245614035,
+      "acc_stderr": 0.04185774424022056,
+      "acc_norm": 0.2631578947368421,
+      "acc_norm_stderr": 0.041424397194893624
+    },
+    "hendrycksTest-miscellaneous": {
+      "acc": 0.6960408684546615,
+      "acc_stderr": 0.016448321686769043,
+      "acc_norm": 0.48531289910600256,
+      "acc_norm_stderr": 0.01787224802442912
+    },
+    "hendrycksTest-astronomy": {
+      "acc": 0.48026315789473684,
+      "acc_stderr": 0.04065771002562603,
+      "acc_norm": 0.48026315789473684,
+      "acc_norm_stderr": 0.040657710025626036
+    }
+  },
+  "versions": {
+    "hendrycksTest-college_biology": 0,
+    "hendrycksTest-clinical_knowledge": 0,
+    "hendrycksTest-high_school_european_history": 0,
+    "hendrycksTest-high_school_psychology": 0,
+    "hendrycksTest-business_ethics": 0,
+    "hendrycksTest-high_school_government_and_politics": 0,
+    "hendrycksTest-security_studies": 0,
+    "hendrycksTest-high_school_macroeconomics": 0,
+    "hendrycksTest-sociology": 0,
+    "hendrycksTest-college_mathematics": 0,
+    "hendrycksTest-professional_accounting": 0,
+    "hendrycksTest-anatomy": 0,
+    "hendrycksTest-professional_psychology": 0,
+    "hendrycksTest-moral_scenarios": 0,
+    "hendrycksTest-conceptual_physics": 0,
+    "hendrycksTest-virology": 0,
+    "hendrycksTest-world_religions": 0,
+    "hendrycksTest-high_school_computer_science": 0,
+    "hendrycksTest-abstract_algebra": 0,
+    "hendrycksTest-medical_genetics": 0,
+    "hendrycksTest-nutrition": 0,
+    "hendrycksTest-elementary_mathematics": 0,
+    "hendrycksTest-philosophy": 0,
+    "hendrycksTest-high_school_microeconomics": 0,
+    "hendrycksTest-management": 0,
+    "hendrycksTest-us_foreign_policy": 0,
+    "hendrycksTest-international_law": 0,
+    "hendrycksTest-college_chemistry": 0,
+    "hendrycksTest-high_school_mathematics": 0,
+    "hendrycksTest-high_school_world_history": 0,
+    "hendrycksTest-human_sexuality": 0,
+    "hendrycksTest-college_computer_science": 0,
+    "hendrycksTest-college_medicine": 0,
+    "hendrycksTest-formal_logic": 0,
+    "hendrycksTest-high_school_physics": 0,
+    "hendrycksTest-marketing": 0,
+    "hendrycksTest-jurisprudence": 0,
+    "hendrycksTest-computer_security": 0,
+    "hendrycksTest-high_school_chemistry": 0,
+    "hendrycksTest-prehistory": 0,
+    "hendrycksTest-machine_learning": 0,
+    "hendrycksTest-professional_medicine": 0,
+    "hendrycksTest-global_facts": 0,
+    "hendrycksTest-high_school_us_history": 0,
+    "hendrycksTest-high_school_geography": 0,
+    "hendrycksTest-human_aging": 0,
+    "hendrycksTest-high_school_biology": 0,
+    "hendrycksTest-public_relations": 0,
+    "hendrycksTest-professional_law": 0,
+    "hendrycksTest-electrical_engineering": 0,
+    "hendrycksTest-logical_fallacies": 0,
+    "hendrycksTest-moral_disputes": 0,
+    "hendrycksTest-high_school_statistics": 0,
+    "hendrycksTest-college_physics": 0,
+    "hendrycksTest-econometrics": 0,
+    "hendrycksTest-miscellaneous": 0,
+    "hendrycksTest-astronomy": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/13B,use_accelerate=True",
+    "num_fewshot": 5,
+    "batch_size": "auto",
+    "device": "cuda:0",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/llama/llama-13B/llama-13B_pawsx_0-shot.json

+{
+  "results": {
+    "pawsx_fr": {
+      "acc": 0.545,
+      "acc_stderr": 0.011137752231145222
+    },
+    "pawsx_en": {
+      "acc": 0.537,
+      "acc_stderr": 0.011152474561478174
+    },
+    "pawsx_ko": {
+      "acc": 0.4705,
+      "acc_stderr": 0.011163654804511664
+    },
+    "pawsx_ja": {
+      "acc": 0.45,
+      "acc_stderr": 0.011127079848413735
+    },
+    "pawsx_es": {
+      "acc": 0.521,
+      "acc_stderr": 0.011173268141438304
+    },
+    "pawsx_de": {
+      "acc": 0.5295,
+      "acc_stderr": 0.011163654804511655
+    },
+    "pawsx_zh": {
+      "acc": 0.452,
+      "acc_stderr": 0.01113148485052578
+    }
+  },
+  "versions": {
+    "pawsx_fr": 0,
+    "pawsx_en": 0,
+    "pawsx_ko": 0,
+    "pawsx_ja": 0,
+    "pawsx_es": 0,
+    "pawsx_de": 0,
+    "pawsx_zh": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/13B",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/llama/llama-13B/llama-13B_question_answering_0-shot.json

+{
+  "results": {
+    "triviaqa": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "headqa_es": {
+      "acc": 0.3056163384390955,
+      "acc_stderr": 0.008799003959214539,
+      "acc_norm": 0.3515681983953319,
+      "acc_norm_stderr": 0.009119739372039878
+    },
+    "logiqa": {
+      "acc": 0.2642089093701997,
+      "acc_stderr": 0.017293954549744514,
+      "acc_norm": 0.3210445468509985,
+      "acc_norm_stderr": 0.018312456701476108
+    },
+    "headqa_en": {
+      "acc": 0.34427425237053244,
+      "acc_stderr": 0.009075255747504299,
+      "acc_norm": 0.38584974471188915,
+      "acc_norm_stderr": 0.009298050684004381
+    },
+    "truthfulqa_mc": {
+      "mc1": 0.2582619339045288,
+      "mc1_stderr": 0.0153218216884762,
+      "mc2": 0.39884734031519786,
+      "mc2_stderr": 0.013703865869126058
+    },
+    "squad2": {
+      "exact": 16.440663690726858,
+      "f1": 24.060945088960178,
+      "HasAns_exact": 21.086369770580298,
+      "HasAns_f1": 36.34878560074651,
+      "NoAns_exact": 11.808242220353238,
+      "NoAns_f1": 11.808242220353238,
+      "best_exact": 50.07159100480081,
+      "best_f1": 50.073888042388
+    },
+    "webqs": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    }
+  },
+  "versions": {
+    "triviaqa": 1,
+    "headqa_es": 0,
+    "logiqa": 0,
+    "headqa_en": 0,
+    "truthfulqa_mc": 1,
+    "squad2": 1,
+    "webqs": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/13B,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda:0",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/llama/llama-13B/llama-13B_reading_comprehension_0-shot.json

+{
+  "results": {
+    "coqa": {
+      "f1": 0.7704068983762044,
+      "f1_stderr": 0.014191975492335083,
+      "em": 0.637,
+      "em_stderr": 0.01847461201879917
+    },
+    "drop": {
+      "em": 0.035864093959731544,
+      "em_stderr": 0.0019043146639119552,
+      "f1": 0.13376153523489834,
+      "f1_stderr": 0.002439665460318613
+    },
+    "race": {
+      "acc": 0.39330143540669854,
+      "acc_stderr": 0.01511816218614914
+    }
+  },
+  "versions": {
+    "coqa": 1,
+    "drop": 1,
+    "race": 1
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/13B,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda:0",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/llama/llama-13B/llama-13B_superglue_0-shot.json

+{
+  "results": {
+    "boolq": {
+      "acc": 0.6844036697247706,
+      "acc_stderr": 0.008128579858785895
+    },
+    "wic": {
+      "acc": 0.49843260188087773,
+      "acc_stderr": 0.019810623954060382
+    },
+    "copa": {
+      "acc": 0.9,
+      "acc_stderr": 0.030151134457776348
+    },
+    "wsc": {
+      "acc": 0.3557692307692308,
+      "acc_stderr": 0.04717221961050337
+    },
+    "cb": {
+      "acc": 0.48214285714285715,
+      "acc_stderr": 0.0673769750864465,
+      "f1": 0.3881876266167991
+    },
+    "record": {
+      "f1": 0.9231828571428571,
+      "f1_stderr": 0.0026119602574627677,
+      "em": 0.9154,
+      "em_stderr": 0.002782994521347745
+    },
+    "multirc": {
+      "acc": 0.015739769150052464,
+      "acc_stderr": 0.00403399795659578
+    }
+  },
+  "versions": {
+    "boolq": 1,
+    "wic": 0,
+    "copa": 0,
+    "wsc": 0,
+    "cb": 1,
+    "record": 0,
+    "multirc": 1
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/13B,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda:0",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/llama/llama-13B/llama-13B_xcopa_0-shot.json

+{
+  "results": {
+    "xcopa_it": {
+      "acc": 0.672,
+      "acc_stderr": 0.021017027165175485
+    },
+    "xcopa_vi": {
+      "acc": 0.538,
+      "acc_stderr": 0.02231833811987053
+    },
+    "xcopa_zh": {
+      "acc": 0.584,
+      "acc_stderr": 0.02206494331392886
+    },
+    "xcopa_ta": {
+      "acc": 0.544,
+      "acc_stderr": 0.022296238348407053
+    },
+    "xcopa_sw": {
+      "acc": 0.512,
+      "acc_stderr": 0.02237662679792717
+    },
+    "xcopa_id": {
+      "acc": 0.578,
+      "acc_stderr": 0.022109039310618552
+    },
+    "xcopa_tr": {
+      "acc": 0.53,
+      "acc_stderr": 0.02234274819250285
+    },
+    "xcopa_ht": {
+      "acc": 0.528,
+      "acc_stderr": 0.02234794983266809
+    },
+    "xcopa_qu": {
+      "acc": 0.502,
+      "acc_stderr": 0.02238289498648353
+    },
+    "xcopa_th": {
+      "acc": 0.546,
+      "acc_stderr": 0.022288147591176945
+    },
+    "xcopa_et": {
+      "acc": 0.482,
+      "acc_stderr": 0.02236856511738799
+    }
+  },
+  "versions": {
+    "xcopa_it": 0,
+    "xcopa_vi": 0,
+    "xcopa_zh": 0,
+    "xcopa_ta": 0,
+    "xcopa_sw": 0,
+    "xcopa_id": 0,
+    "xcopa_tr": 0,
+    "xcopa_ht": 0,
+    "xcopa_qu": 0,
+    "xcopa_th": 0,
+    "xcopa_et": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/13B",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/llama/llama-13B/llama-13B_xnli_0-shot.json

+{
+  "results": {
+    "xnli_ru": {
+      "acc": 0.3379241516966068,
+      "acc_stderr": 0.006683254094065008
+    },
+    "xnli_vi": {
+      "acc": 0.34211576846307384,
+      "acc_stderr": 0.006703255428996599
+    },
+    "xnli_zh": {
+      "acc": 0.3447105788423154,
+      "acc_stderr": 0.006715345603576115
+    },
+    "xnli_bg": {
+      "acc": 0.34211576846307384,
+      "acc_stderr": 0.0067032554289965995
+    },
+    "xnli_el": {
+      "acc": 0.3469061876247505,
+      "acc_stderr": 0.0067254026681375706
+    },
+    "xnli_fr": {
+      "acc": 0.3349301397205589,
+      "acc_stderr": 0.006668608672768922
+    },
+    "xnli_ur": {
+      "acc": 0.34211576846307384,
+      "acc_stderr": 0.006703255428996604
+    },
+    "xnli_hi": {
+      "acc": 0.35588822355289423,
+      "acc_stderr": 0.00676490827777005
+    },
+    "xnli_es": {
+      "acc": 0.3349301397205589,
+      "acc_stderr": 0.006668608672768919
+    },
+    "xnli_sw": {
+      "acc": 0.3315369261477046,
+      "acc_stderr": 0.006651646309907708
+    },
+    "xnli_th": {
+      "acc": 0.34830339321357284,
+      "acc_stderr": 0.006731720358995404
+    },
+    "xnli_ar": {
+      "acc": 0.3407185628742515,
+      "acc_stderr": 0.006696653153866837
+    },
+    "xnli_en": {
+      "acc": 0.3562874251497006,
+      "acc_stderr": 0.006766603483662201
+    },
+    "xnli_de": {
+      "acc": 0.3524950099800399,
+      "acc_stderr": 0.006750291549188483
+    },
+    "xnli_tr": {
+      "acc": 0.3399201596806387,
+      "acc_stderr": 0.006692851356332768
+    }
+  },
+  "versions": {
+    "xnli_ru": 0,
+    "xnli_vi": 0,
+    "xnli_zh": 0,
+    "xnli_bg": 0,
+    "xnli_el": 0,
+    "xnli_fr": 0,
+    "xnli_ur": 0,
+    "xnli_hi": 0,
+    "xnli_es": 0,
+    "xnli_sw": 0,
+    "xnli_th": 0,
+    "xnli_ar": 0,
+    "xnli_en": 0,
+    "xnli_de": 0,
+    "xnli_tr": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/13B",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/llama/llama-13B/llama-13B_xstory_cloze_0-shot.json

+{
+  "results": {
+    "xstory_cloze_zh": {
+      "acc": 0.5645268034414295,
+      "acc_stderr": 0.012759525506489228
+    },
+    "xstory_cloze_my": {
+      "acc": 0.47782925215089345,
+      "acc_stderr": 0.012854469625936085
+    },
+    "xstory_cloze_id": {
+      "acc": 0.5526141628060887,
+      "acc_stderr": 0.012795688167385315
+    },
+    "xstory_cloze_te": {
+      "acc": 0.5334215751158173,
+      "acc_stderr": 0.012838347934731667
+    },
+    "xstory_cloze_ar": {
+      "acc": 0.49702183984116477,
+      "acc_stderr": 0.012866897066011233
+    },
+    "xstory_cloze_sw": {
+      "acc": 0.4990072799470549,
+      "acc_stderr": 0.01286709995542293
+    },
+    "xstory_cloze_hi": {
+      "acc": 0.5234943745863666,
+      "acc_stderr": 0.012852912530051748
+    },
+    "xstory_cloze_eu": {
+      "acc": 0.5069490403706155,
+      "acc_stderr": 0.012865882570960722
+    },
+    "xstory_cloze_en": {
+      "acc": 0.7729980145598941,
+      "acc_stderr": 0.010779920137756025
+    },
+    "xstory_cloze_es": {
+      "acc": 0.6942422236929185,
+      "acc_stderr": 0.011856480568871262
+    },
+    "xstory_cloze_ru": {
+      "acc": 0.6340172071475844,
+      "acc_stderr": 0.012396308684399372
+    }
+  },
+  "versions": {
+    "xstory_cloze_zh": 0,
+    "xstory_cloze_my": 0,
+    "xstory_cloze_id": 0,
+    "xstory_cloze_te": 0,
+    "xstory_cloze_ar": 0,
+    "xstory_cloze_sw": 0,
+    "xstory_cloze_hi": 0,
+    "xstory_cloze_eu": 0,
+    "xstory_cloze_en": 0,
+    "xstory_cloze_es": 0,
+    "xstory_cloze_ru": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/13B",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/llama/llama-13B/llama-13B_xwinograd_0-shot.json

+{
+  "results": {
+    "xwinograd_pt": {
+      "acc": 0.714828897338403,
+      "acc_stderr": 0.02789350966043832
+    },
+    "xwinograd_jp": {
+      "acc": 0.5985401459854015,
+      "acc_stderr": 0.01583743878453324
+    },
+    "xwinograd_en": {
+      "acc": 0.8675268817204301,
+      "acc_stderr": 0.007032136436579812
+    },
+    "xwinograd_ru": {
+      "acc": 0.707936507936508,
+      "acc_stderr": 0.02566084582577463
+    },
+    "xwinograd_zh": {
+      "acc": 0.7003968253968254,
+      "acc_stderr": 0.020424963888406065
+    },
+    "xwinograd_fr": {
+      "acc": 0.6867469879518072,
+      "acc_stderr": 0.051219942106581456
+    }
+  },
+  "versions": {
+    "xwinograd_pt": 0,
+    "xwinograd_jp": 0,
+    "xwinograd_en": 0,
+    "xwinograd_ru": 0,
+    "xwinograd_zh": 0,
+    "xwinograd_fr": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/13B",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/llama/llama-30B/llama-30B_bbh_3-shot.json

+{
+  "results": {
+    "bigbench_hyperbaton": {
+      "multiple_choice_grade": 0.51524,
+      "multiple_choice_grade_stderr": 0.0022350513992069
+    },
+    "bigbench_salient_translation_error_detection": {
+      "multiple_choice_grade": 0.19839679358717435,
+      "multiple_choice_grade_stderr": 0.012629887094728112
+    },
+    "bigbench_geometric_shapes": {
+      "multiple_choice_grade": 0.2785515320334262,
+      "multiple_choice_grade_stderr": 0.023692665345206258,
+      "exact_str_match": 0.0,
+      "exact_str_match_stderr": 0.0
+    },
+    "bigbench_navigate": {
+      "multiple_choice_grade": 0.49,
+      "multiple_choice_grade_stderr": 0.015816135752773193
+    },
+    "bigbench_date_understanding": {
+      "multiple_choice_grade": 0.6991869918699187,
+      "multiple_choice_grade_stderr": 0.023906779002093273
+    },
+    "bigbench_disambiguation_qa": {
+      "multiple_choice_grade": 0.5426356589147286,
+      "multiple_choice_grade_stderr": 0.031075544990472662
+    },
+    "bigbench_tracking_shuffled_objects_three_objects": {
+      "multiple_choice_grade": 0.53,
+      "multiple_choice_grade_stderr": 0.02886365132641709
+    },
+    "bigbench_dyck_languages": {
+      "multiple_choice_grade": 0.212,
+      "multiple_choice_grade_stderr": 0.01293148186493804
+    },
+    "bigbench_formal_fallacies_syllogisms_negation": {
+      "multiple_choice_grade": 0.5058450704225352,
+      "multiple_choice_grade_stderr": 0.004195767817554208
+    },
+    "bigbench_tracking_shuffled_objects_seven_objects": {
+      "multiple_choice_grade": 0.15485714285714286,
+      "multiple_choice_grade_stderr": 0.00865039181414196
+    },
+    "bigbench_causal_judgement": {
+      "multiple_choice_grade": 0.5736842105263158,
+      "multiple_choice_grade_stderr": 0.03597255252302466
+    },
+    "bigbench_movie_recommendation": {
+      "multiple_choice_grade": 0.632,
+      "multiple_choice_grade_stderr": 0.02158898256835354
+    },
+    "bigbench_tracking_shuffled_objects_five_objects": {
+      "multiple_choice_grade": 0.2128,
+      "multiple_choice_grade_stderr": 0.01158102863217863
+    },
+    "bigbench_snarks": {
+      "multiple_choice_grade": 0.4696132596685083,
+      "multiple_choice_grade_stderr": 0.03719891321680327
+    },
+    "bigbench_sports_understanding": {
+      "multiple_choice_grade": 0.6237322515212982,
+      "multiple_choice_grade_stderr": 0.01543581207286162
+    },
+    "bigbench_logical_deduction_seven_objects": {
+      "multiple_choice_grade": 0.25285714285714284,
+      "multiple_choice_grade_stderr": 0.01643996352811702
+    },
+    "bigbench_temporal_sequences": {
+      "multiple_choice_grade": 0.146,
+      "multiple_choice_grade_stderr": 0.011171786285496496
+    },
+    "bigbench_logical_deduction_five_objects": {
+      "multiple_choice_grade": 0.368,
+      "multiple_choice_grade_stderr": 0.021588982568353548
+    },
+    "bigbench_ruin_names": {
+      "multiple_choice_grade": 0.39732142857142855,
+      "multiple_choice_grade_stderr": 0.023145155753004788
+    },
+    "bigbench_logical_deduction_three_objects": {
+      "multiple_choice_grade": 0.53,
+      "multiple_choice_grade_stderr": 0.02886365132641709
+    },
+    "bigbench_reasoning_about_colored_objects": {
+      "multiple_choice_grade": 0.5565,
+      "multiple_choice_grade_stderr": 0.011111507899646487
+    }
+  },
+  "versions": {
+    "bigbench_hyperbaton": 0,
+    "bigbench_salient_translation_error_detection": 0,
+    "bigbench_geometric_shapes": 0,
+    "bigbench_navigate": 0,
+    "bigbench_date_understanding": 0,
+    "bigbench_disambiguation_qa": 0,
+    "bigbench_tracking_shuffled_objects_three_objects": 0,
+    "bigbench_dyck_languages": 0,
+    "bigbench_formal_fallacies_syllogisms_negation": 0,
+    "bigbench_tracking_shuffled_objects_seven_objects": 0,
+    "bigbench_causal_judgement": 0,
+    "bigbench_movie_recommendation": 0,
+    "bigbench_tracking_shuffled_objects_five_objects": 0,
+    "bigbench_snarks": 0,
+    "bigbench_sports_understanding": 0,
+    "bigbench_logical_deduction_seven_objects": 0,
+    "bigbench_temporal_sequences": 0,
+    "bigbench_logical_deduction_five_objects": 0,
+    "bigbench_ruin_names": 0,
+    "bigbench_logical_deduction_three_objects": 0,
+    "bigbench_reasoning_about_colored_objects": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/30B,use_accelerate=True",
+    "num_fewshot": 3,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/llama/llama-30B/llama-30B_common_sense_reasoning_0-shot.json

+{
+  "results": {
+    "hellaswag": {
+      "acc": 0.6264688309101772,
+      "acc_stderr": 0.00482752658488968,
+      "acc_norm": 0.7923720374427405,
+      "acc_norm_stderr": 0.00404779964623464
+    },
+    "copa": {
+      "acc": 0.9,
+      "acc_stderr": 0.030151134457776348
+    },
+    "prost": {
+      "acc": 0.2598740392826644,
+      "acc_stderr": 0.003204110008963041,
+      "acc_norm": 0.2910973526900085,
+      "acc_norm_stderr": 0.003318834364612203
+    },
+    "boolq": {
+      "acc": 0.6840978593272171,
+      "acc_stderr": 0.008130700051380873
+    },
+    "mc_taco": {
+      "em": 0.11411411411411411,
+      "f1": 0.48361974757894227
+    },
+    "winogrande": {
+      "acc": 0.7277032359905288,
+      "acc_stderr": 0.012510697991453936
+    },
+    "arc_challenge": {
+      "acc": 0.46757679180887374,
+      "acc_stderr": 0.014580637569995423,
+      "acc_norm": 0.454778156996587,
+      "acc_norm_stderr": 0.014551507060836352
+    },
+    "wsc273": {
+      "acc": 0.8681318681318682,
+      "acc_stderr": 0.020515321360773595
+    },
+    "openbookqa": {
+      "acc": 0.294,
+      "acc_stderr": 0.020395095484936603,
+      "acc_norm": 0.42,
+      "acc_norm_stderr": 0.02209471322976178
+    },
+    "swag": {
+      "acc": 0.5861241627511746,
+      "acc_stderr": 0.0034822550028030703,
+      "acc_norm": 0.7036389083275018,
+      "acc_norm_stderr": 0.0032286148364766096
+    },
+    "arc_easy": {
+      "acc": 0.7533670033670034,
+      "acc_stderr": 0.008844984581934908,
+      "acc_norm": 0.5896464646464646,
+      "acc_norm_stderr": 0.01009353125576545
+    },
+    "piqa": {
+      "acc": 0.809575625680087,
+      "acc_stderr": 0.009160842206469637,
+      "acc_norm": 0.8008705114254625,
+      "acc_norm_stderr": 0.009317391893706834
+    }
+  },
+  "versions": {
+    "hellaswag": 0,
+    "copa": 0,
+    "prost": 0,
+    "boolq": 1,
+    "mc_taco": 0,
+    "winogrande": 0,
+    "arc_challenge": 0,
+    "wsc273": 0,
+    "openbookqa": 0,
+    "swag": 0,
+    "arc_easy": 0,
+    "piqa": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/30B,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda:0",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/llama/llama-30B/llama-30B_gsm8k_8-shot.json

+{
+  "results": {
+    "gsm8k": {
+      "acc": 0.30477634571645185,
+      "acc_stderr": 0.012679297549515422
+    }
+  },
+  "versions": {
+    "gsm8k": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/30B,use_accelerate=True",
+    "num_fewshot": 8,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/llama/llama-30B/llama-30B_human_alignment_0-shot.json

+{
+  "results": {
+    "crows_pairs_french_disability": {
+      "likelihood_difference": 10.202651515151516,
+      "likelihood_difference_stderr": 1.261700816634343,
+      "pct_stereotype": 0.5606060606060606,
+      "pct_stereotype_stderr": 0.06156009014560979
+    },
+    "crows_pairs_french_religion": {
+      "likelihood_difference": 7.940217391304348,
+      "likelihood_difference_stderr": 0.938898141048901,
+      "pct_stereotype": 0.5652173913043478,
+      "pct_stereotype_stderr": 0.04642922286356427
+    },
+    "crows_pairs_french_sexual_orientation": {
+      "likelihood_difference": 9.890796703296703,
+      "likelihood_difference_stderr": 1.6112974194891465,
+      "pct_stereotype": 0.7142857142857143,
+      "pct_stereotype_stderr": 0.04761904761904758
+    },
+    "toxigen": {
+      "acc": 0.42659574468085104,
+      "acc_stderr": 0.01614008877637632,
+      "acc_norm": 0.4319148936170213,
+      "acc_norm_stderr": 0.016164899004911828
+    },
+    "crows_pairs_english_gender": {
+      "likelihood_difference": 6.6615234375,
+      "likelihood_difference_stderr": 0.4284975339207996,
+      "pct_stereotype": 0.540625,
+      "pct_stereotype_stderr": 0.02790206840430007
+    },
+    "crows_pairs_english_age": {
+      "likelihood_difference": 5.1291208791208796,
+      "likelihood_difference_stderr": 0.5813404620923356,
+      "pct_stereotype": 0.5824175824175825,
+      "pct_stereotype_stderr": 0.05198368783767557
+    },
+    "crows_pairs_english_disability": {
+      "likelihood_difference": 8.886538461538462,
+      "likelihood_difference_stderr": 1.0342476212707912,
+      "pct_stereotype": 0.5384615384615384,
+      "pct_stereotype_stderr": 0.06231481440776789
+    },
+    "crows_pairs_french_age": {
+      "likelihood_difference": 8.925,
+      "likelihood_difference_stderr": 1.01086298976785,
+      "pct_stereotype": 0.4,
+      "pct_stereotype_stderr": 0.05192907868894985
+    },
+    "ethics_utilitarianism": {
+      "acc": 0.5012479201331115,
+      "acc_stderr": 0.00721159934497283
+    },
+    "crows_pairs_english_physical_appearance": {
+      "likelihood_difference": 5.401041666666667,
+      "likelihood_difference_stderr": 0.5913652974915496,
+      "pct_stereotype": 0.5277777777777778,
+      "pct_stereotype_stderr": 0.05924743948371486
+    },
+    "crows_pairs_french_socioeconomic": {
+      "likelihood_difference": 8.312898596938776,
+      "likelihood_difference_stderr": 0.8737467813045966,
+      "pct_stereotype": 0.5255102040816326,
+      "pct_stereotype_stderr": 0.03575911069046443
+    },
+    "crows_pairs_english_nationality": {
+      "likelihood_difference": 5.872829861111111,
+      "likelihood_difference_stderr": 0.3994396285401545,
+      "pct_stereotype": 0.5324074074074074,
+      "pct_stereotype_stderr": 0.03402801581358966
+    },
+    "ethics_cm": {
+      "acc": 0.5750321750321751,
+      "acc_stderr": 0.007932032541825585
+    },
+    "crows_pairs_french_gender": {
+      "likelihood_difference": 13.732768691588785,
+      "likelihood_difference_stderr": 1.1030097530113459,
+      "pct_stereotype": 0.5015576323987538,
+      "pct_stereotype_stderr": 0.027950714088670354
+    },
+    "crows_pairs_french_nationality": {
+      "likelihood_difference": 9.851037549407115,
+      "likelihood_difference_stderr": 0.8908345552184256,
+      "pct_stereotype": 0.38735177865612647,
+      "pct_stereotype_stderr": 0.03068725875850367
+    },
+    "ethics_deontology": {
+      "acc": 0.5417130144605117,
+      "acc_stderr": 0.008310055982844088,
+      "em": 0.06117908787541713
+    },
+    "ethics_utilitarianism_original": {
+      "acc": 0.9396838602329451,
+      "acc_stderr": 0.0034337651785718414
+    },
+    "crows_pairs_english_sexual_orientation": {
+      "likelihood_difference": 7.547715053763441,
+      "likelihood_difference_stderr": 0.7682550004765589,
+      "pct_stereotype": 0.6344086021505376,
+      "pct_stereotype_stderr": 0.05020981279330232
+    },
+    "crows_pairs_english_religion": {
+      "likelihood_difference": 8.075731981981981,
+      "likelihood_difference_stderr": 0.9438303669276185,
+      "pct_stereotype": 0.6216216216216216,
+      "pct_stereotype_stderr": 0.04624128233851482
+    },
+    "ethics_justice": {
+      "acc": 0.5170118343195266,
+      "acc_stderr": 0.009611595027307154,
+      "em": 0.013313609467455622
+    },
+    "ethics_virtue": {
+      "acc": 0.5181909547738693,
+      "acc_stderr": 0.007084831046245509,
+      "em": 0.0814070351758794
+    },
+    "crows_pairs_english_race_color": {
+      "likelihood_difference": 7.68214812992126,
+      "likelihood_difference_stderr": 0.3913516470344277,
+      "pct_stereotype": 0.5610236220472441,
+      "pct_stereotype_stderr": 0.022039775660119297
+    },
+    "crows_pairs_english_autre": {
+      "likelihood_difference": 11.380681818181818,
+      "likelihood_difference_stderr": 3.487665507491904,
+      "pct_stereotype": 0.6363636363636364,
+      "pct_stereotype_stderr": 0.15212000482437738
+    },
+    "crows_pairs_french_race_color": {
+      "likelihood_difference": 10.611684782608696,
+      "likelihood_difference_stderr": 0.7668117638923473,
+      "pct_stereotype": 0.41739130434782606,
+      "pct_stereotype_stderr": 0.023017271312104015
+    },
+    "crows_pairs_french_physical_appearance": {
+      "likelihood_difference": 8.217881944444445,
+      "likelihood_difference_stderr": 1.3267643213128657,
+      "pct_stereotype": 0.5694444444444444,
+      "pct_stereotype_stderr": 0.05876396677084613
+    },
+    "crows_pairs_french_autre": {
+      "likelihood_difference": 6.3798076923076925,
+      "likelihood_difference_stderr": 1.6568389364513447,
+      "pct_stereotype": 0.6153846153846154,
+      "pct_stereotype_stderr": 0.1404416814115811
+    },
+    "crows_pairs_english_socioeconomic": {
+      "likelihood_difference": 7.303947368421053,
+      "likelihood_difference_stderr": 0.5463280290787818,
+      "pct_stereotype": 0.5368421052631579,
+      "pct_stereotype_stderr": 0.036270781985214155
+    }
+  },
+  "versions": {
+    "crows_pairs_french_disability": 0,
+    "crows_pairs_french_religion": 0,
+    "crows_pairs_french_sexual_orientation": 0,
+    "toxigen": 0,
+    "crows_pairs_english_gender": 0,
+    "crows_pairs_english_age": 0,
+    "crows_pairs_english_disability": 0,
+    "crows_pairs_french_age": 0,
+    "ethics_utilitarianism": 0,
+    "crows_pairs_english_physical_appearance": 0,
+    "crows_pairs_french_socioeconomic": 0,
+    "crows_pairs_english_nationality": 0,
+    "ethics_cm": 0,
+    "crows_pairs_french_gender": 0,
+    "crows_pairs_french_nationality": 0,
+    "ethics_deontology": 0,
+    "ethics_utilitarianism_original": 0,
+    "crows_pairs_english_sexual_orientation": 0,
+    "crows_pairs_english_religion": 0,
+    "ethics_justice": 0,
+    "ethics_virtue": 0,
+    "crows_pairs_english_race_color": 0,
+    "crows_pairs_english_autre": 0,
+    "crows_pairs_french_race_color": 0,
+    "crows_pairs_french_physical_appearance": 0,
+    "crows_pairs_french_autre": 0,
+    "crows_pairs_english_socioeconomic": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/30B,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda:0",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/llama/llama-30B/llama-30B_mathematical_reasoning_0-shot.json

+{
+  "results": {
+    "math_prealgebra": {
+      "acc": 0.04133180252583238,
+      "acc_stderr": 0.006748646916387575
+    },
+    "drop": {
+      "em": 0.0382760067114094,
+      "em_stderr": 0.0019648445106113135,
+      "f1": 0.13911493288590454,
+      "f1_stderr": 0.0024846240125468515
+    },
+    "math_intermediate_algebra": {
+      "acc": 0.008859357696566999,
+      "acc_stderr": 0.0031200782932944743
+    },
+    "gsm8k": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "math_num_theory": {
+      "acc": 0.02962962962962963,
+      "acc_stderr": 0.007303608618028771
+    },
+    "math_algebra": {
+      "acc": 0.02948609941027801,
+      "acc_stderr": 0.004912099985374022
+    },
+    "math_precalc": {
+      "acc": 0.018315018315018316,
+      "acc_stderr": 0.005743696731653661
+    },
+    "math_geometry": {
+      "acc": 0.014613778705636743,
+      "acc_stderr": 0.005488713443686309
+    },
+    "math_counting_and_prob": {
+      "acc": 0.04008438818565401,
+      "acc_stderr": 0.009019315660749231
+    },
+    "math_asdiv": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "mathqa": {
+      "acc": 0.30586264656616413,
+      "acc_stderr": 0.00843502782274867,
+      "acc_norm": 0.3088777219430486,
+      "acc_norm_stderr": 0.008458071062361336
+    }
+  },
+  "versions": {
+    "math_prealgebra": 1,
+    "drop": 1,
+    "math_intermediate_algebra": 1,
+    "gsm8k": 0,
+    "math_asdiv": 0,
+    "math_num_theory": 1,
+    "math_algebra": 1,
+    "math_precalc": 1,
+    "math_geometry": 1,
+    "math_counting_and_prob": 1,
+    "mathqa": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/30B,use_accelerate=True",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda:0",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/llama/llama-30B/llama-30B_mathematical_reasoning_few_shot_5-shot.json

+{
+  "results": {
+    "math_prealgebra": {
+      "acc": 0.001148105625717566,
+      "acc_stderr": 0.001148105625717572
+    },
+    "math_intermediate_algebra": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "math_counting_and_prob": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "math_geometry": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "math_precalc": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "drop": {
+      "em": 0.008389261744966443,
+      "em_stderr": 0.0009340543216866975,
+      "f1": 0.016472315436241603,
+      "f1_stderr": 0.001049526866424092
+    },
+    "gsm8k": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "math_num_theory": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "math_algebra": {
+      "acc": 0.0,
+      "acc_stderr": 0.0
+    },
+    "mathqa": {
+      "acc": 0.3474036850921273,
+      "acc_stderr": 0.008716459359487392,
+      "acc_norm": 0.34539363484087104,
+      "acc_norm_stderr": 0.008704580930350191
+    }
+  },
+  "versions": {
+    "math_prealgebra": 1,
+    "math_intermediate_algebra": 1,
+    "math_counting_and_prob": 1,
+    "math_geometry": 1,
+    "math_precalc": 1,
+    "drop": 1,
+    "mathqa": 0,
+    "gsm8k": 0,
+    "math_num_theory": 1,
+    "math_algebra": 1
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/30B,use_accelerate=True",
+    "num_fewshot": 5,
+    "batch_size": "auto",
+    "device": "cuda:0",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/llama/llama-30B/llama-30B_mmlu_5-shot.json

+{
+  "results": {
+    "hendrycksTest-high_school_world_history": {
+      "acc": 0.6962025316455697,
+      "acc_stderr": 0.029936696387138598,
+      "acc_norm": 0.569620253164557,
+      "acc_norm_stderr": 0.032230171959375976
+    },
+    "hendrycksTest-formal_logic": {
+      "acc": 0.42063492063492064,
+      "acc_stderr": 0.04415438226743743,
+      "acc_norm": 0.3968253968253968,
+      "acc_norm_stderr": 0.043758884927270605
+    },
+    "hendrycksTest-human_aging": {
+      "acc": 0.672645739910314,
+      "acc_stderr": 0.03149384670994131,
+      "acc_norm": 0.3632286995515695,
+      "acc_norm_stderr": 0.032277904428505
+    },
+    "hendrycksTest-international_law": {
+      "acc": 0.7024793388429752,
+      "acc_stderr": 0.04173349148083499,
+      "acc_norm": 0.768595041322314,
+      "acc_norm_stderr": 0.03849856098794088
+    },
+    "hendrycksTest-security_studies": {
+      "acc": 0.5714285714285714,
+      "acc_stderr": 0.031680911612338825,
+      "acc_norm": 0.40408163265306124,
+      "acc_norm_stderr": 0.0314147080258659
+    },
+    "hendrycksTest-medical_genetics": {
+      "acc": 0.6,
+      "acc_stderr": 0.049236596391733084,
+      "acc_norm": 0.54,
+      "acc_norm_stderr": 0.05009082659620332
+    },
+    "hendrycksTest-econometrics": {
+      "acc": 0.3508771929824561,
+      "acc_stderr": 0.044895393502707,
+      "acc_norm": 0.3157894736842105,
+      "acc_norm_stderr": 0.043727482902780064
+    },
+    "hendrycksTest-high_school_macroeconomics": {
+      "acc": 0.5153846153846153,
+      "acc_stderr": 0.025339003010106515,
+      "acc_norm": 0.4153846153846154,
+      "acc_norm_stderr": 0.024985354923102332
+    },
+    "hendrycksTest-us_foreign_policy": {
+      "acc": 0.79,
+      "acc_stderr": 0.040936018074033256,
+      "acc_norm": 0.59,
+      "acc_norm_stderr": 0.049431107042371025
+    },
+    "hendrycksTest-logical_fallacies": {
+      "acc": 0.6993865030674846,
+      "acc_stderr": 0.03602511318806771,
+      "acc_norm": 0.5398773006134969,
+      "acc_norm_stderr": 0.039158572914369714
+    },
+    "hendrycksTest-prehistory": {
+      "acc": 0.6635802469135802,
+      "acc_stderr": 0.026289734945952926,
+      "acc_norm": 0.42901234567901236,
+      "acc_norm_stderr": 0.027538925613470867
+    },
+    "hendrycksTest-professional_psychology": {
+      "acc": 0.5882352941176471,
+      "acc_stderr": 0.019910377463105932,
+      "acc_norm": 0.43300653594771243,
+      "acc_norm_stderr": 0.02004544247332422
+    },
+    "hendrycksTest-professional_accounting": {
+      "acc": 0.3971631205673759,
+      "acc_stderr": 0.029189805673587105,
+      "acc_norm": 0.33687943262411346,
+      "acc_norm_stderr": 0.02819553487396673
+    },
+    "hendrycksTest-college_biology": {
+      "acc": 0.6111111111111112,
+      "acc_stderr": 0.04076663253918567,
+      "acc_norm": 0.4236111111111111,
+      "acc_norm_stderr": 0.04132125019723369
+    },
+    "hendrycksTest-high_school_biology": {
+      "acc": 0.6709677419354839,
+      "acc_stderr": 0.02672949906834996,
+      "acc_norm": 0.5451612903225806,
+      "acc_norm_stderr": 0.028327743091561074
+    },
+    "hendrycksTest-philosophy": {
+      "acc": 0.6752411575562701,
+      "acc_stderr": 0.02659678228769704,
+      "acc_norm": 0.5016077170418006,
+      "acc_norm_stderr": 0.02839794490780661
+    },
+    "hendrycksTest-high_school_european_history": {
+      "acc": 0.696969696969697,
+      "acc_stderr": 0.03588624800091707,
+      "acc_norm": 0.5636363636363636,
+      "acc_norm_stderr": 0.03872592983524754
+    },
+    "hendrycksTest-college_medicine": {
+      "acc": 0.5144508670520231,
+      "acc_stderr": 0.03810871630454764,
+      "acc_norm": 0.43352601156069365,
+      "acc_norm_stderr": 0.03778621079092055
+    },
+    "hendrycksTest-professional_medicine": {
+      "acc": 0.5551470588235294,
+      "acc_stderr": 0.03018753206032938,
+      "acc_norm": 0.35661764705882354,
+      "acc_norm_stderr": 0.02909720956841195
+    },
+    "hendrycksTest-moral_scenarios": {
+      "acc": 0.34301675977653634,
+      "acc_stderr": 0.015876912673057724,
+      "acc_norm": 0.27262569832402234,
+      "acc_norm_stderr": 0.014893391735249588
+    },
+    "hendrycksTest-high_school_chemistry": {
+      "acc": 0.39901477832512317,
+      "acc_stderr": 0.03445487686264716,
+      "acc_norm": 0.3694581280788177,
+      "acc_norm_stderr": 0.03395970381998573
+    },
+    "hendrycksTest-high_school_physics": {
+      "acc": 0.31788079470198677,
+      "acc_stderr": 0.038020397601079024,
+      "acc_norm": 0.31125827814569534,
+      "acc_norm_stderr": 0.03780445850526733
+    },
+    "hendrycksTest-high_school_government_and_politics": {
+      "acc": 0.8082901554404145,
+      "acc_stderr": 0.028408953626245282,
+      "acc_norm": 0.6113989637305699,
+      "acc_norm_stderr": 0.03517739796373132
+    },
+    "hendrycksTest-high_school_geography": {
+      "acc": 0.7575757575757576,
+      "acc_stderr": 0.030532892233932026,
+      "acc_norm": 0.5505050505050505,
+      "acc_norm_stderr": 0.0354413249194797
+    },
+    "hendrycksTest-global_facts": {
+      "acc": 0.47,
+      "acc_stderr": 0.05016135580465919,
+      "acc_norm": 0.37,
+      "acc_norm_stderr": 0.04852365870939099
+    },
+    "hendrycksTest-professional_law": {
+      "acc": 0.4002607561929596,
+      "acc_stderr": 0.012513582529136213,
+      "acc_norm": 0.3435462842242503,
+      "acc_norm_stderr": 0.012128961174190158
+    },
+    "hendrycksTest-college_mathematics": {
+      "acc": 0.37,
+      "acc_stderr": 0.048523658709391,
+      "acc_norm": 0.3,
+      "acc_norm_stderr": 0.046056618647183814
+    },
+    "hendrycksTest-college_physics": {
+      "acc": 0.23529411764705882,
+      "acc_stderr": 0.04220773659171452,
+      "acc_norm": 0.29411764705882354,
+      "acc_norm_stderr": 0.04533838195929774
+    },
+    "hendrycksTest-high_school_statistics": {
+      "acc": 0.4351851851851852,
+      "acc_stderr": 0.03381200005643525,
+      "acc_norm": 0.35648148148148145,
+      "acc_norm_stderr": 0.032664783315272714
+    },
+    "hendrycksTest-machine_learning": {
+      "acc": 0.4017857142857143,
+      "acc_stderr": 0.04653333146973646,
+      "acc_norm": 0.30357142857142855,
+      "acc_norm_stderr": 0.04364226155841044
+    },
+    "hendrycksTest-public_relations": {
+      "acc": 0.6454545454545455,
+      "acc_stderr": 0.045820048415054174,
+      "acc_norm": 0.4090909090909091,
+      "acc_norm_stderr": 0.047093069786618966
+    },
+    "hendrycksTest-high_school_computer_science": {
+      "acc": 0.61,
+      "acc_stderr": 0.04902071300001974,
+      "acc_norm": 0.47,
+      "acc_norm_stderr": 0.05016135580465919
+    },
+    "hendrycksTest-high_school_psychology": {
+      "acc": 0.7706422018348624,
+      "acc_stderr": 0.018025349724618684,
+      "acc_norm": 0.5541284403669725,
+      "acc_norm_stderr": 0.021311335009708582
+    },
+    "hendrycksTest-virology": {
+      "acc": 0.4939759036144578,
+      "acc_stderr": 0.03892212195333045,
+      "acc_norm": 0.3433734939759036,
+      "acc_norm_stderr": 0.03696584317010601
+    },
+    "hendrycksTest-marketing": {
+      "acc": 0.8461538461538461,
+      "acc_stderr": 0.023636873317489294,
+      "acc_norm": 0.7649572649572649,
+      "acc_norm_stderr": 0.027778835904935437
+    },
+    "hendrycksTest-human_sexuality": {
+      "acc": 0.7022900763358778,
+      "acc_stderr": 0.04010358942462203,
+      "acc_norm": 0.46564885496183206,
+      "acc_norm_stderr": 0.04374928560599738
+    },
+    "hendrycksTest-sociology": {
+      "acc": 0.7611940298507462,
+      "acc_stderr": 0.03014777593540922,
+      "acc_norm": 0.6616915422885572,
+      "acc_norm_stderr": 0.033455630703391914
+    },
+    "hendrycksTest-college_computer_science": {
+      "acc": 0.43,
+      "acc_stderr": 0.049756985195624284,
+      "acc_norm": 0.34,
+      "acc_norm_stderr": 0.04760952285695236
+    },
+    "hendrycksTest-conceptual_physics": {
+      "acc": 0.5106382978723404,
+      "acc_stderr": 0.03267862331014063,
+      "acc_norm": 0.3276595744680851,
+      "acc_norm_stderr": 0.030683020843231004
+    },
+    "hendrycksTest-anatomy": {
+      "acc": 0.5185185185185185,
+      "acc_stderr": 0.043163785995113245,
+      "acc_norm": 0.4074074074074074,
+      "acc_norm_stderr": 0.04244633238353228
+    },
+    "hendrycksTest-miscellaneous": {
+      "acc": 0.8186462324393359,
+      "acc_stderr": 0.013778693778464062,
+      "acc_norm": 0.6143039591315453,
+      "acc_norm_stderr": 0.017406476619212907
+    },
+    "hendrycksTest-jurisprudence": {
+      "acc": 0.6666666666666666,
+      "acc_stderr": 0.04557239513497751,
+      "acc_norm": 0.5555555555555556,
+      "acc_norm_stderr": 0.04803752235190193
+    },
+    "hendrycksTest-moral_disputes": {
+      "acc": 0.6184971098265896,
+      "acc_stderr": 0.026152198619726792,
+      "acc_norm": 0.4595375722543353,
+      "acc_norm_stderr": 0.026830805998952236
+    },
+    "hendrycksTest-high_school_us_history": {
+      "acc": 0.7205882352941176,
+      "acc_stderr": 0.031493281045079556,
+      "acc_norm": 0.553921568627451,
+      "acc_norm_stderr": 0.03488845451304974
+    },
+    "hendrycksTest-high_school_mathematics": {
+      "acc": 0.25925925925925924,
+      "acc_stderr": 0.026719240783712177,
+      "acc_norm": 0.3148148148148148,
+      "acc_norm_stderr": 0.02831753349606648
+    },
+    "hendrycksTest-high_school_microeconomics": {
+      "acc": 0.5840336134453782,
+      "acc_stderr": 0.032016501007396114,
+      "acc_norm": 0.4831932773109244,
+      "acc_norm_stderr": 0.03246013680375308
+    },
+    "hendrycksTest-astronomy": {
+      "acc": 0.5723684210526315,
+      "acc_stderr": 0.04026097083296564,
+      "acc_norm": 0.5657894736842105,
+      "acc_norm_stderr": 0.04033565667848319
+    },
+    "hendrycksTest-world_religions": {
+      "acc": 0.8128654970760234,
+      "acc_stderr": 0.029913127232368043,
+      "acc_norm": 0.7660818713450293,
+      "acc_norm_stderr": 0.03246721765117825
+    },
+    "hendrycksTest-clinical_knowledge": {
+      "acc": 0.5320754716981132,
+      "acc_stderr": 0.03070948699255654,
+      "acc_norm": 0.4641509433962264,
+      "acc_norm_stderr": 0.030693675018458003
+    },
+    "hendrycksTest-college_chemistry": {
+      "acc": 0.31,
+      "acc_stderr": 0.04648231987117316,
+      "acc_norm": 0.32,
+      "acc_norm_stderr": 0.046882617226215034
+    },
+    "hendrycksTest-abstract_algebra": {
+      "acc": 0.26,
+      "acc_stderr": 0.04408440022768078,
+      "acc_norm": 0.29,
+      "acc_norm_stderr": 0.04560480215720684
+    },
+    "hendrycksTest-business_ethics": {
+      "acc": 0.67,
+      "acc_stderr": 0.04725815626252609,
+      "acc_norm": 0.48,
+      "acc_norm_stderr": 0.050211673156867795
+    },
+    "hendrycksTest-elementary_mathematics": {
+      "acc": 0.4417989417989418,
+      "acc_stderr": 0.02557625706125384,
+      "acc_norm": 0.37037037037037035,
+      "acc_norm_stderr": 0.024870815251057075
+    },
+    "hendrycksTest-management": {
+      "acc": 0.7184466019417476,
+      "acc_stderr": 0.044532548363264673,
+      "acc_norm": 0.5533980582524272,
+      "acc_norm_stderr": 0.04922424153458933
+    },
+    "hendrycksTest-electrical_engineering": {
+      "acc": 0.5172413793103449,
+      "acc_stderr": 0.04164188720169375,
+      "acc_norm": 0.38620689655172413,
+      "acc_norm_stderr": 0.040573247344190336
+    },
+    "hendrycksTest-nutrition": {
+      "acc": 0.6111111111111112,
+      "acc_stderr": 0.02791405551046801,
+      "acc_norm": 0.5032679738562091,
+      "acc_norm_stderr": 0.028629305194003543
+    },
+    "hendrycksTest-computer_security": {
+      "acc": 0.66,
+      "acc_stderr": 0.04760952285695237,
+      "acc_norm": 0.58,
+      "acc_norm_stderr": 0.049604496374885836
+    }
+  },
+  "versions": {
+    "hendrycksTest-high_school_world_history": 0,
+    "hendrycksTest-formal_logic": 0,
+    "hendrycksTest-human_aging": 0,
+    "hendrycksTest-international_law": 0,
+    "hendrycksTest-security_studies": 0,
+    "hendrycksTest-medical_genetics": 0,
+    "hendrycksTest-econometrics": 0,
+    "hendrycksTest-high_school_macroeconomics": 0,
+    "hendrycksTest-us_foreign_policy": 0,
+    "hendrycksTest-logical_fallacies": 0,
+    "hendrycksTest-prehistory": 0,
+    "hendrycksTest-professional_psychology": 0,
+    "hendrycksTest-professional_accounting": 0,
+    "hendrycksTest-college_biology": 0,
+    "hendrycksTest-high_school_biology": 0,
+    "hendrycksTest-philosophy": 0,
+    "hendrycksTest-high_school_european_history": 0,
+    "hendrycksTest-college_medicine": 0,
+    "hendrycksTest-professional_medicine": 0,
+    "hendrycksTest-moral_scenarios": 0,
+    "hendrycksTest-high_school_chemistry": 0,
+    "hendrycksTest-high_school_physics": 0,
+    "hendrycksTest-high_school_government_and_politics": 0,
+    "hendrycksTest-high_school_geography": 0,
+    "hendrycksTest-global_facts": 0,
+    "hendrycksTest-professional_law": 0,
+    "hendrycksTest-college_mathematics": 0,
+    "hendrycksTest-college_physics": 0,
+    "hendrycksTest-high_school_statistics": 0,
+    "hendrycksTest-machine_learning": 0,
+    "hendrycksTest-public_relations": 0,
+    "hendrycksTest-high_school_computer_science": 0,
+    "hendrycksTest-high_school_psychology": 0,
+    "hendrycksTest-virology": 0,
+    "hendrycksTest-marketing": 0,
+    "hendrycksTest-human_sexuality": 0,
+    "hendrycksTest-sociology": 0,
+    "hendrycksTest-college_computer_science": 0,
+    "hendrycksTest-conceptual_physics": 0,
+    "hendrycksTest-anatomy": 0,
+    "hendrycksTest-miscellaneous": 0,
+    "hendrycksTest-jurisprudence": 0,
+    "hendrycksTest-moral_disputes": 0,
+    "hendrycksTest-high_school_us_history": 0,
+    "hendrycksTest-high_school_mathematics": 0,
+    "hendrycksTest-high_school_microeconomics": 0,
+    "hendrycksTest-astronomy": 0,
+    "hendrycksTest-world_religions": 0,
+    "hendrycksTest-clinical_knowledge": 0,
+    "hendrycksTest-college_chemistry": 0,
+    "hendrycksTest-abstract_algebra": 0,
+    "hendrycksTest-business_ethics": 0,
+    "hendrycksTest-elementary_mathematics": 0,
+    "hendrycksTest-management": 0,
+    "hendrycksTest-electrical_engineering": 0,
+    "hendrycksTest-nutrition": 0,
+    "hendrycksTest-computer_security": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/30B,use_accelerate=True",
+    "num_fewshot": 5,
+    "batch_size": "auto",
+    "device": "cuda:0",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

results/llama/llama-30B/llama-30B_pawsx_0-shot.json

+{
+  "results": {
+    "pawsx_fr": {
+      "acc": 0.5285,
+      "acc_stderr": 0.01116495423642881
+    },
+    "pawsx_de": {
+      "acc": 0.582,
+      "acc_stderr": 0.011031720148042082
+    },
+    "pawsx_en": {
+      "acc": 0.5875,
+      "acc_stderr": 0.011010562712487564
+    },
+    "pawsx_ja": {
+      "acc": 0.4675,
+      "acc_stderr": 0.011159486640120933
+    },
+    "pawsx_ko": {
+      "acc": 0.457,
+      "acc_stderr": 0.011141704034140798
+    },
+    "pawsx_zh": {
+      "acc": 0.459,
+      "acc_stderr": 0.011145474902641256
+    },
+    "pawsx_es": {
+      "acc": 0.558,
+      "acc_stderr": 0.011107641056719623
+    }
+  },
+  "versions": {
+    "pawsx_fr": 0,
+    "pawsx_de": 0,
+    "pawsx_en": 0,
+    "pawsx_ja": 0,
+    "pawsx_ko": 0,
+    "pawsx_zh": 0,
+    "pawsx_es": 0
+  },
+  "config": {
+    "model": "hf-causal-experimental",
+    "model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/30B",
+    "num_fewshot": 0,
+    "batch_size": "auto",
+    "device": "cuda",
+    "no_cache": true,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}