llama-7B_blimp_0-shot.json

{
  "results": {
    "blimp_wh_vs_that_no_gap_long_distance": {
      "acc": 0.812,
      "acc_stderr": 0.01236158601510377
    },
    "blimp_ellipsis_n_bar_1": {
      "acc": 0.668,
      "acc_stderr": 0.014899597242811478
    },
    "blimp_distractor_agreement_relative_clause": {
      "acc": 0.314,
      "acc_stderr": 0.014683991951087966
    },
    "blimp_determiner_noun_agreement_with_adjective_1": {
      "acc": 0.577,
      "acc_stderr": 0.01563058909047635
    },
    "blimp_principle_A_reconstruction": {
      "acc": 0.673,
      "acc_stderr": 0.014842213153411245
    },
    "blimp_determiner_noun_agreement_2": {
      "acc": 0.598,
      "acc_stderr": 0.015512467135715077
    },
    "blimp_npi_present_1": {
      "acc": 0.534,
      "acc_stderr": 0.015782683329937618
    },
    "blimp_existential_there_quantifiers_2": {
      "acc": 0.674,
      "acc_stderr": 0.01483050720454104
    },
    "blimp_existential_there_subject_raising": {
      "acc": 0.696,
      "acc_stderr": 0.014553205687950438
    },
    "blimp_tough_vs_raising_1": {
      "acc": 0.298,
      "acc_stderr": 0.014470846741134717
    },
    "blimp_wh_questions_subject_gap_long_distance": {
      "acc": 0.746,
      "acc_stderr": 0.01377220656516854
    },
    "blimp_left_branch_island_echo_question": {
      "acc": 0.835,
      "acc_stderr": 0.011743632866916164
    },
    "blimp_only_npi_licensor_present": {
      "acc": 0.814,
      "acc_stderr": 0.012310790208412803
    },
    "blimp_adjunct_island": {
      "acc": 0.539,
      "acc_stderr": 0.01577110420128319
    },
    "blimp_coordinate_structure_constraint_object_extraction": {
      "acc": 0.629,
      "acc_stderr": 0.015283736211823188
    },
    "blimp_irregular_plural_subject_verb_agreement_1": {
      "acc": 0.653,
      "acc_stderr": 0.01506047203170662
    },
    "blimp_passive_2": {
      "acc": 0.703,
      "acc_stderr": 0.0144568322948011
    },
    "blimp_drop_argument": {
      "acc": 0.701,
      "acc_stderr": 0.014484778521220461
    },
    "blimp_wh_vs_that_with_gap_long_distance": {
      "acc": 0.227,
      "acc_stderr": 0.013253174964763925
    },
    "blimp_existential_there_quantifiers_1": {
      "acc": 0.683,
      "acc_stderr": 0.014721675438880224
    },
    "blimp_coordinate_structure_constraint_complex_left_branch": {
      "acc": 0.682,
      "acc_stderr": 0.014734079309311901
    },
    "blimp_ellipsis_n_bar_2": {
      "acc": 0.794,
      "acc_stderr": 0.012795613612786548
    },
    "blimp_sentential_subject_island": {
      "acc": 0.606,
      "acc_stderr": 0.01545972195749338
    },
    "blimp_determiner_noun_agreement_with_adj_irregular_2": {
      "acc": 0.591,
      "acc_stderr": 0.015555094373257946
    },
    "blimp_npi_present_2": {
      "acc": 0.53,
      "acc_stderr": 0.015790799515836763
    },
    "blimp_wh_questions_subject_gap": {
      "acc": 0.72,
      "acc_stderr": 0.014205696104091519
    },
    "blimp_determiner_noun_agreement_irregular_1": {
      "acc": 0.572,
      "acc_stderr": 0.01565442624502927
    },
    "blimp_superlative_quantifiers_1": {
      "acc": 0.612,
      "acc_stderr": 0.015417317979911076
    },
    "blimp_left_branch_island_simple_question": {
      "acc": 0.74,
      "acc_stderr": 0.013877773329774164
    },
    "blimp_irregular_past_participle_adjectives": {
      "acc": 0.429,
      "acc_stderr": 0.01565899754787024
    },
    "blimp_principle_A_domain_2": {
      "acc": 0.646,
      "acc_stderr": 0.015129868238451772
    },
    "blimp_regular_plural_subject_verb_agreement_1": {
      "acc": 0.645,
      "acc_stderr": 0.015139491543780529
    },
    "blimp_principle_A_case_1": {
      "acc": 0.985,
      "acc_stderr": 0.003845749574502997
    },
    "blimp_principle_A_case_2": {
      "acc": 0.554,
      "acc_stderr": 0.015726771166750357
    },
    "blimp_matrix_question_npi_licensor_present": {
      "acc": 0.117,
      "acc_stderr": 0.010169287802713327
    },
    "blimp_determiner_noun_agreement_with_adj_irregular_1": {
      "acc": 0.563,
      "acc_stderr": 0.015693223928730377
    },
    "blimp_principle_A_domain_1": {
      "acc": 0.962,
      "acc_stderr": 0.006049181150584942
    },
    "blimp_superlative_quantifiers_2": {
      "acc": 0.561,
      "acc_stderr": 0.01570113134540077
    },
    "blimp_wh_island": {
      "acc": 0.275,
      "acc_stderr": 0.014127086556490531
    },
    "blimp_only_npi_scope": {
      "acc": 0.266,
      "acc_stderr": 0.013979965645145156
    },
    "blimp_regular_plural_subject_verb_agreement_2": {
      "acc": 0.705,
      "acc_stderr": 0.014428554438445512
    },
    "blimp_complex_NP_island": {
      "acc": 0.416,
      "acc_stderr": 0.015594460144140603
    },
    "blimp_sentential_negation_npi_scope": {
      "acc": 0.588,
      "acc_stderr": 0.015572363292015093
    },
    "blimp_transitive": {
      "acc": 0.698,
      "acc_stderr": 0.014526080235459543
    },
    "blimp_wh_vs_that_with_gap": {
      "acc": 0.239,
      "acc_stderr": 0.013493000446937587
    },
    "blimp_wh_questions_object_gap": {
      "acc": 0.67,
      "acc_stderr": 0.014876872027456736
    },
    "blimp_sentential_negation_npi_licensor_present": {
      "acc": 0.94,
      "acc_stderr": 0.007513751157474913
    },
    "blimp_expletive_it_object_raising": {
      "acc": 0.659,
      "acc_stderr": 0.0149981313484027
    },
    "blimp_determiner_noun_agreement_1": {
      "acc": 0.636,
      "acc_stderr": 0.015222868840522017
    },
    "blimp_anaphor_gender_agreement": {
      "acc": 0.448,
      "acc_stderr": 0.01573351656634784
    },
    "blimp_irregular_plural_subject_verb_agreement_2": {
      "acc": 0.7,
      "acc_stderr": 0.014498627873361425
    },
    "blimp_passive_1": {
      "acc": 0.702,
      "acc_stderr": 0.014470846741134705
    },
    "blimp_determiner_noun_agreement_irregular_2": {
      "acc": 0.602,
      "acc_stderr": 0.015486634102858913
    },
    "blimp_irregular_past_participle_verbs": {
      "acc": 0.725,
      "acc_stderr": 0.014127086556490523
    },
    "blimp_existential_there_object_raising": {
      "acc": 0.788,
      "acc_stderr": 0.012931481864938055
    },
    "blimp_determiner_noun_agreement_with_adj_2": {
      "acc": 0.54,
      "acc_stderr": 0.015768596914394375
    },
    "blimp_principle_A_domain_3": {
      "acc": 0.501,
      "acc_stderr": 0.015819268290576817
    },
    "blimp_causative": {
      "acc": 0.508,
      "acc_stderr": 0.015817274929209008
    },
    "blimp_tough_vs_raising_2": {
      "acc": 0.768,
      "acc_stderr": 0.01335493745228157
    },
    "blimp_wh_vs_that_no_gap": {
      "acc": 0.848,
      "acc_stderr": 0.011358918303475294
    },
    "blimp_anaphor_number_agreement": {
      "acc": 0.659,
      "acc_stderr": 0.014998131348402704
    },
    "blimp_principle_A_c_command": {
      "acc": 0.39,
      "acc_stderr": 0.01543172505386661
    },
    "blimp_distractor_agreement_relational_noun": {
      "acc": 0.441,
      "acc_stderr": 0.015708779894242676
    },
    "blimp_intransitive": {
      "acc": 0.592,
      "acc_stderr": 0.015549205052920673
    },
    "blimp_animate_subject_passive": {
      "acc": 0.626,
      "acc_stderr": 0.015308767369006363
    },
    "blimp_animate_subject_trans": {
      "acc": 0.761,
      "acc_stderr": 0.01349300044693759
    },
    "blimp_inchoative": {
      "acc": 0.42,
      "acc_stderr": 0.015615500115072957
    }
  },
  "versions": {
    "blimp_wh_vs_that_no_gap_long_distance": 0,
    "blimp_ellipsis_n_bar_1": 0,
    "blimp_distractor_agreement_relative_clause": 0,
    "blimp_determiner_noun_agreement_with_adjective_1": 0,
    "blimp_principle_A_reconstruction": 0,
    "blimp_determiner_noun_agreement_2": 0,
    "blimp_npi_present_1": 0,
    "blimp_existential_there_quantifiers_2": 0,
    "blimp_existential_there_subject_raising": 0,
    "blimp_tough_vs_raising_1": 0,
    "blimp_wh_questions_subject_gap_long_distance": 0,
    "blimp_left_branch_island_echo_question": 0,
    "blimp_only_npi_licensor_present": 0,
    "blimp_adjunct_island": 0,
    "blimp_coordinate_structure_constraint_object_extraction": 0,
    "blimp_irregular_plural_subject_verb_agreement_1": 0,
    "blimp_passive_2": 0,
    "blimp_drop_argument": 0,
    "blimp_wh_vs_that_with_gap_long_distance": 0,
    "blimp_existential_there_quantifiers_1": 0,
    "blimp_coordinate_structure_constraint_complex_left_branch": 0,
    "blimp_ellipsis_n_bar_2": 0,
    "blimp_sentential_subject_island": 0,
    "blimp_determiner_noun_agreement_with_adj_irregular_2": 0,
    "blimp_npi_present_2": 0,
    "blimp_wh_questions_subject_gap": 0,
    "blimp_determiner_noun_agreement_irregular_1": 0,
    "blimp_superlative_quantifiers_1": 0,
    "blimp_left_branch_island_simple_question": 0,
    "blimp_irregular_past_participle_adjectives": 0,
    "blimp_principle_A_domain_2": 0,
    "blimp_regular_plural_subject_verb_agreement_1": 0,
    "blimp_principle_A_case_1": 0,
    "blimp_principle_A_case_2": 0,
    "blimp_matrix_question_npi_licensor_present": 0,
    "blimp_determiner_noun_agreement_with_adj_irregular_1": 0,
    "blimp_principle_A_domain_1": 0,
    "blimp_superlative_quantifiers_2": 0,
    "blimp_wh_island": 0,
    "blimp_only_npi_scope": 0,
    "blimp_regular_plural_subject_verb_agreement_2": 0,
    "blimp_complex_NP_island": 0,
    "blimp_sentential_negation_npi_scope": 0,
    "blimp_transitive": 0,
    "blimp_wh_vs_that_with_gap": 0,
    "blimp_wh_questions_object_gap": 0,
    "blimp_sentential_negation_npi_licensor_present": 0,
    "blimp_expletive_it_object_raising": 0,
    "blimp_determiner_noun_agreement_1": 0,
    "blimp_anaphor_gender_agreement": 0,
    "blimp_irregular_plural_subject_verb_agreement_2": 0,
    "blimp_passive_1": 0,
    "blimp_determiner_noun_agreement_irregular_2": 0,
    "blimp_irregular_past_participle_verbs": 0,
    "blimp_existential_there_object_raising": 0,
    "blimp_determiner_noun_agreement_with_adj_2": 0,
    "blimp_principle_A_domain_3": 0,
    "blimp_causative": 0,
    "blimp_tough_vs_raising_2": 0,
    "blimp_wh_vs_that_no_gap": 0,
    "blimp_anaphor_number_agreement": 0,
    "blimp_principle_A_c_command": 0,
    "blimp_distractor_agreement_relational_noun": 0,
    "blimp_intransitive": 0,
    "blimp_animate_subject_passive": 0,
    "blimp_animate_subject_trans": 0,
    "blimp_inchoative": 0
  },
  "config": {
    "model": "hf-causal-experimental",
    "model_args": "pretrained=/gaueko1/hizkuntza-ereduak/LLaMA/lm/huggingface/7B",
    "num_fewshot": 0,
    "batch_size": "auto",
    "device": "cuda:0",
    "no_cache": true,
    "limit": null,
    "bootstrap_iters": 100000,
    "description_dict": {}
  }
}