mpt-7b_blimp_0-shot.json

{
  "results": {
    "blimp_adjunct_island": {
      "acc": 0.878,
      "acc_stderr": 0.010354864712936724
    },
    "blimp_anaphor_gender_agreement": {
      "acc": 0.995,
      "acc_stderr": 0.002231586874844882
    },
    "blimp_anaphor_number_agreement": {
      "acc": 0.995,
      "acc_stderr": 0.0022315868748448817
    },
    "blimp_animate_subject_passive": {
      "acc": 0.775,
      "acc_stderr": 0.013211720158614753
    },
    "blimp_animate_subject_trans": {
      "acc": 0.884,
      "acc_stderr": 0.010131468138756997
    },
    "blimp_causative": {
      "acc": 0.747,
      "acc_stderr": 0.01375427861358708
    },
    "blimp_complex_NP_island": {
      "acc": 0.528,
      "acc_stderr": 0.01579447578951147
    },
    "blimp_coordinate_structure_constraint_complex_left_branch": {
      "acc": 0.779,
      "acc_stderr": 0.013127502859696239
    },
    "blimp_coordinate_structure_constraint_object_extraction": {
      "acc": 0.848,
      "acc_stderr": 0.011358918303475293
    },
    "blimp_determiner_noun_agreement_1": {
      "acc": 0.992,
      "acc_stderr": 0.002818500300504507
    },
    "blimp_determiner_noun_agreement_2": {
      "acc": 0.974,
      "acc_stderr": 0.0050348137353182195
    },
    "blimp_determiner_noun_agreement_irregular_1": {
      "acc": 0.936,
      "acc_stderr": 0.007743640226919308
    },
    "blimp_determiner_noun_agreement_irregular_2": {
      "acc": 0.928,
      "acc_stderr": 0.008178195576218681
    },
    "blimp_determiner_noun_agreement_with_adj_2": {
      "acc": 0.935,
      "acc_stderr": 0.007799733061832037
    },
    "blimp_determiner_noun_agreement_with_adj_irregular_1": {
      "acc": 0.883,
      "acc_stderr": 0.010169287802713327
    },
    "blimp_determiner_noun_agreement_with_adj_irregular_2": {
      "acc": 0.919,
      "acc_stderr": 0.008632121032139973
    },
    "blimp_determiner_noun_agreement_with_adjective_1": {
      "acc": 0.972,
      "acc_stderr": 0.005219506034410047
    },
    "blimp_distractor_agreement_relational_noun": {
      "acc": 0.889,
      "acc_stderr": 0.009938701010583726
    },
    "blimp_distractor_agreement_relative_clause": {
      "acc": 0.743,
      "acc_stderr": 0.013825416526895042
    },
    "blimp_drop_argument": {
      "acc": 0.787,
      "acc_stderr": 0.012953717566737234
    },
    "blimp_ellipsis_n_bar_1": {
      "acc": 0.79,
      "acc_stderr": 0.012886662332274522
    },
    "blimp_ellipsis_n_bar_2": {
      "acc": 0.928,
      "acc_stderr": 0.008178195576218681
    },
    "blimp_existential_there_object_raising": {
      "acc": 0.834,
      "acc_stderr": 0.01177211037081219
    },
    "blimp_existential_there_quantifiers_1": {
      "acc": 0.988,
      "acc_stderr": 0.003444977194099841
    },
    "blimp_existential_there_quantifiers_2": {
      "acc": 0.27,
      "acc_stderr": 0.014046255632633908
    },
    "blimp_existential_there_subject_raising": {
      "acc": 0.888,
      "acc_stderr": 0.00997775303139724
    },
    "blimp_expletive_it_object_raising": {
      "acc": 0.8,
      "acc_stderr": 0.012655439943366646
    },
    "blimp_inchoative": {
      "acc": 0.673,
      "acc_stderr": 0.01484221315341125
    },
    "blimp_intransitive": {
      "acc": 0.832,
      "acc_stderr": 0.01182860583145426
    },
    "blimp_irregular_past_participle_adjectives": {
      "acc": 0.972,
      "acc_stderr": 0.005219506034410053
    },
    "blimp_irregular_past_participle_verbs": {
      "acc": 0.885,
      "acc_stderr": 0.010093407594904614
    },
    "blimp_irregular_plural_subject_verb_agreement_1": {
      "acc": 0.92,
      "acc_stderr": 0.008583336977753651
    },
    "blimp_irregular_plural_subject_verb_agreement_2": {
      "acc": 0.908,
      "acc_stderr": 0.009144376393151112
    },
    "blimp_left_branch_island_echo_question": {
      "acc": 0.432,
      "acc_stderr": 0.01567232023733621
    },
    "blimp_left_branch_island_simple_question": {
      "acc": 0.897,
      "acc_stderr": 0.009616833339695798
    },
    "blimp_matrix_question_npi_licensor_present": {
      "acc": 0.705,
      "acc_stderr": 0.014428554438445514
    },
    "blimp_npi_present_1": {
      "acc": 0.579,
      "acc_stderr": 0.015620595475301318
    },
    "blimp_npi_present_2": {
      "acc": 0.688,
      "acc_stderr": 0.01465847437050901
    },
    "blimp_only_npi_licensor_present": {
      "acc": 0.916,
      "acc_stderr": 0.008776162089491096
    },
    "blimp_only_npi_scope": {
      "acc": 0.732,
      "acc_stderr": 0.014013292702729491
    },
    "blimp_passive_1": {
      "acc": 0.887,
      "acc_stderr": 0.010016552866696863
    },
    "blimp_passive_2": {
      "acc": 0.895,
      "acc_stderr": 0.009698921026024947
    },
    "blimp_principle_A_c_command": {
      "acc": 0.75,
      "acc_stderr": 0.013699915608779773
    },
    "blimp_principle_A_case_1": {
      "acc": 1.0,
      "acc_stderr": 0.0
    },
    "blimp_principle_A_case_2": {
      "acc": 0.94,
      "acc_stderr": 0.007513751157474928
    },
    "blimp_principle_A_domain_1": {
      "acc": 0.997,
      "acc_stderr": 0.0017303161543469417
    },
    "blimp_principle_A_domain_2": {
      "acc": 0.828,
      "acc_stderr": 0.011939788882495321
    },
    "blimp_principle_A_domain_3": {
      "acc": 0.761,
      "acc_stderr": 0.013493000446937591
    },
    "blimp_principle_A_reconstruction": {
      "acc": 0.41,
      "acc_stderr": 0.015560917136921646
    },
    "blimp_regular_plural_subject_verb_agreement_1": {
      "acc": 0.971,
      "acc_stderr": 0.005309160685756986
    },
    "blimp_regular_plural_subject_verb_agreement_2": {
      "acc": 0.907,
      "acc_stderr": 0.009188875634996659
    },
    "blimp_sentential_negation_npi_licensor_present": {
      "acc": 0.989,
      "acc_stderr": 0.0032999833166078166
    },
    "blimp_sentential_negation_npi_scope": {
      "acc": 0.733,
      "acc_stderr": 0.013996674851796264
    },
    "blimp_sentential_subject_island": {
      "acc": 0.399,
      "acc_stderr": 0.015493193313162906
    },
    "blimp_superlative_quantifiers_1": {
      "acc": 0.822,
      "acc_stderr": 0.012102167676183596
    },
    "blimp_superlative_quantifiers_2": {
      "acc": 0.897,
      "acc_stderr": 0.009616833339695796
    },
    "blimp_tough_vs_raising_1": {
      "acc": 0.69,
      "acc_stderr": 0.014632638658632898
    },
    "blimp_tough_vs_raising_2": {
      "acc": 0.829,
      "acc_stderr": 0.011912216456264606
    },
    "blimp_transitive": {
      "acc": 0.872,
      "acc_stderr": 0.010570133761108658
    },
    "blimp_wh_island": {
      "acc": 0.813,
      "acc_stderr": 0.012336254828074126
    },
    "blimp_wh_questions_object_gap": {
      "acc": 0.763,
      "acc_stderr": 0.01345407046257794
    },
    "blimp_wh_questions_subject_gap": {
      "acc": 0.89,
      "acc_stderr": 0.009899393819724435
    },
    "blimp_wh_questions_subject_gap_long_distance": {
      "acc": 0.893,
      "acc_stderr": 0.009779910359847169
    },
    "blimp_wh_vs_that_no_gap": {
      "acc": 0.946,
      "acc_stderr": 0.007150883521295428
    },
    "blimp_wh_vs_that_no_gap_long_distance": {
      "acc": 0.951,
      "acc_stderr": 0.006829761756140921
    },
    "blimp_wh_vs_that_with_gap": {
      "acc": 0.321,
      "acc_stderr": 0.01477082181793466
    },
    "blimp_wh_vs_that_with_gap_long_distance": {
      "acc": 0.292,
      "acc_stderr": 0.014385511563477347
    }
  },
  "versions": {
    "blimp_adjunct_island": 0,
    "blimp_anaphor_gender_agreement": 0,
    "blimp_anaphor_number_agreement": 0,
    "blimp_animate_subject_passive": 0,
    "blimp_animate_subject_trans": 0,
    "blimp_causative": 0,
    "blimp_complex_NP_island": 0,
    "blimp_coordinate_structure_constraint_complex_left_branch": 0,
    "blimp_coordinate_structure_constraint_object_extraction": 0,
    "blimp_determiner_noun_agreement_1": 0,
    "blimp_determiner_noun_agreement_2": 0,
    "blimp_determiner_noun_agreement_irregular_1": 0,
    "blimp_determiner_noun_agreement_irregular_2": 0,
    "blimp_determiner_noun_agreement_with_adj_2": 0,
    "blimp_determiner_noun_agreement_with_adj_irregular_1": 0,
    "blimp_determiner_noun_agreement_with_adj_irregular_2": 0,
    "blimp_determiner_noun_agreement_with_adjective_1": 0,
    "blimp_distractor_agreement_relational_noun": 0,
    "blimp_distractor_agreement_relative_clause": 0,
    "blimp_drop_argument": 0,
    "blimp_ellipsis_n_bar_1": 0,
    "blimp_ellipsis_n_bar_2": 0,
    "blimp_existential_there_object_raising": 0,
    "blimp_existential_there_quantifiers_1": 0,
    "blimp_existential_there_quantifiers_2": 0,
    "blimp_existential_there_subject_raising": 0,
    "blimp_expletive_it_object_raising": 0,
    "blimp_inchoative": 0,
    "blimp_intransitive": 0,
    "blimp_irregular_past_participle_adjectives": 0,
    "blimp_irregular_past_participle_verbs": 0,
    "blimp_irregular_plural_subject_verb_agreement_1": 0,
    "blimp_irregular_plural_subject_verb_agreement_2": 0,
    "blimp_left_branch_island_echo_question": 0,
    "blimp_left_branch_island_simple_question": 0,
    "blimp_matrix_question_npi_licensor_present": 0,
    "blimp_npi_present_1": 0,
    "blimp_npi_present_2": 0,
    "blimp_only_npi_licensor_present": 0,
    "blimp_only_npi_scope": 0,
    "blimp_passive_1": 0,
    "blimp_passive_2": 0,
    "blimp_principle_A_c_command": 0,
    "blimp_principle_A_case_1": 0,
    "blimp_principle_A_case_2": 0,
    "blimp_principle_A_domain_1": 0,
    "blimp_principle_A_domain_2": 0,
    "blimp_principle_A_domain_3": 0,
    "blimp_principle_A_reconstruction": 0,
    "blimp_regular_plural_subject_verb_agreement_1": 0,
    "blimp_regular_plural_subject_verb_agreement_2": 0,
    "blimp_sentential_negation_npi_licensor_present": 0,
    "blimp_sentential_negation_npi_scope": 0,
    "blimp_sentential_subject_island": 0,
    "blimp_superlative_quantifiers_1": 0,
    "blimp_superlative_quantifiers_2": 0,
    "blimp_tough_vs_raising_1": 0,
    "blimp_tough_vs_raising_2": 0,
    "blimp_transitive": 0,
    "blimp_wh_island": 0,
    "blimp_wh_questions_object_gap": 0,
    "blimp_wh_questions_subject_gap": 0,
    "blimp_wh_questions_subject_gap_long_distance": 0,
    "blimp_wh_vs_that_no_gap": 0,
    "blimp_wh_vs_that_no_gap_long_distance": 0,
    "blimp_wh_vs_that_with_gap": 0,
    "blimp_wh_vs_that_with_gap_long_distance": 0
  },
  "config": {
    "model": "hf-causal-experimental",
    "model_args": "pretrained=mosaicml/mpt-7b,trust_remote_code=True",
    "num_fewshot": 0,
    "batch_size": "auto",
    "device": "cuda",
    "no_cache": true,
    "limit": null,
    "bootstrap_iters": 100000,
    "description_dict": {}
  }
}