Skip to content

GitLab

"lm_eval/tasks/tmlu/default/tmlu_nutritionist.yaml" did not exist on "1ce8c97a4599bce8c73a1f9e412dcd3497c438ba"
Switch branch/tag
Find file Normal viewHistoryPermalink
zhoblimp_group.yaml 4.24 KB
Newer Older
James A. Michaelov's avatar
Add ZhoBLiMP benchmark (#3218)  
James A. Michaelov committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128 
group: zhoblimp
task:
  - zhoblimp_BA_BEI_subj_drop
  - zhoblimp_BA_deletion
  - zhoblimp_BA_duplicate_argument
  - zhoblimp_BA_inversion
  - zhoblimp_BA_meiba
  - zhoblimp_BA_negation
  - zhoblimp_BA_no_progressive
  - zhoblimp_BA_no_stative_verb
  - zhoblimp_BA_suo_adverbial_a
  - zhoblimp_BA_suo_adverbial_b
  - zhoblimp_BA_verb_le_a
  - zhoblimp_BA_verb_le_b
  - zhoblimp_BEI_construction_a
  - zhoblimp_BEI_construction_b
  - zhoblimp_BEI_deletion
  - zhoblimp_BEI_preposition
  - zhoblimp_PN_numP_a
  - zhoblimp_PN_numP_b
  - zhoblimp_adjective_transitive_dui
  - zhoblimp_agent_animacy_adv
  - zhoblimp_agent_animacy_passive
  - zhoblimp_agent_animacy_subj
  - zhoblimp_agent_causative
  - zhoblimp_agent_deletion
  - zhoblimp_anaphor_gender_agreement
  - zhoblimp_anaphor_number_agreement
  - zhoblimp_causative_shi_ba
  - zhoblimp_classifier_noun_agreement
  - zhoblimp_classifier_noun_agreement_no_gap
  - zhoblimp_classifier_noun_subj
  - zhoblimp_control_modal_vs_raising_modal
  - zhoblimp_ellipsis_adj
  - zhoblimp_ellipsis_double_object
  - zhoblimp_ellipsis_n_bar_class
  - zhoblimp_existential_there_subject_raising
  - zhoblimp_fci_renhe_dou
  - zhoblimp_fci_renhe_prepP
  - zhoblimp_fci_renhe_ruguo
  - zhoblimp_fci_renhe_subj
  - zhoblimp_fci_renhe_suoyou
  - zhoblimp_intransitive_double_obj
  - zhoblimp_intransitive_no_obj
  - zhoblimp_left_adverbial_b
  - zhoblimp_left_adverbial_d
  - zhoblimp_left_adverbial_e
  - zhoblimp_left_adverbial_negation
  - zhoblimp_left_dou
  - zhoblimp_modal_raising_hui
  - zhoblimp_modal_raising_topicalization
  - zhoblimp_nominal_definite_men
  - zhoblimp_nominal_modal_insertion
  - zhoblimp_noun_adjective_shi
  - zhoblimp_noun_phrase_conjunction_jian
  - zhoblimp_npi_renhe_A_not_A_question
  - zhoblimp_npi_renhe_conditional
  - zhoblimp_npi_renhe_neg_scope_locP
  - zhoblimp_npi_renhe_neg_scope_subj
  - zhoblimp_npi_renhe_wh_question_obj
  - zhoblimp_npi_renhe_wh_question_subj
  - zhoblimp_passive_agent_deletion_long_left
  - zhoblimp_passive_agent_deletion_long_right_a
  - zhoblimp_passive_agent_deletion_long_right_b
  - zhoblimp_passive_agent_deletion_short
  - zhoblimp_passive_body_part
  - zhoblimp_passive_intransitive
  - zhoblimp_passive_no_adj
  - zhoblimp_passive_suo
  - zhoblimp_plural_cardinal_men_a
  - zhoblimp_plural_cardinal_men_b
  - zhoblimp_preposition_deletion
  - zhoblimp_preposition_insertion
  - zhoblimp_principle_A_c_command
  - zhoblimp_principle_A_c_command_number
  - zhoblimp_principle_A_domain
  - zhoblimp_principle_A_domain_number
  - zhoblimp_question_A_not_A
  - zhoblimp_question_A_not_A_daodi_a
  - zhoblimp_question_A_not_A_daodi_b
  - zhoblimp_question_A_not_A_indirect
  - zhoblimp_question_V_not_VP_1
  - zhoblimp_question_V_not_VP_2
  - zhoblimp_question_daodi_nandao_1
  - zhoblimp_question_daodi_nandao_2
  - zhoblimp_question_daodi_nandao_A_not_A_intran
  - zhoblimp_question_daodi_nandao_A_not_A_tran
  - zhoblimp_question_daodi_negation
  - zhoblimp_question_nandao_negation
  - zhoblimp_question_nandao_raising_1_a
  - zhoblimp_question_nandao_raising_1_b
  - zhoblimp_question_nandao_raising_2
  - zhoblimp_question_nandao_raising_3
  - zhoblimp_question_nandao_scope_1
  - zhoblimp_question_nandao_scope_2
  - zhoblimp_question_particle_daodi_choice_intran
  - zhoblimp_question_particle_daodi_choice_tran
  - zhoblimp_question_particle_nandao
  - zhoblimp_relative_operator_intepretation
  - zhoblimp_relative_operator_who
  - zhoblimp_relativization_movement_no_gap
  - zhoblimp_relativization_movement_when_where
  - zhoblimp_renhe_no_episodic_sentences
  - zhoblimp_renhe_no_superordinate_negation
  - zhoblimp_renhe_non_factive_verb
  - zhoblimp_right_yijing_a
  - zhoblimp_right_yijing_b
  - zhoblimp_singular_PN_but_plural_pron
  - zhoblimp_superlative_quantifiers_1
  - zhoblimp_superlative_quantifiers_2
  - zhoblimp_topicalization_OSV
  - zhoblimp_topicalization_OSV_mei
  - zhoblimp_topicalization_SOV
  - zhoblimp_topicalization_SOV_mei
  - zhoblimp_verb_negation_particle
  - zhoblimp_verb_phrase_left_adverbial
  - zhoblimp_verb_phrase_left_negation
  - zhoblimp_ya_insertion
  - zhoblimp_you_quantifier_adj
  - zhoblimp_you_yige
aggregate_metric_list:
  - metric: acc
    aggregation: mean
    weight_by_size: false
aggregate_metric_list:
  - metric: acc_norm
    aggregation: mean
    weight_by_size: false