flan_held_in.yaml 19.7 KB
Newer Older
lintangsutawika's avatar
lintangsutawika committed
1
group: flan_held_in
2
group_alias: Flan (Held-In)
lintangsutawika's avatar
lintangsutawika committed
3
task:
4
5
6
  # ANLI R1
  - group: anli_r1_flan
    group_alias: ANLI R1
7
    aggregate_metric_list:
lintangsutawika's avatar
lintangsutawika committed
8
9
      - metric: acc
        weight_by_size: True
10
    task:
haileyschoelkopf's avatar
haileyschoelkopf committed
11
      - task: anli_r1_prompt-0
12
13
14
15
        task_alias: prompt-0
        include: _held_in_template_yaml
        doc_to_text: "{{premise}}\n\nChoose your answer: based on the paragraph above can we conclude that \"{{hypothesis}}\"?\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nI think the answer is"
        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
16
      - task: anli_r1_prompt-1
17
18
19
20
        task_alias: prompt-1
        include: _held_in_template_yaml
        doc_to_text: "{{premise}}\n\nBased on that paragraph can we conclude that this sentence is true?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
21
      - task: anli_r1_prompt-2
22
23
24
25
        task_alias: prompt-2
        include: _held_in_template_yaml
        doc_to_text: "{{premise}}\n\nCan we draw the following conclusion?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
26
      - task: anli_r1_prompt-3
27
28
29
30
        task_alias: prompt-3
        include: _held_in_template_yaml
        doc_to_text: "{{premise}}\nDoes this next sentence follow, given the preceding text?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
31
      - task: anli_r1_prompt-4
32
33
34
35
        task_alias: prompt-4
        include: _held_in_template_yaml
        doc_to_text: "{{premise}}\nCan we infer the following?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nThe answer is:"
        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
36
      - task: anli_r1_prompt-5
37
38
39
40
        task_alias: prompt-5
        include: _held_in_template_yaml
        doc_to_text: "Read the following paragraph and determine if the hypothesis is true:\n\n{{premise}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nHypothesis: {{hypothesis}}\n\n\n"
        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
41
      - task: anli_r1_prompt-6
42
43
44
45
        task_alias: prompt-6
        include: _held_in_template_yaml
        doc_to_text: "Read the text and determine if the sentence is true (see options at the end):\n\n{{premise}}\n\nSentence: {{hypothesis}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
46
      - task: anli_r1_prompt-7
47
48
49
50
        task_alias: prompt-7
        include: _held_in_template_yaml
        doc_to_text: "Can we draw the following hypothesis from the context (see options)? \n\nContext:\n\n{{premise}}\n\nHypothesis: {{hypothesis}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
51
      - task: anli_r1_prompt-8
52
53
54
55
56
57
58
        task_alias: prompt-8
        include: _held_in_template_yaml
        doc_to_text: "Choose from options: Determine if the sentence is true based on the text below:\n{{hypothesis}}\n\n{{premise}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
  # ANLI R2
  - group: anli_r2_flan
    group_alias: ANLI R2
59
    aggregate_metric_list:
lintangsutawika's avatar
lintangsutawika committed
60
61
      - metric: acc
        weight_by_size: True
62
    task:
haileyschoelkopf's avatar
haileyschoelkopf committed
63
      - task: anli_r2_prompt-0
64
65
66
67
        task_alias: prompt-0
        include: _held_in_template_yaml
        doc_to_text: "{{premise}}\n\nChoose your answer: based on the paragraph above can we conclude that \"{{hypothesis}}\"?\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nI think the answer is"
        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
68
      - task: anli_r2_prompt-1
69
70
71
72
        task_alias: prompt-1
        include: _held_in_template_yaml
        doc_to_text: "{{premise}}\n\nBased on that paragraph can we conclude that this sentence is true?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
73
      - task: anli_r2_prompt-2
74
75
76
77
        task_alias: prompt-2
        include: _held_in_template_yaml
        doc_to_text: "{{premise}}\n\nCan we draw the following conclusion?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
78
      - task: anli_r2_prompt-3
79
80
81
82
        task_alias: prompt-3
        include: _held_in_template_yaml
        doc_to_text: "{{premise}}\nDoes this next sentence follow, given the preceding text?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
83
      - task: anli_r2_prompt-4
84
85
86
87
        task_alias: prompt-4
        include: _held_in_template_yaml
        doc_to_text: "{{premise}}\nCan we infer the following?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nThe answer is:"
        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
88
      - task: anli_r2_prompt-5
89
90
91
92
        task_alias: prompt-5
        include: _held_in_template_yaml
        doc_to_text: "Read the following paragraph and determine if the hypothesis is true:\n\n{{premise}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nHypothesis: {{hypothesis}}\n\n\n"
        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
93
      - task: anli_r2_prompt-6
94
95
96
97
        task_alias: prompt-6
        include: _held_in_template_yaml
        doc_to_text: "Read the text and determine if the sentence is true (see options at the end):\n\n{{premise}}\n\nSentence: {{hypothesis}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
98
      - task: anli_r2_prompt-7
99
100
101
102
        task_alias: prompt-7
        include: _held_in_template_yaml
        doc_to_text: "Can we draw the following hypothesis from the context (see options)? \n\nContext:\n\n{{premise}}\n\nHypothesis: {{hypothesis}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
103
      - task: anli_r2_prompt-8
104
105
106
107
108
109
110
        task_alias: prompt-8
        include: _held_in_template_yaml
        doc_to_text: "Choose from options: Determine if the sentence is true based on the text below:\n{{hypothesis}}\n\n{{premise}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
  # ANLI R3
  - group: anli_r3_flan
    group_alias: ANLI R3
111
    aggregate_metric_list:
lintangsutawika's avatar
lintangsutawika committed
112
113
      - metric: acc
        weight_by_size: True
114
    task:
haileyschoelkopf's avatar
haileyschoelkopf committed
115
      - task: anli_r3_prompt-0
116
117
118
119
        task_alias: prompt-0
        include: _held_in_template_yaml
        doc_to_text: "{{premise}}\n\nChoose your answer: based on the paragraph above can we conclude that \"{{hypothesis}}\"?\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nI think the answer is"
        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
120
      - task: anli_r3_prompt-1
121
122
123
124
        task_alias: prompt-1
        include: _held_in_template_yaml
        doc_to_text: "{{premise}}\n\nBased on that paragraph can we conclude that this sentence is true?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
125
      - task: anli_r3_prompt-2
126
127
128
129
        task_alias: prompt-2
        include: _held_in_template_yaml
        doc_to_text: "{{premise}}\n\nCan we draw the following conclusion?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
130
      - task: anli_r3_prompt-3
131
132
133
134
        task_alias: prompt-3
        include: _held_in_template_yaml
        doc_to_text: "{{premise}}\nDoes this next sentence follow, given the preceding text?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
135
      - task: anli_r3_prompt-4
136
137
138
139
        task_alias: prompt-4
        include: _held_in_template_yaml
        doc_to_text: "{{premise}}\nCan we infer the following?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nThe answer is:"
        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
140
      - task: anli_r3_prompt-5
141
142
143
144
        task_alias: prompt-5
        include: _held_in_template_yaml
        doc_to_text: "Read the following paragraph and determine if the hypothesis is true:\n\n{{premise}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nHypothesis: {{hypothesis}}\n\n\n"
        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
145
      - task: anli_r3_prompt-6
146
147
148
149
        task_alias: prompt-6
        include: _held_in_template_yaml
        doc_to_text: "Read the text and determine if the sentence is true (see options at the end):\n\n{{premise}}\n\nSentence: {{hypothesis}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
150
      - task: anli_r3_prompt-7
151
152
153
154
        task_alias: prompt-7
        include: _held_in_template_yaml
        doc_to_text: "Can we draw the following hypothesis from the context (see options)? \n\nContext:\n\n{{premise}}\n\nHypothesis: {{hypothesis}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
155
      - task: anli_r3_prompt-8
156
157
158
159
160
161
162
        task_alias: prompt-8
        include: _held_in_template_yaml
        doc_to_text: "Choose from options: Determine if the sentence is true based on the text below:\n{{hypothesis}}\n\n{{premise}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
  # Arc Easy
  - group: arc_easy_flan
    group_alias: Arc Easy
163
    aggregate_metric_list:
lintangsutawika's avatar
lintangsutawika committed
164
165
      - metric: acc
        weight_by_size: True
166
    task:
haileyschoelkopf's avatar
haileyschoelkopf committed
167
      - task: arc_easy_prompt-0
168
169
170
171
        task_alias: prompt-0
        include: _held_in_template_yaml
        doc_to_text: "{{question}}\n\nOPTIONS:\n- {{choices.text|join('\n- ')}}"
        doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
172
      - task: arc_easy_prompt-1
173
174
175
176
        task_alias: prompt-1
        include: _held_in_template_yaml
        doc_to_text: "Question: {{question}}\nOPTIONS:\n- {{choices.text|join('\n- ')}}\nAnswer:"
        doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
177
      - task: arc_easy_prompt-2
178
179
180
181
        task_alias: prompt-2
        include: _held_in_template_yaml
        doc_to_text: "Question: {{question}}\n\nWhat is the correct answer to the question from the following choices?\nOPTIONS:\n- {{choices.text|join('\n- ')}}"
        doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
182
      - task: arc_easy_prompt-3
183
184
185
186
        task_alias: prompt-3
        include: _held_in_template_yaml
        doc_to_text: "Q: {{question}}\nWhat is the correct answer to this question?\nOPTIONS:\n- {{choices.text|join('\n- ')}}...A:"
        doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
187
      - task: arc_easy_prompt-4
188
189
190
191
        task_alias: prompt-4
        include: _held_in_template_yaml
        doc_to_text: "Choose your answer?\n\n{{question}}\n\nOPTIONS:\n- {{choices.text|join('\n- ')}}"
        doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
192
      - task: arc_easy_prompt-5
193
194
195
196
        task_alias: prompt-5
        include: _held_in_template_yaml
        doc_to_text: "Answer the question\n\n{{question}}\nOPTIONS:\n- {{choices.text|join('\n- ')}}"
        doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
197
      - task: arc_easy_prompt-6
198
199
200
201
202
203
204
        task_alias: prompt-6
        include: _held_in_template_yaml
        doc_to_text: "{{question}}\n\nPick the answer from these options\n\nOPTIONS:\n- {{choices.text|join('\n- ')}}"
        doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}"
  # Arc Challenge
  - group: arc_challenge_flan
    group_alias: Arc Challenge
205
    aggregate_metric_list:
lintangsutawika's avatar
lintangsutawika committed
206
207
      - metric: acc
        weight_by_size: True
208
    task:
haileyschoelkopf's avatar
haileyschoelkopf committed
209
      - task: arc_challenge_prompt-0
210
211
212
213
        task_alias: prompt-0
        include: _held_in_template_yaml
        doc_to_text: "{{question}}\n\nOPTIONS:\n- {{choices.text|join('\n- ')}}"
        doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
214
      - task: arc_challenge_prompt-1
215
216
217
218
        task_alias: prompt-1
        include: _held_in_template_yaml
        doc_to_text: "Question: {{question}}\nOPTIONS:\n- {{choices.text|join('\n- ')}}\nAnswer:"
        doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
219
      - task: arc_challenge_prompt-2
220
221
222
223
        task_alias: prompt-2
        include: _held_in_template_yaml
        doc_to_text: "Question: {{question}}\n\nWhat is the correct answer to the question from the following choices?\nOPTIONS:\n- {{choices.text|join('\n- ')}}"
        doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
224
      - task: arc_challenge_prompt-3
225
226
227
228
        task_alias: prompt-3
        include: _held_in_template_yaml
        doc_to_text: "Q: {{question}}\nWhat is the correct answer to this question?\nOPTIONS:\n- {{choices.text|join('\n- ')}}...A:"
        doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
229
      - task: arc_challenge_prompt-4
230
231
232
233
        task_alias: prompt-4
        include: _held_in_template_yaml
        doc_to_text: "Choose your answer?\n\n{{question}}\n\nOPTIONS:\n- {{choices.text|join('\n- ')}}"
        doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
234
      - task: arc_challenge_prompt-5
235
236
237
238
        task_alias: prompt-5
        include: _held_in_template_yaml
        doc_to_text: "Answer the question\n\n{{question}}\nOPTIONS:\n- {{choices.text|join('\n- ')}}"
        doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
239
      - task: arc_challenge_prompt-6
240
241
242
243
244
245
246
        task_alias: prompt-6
        include: _held_in_template_yaml
        doc_to_text: "{{question}}\n\nPick the answer from these options\n\nOPTIONS:\n- {{choices.text|join('\n- ')}}"
        doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}"
  # BoolQ
  - group: boolq_flan
    group_alias: BoolQ
247
    aggregate_metric_list:
lintangsutawika's avatar
lintangsutawika committed
248
249
      - metric: acc
        weight_by_size: True
250
    task:
haileyschoelkopf's avatar
haileyschoelkopf committed
251
      - task: boolq_prompt-0
252
253
254
255
        task_alias: prompt-0
        include: _held_in_template_yaml
        doc_to_text: "{{passage}}\n\nCan we conclude that {{question}}?\n\nOPTIONS:\n- no\n- yes"
        doc_to_target: "{{['no', 'yes'][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
256
      - task: boolq_prompt-1
257
258
259
260
        task_alias: prompt-1
        include: _held_in_template_yaml
        doc_to_text: "{{passage}}\n\nIs it true that {{question}}?\n\nOPTIONS:\n- no\n- yes"
        doc_to_target: "{{['no', 'yes'][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
261
      - task: boolq_prompt-2
262
263
264
265
        task_alias: prompt-2
        include: _held_in_template_yaml
        doc_to_text: "{{passage}}\n\n{{question}}?\n\nOPTIONS:\n- no\n- yes"
        doc_to_target: "{{['no', 'yes'][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
266
      - task: boolq_prompt-3
267
268
269
270
        task_alias: prompt-3
        include: _held_in_template_yaml
        doc_to_text: "Text: {{passage}}\n\nQuestion: {{question}}?\n\nOPTIONS:\n- no\n- yes"
        doc_to_target: "{{['no', 'yes'][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
271
      - task: boolq_prompt-4
272
273
274
275
        task_alias: prompt-4
        include: _held_in_template_yaml
        doc_to_text: "{{passage}}\n\nWhat's the best answer to this question: {{question}}?\n\nOPTIONS:\n- no\n- yes"
        doc_to_target: "{{['no', 'yes'][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
276
      - task: boolq_prompt-5
277
278
279
280
        task_alias: prompt-5
        include: _held_in_template_yaml
        doc_to_text: "{{passage}}\nBased on the above text what's the best answer to this question: {{question}}?\n\nOPTIONS:\n- no\n- yes"
        doc_to_target: "{{['no', 'yes'][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
281
      - task: boolq_prompt-6
282
283
284
285
        task_alias: prompt-6
        include: _held_in_template_yaml
        doc_to_text: "{{passage}}\nAnswer this question making sure that the answer is supposed by the text: {{question}}?\n\nOPTIONS:\n- no\n- yes"
        doc_to_target: "{{['no', 'yes'][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
286
      - task: boolq_prompt-7
287
288
289
290
        task_alias: prompt-7
        include: _held_in_template_yaml
        doc_to_text: "{{passage}}\n\nIs the following statement correct based on the text\n\n{{question}}\n\nOPTIONS:\n- no\n- yes"
        doc_to_target: "{{['no', 'yes'][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
291
      - task: boolq_prompt-8
292
293
294
295
        task_alias: prompt-8
        include: _held_in_template_yaml
        doc_to_text: "{{passage}}\n\nIs this statement correct \"{{question}}\"?\n\nOPTIONS:\n- no\n- yes"
        doc_to_target: "{{['no', 'yes'][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
296
      - task: boolq_prompt-9
297
298
299
300
301
302
303
        task_alias: prompt-9
        include: _held_in_template_yaml
        doc_to_text: "Is it true that {{question}} based on the following text?\n\n{{passage}}\n\nOPTIONS:\n- no\n- yes"
        doc_to_target: "{{['no', 'yes'][label]}}"
  # RTE
  - group: rte_flan
    group_alias: RTE
304
    aggregate_metric_list:
lintangsutawika's avatar
lintangsutawika committed
305
306
      - metric: acc
        weight_by_size: True
307
    task:
haileyschoelkopf's avatar
haileyschoelkopf committed
308
      - task: rte_prompt-0
309
310
311
312
        task_alias: prompt-0
        include: _held_in_template_yaml
        doc_to_text: "{{sentence1}}\n\nQuestion with options: Based on the paragraph above can we conclude that \"{{sentence2}}\"?\n\nOPTIONS:\n- yes\n- no"
        doc_to_target: "{{['yes', 'no'][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
313
      - task: rte_prompt-1
314
315
316
317
        task_alias: prompt-1
        include: _held_in_template_yaml
        doc_to_text: "{{sentence1}}\n\nBased on that paragraph can we conclude that the sentence below is true?\n{{sentence2}}\n\nOPTIONS:\n- yes\n- no"
        doc_to_target: "{{['yes', 'no'][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
318
      - task: rte_prompt-1
319
320
321
322
        task_alias: prompt-2
        include: _held_in_template_yaml
        doc_to_text: "{{sentence1}}\n\nQ with options: Can we draw the following conclusion?\n{{sentence2}}\n\nOPTIONS:\n- yes\n- no"
        doc_to_target: "{{['yes', 'no'][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
323
      - task: rte_prompt-3
324
325
326
327
        task_alias: prompt-3
        include: _held_in_template_yaml
        doc_to_text: "{{sentence1}}\nDoes this next sentence follow, given the preceding text?\n{{sentence2}}\n\nOPTIONS:\n- yes\n- no"
        doc_to_target: "{{['yes', 'no'][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
328
      - task: rte_prompt-4
329
330
331
332
        task_alias: prompt-4
        include: _held_in_template_yaml
        doc_to_text: "{{sentence1}}\nOPTIONS:\n- yes\n- no\nQuestion: Can we infer the following?\n{{sentence2}}"
        doc_to_target: "{{['yes', 'no'][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
333
      - task: rte_prompt-5
334
335
336
337
        task_alias: prompt-5
        include: _held_in_template_yaml
        doc_to_text: "Read the following paragraph and determine if the hypothesis is true. Select from options at the end:\n\n{{sentence1}}\n\nHypothesis: {{sentence2}}\nOPTIONS:\n- yes\n- no\nThe answer is"
        doc_to_target: "{{['yes', 'no'][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
338
      - task: rte_prompt-6
339
340
341
342
        task_alias: prompt-6
        include: _held_in_template_yaml
        doc_to_text: "Read the text and determine if the sentence is true:\n\n{{sentence1}}\n\nSentence: {{sentence2}}\nOPTIONS:\n- yes\n- no\nA:"
        doc_to_target: "{{['yes', 'no'][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
343
      - task: rte_prompt-7
344
345
346
347
        task_alias: prompt-7
        include: _held_in_template_yaml
        doc_to_text: "Question with options: can we draw the following hypothesis from the context? \n\nContext:\n\n{{sentence1}}\n\nHypothesis: {{sentence2}}\nOPTIONS:\n- yes\n- no\nA:"
        doc_to_target: "{{['yes', 'no'][label]}}"
haileyschoelkopf's avatar
haileyschoelkopf committed
348
      - task: rte_prompt-8
349
350
351
352
        task_alias: prompt-8
        include: _held_in_template_yaml
        doc_to_text: "Determine if the sentence is true based on the text below. Choose from options.\n{{sentence2}}\n\n{{sentence1}}\nOPTIONS:\n- yes\n- no"
        doc_to_target: "{{['yes', 'no'][label]}}"