return"Given a premise and a hypothesis, classify whether the author of the premise is committed to the truth of the hypothesis. The three possible labels are true, false or neither."
return"Given a premise and a hypothesis, classify whether the author of the premise is committed to the truth of the hypothesis. The three possible labels are true, false or neither."
defdoc_to_text(self,doc):
defdoc_to_text(self,doc):
return"{}\nquestion:\t{}\ttrue, false or neither?\nanswer:".format(
return"{}\nquestion: {} true, false or neither?\nanswer:".format(
doc["premise"],
doc["premise"],
doc["hypothesis"],
doc["hypothesis"],
)
)
...
@@ -102,7 +102,8 @@ class CommitmentBank(HFTask):
...
@@ -102,7 +102,8 @@ class CommitmentBank(HFTask):
defhigher_is_better(self):
defhigher_is_better(self):
return{
return{
"acc":True
"acc":True,
"f1":True
}
}
defaggregation(self):
defaggregation(self):
...
@@ -242,7 +243,7 @@ class WordsInContext(HFTask):
...
@@ -242,7 +243,7 @@ class WordsInContext(HFTask):
returnTrue
returnTrue
defdoc_to_text(self,doc):
defdoc_to_text(self,doc):
return"{}\n{}\nquestion\tIs the word '{}' used in the same way in the" \
return"{}\n{}\nQuestion: Is the word '{}' used in the same way in the" \