fixes to pythia benchmark, boolqseq2seq

1428ad57 · haileyschoelkopf · d9b4ca7f · 1428ad57 · 1428ad57
Commit 1428ad57 authored Oct 19, 2023 by haileyschoelkopf
Hide whitespace changes
Inline Side-by-side

Showing with 3 additions and 2 deletions

lm_eval/tasks/benchmarks/pythia.yaml lm_eval/tasks/benchmarks/pythia.yaml +1 -1

lm_eval/tasks/super_glue/boolq/seq2seq.yaml lm_eval/tasks/super_glue/boolq/seq2seq.yaml +2 -1

No files found.
--- a/lm_eval/tasks/benchmarks/pythia.yaml
+++ b/lm_eval/tasks/benchmarks/pythia.yaml
@@ -9,4 +9,4 @@ task:
  - wsc
  - ai2_arc
  - blimp
-  - hendrycksTest*
+  - mmlu
--- a/lm_eval/tasks/super_glue/boolq/seq2seq.yaml
+++ b/lm_eval/tasks/super_glue/boolq/seq2seq.yaml
@@ -8,7 +8,8 @@ training_split: train
 validation_split: validation
 doc_to_text: "{{passage}}\nQuestion: {{question}}?\nAnswer:"
 doc_to_target: label
-doc_to_choice: ['no', 'yes']
+doc_to_choice: [' no', ' yes']
+target_delimiter: ""
 generation_kwargs:
  until:
    - "\n\n"