fix do_sample

lm_eval/tasks/longbench/2wikimqa.yaml

@@ -10,7 +10,7 @@ doc_to_target: '{{answers}}'
 generation_kwargs:
   max_gen_toks: 32
   temperature: 1
-  do_sample: False
+  do_sample: True
 metric_list:
   - metric: !function metrics.qa_f1_score
     aggregation: mean

lm_eval/tasks/longbench/2wikimqa_e.yaml

@@ -10,7 +10,7 @@ doc_to_target: '{{answers}}'
 generation_kwargs:
   max_gen_toks: 32
   temperature: 1
-  do_sample: False
+  do_sample: True
 metric_list:
   - metric: !function metrics.qa_f1_score
     aggregation: mean

lm_eval/tasks/longbench/dureader.yaml

@@ -10,7 +10,7 @@ doc_to_target: '{{answers}}'
 generation_kwargs:
   max_gen_toks: 128
   temperature: 1
-  do_sample: False
+  do_sample: True
 metric_list:
   - metric: !function metrics.rouge_zh_score
     aggregation: mean

lm_eval/tasks/longbench/gov_report.yaml

@@ -10,7 +10,7 @@ doc_to_target: '{{answers}}'
 generation_kwargs:
   max_gen_toks: 512
   temperature: 1
-  do_sample: False
+  do_sample: True
 metric_list:
   - metric: !function metrics.rouge_score
     aggregation: mean

lm_eval/tasks/longbench/gov_report_e.yaml

@@ -10,7 +10,7 @@ doc_to_target: '{{answers}}'
 generation_kwargs:
   max_gen_toks: 512
   temperature: 1
-  do_sample: False
+  do_sample: True
 metric_list:
   - metric: !function metrics.rouge_score
     aggregation: mean

lm_eval/tasks/longbench/hotpotqa.yaml

@@ -10,7 +10,7 @@ doc_to_target: '{{answers}}'
 generation_kwargs:
   max_gen_toks: 32
   temperature: 1
-  do_sample: False
+  do_sample: True
 metric_list:
   - metric: !function metrics.qa_f1_score
     aggregation: mean

lm_eval/tasks/longbench/hotpotqa_e.yaml

@@ -6,14 +6,14 @@ dataset_path: THUDM/LongBench
 test_split: test
 dataset_name: hotpotqa_e
 doc_to_text: 'Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nThe following are given passages.\n{{context}}\n\nAnswer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {{input}}\nAnswer:'
-doc_to_target: "{{answers}}"
+doc_to_target: '{{answers}}'
 generation_kwargs:
   max_gen_toks: 32
   temperature: 1
-  do_sample: False
+  do_sample: True
 metric_list:
   - metric: !function metrics.qa_f1_score
     aggregation: mean
-    higher_is_better: true
+    higher_is_better: True
 metadata:
   version: 1.0

lm_eval/tasks/longbench/lcc.yaml

@@ -10,7 +10,7 @@ doc_to_target: '{{answers}}'
 generation_kwargs:
   max_gen_toks: 64
   temperature: 1
-  do_sample: False
+  do_sample: True
 metric_list:
   - metric: !function metrics.code_sim_score
     aggregation: mean

lm_eval/tasks/longbench/lcc_e.yaml

@@ -10,7 +10,7 @@ doc_to_target: '{{answers}}'
 generation_kwargs:
   max_gen_toks: 64
   temperature: 1
-  do_sample: False
+  do_sample: True
 metric_list:
   - metric: !function metrics.code_sim_score
     aggregation: mean

lm_eval/tasks/longbench/lsht.yaml

@@ -10,7 +10,7 @@ doc_to_target: '{{answers}}'
 generation_kwargs:
   max_gen_toks: 64
   temperature: 1
-  do_sample: False
+  do_sample: True
 metric_list:
   - metric: !function metrics.classification_score
     aggregation: mean

lm_eval/tasks/longbench/multi_news.yaml

@@ -10,7 +10,7 @@ doc_to_target: '{{answers}}'
 generation_kwargs:
   max_gen_toks: 512
   temperature: 1
-  do_sample: False
+  do_sample: True
 metric_list:
   - metric: !function metrics.rouge_score
     aggregation: mean

lm_eval/tasks/longbench/multi_news_e.yaml

@@ -10,7 +10,7 @@ doc_to_target: '{{answers}}'
 generation_kwargs:
   max_gen_toks: 512
   temperature: 1
-  do_sample: False
+  do_sample: True
 metric_list:
   - metric: !function metrics.rouge_score
     aggregation: mean

lm_eval/tasks/longbench/multifieldqa_en.yaml

@@ -10,7 +10,7 @@ doc_to_target: '{{answers}}'
 generation_kwargs:
   max_gen_toks: 64
   temperature: 1
-  do_sample: False
+  do_sample: True
 metric_list:
   - metric: !function metrics.qa_f1_score
     aggregation: mean

lm_eval/tasks/longbench/multifieldqa_en_e.yaml

@@ -10,7 +10,7 @@ doc_to_target: '{{answers}}'
 generation_kwargs:
   max_gen_toks: 64
   temperature: 1
-  do_sample: False
+  do_sample: True
 metric_list:
   - metric: !function metrics.qa_f1_score
     aggregation: mean

lm_eval/tasks/longbench/multifieldqa_zh.yaml

@@ -10,7 +10,7 @@ doc_to_target: '{{answers}}'
 generation_kwargs:
   max_gen_toks: 64
   temperature: 1
-  do_sample: False
+  do_sample: True
 metric_list:
   - metric: !function metrics.qa_f1_zh_score
     aggregation: mean

lm_eval/tasks/longbench/musique.yaml

@@ -10,7 +10,7 @@ doc_to_target: '{{answers}}'
 generation_kwargs:
   max_gen_toks: 32
   temperature: 1
-  do_sample: False
+  do_sample: True
 metric_list:
   - metric: !function metrics.qa_f1_score
     aggregation: mean

lm_eval/tasks/longbench/narrativeqa.yaml

@@ -10,7 +10,7 @@ doc_to_target: '{{answers}}'
 generation_kwargs:
   max_gen_toks: 128
   temperature: 1
-  do_sample: False
+  do_sample: True
 metric_list:
   - metric: !function metrics.qa_f1_score
     aggregation: mean

lm_eval/tasks/longbench/passage_count.yaml

@@ -10,7 +10,7 @@ doc_to_target: '{{answers}}'
 generation_kwargs:
   max_gen_toks: 32
   temperature: 1
-  do_sample: False
+  do_sample: True
 metric_list:
   - metric: !function metrics.count_score
     aggregation: mean

lm_eval/tasks/longbench/passage_count_e.yaml

@@ -10,7 +10,7 @@ doc_to_target: '{{answers}}'
 generation_kwargs:
   max_gen_toks: 32
   temperature: 1
-  do_sample: False
+  do_sample: True
 metric_list:
   - metric: !function metrics.count_score
     aggregation: mean

lm_eval/tasks/longbench/passage_retrieval_en.yaml

@@ -10,7 +10,7 @@ doc_to_target: '{{answers}}'
 generation_kwargs:
   max_gen_toks: 32
   temperature: 1
-  do_sample: False
+  do_sample: True
 metric_list:
   - metric: !function metrics.retrieval_score
     aggregation: mean