Merge remote-tracking branch 'origin/big-refactor' into calibration

lm_eval/tasks/bbh/flan_fewshot/web_of_lies.yaml

+"dataset_name": "web_of_lies"
+"description": "Evaluate a random boolean function expressed as a word problem.\n\n"
+"doc_to_text": "Q: Question: Fidel tells the truth. Jerry says Fidel tells the truth. Vina says Jerry tells the truth. Millicent says Vina lies. Raymond says Millicent lies. Does Raymond tell the truth?\nA: Yes\n\nQ: Question: Kristian lies. Millie says Kristian lies. Maybelle says Millie tells the truth. Fidel says Maybelle lies. Leda says Fidel lies. Does Leda tell the truth?\nA: Yes\n\nQ: Question: Kristian tells the truth. Michaela says Kristian lies. Raymond says Michaela tells the truth. Osvaldo says Raymond tells the truth. Jamey says Osvaldo tells the truth. Does Jamey tell the truth?\nA: No\n\nQ: {{input}}\nA:"
+"include": "_flan_fewshot_template_yaml"
+"task": "bbh_flan_fewshot_web_of_lies"

lm_eval/tasks/bbh/flan_fewshot/word_sorting.yaml

+"dataset_name": "word_sorting"
+"description": "Sort a list of words.\n\n"
+"doc_to_text": "Q: Sort the following words alphabetically: List: oven costume counterpart\nA: costume counterpart oven\n\nQ: Sort the following words alphabetically: List: hypochlorite ponderosa phone credulity\nA: credulity hypochlorite phone ponderosa\n\nQ: Sort the following words alphabetically: List: newt arson parthia seismography mugho aspect census\nA: arson aspect census mugho newt parthia seismography\n\nQ: {{input}}\nA:"
+"include": "_flan_fewshot_template_yaml"
+"task": "bbh_flan_fewshot_word_sorting"

lm_eval/tasks/bbh/flan_zeroshot/_flan_zeroshot_template_yaml

+group: bbh_flan_zeroshot
+dataset_path: lukaemon/bbh
+output_type: generate_until
+test_split: test
+doc_to_target: "{{target}}"
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    # ignore_case: true
+    # ignore_punctuation: true
+generation_kwargs:
+  until:
+    - "</s>"
+  do_sample: false
+  temperature: 0.0

lm_eval/tasks/bbh/flan_zeroshot/boolean_expressions.yaml

+"dataset_name": "boolean_expressions"
+"description": "Evaluate the result of a random Boolean expression.\n\n"
+"doc_to_text": "Q: {{input}}\nA:"
+"include": "_flan_zeroshot_template_yaml"
+"task": "bbh_flan_zeroshot_boolean_expressions"

lm_eval/tasks/bbh/flan_zeroshot/causal_judgement.yaml

+"dataset_name": "causal_judgement"
+"description": "Answer questions about causal attribution.\n\n"
+"doc_to_text": "Q: {{input}}\nA:"
+"include": "_flan_zeroshot_template_yaml"
+"task": "bbh_flan_zeroshot_causal_judgement"

lm_eval/tasks/bbh/flan_zeroshot/date_understanding.yaml

+"dataset_name": "date_understanding"
+"description": "Infer the date from context.\n\n"
+"doc_to_text": "Q: {{input}}\nA:"
+"include": "_flan_zeroshot_template_yaml"
+"task": "bbh_flan_zeroshot_date_understanding"

lm_eval/tasks/bbh/flan_zeroshot/disambiguation_qa.yaml

+"dataset_name": "disambiguation_qa"
+"description": "Clarify the meaning of sentences with ambiguous pronouns.\n\n"
+"doc_to_text": "Q: {{input}}\nA:"
+"include": "_flan_zeroshot_template_yaml"
+"task": "bbh_flan_zeroshot_disambiguation_qa"

lm_eval/tasks/bbh/flan_zeroshot/dyck_languages.yaml

+"dataset_name": "dyck_languages"
+"description": "Correctly close a Dyck-n word.\n\n"
+"doc_to_text": "Q: {{input}}\nA:"
+"include": "_flan_zeroshot_template_yaml"
+"task": "bbh_flan_zeroshot_dyck_languages"

lm_eval/tasks/bbh/flan_zeroshot/formal_fallacies.yaml

+"dataset_name": "formal_fallacies"
+"description": "Distinguish deductively valid arguments from formal fallacies.\n\n"
+"doc_to_text": "Q: {{input}}\nA:"
+"include": "_flan_zeroshot_template_yaml"
+"task": "bbh_flan_zeroshot_formal_fallacies"

lm_eval/tasks/bbh/flan_zeroshot/geometric_shapes.yaml

+"dataset_name": "geometric_shapes"
+"description": "Name geometric shapes from their SVG paths.\n\n"
+"doc_to_text": "Q: {{input}}\nA:"
+"include": "_flan_zeroshot_template_yaml"
+"task": "bbh_flan_zeroshot_geometric_shapes"

lm_eval/tasks/bbh/flan_zeroshot/hyperbaton.yaml

+"dataset_name": "hyperbaton"
+"description": "Order adjectives correctly in English sentences.\n\n"
+"doc_to_text": "Q: {{input}}\nA:"
+"include": "_flan_zeroshot_template_yaml"
+"task": "bbh_flan_zeroshot_hyperbaton"

lm_eval/tasks/bbh/flan_zeroshot/logical_deduction_five_objects.yaml

+"dataset_name": "logical_deduction_five_objects"
+"description": "A logical deduction task which requires deducing the order of a sequence of objects.\n\n"
+"doc_to_text": "Q: {{input}}\nA:"
+"include": "_flan_zeroshot_template_yaml"
+"task": "bbh_flan_zeroshot_logical_deduction_five_objects"

lm_eval/tasks/bbh/flan_zeroshot/logical_deduction_seven_objects.yaml

+"dataset_name": "logical_deduction_seven_objects"
+"description": "A logical deduction task which requires deducing the order of a sequence of objects.\n\n"
+"doc_to_text": "Q: {{input}}\nA:"
+"include": "_flan_zeroshot_template_yaml"
+"task": "bbh_flan_zeroshot_logical_deduction_seven_objects"

lm_eval/tasks/bbh/flan_zeroshot/logical_deduction_three_objects.yaml

+"dataset_name": "logical_deduction_three_objects"
+"description": "A logical deduction task which requires deducing the order of a sequence of objects.\n\n"
+"doc_to_text": "Q: {{input}}\nA:"
+"include": "_flan_zeroshot_template_yaml"
+"task": "bbh_flan_zeroshot_logical_deduction_three_objects"

lm_eval/tasks/bbh/flan_zeroshot/movie_recommendation.yaml

+"dataset_name": "movie_recommendation"
+"description": "Recommend movies similar to the given list of movies.\n\n"
+"doc_to_text": "Q: {{input}}\nA:"
+"include": "_flan_zeroshot_template_yaml"
+"task": "bbh_flan_zeroshot_movie_recommendation"

lm_eval/tasks/bbh/flan_zeroshot/multistep_arithmetic_two.yaml

+"dataset_name": "multistep_arithmetic_two"
+"description": "Solve multi-step arithmetic problems.\n\n"
+"doc_to_text": "Q: {{input}}\nA:"
+"include": "_flan_zeroshot_template_yaml"
+"task": "bbh_flan_zeroshot_multistep_arithmetic_two"

lm_eval/tasks/bbh/flan_zeroshot/navigate.yaml

+"dataset_name": "navigate"
+"description": "Given a series of navigation instructions, determine whether one would end up back at the starting point.\n\n"
+"doc_to_text": "Q: {{input}}\nA:"
+"include": "_flan_zeroshot_template_yaml"
+"task": "bbh_flan_zeroshot_navigate"

lm_eval/tasks/bbh/flan_zeroshot/object_counting.yaml

+"dataset_name": "object_counting"
+"description": "Questions that involve enumerating objects and asking the model to count them.\n\n"
+"doc_to_text": "Q: {{input}}\nA:"
+"include": "_flan_zeroshot_template_yaml"
+"task": "bbh_flan_zeroshot_object_counting"

lm_eval/tasks/bbh/flan_zeroshot/penguins_in_a_table.yaml

+"dataset_name": "penguins_in_a_table"
+"description": "Answer questions about a table of penguins and their attributes.\n\n"
+"doc_to_text": "Q: {{input}}\nA:"
+"include": "_flan_zeroshot_template_yaml"
+"task": "bbh_flan_zeroshot_penguins_in_a_table"

lm_eval/tasks/bbh/flan_zeroshot/reasoning_about_colored_objects.yaml

+"dataset_name": "reasoning_about_colored_objects"
+"description": "Answer extremely simple questions about the colors of objects on a surface.\n\n"
+"doc_to_text": "Q: {{input}}\nA:"
+"include": "_flan_zeroshot_template_yaml"
+"task": "bbh_flan_zeroshot_reasoning_about_colored_objects"