logieval.yaml

#group:
#  - greedy_until
task: logieval
dataset_path: lm_eval/tasks/logiqav2/logiqa2.py
dataset_name: logieval
output_type: greedy_until
training_split: train
#validation_split: validation
test_split: test
# Instructions + {content}
doc_to_text: "Instructions: You will be presented with a passage and a question about that passage. There are four options to be chosen from, you need to choose the only correct option to answer that question. If the first option is right, you generate the answer 'A', if the second option is right, you generate the answer 'B', if the third option is right, you generate the answer 'C', if the fourth option is right, you generate the answer 'D'. Read the question and options thoroughly and select the correct answer from the four answer labels. Read the passage thoroughly to ensure you know what the passage entails.\n{{content}}"
doc_to_target: "{{ideal}}"
metric_list:
  - metric: exact_match
    aggregation: mean
    higher_is_better: true
generation_kwargs:
  do_sample: false
num_fewshot: 1
filter_list:
  - name: "get-answer"
    filter:
      - function: "regex"
        # starts with A-D excluding leading spaces
        # original implementation uses a.startswith(b)
        # https://github.com/openai/evals/blob/305b237cdb3884c7ddb6a5d12cb184a83551fcba/evals/api.py#L84
        regex_pattern: "^\\s*([A-D])"
      - function: "take_first"