Merge branch 'main' into metrics

# Conflicts:
#	.pre-commit-config.yaml
#	lm_eval/api/task.py
#	lm_eval/models/huggingface.py
#	lm_eval/models/vllm_causallms.py
#	pyproject.toml

lm_eval/tasks/libra/ru_sci_passage_count.yaml

+tag:
+- libra
+task: ru_sci_passage_count
+task_alias: ruSciPassageCount
+dataset_name: ru_sci_passage_count
+dataset_kwargs:
+  dataset_name: ru_sci_passage_count
+include: _template_yaml
+doc_to_text: 'Ниже приведены несколько параграфов. Прочти их и определи количество
+  уникальных параграфов.
+
+  {{context}}
+
+  Определи количество уникальных параграфов. Ответ должен содержать только одно число.
+
+  Вопрос:{{input}}
+
+  Ответ:'
+generation_kwargs:
+  do_sample: false
+  temperature: 0.0
+  max_gen_toks: 8
+metric_list:
+- metric: libra_score
+  aggregation: !function utils.aggregate_results_count_score
+  higher_is_better: true
+  weight_by_size: true
+metadata:
+  version: 0.0

lm_eval/tasks/libra/utils.py

+import re
+from collections import Counter, defaultdict
+from dataclasses import dataclass
+from typing import Callable, Dict, List
+
+import datasets
+
+
+try:
+    import pymorphy2
+
+    normalizer = pymorphy2.MorphAnalyzer()
+except ImportError:
+    print(
+        "Can not import pymorphy2. If you try to score libra, do `pip install pymorphy2`"
+    )
+
+
+@dataclass
+class PredictionResult:
+    pred_answer: str
+    answers: List[str]
+    length: str
+
+
+def filter_dataset_by_page_lengths(*args, **kwargs) -> Dict[str, datasets.Dataset]:
+    """Filter dataset by page lengths for Libra task.
+
+    in CLI metadata --metadata '{"valid_pages": ["8p", "32p"], "dataset_repo_name": "ai-forever/LIBRA"}'
+    """
+    valid_pages = kwargs.get("valid_pages", [])
+
+    dataset_repo_name = kwargs.get("dataset_repo_name", "ai-forever/LIBRA")
+    dataset_name = kwargs.get("dataset_name", None)
+    filter_colname = kwargs.get("filter_colname", "length")
+    token = kwargs.get("token", None)
+
+    dataset_columns = list(
+        datasets.load_dataset(dataset_repo_name, dataset_name, token=token)[
+            "test"
+        ].features.keys()
+    )
+    if filter_colname not in dataset_columns:
+        raise ValueError(f"Column {filter_colname} not found in dataset {dataset_name}")
+
+    if valid_pages:
+        dataset_filtered = datasets.load_dataset(
+            dataset_repo_name, dataset_name, token=token
+        )["test"].filter(lambda doc: doc.get(filter_colname) in valid_pages)
+    else:
+        dataset_filtered = datasets.load_dataset(
+            dataset_repo_name, dataset_name, token=token
+        )["test"]
+    return {"test": dataset_filtered}
+
+
+def normalize_answer(sentence: str) -> str:
+    """Normalize an input sentence by removing punctuation and converting words to their base (lemmatized) form.
+    :param sentence: str
+        Input sentence.
+    :return: str
+        A normalized sentence where:
+        - All characters except letters, digits, and underscores are removed.
+        - All words are converted to lowercase.
+        - Words are lemmatized using `normalizer`.
+    :raises ValueError:
+        If `sentence` is not a string.
+    :example:
+    >>> normalize_answer("Hello, world! This is a test sentence.")
+    'hello world this is a test sentence'
+    """
+    sentence = str(sentence)
+    new_sentence = []
+    for word in sentence.split():
+        token = re.sub(r"[^a-zа-яй0-9_]+", "", word.lower())
+        token = normalizer.parse(token)[0].normal_form.lower()
+        new_sentence.append(token)
+    return " ".join(new_sentence)
+
+
+def process_results(doc: List, results: List[str]) -> Dict:
+    """Processes evaluation results by extracting prediction and relevant metadata.
+
+    :param doc: A single instance from the evaluation dataset, containing reference answers and metadata.
+    :param results: A list containing the predicted answer(s). The first element is used as the main prediction.
+    :return: A dictionary where the key is the metric name ("libra_score") and the value is a dictionary
+             with the predicted answer, reference answers, and context length.
+    """
+    prediction = results[0]
+
+    data_dict = {
+        "pred_answer": prediction,
+        "answers": doc["positive_outputs"],
+        "length": doc["length"],
+    }
+
+    return {"libra_score": data_dict}
+
+
+def exact_match_score(prediction: str, ground_truth: str) -> float:
+    result = 0.0
+    if normalize_answer(ground_truth) in normalize_answer(prediction):
+        result = 1.0
+    return result
+
+
+def f1_score(prediction: str, ground_truth: str) -> float:
+    common = Counter(prediction) & Counter(ground_truth)
+    num_same = sum(common.values())
+    if num_same == 0:
+        return 0
+    precision = 1.0 * num_same / len(prediction)
+    recall = 1.0 * num_same / len(ground_truth)
+    f1 = (2 * precision * recall) / (precision + recall)
+    return f1
+
+
+def count_score(prediction: str, ground_truth: str) -> float:
+    numbers = re.findall(r"\d+", prediction)
+    right_num = 0
+    for number in numbers:
+        if str(number) == str(ground_truth):
+            right_num += 1
+    final_score = 0.0 if len(numbers) == 0 else right_num / len(numbers)
+    return float(final_score)
+
+
+def aggregate_results(
+    results: List[PredictionResult], scoring_function: Callable
+) -> Dict[str, float]:
+    """Aggregates score by 'length' by scoring_function.
+
+    :param results: List of dictionaries containing 'pred_answer', 'answers', and 'length'.
+    :return: Dictionary with 'length' as keys and average score as values.
+
+    :example:
+    >>> results = [
+    ...     {"pred_answer": "1", "answers": ["1", "one"], "length": "8p"},
+    ...     {"pred_answer": "0", "answers": ["zero", "none"], "length": "8p"},
+    ...     {"pred_answer": "one", "answers": ["1", "one"], "length": "16p"}
+    ... ]
+    >>> aggregate_results(results=results)
+    {'8p': 0.5, '16p': 1.0}
+    """
+    scores = defaultdict(lambda: [0, 0])
+
+    for result in results:
+        length = result["length"]
+        pred_answer = normalize_answer(result["pred_answer"])
+        answers = set([normalize_answer(text) for text in result["answers"]])
+
+        scores[length][1] += 1
+        for answer in answers:
+            metric = scoring_function(prediction=pred_answer, ground_truth=answer)
+            if metric > 0:
+                scores[length][0] += metric
+                break
+    return {key: correct / total for key, (correct, total) in scores.items()}
+
+
+def aggregate_results_em(results: List[PredictionResult]) -> Dict[str, float]:
+    return aggregate_results(results, exact_match_score)
+
+
+def aggregate_results_f1(results: List[PredictionResult]) -> Dict[str, float]:
+    return aggregate_results(results, f1_score)
+
+
+def aggregate_results_count_score(results: List[PredictionResult]) -> Dict[str, float]:
+    return aggregate_results(results, count_score)

lm_eval/tasks/llama3/instruct/mmlu/_continuation_template_yaml

@@ -29,5 +29,3 @@ filter_list:
       - function: take_first
 metadata:
   version: 1.0
-dataset_kwargs:
-  trust_remote_code: true

lm_eval/tasks/llama3/instruct/mmlu_cot/_mmlu_cot_llama_template_yaml

@@ -23,6 +23,4 @@ metric_list:
     ignore_punctuation: true
 metadata:
   version: 1.0
-dataset_kwargs:
-  trust_remote_code: true
 num_fewshot: 0

lm_eval/tasks/llama3/instruct/mmlu_de/_continuation_template_yaml

@@ -28,5 +28,3 @@ filter_list:
       - function: take_first
 metadata:
   version: 1.0
-dataset_kwargs:
-  trust_remote_code: true

lm_eval/tasks/llama3/instruct/mmlu_es/_continuation_template_yaml

@@ -28,5 +28,3 @@ filter_list:
       - function: take_first
 metadata:
   version: 1.0
-dataset_kwargs:
-  trust_remote_code: true

lm_eval/tasks/llama3/instruct/mmlu_fr/_continuation_template_yaml

@@ -28,5 +28,3 @@ filter_list:
       - function: take_first
 metadata:
   version: 1.0
-dataset_kwargs:
-  trust_remote_code: true

lm_eval/tasks/llama3/instruct/mmlu_hi/_continuation_template_yaml

@@ -28,5 +28,3 @@ filter_list:
       - function: take_first
 metadata:
   version: 1.0
-dataset_kwargs:
-  trust_remote_code: true

lm_eval/tasks/llama3/instruct/mmlu_it/_continuation_template_yaml

@@ -28,5 +28,3 @@ filter_list:
       - function: take_first
 metadata:
   version: 1.0
-dataset_kwargs:
-  trust_remote_code: true

lm_eval/tasks/llama3/instruct/mmlu_pro/_default_template_yaml

@@ -31,5 +31,3 @@ filter_list:
         - function: take_first
 metadata:
   version: 1.0
-dataset_kwargs:
-  trust_remote_code: true

lm_eval/tasks/llama3/instruct/mmlu_pt/_continuation_template_yaml

@@ -28,5 +28,3 @@ filter_list:
       - function: take_first
 metadata:
   version: 1.0
-dataset_kwargs:
-  trust_remote_code: true

lm_eval/tasks/llama3/instruct/mmlu_th/_continuation_template_yaml

@@ -28,5 +28,3 @@ filter_list:
       - function: take_first
 metadata:
   version: 1.0
-dataset_kwargs:
-  trust_remote_code: true

lm_eval/tasks/logiqa/logiqa.yaml

@@ -19,5 +19,3 @@ metric_list:
     higher_is_better: true
 metadata:
   version: 1.0
-dataset_kwargs:
-  trust_remote_code: true

lm_eval/tasks/logiqa2/logieval.yaml

@@ -25,5 +25,3 @@ filter_list:
       - function: "take_first"
 metadata:
   version: 0.0
-dataset_kwargs:
-  trust_remote_code: true

lm_eval/tasks/mediqa_qa2019/mediqa_qa2019_perplexity.yaml

@@ -23,5 +23,3 @@ metric_list:
     higher_is_better: false
 metadata:
   version: 1.0
-dataset_kwargs:
-  trust_remote_code: true

lm_eval/tasks/minerva_math/minerva_math_algebra.yaml

@@ -7,7 +7,7 @@ dataset_name: algebra
 output_type: generate_until
 training_split: train
 test_split: test
-doc_to_text:  !function utils.doc_to_text
+doc_to_text: !function utils.doc_to_text
 process_results: !function utils.process_results
 doc_to_target: "{{answer if few_shot is undefined else solution}}"
 generation_kwargs:
@@ -25,8 +25,6 @@ metric_list:
 num_fewshot: 4
 metadata:
   version: 2.0
-dataset_kwargs:
-  trust_remote_code: true
 fewshot_config:
   sampler: first_n
   samples: !function utils.list_fewshot_samples

lm_eval/tasks/mlqa/mlqa_common_yaml

 dataset_path: facebook/mlqa
-dataset_kwargs:
-  trust_remote_code: true
 test_split: test
 validation_split: validation
 output_type: generate_until

lm_eval/tasks/mmlu/continuation/_continuation_template_yaml

@@ -9,5 +9,3 @@ doc_to_choice: "{{choices}}"
 doc_to_target: "{{answer}}"
 metadata:
   version: 1.0
-dataset_kwargs:
-  trust_remote_code: true

lm_eval/tasks/mmlu/default/_default_template_yaml

@@ -13,5 +13,3 @@ metric_list:
     higher_is_better: true
 metadata:
   version: 1.0
-dataset_kwargs:
-  trust_remote_code: true

lm_eval/tasks/mmlu/flan_cot_fewshot/_mmlu_flan_cot_fewshot_template_yaml

@@ -26,5 +26,3 @@ metric_list:
     ignore_punctuation: true
 metadata:
   version: 2.0
-dataset_kwargs:
-  trust_remote_code: true