Merge branch 'main' into multimodal-prototyping

lm_eval/tasks/unitxt/law_stack_exchange.yaml

-include: unitxt_tasks.classification.multi_class
 task: law_stack_exchange
-dataset_name: card=cards.law_stack_exchange,template=templates.classification.multi_class.title
+include: unitxt
+recipe: card=cards.law_stack_exchange,template=templates.classification.multi_class.title

lm_eval/tasks/unitxt/ledgar.yaml

-include: unitxt_tasks.classification.multi_class
 task: ledgar
-dataset_name: card=cards.ledgar,template=templates.classification.multi_class.title
+include: unitxt
+recipe: card=cards.ledgar,template=templates.classification.multi_class.title

lm_eval/tasks/unitxt/medical_abstracts.yaml

-include: unitxt_tasks.classification.multi_class
 task: medical_abstracts
-dataset_name: card=cards.medical_abstracts,template=templates.classification.multi_class.title
+include: unitxt
+recipe: card=cards.medical_abstracts,template=templates.classification.multi_class.title

lm_eval/tasks/unitxt/stsb.yaml

-include: unitxt_tasks.regression.two_texts
 task: stsb
-dataset_name: card=cards.stsb,template=templates.regression.two_texts.simple
+include: unitxt
+recipe: card=cards.stsb,template=templates.regression.two_texts.simple

lm_eval/tasks/unitxt/task.py

+"""
+In the dynamic landscape of generative NLP, traditional text processing pipelines limit research flexibility and reproducibility, as they are tailored to specific dataset, task, and model combinations. The escalating complexity, involving system prompts, model-specific formats, instructions, and more, calls for a shift to a structured, modular, and customizable solution.
+
+Addressing this need, we present Unitxt, an innovative library for customizable textual data preparation and evaluation tailored to generative language models. Unitxt natively integrates with common libraries like HuggingFace and LM-eval-harness and deconstructs processing flows into modular components, enabling easy customization and sharing between practitioners. These components encompass model-specific formats, task prompts, and many other comprehensive dataset processing definitions. The Unitxt-Catalog centralizes these components, fostering collaboration and exploration in modern textual data workflows. Beyond being a tool, Unitxt is a community-driven platform, empowering users to build, share, and advance their pipelines collaboratively.
+"""
+
+from functools import partial
+from typing import Optional
+
+import evaluate
+
+from lm_eval.api.instance import Instance
+from lm_eval.api.task import ConfigurableTask
+
+
+_CITATION = """
+@misc{bandel2024unitxt,
+      title={Unitxt: Flexible, Shareable and Reusable Data Preparation and Evaluation for Generative AI},
+      author={Elron Bandel and Yotam Perlitz and Elad Venezian and Roni Friedman-Melamed and Ofir Arviv and Matan Orbach and Shachar Don-Yehyia and Dafna Sheinwald and Ariel Gera and Leshem Choshen and Michal Shmueli-Scheuer and Yoav Katz},
+      year={2024},
+      eprint={2401.14019},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+"""
+
+
+def score(items, metric):
+    predictions, references = zip(*items)
+    evaluator = evaluate.load("unitxt/metric")
+    for reference in references:
+        reference["metrics"] = [metric]
+    results = evaluator.compute(predictions=predictions, references=references)
+    return results[0]["score"]["global"]["score"]
+
+
+class Unitxt(ConfigurableTask):
+    VERSION = 0
+
+    def __init__(
+        self,
+        config: Optional[dict] = None,
+    ) -> None:
+        assert "recipe" in config, "Unitxt task must have a 'recipe' string."
+        super().__init__(
+            config={
+                "metadata": {"version": self.VERSION},
+                "dataset_kwargs": {"trust_remote_code": True},
+                "dataset_name": config["recipe"],
+                "dataset_path": "unitxt/data",
+            }
+        )
+        self.metrics = self.dataset["test"][0]["metrics"]
+
+    def has_training_docs(self):
+        return "train" in self.dataset
+
+    def has_validation_docs(self):
+        return "validation" in self.dataset
+
+    def has_test_docs(self):
+        return "test" in self.dataset
+
+    def training_docs(self):
+        return self.dataset["train"]
+
+    def validation_docs(self):
+        return self.dataset["validation"]
+
+    def test_docs(self):
+        return self.dataset["test"]
+
+    def doc_to_text(self, doc):
+        return doc["source"]
+
+    def should_decontaminate(self):
+        return False
+
+    def doc_to_target(self, doc):
+        doc["target"]
+
+    def construct_requests(self, doc, ctx, **kwargs):
+        """Uses RequestFactory to construct Requests and returns an iterable of
+        Requests which will be sent to the LM.
+
+        :param doc:
+            The document as returned from training_docs, validation_docs, or test_docs.
+        :param ctx: str
+            The context string, generated by fewshot_context. This includes the natural
+            language description, as well as the few shot examples, and the question
+            part of the document for `doc`.
+        """
+
+        return [
+            Instance(
+                request_type="generate_until",
+                doc=doc,
+                arguments=(ctx, {"until": ["\n"]}),
+                idx=0,
+                **kwargs,
+            )
+        ]
+
+    def process_results(self, doc, results):
+        """Take a single document and the LM results and evaluates, returning a
+        dict where keys are the names of submetrics and values are the values of
+        the metric for that one document
+
+        :param doc:
+            The document as returned from training_docs, validation_docs, or test_docs.
+        :param results:
+            The results of the requests created in construct_requests.
+        """
+
+        continuation = results[0]
+
+        predictions = continuation
+
+        references = doc
+        return {
+            metric.replace("metrics.", ""): (predictions, references)
+            for metric in self.metrics
+        }
+
+    def aggregation(self):
+        """
+        :returns: {str: [float] -> float}
+            A dictionary where keys are the names of submetrics and values are
+            functions that aggregate a list of metrics
+        """
+        return {
+            metric.replace("metrics.", ""): partial(score, metric=metric)
+            for metric in self.metrics
+        }
+
+    def higher_is_better(self):
+        """
+        :returns: {str: bool}
+            A dictionary where keys are the names of submetrics and values are
+            whether a higher value of the submetric is better
+        """
+        return {metric.replace("metrics.", ""): True for metric in self.metrics}

lm_eval/tasks/unitxt/unfair_tos.yaml

-include: unitxt_tasks.classification.multi_label
 task: unfair_tos
-dataset_name: card=cards.unfair_tos,template=templates.classification.multi_label.title
+include: unitxt
+recipe: card=cards.unfair_tos,template=templates.classification.multi_label.title

lm_eval/tasks/unitxt/unitxt

+class: !function task.Unitxt

lm_eval/tasks/unitxt/unitxt_datasets

-coedit_gec
-atis
-20_newsgroups
-ag_news
-argument_topic
-banking77
-claim_stance_topic
-cnn_dailymail
-dbpedia_14
-ethos_binary
-financial_tweets
-law_stack_exchange
-ledgar
-medical_abstracts
-stsb
-unfair_tos
-xsum
-yahoo_answers_topics

lm_eval/tasks/unitxt/unitxt_tasks.classification.multi_class

-group:
-- unitxt
-dataset_path: unitxt/data
-output_type: generate_until
-training_split: train
-validation_split: test
-doc_to_text: '{{source}}'
-doc_to_target: target
-process_results: !function 'unitxt_wrapper.process_results'
-generation_kwargs:
-  until:
-  - </s>
-metric_list:
-- metric: unitxt_f1_micro
-  aggregation: unitxt
-  higher_is_better: true
-- metric: unitxt_accuracy
-  aggregation: unitxt
-  higher_is_better: true
-- metric: unitxt_f1_macro
-  aggregation: unitxt
-  higher_is_better: true
-metadata:
-  verison: 1.0

lm_eval/tasks/unitxt/unitxt_tasks.classification.multi_label

-group:
-- unitxt
-dataset_path: unitxt/data
-output_type: generate_until
-training_split: train
-validation_split: test
-doc_to_text: '{{source}}'
-doc_to_target: target
-process_results: !function 'unitxt_wrapper.process_results'
-generation_kwargs:
-  until:
-  - </s>
-metric_list:
-- metric: unitxt_f1_micro_multi_label
-  aggregation: unitxt
-  higher_is_better: true
-- metric: unitxt_accuracy
-  aggregation: unitxt
-  higher_is_better: true
-- metric: unitxt_f1_macro_multi_label
-  aggregation: unitxt
-  higher_is_better: true
-metadata:
-  verison: 1.0

lm_eval/tasks/unitxt/unitxt_tasks.grammatical_error_correction

-group:
-- unitxt
-dataset_path: unitxt/data
-output_type: generate_until
-training_split: train
-validation_split: test
-doc_to_text: '{{source}}'
-doc_to_target: target
-process_results: !function 'unitxt_wrapper.process_results'
-generation_kwargs:
-  until:
-  - </s>
-metric_list:
-- metric: unitxt_char_edit_dist_accuracy
-  aggregation: unitxt
-  higher_is_better: true
-- metric: unitxt_rouge
-  aggregation: unitxt
-  higher_is_better: true
-- metric: unitxt_char_edit_distance[reference_field=original_text]
-  aggregation: unitxt
-  higher_is_better: true
-metadata:
-  verison: 1.0

lm_eval/tasks/unitxt/unitxt_tasks.qa.with_context.extractive

-group:
-- unitxt
-dataset_path: unitxt/data
-output_type: generate_until
-training_split: train
-validation_split: test
-doc_to_text: '{{source}}'
-doc_to_target: target
-process_results: !function 'unitxt_wrapper.process_results'
-generation_kwargs:
-  until:
-  - </s>
-metric_list:
-- metric: unitxt_squad
-  aggregation: unitxt
-  higher_is_better: true
-metadata:
-  verison: 1.0

lm_eval/tasks/unitxt/unitxt_tasks.regression.two_texts

-group:
-- unitxt
-dataset_path: unitxt/data
-output_type: generate_until
-training_split: train
-validation_split: test
-doc_to_text: '{{source}}'
-doc_to_target: target
-process_results: !function 'unitxt_wrapper.process_results'
-generation_kwargs:
-  until:
-  - </s>
-metric_list:
-- metric: unitxt_spearman
-  aggregation: unitxt
-  higher_is_better: true
-metadata:
-  verison: 1.0

lm_eval/tasks/unitxt/unitxt_tasks.span_labeling.extraction

-group:
-- unitxt
-dataset_path: unitxt/data
-output_type: generate_until
-training_split: train
-validation_split: test
-doc_to_text: '{{source}}'
-doc_to_target: target
-process_results: !function 'unitxt_wrapper.process_results'
-generation_kwargs:
-  until:
-  - </s>
-metric_list:
-- metric: unitxt_ner
-  aggregation: unitxt
-  higher_is_better: true
-metadata:
-  verison: 1.0

lm_eval/tasks/unitxt/unitxt_tasks.summarization.abstractive

-group:
-- unitxt
-dataset_path: unitxt/data
-output_type: generate_until
-training_split: train
-validation_split: test
-doc_to_text: '{{source}}'
-doc_to_target: target
-process_results: !function 'unitxt_wrapper.process_results'
-generation_kwargs:
-  until:
-  - </s>
-metric_list:
-- metric: unitxt_rouge
-  aggregation: unitxt
-  higher_is_better: true
-metadata:
-  verison: 1.0

lm_eval/tasks/unitxt/unitxt_wrapper.py

-try:
-    from unitxt import evaluate
-except ImportError:
-    raise ImportError(
-        "Package 'unitxt' is not installed. To install it, use `pip install 'lm_eval[unitxt]'`"
-    )
-
-from lm_eval.api.registry import AGGREGATION_REGISTRY, METRIC_REGISTRY, register_metric
-
-
-def unitxt_agg_metric(items):
-    preds = [pred[0] for pred, _, _ in items]
-    refs = [ref for _, ref, _ in items]
-    metric_name = items[0][2].replace("unitxt_", "metrics.")
-    for ref in refs:
-        ref["metrics"] = [metric_name]
-
-    result_metrics = evaluate(preds, refs)
-    return result_metrics[0]["score"]["global"]["score"]
-
-
-AGGREGATION_REGISTRY["unitxt"] = unitxt_agg_metric
-
-
-def unitxt_metric(items):  # This is a passthrough function
-    return items
-
-
-def process_results(doc, results):
-    metrics = doc["metrics"]
-    scores = {}
-    for metric in metrics:
-        metric = metric.replace("metrics.", "unitxt_")
-        scores[metric] = (results, doc, metric)
-
-        if metric not in METRIC_REGISTRY:
-            register_metric(
-                metric=metric,
-                higher_is_better=True,
-                output_type="generate_until",
-                aggregation="unitxt",
-            )(unitxt_metric)
-    return scores
-
-
-#

lm_eval/tasks/unitxt/xsum.yaml

-include: unitxt_tasks.summarization.abstractive
 task: xsum
-dataset_name: card=cards.xsum,template=templates.summarization.abstractive.full
+include: unitxt
+recipe: card=cards.xsum,template=templates.summarization.abstractive.full

lm_eval/tasks/unitxt/yahoo_answers_topics.yaml

-include: unitxt_tasks.classification.multi_class
 task: yahoo_answers_topics
-dataset_name: card=cards.yahoo_answers_topics,template=templates.classification.multi_class.title
+include: unitxt
+recipe: card=cards.yahoo_answers_topics,template=templates.classification.multi_class.title

lm_eval/tasks/unscramble/anagrams1.yaml

-group:
+tag:
   - unscramble
 task: anagrams1
 dataset_path: EleutherAI/unscramble

lm_eval/tasks/unscramble/anagrams2.yaml

-group:
+tag:
   - unscramble
 task: anagrams2
 dataset_path: EleutherAI/unscramble