Add support for OpenVINO text2text generation models (#3101)

* Add support for OVModelForSeq2SeqLM

* Add test

lm_eval/models/optimum_lm.py

@@ -28,9 +28,8 @@ class OptimumLM(HFLM):
         **kwargs,
     ) -> None:
         if "backend" in kwargs:
-            # optimum currently only supports causal models
-            assert kwargs["backend"] == "causal", (
-                "Currently, only OVModelForCausalLM is supported."
+            assert kwargs["backend"] in ["causal", "seq2seq"], (
+                "Currently, only OVModelForCausalLM or OVModelForSeq2SeqLM are supported."
             )
 
         self.openvino_device = device
@@ -54,7 +53,7 @@ class OptimumLM(HFLM):
                 "package `optimum` is not installed. Please install it via `pip install optimum[openvino]`"
             )
         else:
-            from optimum.intel.openvino import OVModelForCausalLM
+            from optimum.intel.openvino import OVModelForCausalLM, OVModelForSeq2SeqLM
 
         model_kwargs = kwargs if kwargs else {}
         if "ov_config" in model_kwargs:
@@ -76,17 +75,12 @@ class OptimumLM(HFLM):
                 model_kwargs["ov_config"]["MODEL_DISTRIBUTION_POLICY"] = (
                     "PIPELINE_PARALLEL"
                 )
-        model_file = Path(pretrained) / "openvino_model.xml"
-        if model_file.exists():
-            export = False
-        else:
-            export = True
 
-        self._model = OVModelForCausalLM.from_pretrained(
+        model_cls = OVModelForCausalLM if self.backend == "causal" else OVModelForSeq2SeqLM
+        self._model = model_cls.from_pretrained(
             pretrained,
             revision=revision,
             trust_remote_code=trust_remote_code,
-            export=export,
             device=self.openvino_device.upper(),
             **model_kwargs,
         )

tests/models/test_openvino.py

@@ -3,23 +3,25 @@ import tempfile
 from pathlib import Path
 
 import pytest
-from optimum.intel import OVModelForCausalLM
+from optimum.intel import OVModelForCausalLM, OVModelForSeq2SeqLM
 from transformers import AutoTokenizer
 
 from lm_eval import evaluator
 from lm_eval.api.registry import get_model
 
 
-SUPPORTED_ARCHITECTURES_TASKS = {
-    "facebook/opt-125m": "lambada_openai",
-    "hf-internal-testing/tiny-random-gpt2": "wikitext",
-}
+SUPPORTED_ARCHITECTURES_TASKS = [
+    ("causal", "facebook/opt-125m", "lambada_openai",),
+    ("causal", "hf-internal-testing/tiny-random-gpt2", "wikitext",),
+    ("seq2seq", "hf-internal-testing/tiny-random-t5", "sst2",),
+]
 
 
-@pytest.mark.parametrize("model_id,task", SUPPORTED_ARCHITECTURES_TASKS.items())
-def test_evaluator(model_id, task):
+@pytest.mark.parametrize("backend,model_id,task", SUPPORTED_ARCHITECTURES_TASKS)
+def test_evaluator(backend, model_id, task):
     with tempfile.TemporaryDirectory() as tmpdirname:
-        model = OVModelForCausalLM.from_pretrained(
+        model_cls = OVModelForCausalLM if backend == "causal" else OVModelForSeq2SeqLM
+        model = model_cls.from_pretrained(
             model_id, export=True, use_cache=True
         )
         model.save_pretrained(tmpdirname)
@@ -27,7 +29,7 @@ def test_evaluator(model_id, task):
         tokenizer.save_pretrained(tmpdirname)
 
         lm = get_model("openvino").create_from_arg_string(
-            f"pretrained={tmpdirname}",
+            f"pretrained={tmpdirname},backend={backend}",
             {
                 "batch_size": 1,
                 "device": "cpu",