Add option to set OpenVINO config (#1730)

* Add option to set OpenVINO config

* Use utils.eval_logger for logging

lm_eval/models/optimum_lm.py

+import json
 from importlib.util import find_spec
 from pathlib import Path
 
+from lm_eval import utils
 from lm_eval.api.registry import register_model
 from lm_eval.models.huggingface import HFLM
 
 
+eval_logger = utils.eval_logger
+
+
 @register_model("openvino")
 class OptimumLM(HFLM):
     """
     Optimum Intel provides a simple interface to optimize Transformer models and convert them to \
     OpenVINO™ Intermediate Representation (IR) format to accelerate end-to-end pipelines on \
     Intel® architectures using OpenVINO™ runtime.
+
+    To use an OpenVINO config, use `--model_args ov_config` to point to a json file with an OpenVINO config:
+    `lm_eval --model openvino --model_args pretrained=gpt2,ov_config=config.json --task lambada_openai`
+    Example json file contents: {"INFERENCE_PRECISION_HINT": "f32", "CACHE_DIR": "model_cache"}
     """
 
     def __init__(
@@ -48,16 +57,25 @@ class OptimumLM(HFLM):
             from optimum.intel.openvino import OVModelForCausalLM
 
         model_kwargs = kwargs if kwargs else {}
+        if "ov_config" in model_kwargs:
+            if not Path(model_kwargs["ov_config"]).exists():
+                raise ValueError(
+                    "ov_config should point to a .json file containing an OpenVINO config"
+                )
+            with open(model_kwargs["ov_config"]) as f:
+                model_kwargs["ov_config"] = json.load(f)
+                eval_logger.info(
+                    f"Using custom OpenVINO config: {model_kwargs['ov_config']}"
+                )
+
+        else:
+            model_kwargs["ov_config"] = {}
+        model_kwargs["ov_config"].setdefault("CACHE_DIR", "")
         model_file = Path(pretrained) / "openvino_model.xml"
         if model_file.exists():
             export = False
         else:
             export = True
-        kwargs["ov_config"] = {
-            "PERFORMANCE_HINT": "LATENCY",
-            "NUM_STREAMS": "1",
-            "CACHE_DIR": "",
-        }
 
         self._model = OVModelForCausalLM.from_pretrained(
             pretrained,

tests/models/test_openvino.py

 import random
 import tempfile
+from pathlib import Path
 
 import pytest
 from optimum.intel import OVModelForCausalLM
@@ -71,3 +72,21 @@ def test_evaluator(model_id, task):
             limit=limit,
             bootstrap_iters=10,
         )
+
+
+def test_ov_config():
+    """Test that if specified, a custom OpenVINO config is loaded correctly"""
+    model_id = "hf-internal-testing/tiny-random-gpt2"
+    with tempfile.TemporaryDirectory() as tmpdirname:
+        config_file = str(Path(tmpdirname) / "ov_config.json")
+        with open(Path(config_file), "w") as f:
+            f.write('{"DYNAMIC_QUANTIZATION_GROUP_SIZE" : "32"}')
+        lm = get_model("openvino").create_from_arg_string(
+            f"pretrained={model_id},ov_config={config_file}"
+        )
+    assert (
+        lm.model.request.get_compiled_model().get_property(
+            "DYNAMIC_QUANTIZATION_GROUP_SIZE"
+        )
+        == 32
+    )