Merge branch 'main' into metrics

# Conflicts:
#	lm_eval/api/model.py
#	tests/test_metrics.py

.coveragerc

-[run]
-
-# tasks that aren't wired up.
-omit =
-    lm_eval/tasks/quac.py
-    lm_eval/tasks/storycloze.py
-    lm_eval/tasks/cbt.py
-    lm_eval/tasks/sat.py
-    lm_eval/tasks/triviaqa.py
-    lm_eval/tasks/naturalqs.py
-    lm_eval/models/dummy.py
-
-[report]
-exclude_lines =
-    # Skip any pass lines such as may be used for @abstractmethod
-    pass
-
-    # Have to re-enable the standard pragma
-    pragma: no cover
-
-    # Don't complain about missing debug-only code:
-    def __repr__
-    if self\.debug
-
-    # Don't complain if tests don't hit defensive assertion code:
-    raise AssertionError
-    raise NotImplementedError
-    return NotImplemented

.flake8

-[flake8]
-ignore = E203, E266, E501, W503, F403, F401, C901
-max-line-length = 127
-max-complexity = 10
-select = B,C,E,F,W,T4,B9

.github/workflows/new_tasks.yml

@@ -50,12 +50,12 @@ jobs:
         with:
           python-version: 3.9
           cache: 'pip'
-          cache-dependency-path: setup.py
+          cache-dependency-path: pyproject.toml
       - name: Install dependencies
         if: steps.changed-tasks.outputs.tasks_any_modified == 'true' || steps.changed-tasks.outputs.api_any_modified == 'true'
         run: |
             python -m pip install --upgrade pip
-            pip install -e '.[dev,ifeval]' --extra-index-url https://download.pytorch.org/whl/cpu
+            pip install -e '.[dev,ifeval,unitxt]' --extra-index-url https://download.pytorch.org/whl/cpu
     #   Install optional git dependencies
     #       pip install bleurt@https://github.com/google-research/bleurt/archive/b610120347ef22b494b6d69b4316e303f5932516.zip#egg=bleurt
     #       if [ -f requirements.txt ]; then pip install -r requirements.txt; fi

.github/workflows/unit_tests.yml

@@ -53,7 +53,7 @@ jobs:
 
       # Cache HuggingFace cache directory for CPU tests
       - name: Cache HuggingFace cache (CPU tests)
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         id: cache-hf-cpu
         with:
           path: ~/.cache/huggingface
@@ -64,11 +64,11 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -e '.[dev]' --extra-index-url https://download.pytorch.org/whl/cpu
+          pip install -e '.[dev,unitxt]' --extra-index-url https://download.pytorch.org/whl/cpu
           pip install hf_xet
 
       - name: Test with pytest
-        run: python -m pytest --showlocals -s -vv -n=auto --ignore=tests/models/test_neuralmagic.py --ignore=tests/models/test_openvino.py --ignore=tests/models/test_hf_steered.py
+        run: python -m pytest --showlocals -s -vv -n=auto --ignore=tests/models/test_openvino.py --ignore=tests/models/test_hf_steered.py
         continue-on-error: true  # Continue workflow even if tests fail
 
       # Save test artifacts
@@ -106,7 +106,7 @@ jobs:
 #      - name: Install dependencies
 #        run: |
 #          python -m pip install --upgrade pip
-#          pip install -e '.[dev,optimum,deepsparse,sparseml,api]' --extra-index-url https://download.pytorch.org/whl/cpu
+#          pip install -e '.[dev,optimum,api]' --extra-index-url https://download.pytorch.org/whl/cpu
 #          pip install -U transformers peft accelerate
 #
 #      - name: Test with pytest

.pre-commit-config.yaml

@@ -29,7 +29,7 @@ repos:
       - id: mixed-line-ending
         args: [--fix=lf]
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.11.10
+    rev: v0.12.2
     hooks:
       # Run the linter.
       - id: ruff
@@ -47,7 +47,7 @@ repos:
           )$
         args: [--check-filenames, --check-hidden, --ignore-words=ignore.txt]
   - repo: https://github.com/jackdewinter/pymarkdown
-    rev: v0.9.29
+    rev: v0.9.30
     hooks:
       - id: pymarkdown
         exclude: ^(lm_eval/tasks/.*|docs/footguns\.md)$

README.md

@@ -364,7 +364,7 @@ lm_eval --model local-completions --tasks gsm8k --model_args model=facebook/opt-
 Note that for externally hosted models, configs such as `--device` which relate to where to place a local model should not be used and do not function. Just like you can use `--model_args` to pass arbitrary arguments to the model constructor for local models, you can use it to pass arbitrary arguments to the model API for hosted models. See the documentation of the hosting service for information on what arguments they support.
 
 | API or Inference Server                                                                                                   | Implemented?                                                                                            | `--model <xxx>` name                                | Models supported:                                                                                                                                                                                                                                                                                                                                          | Request Types:                                                                 |
-| --------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------|-----------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------|
+|---------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------|-----------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------|
 | OpenAI Completions                                                                                                        | :heavy_check_mark:                                                                                      | `openai-completions`, `local-completions`           | All OpenAI Completions API models                                                                                                                                                                                                                                                                                                                          | `generate_until`, `loglikelihood`, `loglikelihood_rolling`                     |
 | OpenAI ChatCompletions                                                                                                    | :heavy_check_mark:                                                                                      | `openai-chat-completions`, `local-chat-completions` | [All ChatCompletions API models](https://platform.openai.com/docs/guides/gpt)                                                                                                                                                                                                                                                                              | `generate_until` (no logprobs)                                                 |
 | Anthropic                                                                                                                 | :heavy_check_mark:                                                                                      | `anthropic`                                         | [Supported Anthropic Engines](https://docs.anthropic.com/claude/reference/selecting-a-model)                                                                                                                                                                                                                                                               | `generate_until` (no logprobs)                                                 |
@@ -377,8 +377,6 @@ Note that for externally hosted models, configs such as `--device` which relate
 | Huggingface Optimum (Causal LMs)                                                                                          | :heavy_check_mark:                                                                                      | `openvino`                                          | Any decoder-only AutoModelForCausalLM converted with Huggingface Optimum into OpenVINO™ Intermediate Representation (IR) format                                                                                                                                                                                                                            | `generate_until`, `loglikelihood`, `loglikelihood_rolling`                     |
 | Huggingface Optimum-intel IPEX (Causal LMs)                                                                               | :heavy_check_mark:                                                                                      | `ipex`                                              | Any decoder-only AutoModelForCausalLM                                                                                                                                                                                                                                                                                                                      | `generate_until`, `loglikelihood`, `loglikelihood_rolling`                     |
 | Neuron via AWS Inf2 (Causal LMs)                                                                                          | :heavy_check_mark:                                                                                      | `neuronx`                                           | Any decoder-only AutoModelForCausalLM supported to run on [huggingface-ami image for inferentia2](https://aws.amazon.com/marketplace/pp/prodview-gr3e6yiscria2)                                                                                                                                                                                            | `generate_until`, `loglikelihood`, `loglikelihood_rolling`                     |
-| [Neural Magic DeepSparse](https://github.com/neuralmagic/deepsparse)                                                      | :heavy_check_mark:                                                                                      | `deepsparse`                                        | Any LM from [SparseZoo](https://sparsezoo.neuralmagic.com/) or on [HF Hub with the "deepsparse" tag](https://huggingface.co/models?other=deepsparse)                                                                                                                                                                                                       | `generate_until`, `loglikelihood`                                              |
-| [Neural Magic SparseML](https://github.com/neuralmagic/sparseml)                                                          | :heavy_check_mark:                                                                                      | `sparseml`                                          | Any decoder-only AutoModelForCausalLM from [SparseZoo](https://sparsezoo.neuralmagic.com/) or on [HF Hub](https://huggingface.co/neuralmagic). Especially useful for models with quantization like [`zoo:llama2-7b-gsm8k_llama2_pretrain-pruned60_quantized`](https://sparsezoo.neuralmagic.com/models/llama2-7b-gsm8k_llama2_pretrain-pruned60_quantized) | `generate_until`, `loglikelihood`, `loglikelihood_rolling`                     |
 | NVIDIA NeMo                                                                                                               | :heavy_check_mark:                                                                                      | `nemo_lm`                                           | [All supported models](https://docs.nvidia.com/nemo-framework/user-guide/24.09/nemotoolkit/core/core.html#nemo-models)                                                                                                                                                                                                                                     | `generate_until`, `loglikelihood`, `loglikelihood_rolling`                     |
 | Watsonx.ai                                                                                                                | :heavy_check_mark:                                                                                      | `watsonx_llm`                                       | [Supported Watsonx.ai Engines](https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx)                                                                                                                                                                                                                                 | `generate_until` `loglikelihood`                                               |
 | [Your local inference server!](docs/API_guide.md)                                                                         | :heavy_check_mark:                                                                                      | `local-completions` or `local-chat-completions`     | Support for OpenAI API-compatible servers, with easy customization for other APIs.                                                                                                                                                                                                                                                                         | `generate_until`, `loglikelihood`, `loglikelihood_rolling`                     |
@@ -572,9 +570,19 @@ lm_eval \
 
 In the stdout, you will find the link to the W&B run page as well as link to the generated report. You can find an example of this workflow in [examples/visualize-wandb.ipynb](examples/visualize-wandb.ipynb), and an example of how to integrate it beyond the CLI.
 
-## How to Contribute or Learn More?
+## Contributing
 
-For more information on the library and how everything fits together, check out all of our [documentation pages](https://github.com/EleutherAI/lm-evaluation-harness/tree/main/docs)! We plan to post a larger roadmap of desired + planned library improvements soon, with more information on how contributors can help.
+Check out our [open issues](https://github.com/EleutherAI/lm-evaluation-harness/issues) and feel free to submit pull requests!
+
+For more information on the library and how everything fits together, see our [documentation pages](https://github.com/EleutherAI/lm-evaluation-harness/tree/main/docs).
+
+To get started with development, first clone the repository and install the dev dependencies:
+
+```bash
+git clone https://github.com/EleutherAI/lm-evaluation-harness
+cd lm-evaluation-harness
+pip install -e ".[dev]"
+````
 
 ### Implementing new tasks
 
@@ -599,37 +607,24 @@ The best way to get support is to open an issue on this repo or join the [Eleuth
 
 Extras dependencies can be installed via `pip install -e ".[NAME]"`
 
-| Name                 | Use                                                   |
-| -------------------- | ----------------------------------------------------- |
-| api                  | For using api models (Anthropic, OpenAI API)          |
-| audiolm_qwen         | For running Qwen2 audio models                        |
-| deepsparse           | For running NM's DeepSparse models                    |
-| dev                  | For linting PRs and contributions                     |
-| gptq                 | For loading models with AutoGPTQ                      |
-| gptqmodel            | For loading models with GPTQModel                     |
-| hf_transfer          | For speeding up HF Hub file downloads                 |
-| ibm_watsonx_ai       | For using IBM watsonx.ai model apis                   |
-| ifeval               | For running the IFEval task                           |
-| ipex                 | For running on optimum-intel ipex backend             |
-| japanese_leaderboard | For running Japanese LLM Leaderboard tasks            |
-| longbench            | For running LongBench tasks                           |
-| mamba                | For loading Mamba SSM models                          |
-| math                 | For running math task answer checking                 |
-| multilingual         | For multilingual tokenizers                           |
-| neuronx              | For running on AWS inf2 instances                     |
-| optimum              | For running Intel OpenVINO models                     |
-| promptsource         | For using PromptSource prompts                        |
-| ruler                | For running RULER tasks                               |
-| sae_lens             | For using SAELens to steer models                     |
-| sentencepiece        | For using the sentencepiece tokenizer                 |
-| sparseml             | For using NM's SparseML models                        |
-| sparsify             | For using Sparsify to steer models                    |
-| testing              | For running library test suite                        |
-| vllm                 | For loading models with vLLM                          |
-| wandb                | For integration with `Weights and Biases` platform    |
-| zeno                 | For visualizing results with Zeno                     |
-| -------------------- | ----------------------------------------------------- |
-| all                  | Loads all extras (not recommended)                    |
+| NAME                 | Description                    | NAME           | Description                           |
+|----------------------|--------------------------------|----------------|---------------------------------------|
+| tasks                | All task-specific dependencies | api            | API models (Anthropic, OpenAI, local) |
+| acpbench             | ACP Bench tasks                | audiolm_qwen   | Qwen2 audio models                    |
+| ifeval               | IFEval task                    |                |                                       |
+| japanese_leaderboard | Japanese LLM tasks             | gptq           | AutoGPTQ models                       |
+| longbench            | LongBench tasks                | gptqmodel      | GPTQModel models                      |
+| math                 | Math answer checking           | hf_transfer    | Speed up HF downloads                 |
+| multilingual         | Multilingual tokenizers        | ibm_watsonx_ai | IBM watsonx.ai models                 |
+| ruler                | RULER tasks                    | ipex           | Intel IPEX backend                    |
+|                      |                                |                |                                       |
+| dev                  | Linting & contributions        | mamba          | Mamba SSM models                      |
+| promptsource         | PromptSource prompts           | neuronx        | AWS inf2 instances                    |
+| sentencepiece        | Sentencepiece tokenizer        | optimum        | Intel OpenVINO models                 |
+| testing              | Run test suite                 | sae_lens       | SAELens model steering                |
+| unitxt               | Run unitxt tasks               |                |                                       |
+| wandb                | Weights & Biases               | sparsify       | Sparsify model steering               |
+| zeno                 | Result visualization           | vllm           | vLLM models                           |
 
 ## Cite as
 

docs/CONTRIBUTING.md

@@ -30,7 +30,7 @@ in order to ensure linters and other checks will be run upon committing.
 We use [pytest](https://docs.pytest.org/en/latest/) for running unit tests. All library unit tests can be run via:
 
 ```bash
-python -m pytest --showlocals -s -vv -n=auto --ignore=tests/models/test_neuralmagic.py --ignore=tests/models/test_openvino.py
+python -m pytest --showlocals -s -vv -n=auto --ignore=tests/models/test_openvino.py
 ```
 
 ## Contributor License Agreement

lm_eval/api/metrics.py

 import logging
 import math
+import os
 import random
 import re
 import string
 from collections.abc import Iterable
-from typing import List
+from typing import Callable, List, Optional, Sequence, TypeVar
 
 import numpy as np
 
 from lm_eval.api.registry import register_aggregation, register_metric
 
 
+T = TypeVar("T")
+
 eval_logger = logging.getLogger(__name__)
 
 
@@ -292,7 +295,7 @@ def pop_stddev(arr):
     return math.sqrt(sum([(x - mu) ** 2 for x in arr]) / len(arr))
 
 
-def sample_stddev(arr):
+def sample_stddev(arr: Sequence[T]) -> float:
     mu = mean(arr)
     return math.sqrt(sum([(x - mu) ** 2 for x in arr]) / (len(arr) - 1))
 
@@ -454,11 +457,16 @@ def _sacreformat(refs, preds):
 
 
 class _bootstrap_internal:
-    def __init__(self, f, n) -> None:
+    """
+    Pool worker: `(i, xs)` → `n` bootstrap replicates
+    of `f(xs)`using a RNG seeded with `i`.
+    """
+
+    def __init__(self, f: Callable[[Sequence[T]], float], n: int) -> None:
         self.f = f
         self.n = n
 
-    def __call__(self, v):
+    def __call__(self, v: tuple[int, Sequence[T]]) -> list[float]:
         i, xs = v
         rnd = random.Random()
         rnd.seed(i)
@@ -468,36 +476,81 @@ class _bootstrap_internal:
         return res
 
 
-def bootstrap_stderr(f, xs, iters):
-    import multiprocessing as mp
-
-    pool = mp.Pool(mp.cpu_count())
-    # this gives a biased estimate of the stderr (i.e w/ the mean, it gives something
-    # equivalent to stderr calculated without Bessel's correction in the stddev.
-    # Unfortunately, I haven't been able to figure out what the right correction is
-    # to make the bootstrap unbiased - i considered multiplying by sqrt(n/(n-1)) but
-    # that would be ad-hoc and I can't prove that that would actually be an unbiased estimator)
-    # Thankfully, shouldn't matter because our samples are pretty big usually anyways
+def _bootstrap_internal_no_mp(
+    f: Callable[[Sequence[T]], float], xs: Sequence[T], iters: int
+) -> list[float]:
+    """
+    Single-process fallback: compute `iters` bootstrap replicates
+    of statistic`f(xs)`, chunked (≤ 1000 draws).
+    """
     res = []
     chunk_size = min(1000, iters)
     from tqdm import tqdm
 
-    print("bootstrapping for stddev:", f.__name__)
-    for bootstrap in tqdm(
-        pool.imap(
-            _bootstrap_internal(f, chunk_size),
-            [(i, xs) for i in range(iters // chunk_size)],
-        ),
-        total=iters // chunk_size,
-    ):
-        # sample w replacement
-        res.extend(bootstrap)
-
-    pool.close()
+    print(f"bootstrapping for stddev: {f.__name__}")
+
+    # A single loop replaces the multiprocessing pool.
+    for i in tqdm(range(iters // chunk_size)):
+        rnd = random.Random(i)
+        for _ in range(chunk_size):
+            res.append(f(rnd.choices(xs, k=len(xs))))
+
+    return res
+
+
+def bootstrap_stderr(
+    f: Callable[[Sequence[T]], float], xs: Sequence[T], iters: int
+) -> float:
+    """
+    Bootstrap estimate of the standard error of statistic `f(xs)`
+    using up to `iters` resamples, chunked (≤ 1000 draws)
+
+    Executes in parallel unless the env-var `DISABLE_MULTIPROC` is set;
+    """
+    if not os.getenv("DISABLE_MULTIPROC"):
+        import multiprocessing as mp
+
+        pool = mp.Pool(mp.cpu_count())
+        # this gives a biased estimate of the stderr (i.e w/ the mean, it gives something
+        # equivalent to stderr calculated without Bessel's correction in the stddev.
+        # Unfortunately, I haven't been able to figure out what the right correction is
+        # to make the bootstrap unbiased - i considered multiplying by sqrt(n/(n-1)) but
+        # that would be ad-hoc and I can't prove that that would actually be an unbiased estimator)
+        # Thankfully, shouldn't matter because our samples are pretty big usually anyways
+        res = []
+        chunk_size = min(1000, iters)
+        from tqdm import tqdm
+
+        print("bootstrapping for stddev:", f.__name__)
+        for bootstrap in tqdm(
+            pool.imap(
+                _bootstrap_internal(f, chunk_size),
+                [(i, xs) for i in range(iters // chunk_size)],
+            ),
+            total=iters // chunk_size,
+        ):
+            # sample w replacement
+            res.extend(bootstrap)
+
+        pool.close()
+    else:
+        res = _bootstrap_internal_no_mp(f, xs, iters)
+
     return sample_stddev(res)
 
 
-def stderr_for_metric(metric, bootstrap_iters: int):
+def stderr_for_metric(
+    metric: Callable[[Sequence[T]], float], bootstrap_iters: int
+) -> Optional[Callable[[Sequence[T]], float]]:
+    """
+    Return a function that estimates the standard error of `metric(xs)`.
+
+    * If `bootstrap_iters > 0` and the metric is in the pre-approved
+      bootstrappable list, use `bootstrap_stderr` with that many draws.
+    * If the metric has a closed-form SE (e.g. `mean`, `acc_all`), use it.
+    * Otherwise, return `None`.
+    """
+
     if bootstrap_iters <= 0:
         # return no function (don't compute stderr) if bootstrap iters = 0
         return None

lm_eval/api/model.py

@@ -4,14 +4,19 @@ import json
 import logging
 import os
 from typing import TYPE_CHECKING, Optional, Type, TypeVar, Union
+from typing import TYPE_CHECKING, Any, Iterable, Optional, Type, TypeVar, Union
 
-import transformers
-from sqlitedict import SqliteDict
 from tqdm import tqdm
 
 from lm_eval import utils
 
 
+if TYPE_CHECKING:
+    from sqlitedict import SqliteDict
+
+    from lm_eval.api.instance import Instance
+
+
 if TYPE_CHECKING:
     from lm_eval.api.instance import Instance
 
@@ -31,10 +36,10 @@ class LM(abc.ABC):
         # set rank and world size to a single process, by default.
         self._rank = 0
         self._world_size = 1
-        self.cache_hook = CacheHook(None)
+        self.cache_hook: "CacheHook" = CacheHook(None)
 
     @abc.abstractmethod
-    def loglikelihood(self, requests: list[Instance]) -> list[tuple[float, bool]]:
+    def loglikelihood(self, requests) -> list[tuple[float, bool]]:
         """Compute log-likelihood of generating a continuation from a context.
         Downstream tasks should attempt to use loglikelihood instead of other
         LM calls whenever possible.
@@ -59,7 +64,7 @@ class LM(abc.ABC):
         pass
 
     @abc.abstractmethod
-    def loglikelihood_rolling(self, requests: list[Instance]) -> list[float]:
+    def loglikelihood_rolling(self, requests) -> list[float]:
         """Compute full log-likelihood of a string, with no truncation, for perplexity computation
         - We will use the full max context length of the model.
         - For inputs that exceed the max context length, we divide the tokenized string into chunks of up to
@@ -169,8 +174,7 @@ class LM(abc.ABC):
         - Instance of the LM class.
         """
 
-        additional_config = {} if additional_config is None else additional_config
-        additional_config = {
+        additional_config = additional_config or {} | {
             k: v for k, v in additional_config.items() if v is not None
         }
 
@@ -216,21 +220,21 @@ class LM(abc.ABC):
 
 
 ### SQLite-based caching of LM responses
-def hash_args(attr, args):
+def hash_args(attr: str, args: Iterable[Any]) -> str:
     dat = json.dumps([attr] + list(args))
     return hashlib.sha256(dat.encode("utf-8")).hexdigest()
 
 
 class CacheHook:
-    def __init__(self, cachinglm) -> None:
+    def __init__(self, cachinglm: Optional["CachingLM"]) -> None:
         """CacheHook is used to cache responses from the LM."""
         if cachinglm is None:
-            self.dbdict = None
+            self.dbdict: Optional["SqliteDict"] = None
             return
 
         self.dbdict = cachinglm.dbdict
 
-    def add_partial(self, attr, req, res) -> None:
+    def add_partial(self, attr: str, req: Iterable[Any], res: Any) -> None:
         """Adds a partial result to the cache."""
         if self.dbdict is None:
             return
@@ -247,8 +251,10 @@ class CachingLM:
         :param cache_db: str
             Path to cache db
         """
-        self.lm = lm
-        self.cache_db = cache_db
+        from sqlitedict import SqliteDict
+
+        self.lm: LM = lm
+        self.cache_db: str = cache_db
         if os.path.dirname(cache_db):
             os.makedirs(os.path.dirname(cache_db), exist_ok=True)
         self.dbdict = SqliteDict(cache_db, autocommit=True)
@@ -256,13 +262,13 @@ class CachingLM:
         # add hook to lm
         lm.set_cache_hook(self.get_cache_hook())
 
-    def __getattr__(self, attr: str):
+    def __getattr__(self, attr: str) -> Any:
         lm_attr = getattr(self.lm, attr)
         if attr not in ["loglikelihood", "loglikelihood_rolling", "generate_until"]:
             eval_logger.debug(f"Passing through attribute '{attr}' to underlying LM")
             return lm_attr
 
-        def fn(requests: list["Instance"]) -> list["Instance"]:
+        def _fn(requests: list["Instance"]) -> list["Instance"]:
             res = []
             remaining_reqs = []
             warned = False
@@ -315,9 +321,9 @@ class CachingLM:
 
             return res
 
-        return fn
+        return _fn
 
-    def get_cache_hook(self):
+    def get_cache_hook(self) -> "CacheHook":
         return CacheHook(self)
 
 
@@ -372,10 +378,8 @@ class TemplateLM(LM):
     def _encode_pair(
         self, context: str, continuation: str
     ) -> tuple[list[int], list[int]]:
-        """Encodes a pair of context and continuation strings into token IDs.
+        import transformers
 
-        We encode using encode(context+continuation) and then split into context and continuation.
-        """
         n_spaces = len(context) - len(context.rstrip())
         if n_spaces > 0:
             continuation = context[-n_spaces:] + continuation
@@ -428,7 +432,7 @@ class TemplateLM(LM):
         pass
 
     @abc.abstractmethod
-    def generate_until(self, requests, disable_tqdm: bool = False) -> list[str]:
+    def generate_until(self, requests: list[Instance], disable_tqdm: bool = False) -> list[str]:
         """Generate until a stopping sequence.
 
         Args:

lm_eval/evaluator.py

 import itertools
 import json
 import logging
+import os
 import random
 import time
 from collections import defaultdict
@@ -29,6 +30,7 @@ from lm_eval.loggers.utils import add_env_info, add_tokenizer_info, get_git_comm
 from lm_eval.tasks import TaskManager, get_task_dict
 from lm_eval.utils import (
     handle_non_serializable,
+    hash_dict_images,
     hash_string,
     positional_deprecated,
     setup_logging,
@@ -140,7 +142,6 @@ def simple_evaluate(
         Random seed for fewshot sampler random generator. If set to None, the seed of generator will be set to None.
     :param metadata: dict
         Additional metadata to be added to the task manager. Will get passed to the download function of the task.
-
     return
         Dictionary of results
     """
@@ -747,6 +748,12 @@ def evaluate(
             },
         }
         if log_samples:
+            # default: hash images
+            samples = (
+                hash_dict_images(samples)
+                if os.environ.get("LMEVAL_HASHMM", "1") != "0"
+                else samples
+            )
             results_dict["samples"] = dict(samples)
 
         return results_dict

lm_eval/models/__init__.py

@@ -10,7 +10,6 @@ from . import (
     ibm_watsonx_ai,
     mamba_lm,
     nemo_lm,
-    neuralmagic,
     neuron_optimum,
     openai_completions,
     optimum_ipex,

lm_eval/models/neuralmagic.py

-import copy
-import logging
-from typing import List, Optional, Tuple, Union
-
-import numpy
-import transformers
-from tqdm import tqdm
-
-import lm_eval.models.utils
-from lm_eval import utils
-from lm_eval.api.instance import Instance
-from lm_eval.api.model import LM
-from lm_eval.api.registry import register_model
-from lm_eval.models.huggingface import HFLM
-
-
-eval_logger = logging.getLogger(__name__)
-
-
-@register_model("sparseml")
-class SparseMLLM(HFLM):
-    """
-    SparseML is an open-source model optimization toolkit that enables you to create
-    inference-optimized sparse models using pruning, quantization, and distillation
-    algorithms. Models optimized with SparseML can then be exported to the ONNX format and
-    deployed with DeepSparse for GPU-class performance on CPU hardware.
-
-    This class is a wrapper around the HuggingFace LM class to enable SparseML
-    integration with the lm-evaluation-harness.
-    """
-
-    def _create_model(
-        self,
-        pretrained: str,
-        revision: Optional[str] = "main",
-        dtype: Optional[str] = "auto",
-        trust_remote_code: Optional[bool] = False,
-        **kwargs,
-    ) -> None:
-        try:
-            from sparseml.transformers import SparseAutoModelForCausalLM
-        except ModuleNotFoundError as exception:
-            raise type(exception)(
-                "Package `sparseml` is not installed. "
-                "Please install it via `pip install sparseml[transformers]`"
-            )
-
-        model_kwargs = kwargs if kwargs else {}
-
-        if "device_map" not in model_kwargs:
-            # set a device_map to initialize model on the right GPU.
-            # this is needed because it seems that the default behavior
-            # for quantized models now seems to be device_map="auto"
-            # which breaks data-parallel mode.
-            if hasattr(self, "accelerator"):
-                model_kwargs.update(
-                    {"device_map": {"": f"cuda:{self.accelerator.local_process_index}"}}
-                )
-            else:
-                model_kwargs.update({"device_map": {"": str(self.device)}})
-
-        relevant_kwarg_names = [
-            "offload_folder",
-            "device_map",
-        ]
-        relevant_kwargs = {
-            k: v for k, v in model_kwargs.items() if k in relevant_kwarg_names
-        }
-
-        # Log the difference between model_kwargs and relevant_kwargs so we can see
-        # what is being ignored
-        ignored_kwargs = {}
-        for k, v in model_kwargs.items():
-            if k not in relevant_kwargs.keys():
-                ignored_kwargs[k] = v
-        eval_logger.warning(
-            f"The sparseml integration is ignoring the following kwargs that are specified: {ignored_kwargs}"
-        )
-
-        model = SparseAutoModelForCausalLM.from_pretrained(
-            pretrained,
-            revision=revision,
-            torch_dtype=lm_eval.models.utils.get_dtype(dtype),
-            trust_remote_code=trust_remote_code,
-            **relevant_kwargs,
-        )
-        self._model = model
-
-    def _get_config(self, pretrained: str, **kwargs) -> None:
-        try:
-            from sparseml.transformers import SparseAutoConfig
-        except ModuleNotFoundError as exception:
-            raise type(exception)(
-                "Package `sparseml` is not installed. "
-                "Please install it via `pip install sparseml[transformers]`"
-            )
-
-        self._config = SparseAutoConfig.from_pretrained(
-            pretrained_model_name_or_path=pretrained, **kwargs
-        )
-
-    def _create_tokenizer(
-        self,
-        pretrained: Union[str, transformers.PreTrainedModel],
-        tokenizer: Optional[
-            Union[
-                str,
-                transformers.PreTrainedTokenizer,
-                transformers.PreTrainedTokenizerFast,
-            ]
-        ],
-        **kwargs,
-    ) -> None:
-        try:
-            from sparseml.transformers import SparseAutoTokenizer
-        except ModuleNotFoundError as exception:
-            raise type(exception)(
-                "Package `sparseml` is not installed. "
-                "Please install it via `pip install sparseml[transformers]`"
-            )
-
-        if tokenizer:
-            if isinstance(tokenizer, str):
-                self.tokenizer = SparseAutoTokenizer.from_pretrained(
-                    tokenizer,
-                    **kwargs,
-                )
-            else:
-                assert isinstance(
-                    tokenizer, transformers.PreTrainedTokenizer
-                ) or isinstance(tokenizer, transformers.PreTrainedTokenizerFast)
-                self.tokenizer = tokenizer
-        else:
-            # Get tokenizer based on 'pretrained'
-            if isinstance(pretrained, str):
-                model_name = pretrained
-            else:
-                # get the HF hub name via accessor on model
-                model_name = self.model.name_or_path
-            self.tokenizer = SparseAutoTokenizer.from_pretrained(
-                model_name,
-                **kwargs,
-            )
-        return None
-
-
-@register_model("deepsparse")
-class DeepSparseLM(LM):
-    """
-    Wrapper around DeepSparse, a sparsity-aware deep learning
-    inference runtime for CPUs, to make it compatible with the
-    lm-evaluation-harness.
-    """
-
-    _DEFAULT_MAX_LENGTH = 2048
-
-    def __init__(
-        self,
-        pretrained: str,
-        tokenizer: Optional[
-            Union[
-                str,
-                transformers.PreTrainedTokenizer,
-                transformers.PreTrainedTokenizerFast,
-            ]
-        ] = None,
-        batch_size: Optional[Union[int, str]] = 1,
-        max_gen_toks: Optional[int] = 256,
-        max_length: Optional[int] = None,
-    ):
-        super().__init__()
-
-        try:
-            import deepsparse
-        except ModuleNotFoundError as exception:
-            raise type(exception)(
-                "Package `deepsparse` is not installed. "
-                "Please install it via `pip install deepsparse[transformers]`"
-            )
-
-        if isinstance(batch_size, str) and not batch_size.isdigit():
-            eval_logger.warning(
-                f"batch_size={batch_size} is not valid for deepsparse because it is not an integer. "
-                "Ignoring and using the default of 1."
-            )
-            batch_size = 1
-
-        self.batch_size = int(batch_size)
-        self._max_length = max_length if max_length else self._DEFAULT_MAX_LENGTH
-        self._max_gen_toks = max_gen_toks
-        self.batch_sizes = {}
-
-        # Initialize new model and tokenizer instances
-        self.model = deepsparse.TextGeneration(
-            model_path=pretrained,
-            sequence_length=self._max_length,
-            batch_size=batch_size,
-        )
-        self.tokenizer = tokenizer if tokenizer else self.model.tokenizer
-        self.config = self.model.config
-
-    def tok_encode(self, string: str) -> List[int]:
-        return self.tokenizer.encode(string)
-
-    def tok_decode(self, tokens: List[int]) -> str:
-        return self.tokenizer.decode(tokens)
-
-    @property
-    def eot_token_id(self):
-        # we use EOT because end of *text* is more accurate for what we're doing than end of *sentence*
-        return self.tokenizer.eos_token_id
-
-    @property
-    def prefix_token_id(self):
-        # it is used as prefix for loglikelihood
-        if self.tokenizer.bos_token_id is not None:
-            return self.tokenizer.bos_token_id
-        return self.tokenizer.eos_token_id
-
-    @property
-    def max_length(self) -> int:
-        return self._max_length
-
-    @property
-    def max_gen_toks(self) -> int:
-        return self._max_gen_toks
-
-    def loglikelihood(self, requests) -> List[Tuple[float, bool]]:
-        """
-        Copied directly from
-        https://github.com/EleutherAI/lm-evaluation-harness/blob/main/lm_eval/models/huggingface.py
-        """
-        new_reqs = []
-        for context, continuation in [req.args for req in requests]:
-            if context == "":
-                raise NotImplementedError(
-                    "Implementing empty context is not supported yet"
-                )
-            context_enc, continuation_enc = self._encode_pair(context, continuation)
-
-            new_reqs.append(((context, continuation), context_enc, continuation_enc))
-
-        return self._loglikelihood_tokens(new_reqs)
-
-    def _loglikelihood_tokens(
-        self,
-        requests: List[Tuple[Tuple[str, str], List[int], List[int]]],
-        disable_tqdm: bool = False,
-    ) -> List[Tuple[float, bool]]:
-        """
-        The function to compute the loglikelihood of the continuation
-        tokens given the context tokens.
-
-        This function is an adapted version of the original function from
-        https://github.com/EleutherAI/lm-evaluation-harness/blob/main/lm_eval/models/huggingface.py
-        """
-        res = []
-
-        def _collate(x):
-            """Defines the key for the sorted method"""
-            toks = x[1] + x[2]
-            return -len(toks), tuple(toks)
-
-        re_ord = utils.Reorderer(requests, _collate)
-
-        for chunk in tqdm(
-            list(lm_eval.models.utils.chunks(re_ord.get_reordered(), self.batch_size)),
-            disable=disable_tqdm,
-        ):
-            batch_inp = []
-            batch_cache_key = []
-            batch_continuation_enc = []
-            # len(chunk) is the batch_size
-            for cache_key, context_enc, continuation_enc in chunk:
-                # how this all works (illustrated on a causal decoder-only setup):
-                #          CTX      CONT
-                # inp    0 1 2 3|4 5 6 7 8 9   <- last token is deleted by inp[:, :-1]
-                # model  \               \
-                # logits   1 2 3|4 5 6 7 8 9   <- the ctx half gets tossed out by the
-                # cont_toks      4 5 6 7 8 9      [:, -len(continuation_enc):, :self.vocab_size] slice # noqa: E501
-
-                inp = (context_enc + continuation_enc)[-(self.max_length + 1) :][:-1]
-
-                batch_inp.append(self.tokenizer.decode(inp))
-                batch_cache_key.append(cache_key)
-                batch_continuation_enc.append(continuation_enc)
-
-            response = self.model(
-                prompt=batch_inp,
-                max_new_tokens=0,
-                output_scores=True,
-                include_prompt_logits=True,
-            )
-
-            for resp, continuation_enc, cache_key in zip(
-                response.generations, batch_continuation_enc, batch_cache_key
-            ):
-                # (seq_len, vocab_size)
-                multi_scores = resp.score
-
-                from deepsparse.utils.data import numpy_log_softmax
-
-                # (seq_len, vocab_size) but with softmax applied
-                multi_logits = numpy_log_softmax(multi_scores, axis=1)
-                # toss out the context half of the sequence
-                # (cont_len, vocab_size)
-                continuation_multi_logits = multi_logits[-len(continuation_enc) :]
-
-                # pick out the logits for the continuation tokens
-                # (cont_len,)
-                continuation_logits = continuation_multi_logits[
-                    numpy.arange(len(continuation_enc)), continuation_enc
-                ]
-                # check if the tokens generated greedly are the same
-                # as the expected continuation
-                greedy_tokens = continuation_multi_logits.argmax(axis=1)
-                max_equal = greedy_tokens.tolist() == continuation_enc
-
-                # Answer: (log prob, is-exact-match)
-                answer = (float(continuation_logits.sum()), bool(max_equal))
-
-                res.append(answer)
-
-                if cache_key is not None:
-                    # special case: loglikelihood_rolling produces a number of loglikelihood requests
-                    # all with cache key None. instead do add_partial on the per-example level
-                    # in the loglikelihood_rolling() function for those.
-                    self.cache_hook.add_partial("loglikelihood", cache_key, answer)
-
-        return re_ord.get_original(res)
-
-    def loglikelihood_rolling(self, requests: List[Instance]) -> List[float]:
-        raise NotImplementedError(
-            "The method not required by any of our current task integrations so far"
-        )
-
-    def generate_until(self, requests: List[Instance]) -> List[str]:
-        """
-        The function to generate a certain number of new tokens
-        given a context.
-
-        This function is an adapted version of the original function from
-        https://github.com/EleutherAI/lm-evaluation-harness/blob/main/lm_eval/models/openai_completions.py
-        """
-        if not requests:
-            return []
-        res = []
-        requests = [req.args for req in requests]
-
-        def _collate(x):
-            toks = self.tok_encode(x[0])
-            return len(toks), x[0]
-
-        re_ord = utils.Reorderer(requests, _collate)
-
-        def sameuntil_chunks(xs, size):
-            ret = []
-            lastuntil = xs[0][1]
-            for x in xs:
-                if len(ret) >= size or x[1] != lastuntil:
-                    yield ret, lastuntil
-                    ret = []
-                    lastuntil = x[1]
-                ret.append(x)
-
-            if ret:
-                yield ret, lastuntil
-
-        pbar = tqdm(total=len(requests))
-        for chunk, request_args in tqdm(
-            list(sameuntil_chunks(re_ord.get_reordered(), self.batch_size))
-        ):
-            inps = []
-
-            # make a deepcopy since we are changing arguments
-            request_args = copy.deepcopy(request_args)
-
-            self._max_gen_toks = request_args.pop("max_gen_toks", self.max_gen_toks)
-
-            for context, _ in chunk:
-                # add context (prompts) to the list
-                inps.append(context)
-
-            until = request_args.pop("until", ["<|endoftext|>"])
-            request_args.pop("do_sample", None)
-            request_args["temperature"] = request_args.get("temperature", 0)
-
-            # run inference (generate max_gen_toks tokens)
-            out = self.model(
-                sequences=inps,
-                max_new_tokens=self.max_gen_toks - 1,
-                stop=until,
-                **request_args,
-            )
-
-            for resp, (context, args_) in zip(out.generations, chunk):
-                text = resp.text
-                until_ = until
-                # split the text at the first occurrence of any of the until tokens
-                for term in until_:
-                    if len(term) > 0:
-                        text = text.split(term)[0]
-
-                res.append(text)
-
-                self.cache_hook.add_partial(
-                    "generate_until", (context, {"until": until_}), text
-                )
-                pbar.update(1)
-
-        pbar.close()
-
-        return re_ord.get_original(res)
-
-    def _encode_pair(
-        self, context: str, continuation: str
-    ) -> Tuple[List[int], List[int]]:
-        """
-        Copied directly from
-        https://github.com/EleutherAI/lm-evaluation-harness/blob/main/lm_eval/models/huggingface.py
-        """
-        n_spaces = len(context) - len(context.rstrip())
-        if n_spaces > 0:
-            continuation = context[-n_spaces:] + continuation
-            context = context[:-n_spaces]
-        whole_enc = self.tok_encode(context + continuation)
-        context_enc = self.tok_encode(context)
-        context_enc_len = len(context_enc)
-        continuation_enc = whole_enc[context_enc_len:]
-        return context_enc, continuation_enc

lm_eval/utils.py

@@ -553,3 +553,54 @@ def weighted_f1_score(items):
     preds = unzipped_list[1]
     fscore = f1_score(golds, preds, average="weighted")
     return fscore
+
+
+def convert_pil_to_hash(value):
+    from io import BytesIO
+
+    img_bytes = BytesIO()
+    value.save(img_bytes, format="PNG")
+    return hashlib.sha256(str(img_bytes).encode()).hexdigest()
+
+
+def convert_bytes_to_hash(value):
+    return hashlib.sha256(str(value).encode()).hexdigest()
+
+
+def hash_dict_images(data_dict):
+    """
+    Create a deep copy of `data_dict` where all bytes and PIL.Image.Image values
+    are replaced by their respective hashes using the provided converter functions.
+
+    Parameters:
+        data_dict (dict): The input dictionary with arbitrary nesting of dicts and lists.
+
+    Returns:
+        dict: A new dictionary with the same structure as `data_dict`, but with all
+              bytes and PIL.Image.Image objects replaced by their hashes.
+    """
+    from PIL import Image
+
+    def _process_value(value):
+        # Bytes -> hash
+        if isinstance(value, (bytes, bytearray)):
+            return convert_bytes_to_hash(value)
+        # PIL Image -> hash
+        if isinstance(value, Image.Image):
+            return convert_pil_to_hash(value)
+        # Nested dictionary -> recurse
+        if isinstance(value, dict):
+            return {k: _process_value(v) for k, v in value.items()}
+        # List or tuple -> recurse, preserving type
+        if isinstance(value, list):
+            return [_process_value(v) for v in value]
+        if isinstance(value, tuple):
+            return tuple(_process_value(v) for v in value)
+        # Other types remain unchanged
+        return value
+
+    # Ensure the top-level is a dict
+    if not isinstance(data_dict, dict):
+        raise TypeError("Input must be a dictionary")
+
+    return {key: _process_value(val) for key, val in data_dict.items()}

mypy.ini

-[mypy]
-python_version = 3.8
-show_traceback = True
-check_untyped_defs = True
-no_implicit_reexport = True
-warn_unreachable = True
-warn_unused_configs = True
-warn_unused_ignores = True
-warn_redundant_casts = True
-
-# We ignore errors everywhere to gradually add type annotations
-
-[mypy-lm_eval.*]
-ignore_errors = True
-
-[mypy-lm_eval.api.*]
-ignore_errors = True
-
-[mypy-lm_eval.prompts.*]
-ignore_errors = True
-
-[mypy-lm_eval.models.*]
-ignore_errors = True
-
-[mypy-scripts.*]
-ignore_errors = True
-
-[mypy-main]
-ignore_errors = True

pyproject.toml

@@ -60,8 +60,7 @@ Repository = "https://github.com/EleutherAI/lm-evaluation-harness"
 acpbench = ["lark>=1.1.9", "tarski[clingo]==0.8.2", "pddl==0.4.2", "kstar-planner==1.4.2"]
 api = ["requests", "aiohttp", "tenacity", "tqdm", "tiktoken"]
 audiolm_qwen = ["librosa", "soundfile"]
-deepsparse = ["deepsparse-nightly[llm]>=1.8.0.20240404"]
-dev = ["pytest", "pytest-cov", "pytest-xdist", "pre-commit", "mypy", "unitxt==1.22.0", "requests", "aiohttp", "tenacity", "tqdm", "tiktoken", "sentencepiece"]
+dev = ["pytest", "pytest-cov", "pytest-xdist", "pre-commit", "requests", "aiohttp", "tenacity", "tqdm", "tiktoken", "sentencepiece"]
 gptq = ["auto-gptq[triton]>=0.6.0"]
 gptqmodel = ["gptqmodel>=1.0.9"]
 hf_transfer = ["hf_transfer"]
@@ -79,41 +78,20 @@ promptsource = ["promptsource>=0.2.3"]
 ruler = ["nltk", "wonderwords", "scipy"]
 sae_lens = ["sae_lens"]
 sentencepiece = ["sentencepiece>=0.1.98"]
-sparseml = ["sparseml-nightly[llm]>=1.8.0.20240404"]
 sparsify = ["sparsify"]
 testing = ["pytest", "pytest-cov", "pytest-xdist"]
+unitxt = ["unitxt==1.22.0"]
 vllm = ["vllm>=0.4.2"]
 wandb = ["wandb>=0.16.3", "pandas", "numpy"]
 zeno = ["pandas", "zeno-client"]
-all = [
+tasks = [
     "lm_eval[acpbench]",
-    "lm_eval[api]",
-    "lm_eval[audiolm_qwen]",
-    "lm_eval[deepsparse]",
-    "lm_eval[dev]",
-    "lm_eval[gptq]",
-    "lm_eval[gptqmodel]",
-    "lm_eval[hf_transfer]",
-    "lm_eval[ibm_watsonx_ai]",
     "lm_eval[ifeval]",
-    "lm_eval[ipex]",
     "lm_eval[japanese_leaderboard]",
     "lm_eval[longbench]",
-    "lm_eval[mamba]",
     "lm_eval[math]",
     "lm_eval[multilingual]",
-    "lm_eval[neuronx]",
-    "lm_eval[optimum]",
-    "lm_eval[promptsource]",
     "lm_eval[ruler]",
-    "lm_eval[sae_lens]",
-    "lm_eval[sentencepiece]",
-    "lm_eval[sparseml]",
-    "lm_eval[sparsify]",
-    "lm_eval[testing]",
-    "lm_eval[vllm]",
-    "lm_eval[wandb]",
-    "lm_eval[zeno]",
 ]
 
 [tool.pymarkdown]

scripts/zeno_visualize.py

@@ -4,6 +4,7 @@ import logging
 import os
 import re
 from pathlib import Path
+from typing import Union
 
 import pandas as pd
 from zeno_client import ZenoClient, ZenoMetric
@@ -35,6 +36,22 @@ def parse_args():
     return parser.parse_args()
 
 
+def sanitize_string(model_args_raw: Union[str, dict]) -> str:
+    """Sanitize the model_args string or dict"""
+    # Convert to string if it's a dictionary
+    model_args_str = (
+        json.dumps(model_args_raw)
+        if isinstance(model_args_raw, dict)
+        else model_args_raw
+    )
+    # Apply the sanitization
+    return re.sub(
+        r"[\"<>:/|\\?*\[\]]+",
+        "__",
+        model_args_str,
+    )
+
+
 def main():
     """Upload the results of your benchmark tasks to the Zeno AI evaluation platform.
 
@@ -87,13 +104,16 @@ def main():
             latest_sample_results = get_latest_filename(
                 [Path(f).name for f in model_sample_filenames if task in f]
             )
-            model_args = re.sub(
-                r"[\"<>:/\|\\?\*\[\]]+",
-                "__",
+            # Load the model_args, which can be either a string or a dictionary
+            model_args = sanitize_string(
                 json.load(
-                    open(Path(args.data_path, model, latest_results), encoding="utf-8")
-                )["config"]["model_args"],
+                    open(
+                        Path(args.data_path, model, latest_results),
+                        encoding="utf-8",
+                    )
+                )["config"]["model_args"]
             )
+
             print(model_args)
             data = []
             with open(

tests/models/test_neuralmagic.py

-import pytest
-
-from lm_eval import evaluator
-from lm_eval.api.registry import get_model
-
-
-SPARSEML_MODELS_TASKS = [
-    # loglikelihood
-    ("facebook/opt-125m", "lambada_openai"),
-    # loglikelihood_rolling
-    ("hf-internal-testing/tiny-random-gpt2", "wikitext"),
-    # generate_until
-    ("mgoin/tiny-random-llama-2-quant", "gsm8k"),
-]
-
-DEEPSPARSE_MODELS_TASKS = [
-    # loglikelihood
-    ("hf:mgoin/llama2.c-stories15M-quant-ds", "lambada_openai"),
-    # loglikelihood_rolling (not supported yet)
-    # ("hf:mgoin/llama2.c-stories15M-quant-ds", "wikitext"),
-    # generate_until
-    ("hf:mgoin/llama2.c-stories15M-quant-ds", "gsm8k"),
-]
-
-
-@pytest.mark.skip(reason="test failing")
-@pytest.mark.parametrize("model_id,task", SPARSEML_MODELS_TASKS)
-def test_sparseml_eval(model_id, task):
-    lm = get_model("sparseml").create_from_arg_string(
-        f"pretrained={model_id}",
-        {
-            "batch_size": 1,
-            "device": "cpu",
-            "dtype": "float32",
-        },
-    )
-
-    limit = 5
-    evaluator.simple_evaluate(
-        model=lm,
-        tasks=[task],
-        num_fewshot=0,
-        limit=limit,
-    )
-
-
-@pytest.mark.parametrize("model_id,task", DEEPSPARSE_MODELS_TASKS)
-def test_deepsparse_eval(model_id, task):
-    lm = get_model("deepsparse").create_from_arg_string(
-        f"pretrained={model_id}",
-        {
-            "batch_size": 1,
-        },
-    )
-
-    limit = 5
-    evaluator.simple_evaluate(
-        model=lm,
-        tasks=[task],
-        num_fewshot=0,
-        limit=limit,
-    )

tests/scripts/test_zeno_visualize.py

+import json
+import re
+
+import pytest
+
+from scripts.zeno_visualize import sanitize_string
+
+
+@pytest.skip("requires zeno_client dependency")
+def test_zeno_sanitize_string():
+    """
+    Test that the model_args handling logic in zeno_visualize.py properly handles
+    different model_args formats (string and dictionary).
+    """
+
+    # Define the process_model_args function that replicates the fixed logic in zeno_visualize.py
+    # Test case 1: model_args as a string
+    string_model_args = "pretrained=EleutherAI/pythia-160m,dtype=float32"
+    result_string = sanitize_string(string_model_args)
+    expected_string = re.sub(r"[\"<>:/\|\\?\*\[\]]+", "__", string_model_args)
+
+    # Test case 2: model_args as a dictionary
+    dict_model_args = {"pretrained": "EleutherAI/pythia-160m", "dtype": "float32"}
+    result_dict = sanitize_string(dict_model_args)
+    expected_dict = re.sub(r"[\"<>:/\|\\?\*\[\]]+", "__", json.dumps(dict_model_args))
+
+    # Verify the results
+    assert result_string == expected_string
+    assert result_dict == expected_dict
+
+    # Also test that the sanitization works as expected
+    assert ":" not in result_string  # No colons in sanitized output
+    assert ":" not in result_dict  # No colons in sanitized output
+    assert "/" not in result_dict  # No slashes in sanitized output
+    assert "<" not in result_dict  # No angle brackets in sanitized output
+
+
+if __name__ == "__main__":
+    test_zeno_sanitize_string()
+    print("All tests passed.")

tests/test_metrics.py

+import unittest.mock as mock
+
+from lm_eval.api.metrics import _bootstrap_internal_no_mp, mean
 from lm_eval.api.task import ConfigurableTask
 from lm_eval.config.task import TaskConfig
 
@@ -150,8 +153,34 @@ def test_acc_mutual_info_without_metric():
     assert result_dict["acc"] == 1.0
 
 
+def test_bootstrap_internal_no_mp():
+    """Test basic functionality of _bootstrap_internal_no_mp"""
+
+    data = [1, 2, 3, 4, 5]
+
+    # Mock tqdm to avoid progress bar output during testing
+    with mock.patch("tqdm.tqdm") as mock_tqdm:
+        mock_tqdm.return_value = range(1)  # Single chunk
+
+        # Mock print to avoid output during testing
+        with mock.patch("builtins.print"):
+            result = _bootstrap_internal_no_mp(mean, data, 100)
+
+    # Should return 100 bootstrap replicates
+    assert len(result) == 100
+
+    # All results should be numbers (means)
+    assert all(isinstance(x, (int, float)) for x in result)
+
+    # Bootstrap means should be close to original mean
+    bootstrap_mean = mean(result)
+    original_mean = mean(data)
+    assert abs(bootstrap_mean - original_mean) < 0.5  # Should be reasonably close
+
+
 if __name__ == "__main__":
     test_acc_mutual_info_slicing()
     test_acc_mutual_info_different_predictions()
     test_acc_mutual_info_without_metric()
+    test_bootstrap_internal_no_mp()
     print("All tests passed!")