add __version__ (#2808)

* add __version__

* add version consistency check to publish action

.github/workflows/publish.yml

@@ -17,6 +17,25 @@ jobs:
       with:
         python-version: "3.x"
 
+    - name: Check version consistency
+      run: |
+        # Extract version from pyproject.toml
+        PYPROJECT_VERSION=$(grep 'version = ' pyproject.toml | head -1 | cut -d'"' -f2)
+
+        # Extract version from __init__.py
+        INIT_VERSION=$(grep '__version__ = ' lm_eval/__init__.py | head -1 | cut -d'"' -f2)
+
+        echo "Version in pyproject.toml: $PYPROJECT_VERSION"
+        echo "Version in __init__.py: $INIT_VERSION"
+
+        # Check if versions match
+        if [ "$PYPROJECT_VERSION" != "$INIT_VERSION" ]; then
+          echo "Error: Version mismatch between pyproject.toml ($PYPROJECT_VERSION) and __init__.py ($INIT_VERSION)"
+          exit 1
+        fi
+
+        echo "Version check passed: $PYPROJECT_VERSION"
+
     - name: Install pypa/build
       run: >-
         python3 -m

.github/workflows/unit_tests.yml

@@ -31,11 +31,7 @@ jobs:
     - name: Pre-Commit
       env:
         SKIP: "no-commit-to-branch,mypy"
-
       uses: pre-commit/action@v3.0.1
-#       # mypy turned off for now
-#    - name: Lint with mypy
-#      run: mypy . --ignore-missing-imports --check-untyped-defs --explicit-package-bases --warn-unreachable
 # Job 2
   testcpu:
     name: CPU Tests
@@ -57,9 +53,6 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install -e '.[dev,sentencepiece,api]' --extra-index-url https://download.pytorch.org/whl/cpu
-#         Install optional git dependencies
-#                pip install bleurt@https://github.com/google-research/bleurt/archive/b610120347ef22b494b6d69b4316e303f5932516.zip#egg=bleurt
-#        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
     - name: Test with pytest
       run: python -m pytest --showlocals -s -vv -n=auto --ignore=tests/models/test_neuralmagic.py --ignore=tests/models/test_openvino.py --ignore=tests/models/test_hf_steered.py
     - name: Archive artifacts

lm_eval/__init__.py

@@ -2,3 +2,6 @@ import logging
 import os
 
 from .evaluator import evaluate, simple_evaluate
+
+
+__version__ = "0.4.8"

lm_eval/loggers/utils.py

@@ -2,6 +2,7 @@ import logging
 import os
 import re
 import subprocess
+from importlib.metadata import version
 from pathlib import Path
 from typing import Any, Dict, Optional, Tuple, Union
 
@@ -100,6 +101,10 @@ def add_env_info(storage: Dict[str, Any]):
         pretty_env_info = get_pretty_env_info()
     except Exception as err:
         pretty_env_info = str(err)
+    try:
+        lm_eval_version = version("lm_eval")
+    except Exception as err:
+        lm_eval_version = str(err)
     transformers_version = trans_version
     upper_dir_commit = get_commit_from_path(
         Path(os.getcwd(), "..")
@@ -107,6 +112,7 @@ def add_env_info(storage: Dict[str, Any]):
     added_info = {
         "pretty_env_info": pretty_env_info,
         "transformers_version": transformers_version,
+        "lm_eval_version": lm_eval_version,
         "upper_git_hash": upper_dir_commit,  # in case this repo is submodule
     }
     storage.update(added_info)