Merge pull request #542 from EleutherAI/promptsource-extra

[Refactor] Make promptsource an extra / not required for installation

lm_eval/prompts/__init__.py

 from lm_eval.logger import eval_logger
-from promptsource.templates import DatasetTemplates
 
-# TODO: decide whether we want jinja2 or f-string prompts. would it be cursed to support both?
 # Prompt library.
 # Stores prompts in a dictionary indexed by 2 levels:
 # prompt category name, and prompt name.
@@ -23,6 +21,13 @@ def get_prompt(prompt_id: str, dataset_name=None, subset_name=None):
         dataset_full_name = f"{dataset_name}-{subset_name}"
     eval_logger.info(f"Loading prompt from {category_name} for {dataset_full_name}")
     if category_name == "promptsource":
+        try:
+            from promptsource.templates import DatasetTemplates
+        except ModuleNotFoundError:
+            raise Exception(
+                "Tried to load a Promptsource template, but promptsource is not installed ",
+                "please install promptsource via pip install lm-eval[promptsource] or pip install -e .[promptsource]",
+            )
         try:
             if subset_name is None:
                 prompts = DatasetTemplates(dataset_name=dataset_name)

setup.py

@@ -44,5 +44,8 @@ setuptools.setup(
         "dev": ["black", "flake8", "pre-commit", "pytest", "pytest-cov"],
         "multilingual": ["nagisa>=0.2.7", "jieba>=0.42.1"],
         "sentencepiece": ["sentencepiece>=0.1.98", "protobuf>=4.22.1"],
+        "promptsource": [
+            "promptsource @ git+https://github.com/bigscience-workshop/promptsource.git#egg=promptsource"
+        ],
     },
 )