add CI test case if api/* files changed then pytest more tasks

.github/workflows/new_tasks.yml

@@ -27,22 +27,24 @@ jobs:
           files_yaml: |
             tasks:
               - lm_eval/tasks/**
+            api:
+              - lm_eval/api/**
           write_output_files: true
 
       - name: Run Tests
-        if: steps.changed-tasks.outputs.tasks_any_modified == 'true'
+        if: steps.changed-tasks.outputs.tasks_any_modified == 'true' || steps.changed-tasks.outputs.api_any_modified == 'true'
         run: |
           echo .github/outputs/tasks_all_changed_and_modified_files.txt >> 'GITHUB_ENV'
           echo "One or more test file(s) has changed."
           echo "List of all the files that have changed: ${{ steps.changed-tasks.outputs.tasks_all_modified_files }}"
       - name: Set up Python 3.9
-        if: steps.changed-tasks.outputs.tasks_any_modified == 'true'
+        if: steps.changed-tasks.outputs.tasks_any_modified == 'true' || steps.changed-tasks.outputs.api_any_modified == 'true'
         uses: actions/setup-python@v4
         with:
           python-version: 3.9
           cache: 'pip'
       - name: Install dependencies
-        if: steps.changed-tasks.outputs.tasks_any_modified == 'true'
+        if: steps.changed-tasks.outputs.tasks_any_modified == 'true' || steps.changed-tasks.outputs.api_any_modified == 'true'
         run: |
             python -m pip install --upgrade pip
             pip install -e '.[testing]' --extra-index-url https://download.pytorch.org/whl/cpu
@@ -52,3 +54,8 @@ jobs:
       - name: Test with pytest
         if: steps.changed-tasks.outputs.tasks_any_modified == 'true'
         run: python -m pytest tests/test_tasks.py -s -vv -n=auto --new_task
+      - name: Test more tasks with pytest
+        env:
+          API: true
+        if: steps.changed-tasks.outputs.api_any_modified == 'true'
+        run: python -m pytest tests/test_api.py -s -vv -n=auto --new_task

tests/test_tasks.py

@@ -14,11 +14,12 @@ def any_new_tasks(request) -> bool:
 # ["arc_easy] else get list of new tasks
 def new_tasks(any_new_tasks: bool) -> List[str]:
     FILENAME = ".github/outputs/tasks_all_changed_and_modified_files.txt"
-    return [
-        parser(load_changed_files(FILENAME))
-        if any_new_tasks and os.path.exists(FILENAME)
-        else "arc_easy"
-    ]
+    if any_new_tasks and os.path.exists(FILENAME):
+        return [parser(load_changed_files(FILENAME))]
+    elif "API" in os.getenv():
+        return ["arc_easy", "hellaswag", "piqa", "wikitext"]
+    else:
+        return ["arc_easy"]
 
 
 @pytest.fixture(params=new_tasks(any_new_tasks))