name: Tasks Modified

on:
  pull_request:
    branches:
      - 'main'
  workflow_dispatch:
# comment/edit out the above to stop/change the triggers
jobs:
  changed_files:
    runs-on: ubuntu-latest  # windows-latest || macos-latest
    timeout-minutes: 240
    name: Scan for changed tasks
    steps:
      - name: checkout
        uses: actions/checkout@v3
        with:
          fetch-depth: 2  # OR "2" -> To retrieve the preceding commit.

      # Uses the tj-actions/changed-files@v37 action to check for changes.
      # Outputs provided here: https://github.com/tj-actions/changed-files#outputs
      # The `files_yaml` input optionally takes a yaml string to specify filters,
      # and prepends the filter name to the standard output names.
      - name: Check task folders
        id: changed-tasks
        uses: tj-actions/changed-files@v37.1.2
        with:
          # tasks checks the tasks folder and api checks the api folder for changes
          files_yaml: |
            tasks:
              - lm_eval/tasks/**
            api:
              - lm_eval/api/**
          write_output_files: true

    # The next step is optional; the files are written to the workspace by default (above).
    # so it's just for debugging
      - name: Run Tests
        if: steps.changed-tasks.outputs.tasks_any_modified == 'true' || steps.changed-tasks.outputs.api_any_modified == 'true'
        run: |
          echo .github/outputs/tasks_all_changed_and_modified_files.txt >> 'GITHUB_ENV'
          echo "One or more test file(s) has changed."
          echo "List of all the files that have changed: ${{ steps.changed-tasks.outputs.tasks_all_modified_files }}"

      - name: Set up Python 3.9
        if: steps.changed-tasks.outputs.tasks_any_modified == 'true' || steps.changed-tasks.outputs.api_any_modified == 'true'
        uses: actions/setup-python@v4
        with:
          python-version: 3.9
          cache: 'pip'
          cache-dependency-path: setup.py
      - name: Install dependencies
        if: steps.changed-tasks.outputs.tasks_any_modified == 'true' || steps.changed-tasks.outputs.api_any_modified == 'true'
        run: |
            python -m pip install --upgrade pip
            pip install -e '.[dev]' --extra-index-url https://download.pytorch.org/whl/cpu
      - name: Test with pytest
        id: test_pytest
        # if new tasks are added, run tests on them
        if: steps.changed-tasks.outputs.tasks_any_modified == 'true'
        run: python -m pytest tests/test_tasks.py -s -vv
        # Edit task table if new tasks added
        # copied from gpt-neox repo
      - name: Update Docs
        if: steps.changed-tasks.outputs.tasks_any_modified == 'true' && steps.test_pytest.outcome == 'success'
        run: |
          python scripts/make_table_tasks.py
          git config user.name github-actions
          git config user.email github-actions@github.com
          git add docs/task_table.md
          git add docs/task_table.csv
          git commit -m "Update Task Table automatically"
          git push
        # if api is modified, run tests on it
      - name: Test more tasks with pytest
        env:
          API: true
        if: steps.changed-tasks.outputs.api_any_modified == 'true'
        run: python -m pytest tests/test_tasks.py -s -vv