Properly use test_fetcher for examples (#13604)

* Properly use test_fetcher for examples

* Fake example modification

* Fake modeling file modification

* Clean fake modifications

* Run example tests for any modification.

.circleci/config.yml

@@ -564,18 +564,47 @@ jobs:
                   key: v0.4-torch_examples-{{ checksum "setup.py" }}
                   paths:
                       - '~/.cache/pip'
-            - run: python utils/tests_fetcher.py | tee test_preparation.txt
+            - run: python utils/tests_fetcher.py --filters examples tests | tee test_preparation.txt
             - store_artifacts:
                   path: ~/transformers/test_preparation.txt
             - run: |
                   if [ -f test_list.txt ]; then
-                    TRANSFORMERS_IS_CI=1 python -m pytest -n 8 --dist=loadfile -s --make-reports=examples_torch ./examples/pytorch/ | tee examples_output.txt
+                    python -m pytest -n 8 --dist=loadfile -s --make-reports=examples_torch ./examples/pytorch/ | tee tests_output.txt
                   fi
             - store_artifacts:
                   path: ~/transformers/examples_output.txt
             - store_artifacts:
                   path: ~/transformers/reports
     
+    run_examples_torch_all:
+        working_directory: ~/transformers
+        docker:
+            - image: circleci/python:3.6
+        environment:
+            OMP_NUM_THREADS: 1
+            TRANSFORMERS_IS_CI: yes
+        resource_class: xlarge
+        parallelism: 1
+        steps:
+            - checkout
+            - restore_cache:
+                  keys:
+                      - v0.4-torch_examples-{{ checksum "setup.py" }}
+                      - v0.4-{{ checksum "setup.py" }}
+            - run: pip install --upgrade pip
+            - run: pip install .[sklearn,torch,sentencepiece,testing]
+            - run: pip install -r examples/pytorch/_tests_requirements.txt
+            - save_cache:
+                  key: v0.4-torch_examples-{{ checksum "setup.py" }}
+                  paths:
+                      - '~/.cache/pip'
+            - run: |
+                  TRANSFORMERS_IS_CI=1 python -m pytest -n 8 --dist=loadfile -s --make-reports=examples_torch ./examples/pytorch/ | tee examples_output.txt
+            - store_artifacts:
+                  path: ~/transformers/examples_output.txt
+            - store_artifacts:
+                  path: ~/transformers/reports
+
     run_tests_hub:
         working_directory: ~/transformers
         docker:
@@ -901,6 +930,7 @@ workflows:
                         only:
                             - master
         jobs:
+            - run_examples_torch_all
             - run_tests_torch_and_tf_all
             - run_tests_torch_and_flax_all
             - run_tests_torch_all

utils/tests_fetcher.py

@@ -426,6 +426,9 @@ def infer_tests_to_run(output_file, diff_with_last_commit=False, filters=None):
             # Modified test files are always added
             if f.startswith("tests/"):
                 test_files_to_run.append(f)
+            # Example files are tested separately
+            elif f.startswith("examples/pytorch"):
+                test_files_to_run.append("examples/pytorch/test_examples.py")
             else:
                 new_tests = module_to_test_file(f)
                 if new_tests is not None:
@@ -462,7 +465,11 @@ if __name__ == "__main__":
         help="To fetch the tests between the current commit and the last commit",
     )
     parser.add_argument(
-        "--filters", type=str, nargs="*", help="Only keep the test files matching one of those filters."
+        "--filters",
+        type=str,
+        nargs="*",
+        default=["tests"],
+        help="Only keep the test files matching one of those filters.",
     )
     args = parser.parse_args()
     if args.sanity_check: