Run doctest for new files (#25588)

fix
Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>

utils/check_doctest_list.py

@@ -37,7 +37,7 @@ import os
 # All paths are set with the intent you should run this script from the root of the repo with the command
 # python utils/check_doctest_list.py
 REPO_PATH = "."
-DOCTEST_FILE_PATHS = ["documentation_tests.txt", "slow_documentation_tests.txt"]
+DOCTEST_FILE_PATHS = ["not_doctested.txt", "slow_documentation_tests.txt"]
 
 
 def clean_doctest_list(doctest_file: str, overwrite: bool = False):
@@ -62,7 +62,7 @@ def clean_doctest_list(doctest_file: str, overwrite: bool = False):
 
     if len(non_existent_paths) > 0:
         non_existent_paths = "\n".join([f"- {f}" for f in non_existent_paths])
-        raise ValueError(f"`utils/documentation_tests.txt` contains non-existent paths:\n{non_existent_paths}")
+        raise ValueError(f"`{doctest_file}` contains non-existent paths:\n{non_existent_paths}")
 
     sorted_paths = sorted(all_paths)
     if all_paths != sorted_paths:

utils/tests_fetcher.py

@@ -396,17 +396,22 @@ def get_doctest_files(diff_with_last_commit: bool = False) -> List[str]:
             print(f"Parent commit: {commit}")
         test_files_to_run = get_diff_for_doctesting(repo, repo.head.commit, parent_commits)
 
-    # This is the full list of doctest tests
-    with open("utils/documentation_tests.txt") as fp:
-        documentation_tests = set(fp.read().strip().split("\n"))
+    # These are files not doctested yet.
+    with open("utils/not_doctested.txt") as fp:
+        not_doctested = set(fp.read().strip().split("\n"))
     # Do not run slow doctest tests
     with open("utils/slow_documentation_tests.txt") as fp:
         slow_documentation_tests = set(fp.read().strip().split("\n"))
 
     # So far we don't have 100% coverage for doctest. This line will be removed once we achieve 100%.
-    test_files_to_run = [
-        x for x in test_files_to_run if x in documentation_tests and x not in slow_documentation_tests
-    ]
+    test_files_to_run = [x for x in test_files_to_run if x not in not_doctested and x not in slow_documentation_tests]
+
+    # The file `utils/not_doctested.txt` doesn't contain all files that are not doc-tested, so we need more filters.
+    # 1. only include files in `src` or `docs/source/en/`
+    test_files_to_run = [x for x in test_files_to_run if x.startswith(("src/", "docs/source/en/"))]
+    # 2. not include init files
+    test_files_to_run = [x for x in test_files_to_run if not x.endswith(("__init__.py",))]
+
     # Make sure we did not end up with a test file that was removed
     test_files_to_run = [f for f in test_files_to_run if (PATH_TO_REPO / f).exists()]