"docs/vscode:/vscode.git/clone" did not exist on "71ca79448cd334970fa2893f4faaa094ca13ca6f"
Unverified Commit 9ac586b3 authored by Sylvain Gugger's avatar Sylvain Gugger Committed by GitHub
Browse files

Rework pipeline tests (#19366)

* Rework pipeline tests

* Try to fix Flax tests

* Try to put it before

* Use a new decorator instead

* Remove ignore marker since it doesn't work

* Filter pipeline tests

* Woopsie

* Use the fitlered list

* Clean up and fake modif

* Remove init

* Revert fake modif
parent 983451a1
...@@ -16,17 +16,17 @@ import unittest ...@@ -16,17 +16,17 @@ import unittest
from transformers import MODEL_FOR_CAUSAL_LM_MAPPING, TF_MODEL_FOR_CAUSAL_LM_MAPPING, TextGenerationPipeline, pipeline from transformers import MODEL_FOR_CAUSAL_LM_MAPPING, TF_MODEL_FOR_CAUSAL_LM_MAPPING, TextGenerationPipeline, pipeline
from transformers.testing_utils import ( from transformers.testing_utils import (
is_pipeline_test,
require_accelerate, require_accelerate,
require_tf, require_tf,
require_torch, require_torch,
require_torch_gpu, require_torch_gpu,
require_torch_or_tf,
) )
from .test_pipelines_common import ANY, PipelineTestCaseMeta from .test_pipelines_common import ANY, PipelineTestCaseMeta
@is_pipeline_test @require_torch_or_tf
class TextGenerationPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta): class TextGenerationPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta):
model_mapping = MODEL_FOR_CAUSAL_LM_MAPPING model_mapping = MODEL_FOR_CAUSAL_LM_MAPPING
tf_model_mapping = TF_MODEL_FOR_CAUSAL_LM_MAPPING tf_model_mapping = TF_MODEL_FOR_CAUSAL_LM_MAPPING
......
...@@ -25,14 +25,7 @@ from transformers import ( ...@@ -25,14 +25,7 @@ from transformers import (
pipeline, pipeline,
) )
from transformers.pipelines import AggregationStrategy, TokenClassificationArgumentHandler from transformers.pipelines import AggregationStrategy, TokenClassificationArgumentHandler
from transformers.testing_utils import ( from transformers.testing_utils import nested_simplify, require_tf, require_torch, require_torch_gpu, slow
is_pipeline_test,
nested_simplify,
require_tf,
require_torch,
require_torch_gpu,
slow,
)
from .test_pipelines_common import ANY, PipelineTestCaseMeta from .test_pipelines_common import ANY, PipelineTestCaseMeta
...@@ -40,7 +33,6 @@ from .test_pipelines_common import ANY, PipelineTestCaseMeta ...@@ -40,7 +33,6 @@ from .test_pipelines_common import ANY, PipelineTestCaseMeta
VALID_INPUTS = ["A simple string", ["list of strings", "A simple string that is quite a bit longer"]] VALID_INPUTS = ["A simple string", ["list of strings", "A simple string that is quite a bit longer"]]
@is_pipeline_test
class TokenClassificationPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta): class TokenClassificationPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta):
model_mapping = MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING model_mapping = MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING
tf_model_mapping = TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING tf_model_mapping = TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING
...@@ -770,7 +762,6 @@ class TokenClassificationPipelineTests(unittest.TestCase, metaclass=PipelineTest ...@@ -770,7 +762,6 @@ class TokenClassificationPipelineTests(unittest.TestCase, metaclass=PipelineTest
) )
@is_pipeline_test
class TokenClassificationArgumentHandlerTestCase(unittest.TestCase): class TokenClassificationArgumentHandlerTestCase(unittest.TestCase):
def setUp(self): def setUp(self):
self.args_parser = TokenClassificationArgumentHandler() self.args_parser = TokenClassificationArgumentHandler()
......
...@@ -25,12 +25,11 @@ from transformers import ( ...@@ -25,12 +25,11 @@ from transformers import (
TranslationPipeline, TranslationPipeline,
pipeline, pipeline,
) )
from transformers.testing_utils import is_pipeline_test, require_tf, require_torch, slow from transformers.testing_utils import require_tf, require_torch, slow
from .test_pipelines_common import ANY, PipelineTestCaseMeta from .test_pipelines_common import ANY, PipelineTestCaseMeta
@is_pipeline_test
class TranslationPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta): class TranslationPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta):
model_mapping = MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING model_mapping = MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING
tf_model_mapping = TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING tf_model_mapping = TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING
...@@ -118,7 +117,6 @@ class TranslationPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta ...@@ -118,7 +117,6 @@ class TranslationPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta
) )
@is_pipeline_test
class TranslationNewFormatPipelineTests(unittest.TestCase): class TranslationNewFormatPipelineTests(unittest.TestCase):
@require_torch @require_torch
@slow @slow
......
...@@ -16,14 +16,7 @@ import unittest ...@@ -16,14 +16,7 @@ import unittest
from transformers import MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING, is_vision_available from transformers import MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING, is_vision_available
from transformers.pipelines import pipeline from transformers.pipelines import pipeline
from transformers.testing_utils import ( from transformers.testing_utils import nested_simplify, require_tf, require_torch, require_vision, slow
is_pipeline_test,
nested_simplify,
require_tf,
require_torch,
require_vision,
slow,
)
from .test_pipelines_common import ANY, PipelineTestCaseMeta from .test_pipelines_common import ANY, PipelineTestCaseMeta
...@@ -38,7 +31,6 @@ else: ...@@ -38,7 +31,6 @@ else:
pass pass
@is_pipeline_test
@require_torch @require_torch
@require_vision @require_vision
class VisualQuestionAnsweringPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta): class VisualQuestionAnsweringPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta):
......
...@@ -21,12 +21,11 @@ from transformers import ( ...@@ -21,12 +21,11 @@ from transformers import (
ZeroShotClassificationPipeline, ZeroShotClassificationPipeline,
pipeline, pipeline,
) )
from transformers.testing_utils import is_pipeline_test, nested_simplify, require_tf, require_torch, slow from transformers.testing_utils import nested_simplify, require_tf, require_torch, slow
from .test_pipelines_common import ANY, PipelineTestCaseMeta from .test_pipelines_common import ANY, PipelineTestCaseMeta
@is_pipeline_test
class ZeroShotClassificationPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta): class ZeroShotClassificationPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta):
model_mapping = MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING model_mapping = MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING
tf_model_mapping = TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING tf_model_mapping = TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING
......
...@@ -16,14 +16,7 @@ import unittest ...@@ -16,14 +16,7 @@ import unittest
from transformers import is_vision_available from transformers import is_vision_available
from transformers.pipelines import pipeline from transformers.pipelines import pipeline
from transformers.testing_utils import ( from transformers.testing_utils import nested_simplify, require_tf, require_torch, require_vision, slow
is_pipeline_test,
nested_simplify,
require_tf,
require_torch,
require_vision,
slow,
)
from .test_pipelines_common import ANY, PipelineTestCaseMeta from .test_pipelines_common import ANY, PipelineTestCaseMeta
...@@ -39,7 +32,6 @@ else: ...@@ -39,7 +32,6 @@ else:
@require_vision @require_vision
@is_pipeline_test
class ZeroShotImageClassificationPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta): class ZeroShotImageClassificationPipelineTests(unittest.TestCase, metaclass=PipelineTestCaseMeta):
# Deactivating auto tests since we don't have a good MODEL_FOR_XX mapping, # Deactivating auto tests since we don't have a good MODEL_FOR_XX mapping,
# and only CLIP would be there for now. # and only CLIP would be there for now.
......
...@@ -619,6 +619,25 @@ def infer_tests_to_run(output_file, diff_with_last_commit=False, filters=None, j ...@@ -619,6 +619,25 @@ def infer_tests_to_run(output_file, diff_with_last_commit=False, filters=None, j
json.dump(test_map, fp, ensure_ascii=False) json.dump(test_map, fp, ensure_ascii=False)
def filter_pipeline_tests(output_file):
if not os.path.isfile(output_file):
print("No test file found.")
return
with open(output_file, "r", encoding="utf-8") as f:
test_files = f.read().split(" ")
if len(test_files) == 0:
print("No tests to filter.")
return
if test_files == ["tests"]:
test_files = [os.path.join("tests", f) for f in os.listdir("tests") if f not in ["__init__.py", "pipelines"]]
else:
test_files = [f for f in test_files if not f.startswith(os.path.join("tests", "pipelines"))]
with open(output_file, "w", encoding="utf-8") as f:
f.write(" ".join(test_files))
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument( parser.add_argument(
...@@ -645,6 +664,11 @@ if __name__ == "__main__": ...@@ -645,6 +664,11 @@ if __name__ == "__main__":
default=["tests"], default=["tests"],
help="Only keep the test files matching one of those filters.", help="Only keep the test files matching one of those filters.",
) )
parser.add_argument(
"--filter_pipeline_tests",
action="store_true",
help="Will filter the pipeline tests outside of the generated list of tests.",
)
parser.add_argument( parser.add_argument(
"--print_dependencies_of", "--print_dependencies_of",
type=str, type=str,
...@@ -656,6 +680,8 @@ if __name__ == "__main__": ...@@ -656,6 +680,8 @@ if __name__ == "__main__":
print_tree_deps_of(args.print_dependencies_of) print_tree_deps_of(args.print_dependencies_of)
elif args.sanity_check: elif args.sanity_check:
sanity_check() sanity_check()
elif args.filter_pipeline_tests:
filter_pipeline_tests(args.output_file)
else: else:
repo = Repo(PATH_TO_TRANFORMERS) repo = Repo(PATH_TO_TRANFORMERS)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment