Skip to content

GitLab

Commit d2afb65b authored by wooway777's avatar wooway777
Browse files

issue/593 - added verbose test mode

parent 576b7552
Show whitespace changes
Inline Side-by-side
Showing with 102 additions and 36 deletions
test/infinicore/framework/base.py
View file @ d2afb65b
import torch
import infinicore
import traceback # Add import for traceback
from abc import ABC, abstractmethod
from typing import List, Dict, Any, Optional
......@@ -216,14 +217,19 @@ class TestCase:
class TestConfig:
"""Test configuration"""
def __init__(self, debug=False, bench=False, num_prerun=10, num_iterations=1000):
def __init__(
self,
debug=False,
bench=False,
num_prerun=10,
num_iterations=1000,
verbose=False,
):
self.debug = debug
self.bench = bench
self.num_prerun = num_prerun
self.num_iterations = num_iterations
# In base.py - update the TestRunner class
self.verbose = verbose
class TestRunner:
......@@ -273,16 +279,11 @@ class TestRunner:
# Test was skipped due to both operators not being implemented
skip_msg = f"{test_case} - {InfiniDeviceNames[device]} - Both operators not implemented"
self.skipped_tests.append(skip_msg)
print(
f"\033[93m⚠\033[0m Skipped - both operators not implemented"
)
elif status == "partial":
# Test was partially executed (one operator not implemented)
partial_msg = f"{test_case} - {InfiniDeviceNames[device]} - One operator not implemented"
self.partial_tests.append(partial_msg)
print(
f"\033[93m⚠\033[0m Partial - one operator not implemented"
)
# Failed tests are handled in the exception handler below
except Exception as e:
......@@ -291,6 +292,12 @@ class TestRunner:
)
print(f"\033[91m✗\033[0m {error_msg}")
self.failed_tests.append(error_msg)
# In verbose mode, print full traceback and stop execution
if self.config.verbose:
traceback.print_exc()
return False # Stop test execution immediately
if self.config.debug:
raise
......@@ -312,34 +319,16 @@ class TestRunner:
print(f"\n{'='*60}")
print("TEST SUMMARY")
print(f"{'='*60}")
print(f"Total tests: {total_tests}")
print(f"\033[92mPassed: {passed_count}\033[0m")
# Display partial tests (one operator not implemented)
if self.partial_tests:
print(
f"\033[93mPartial (one operator not implemented): {partial_count}\033[0m"
)
for test in self.partial_tests:
print(f" - {test}")
# Display skipped tests (both operators not implemented)
if self.skipped_tests:
print(
f"\033[93mSkipped (both operators not implemented): {skipped_count}\033[0m"
)
for test in self.skipped_tests:
print(f" - {test}")
result = True
# Display failed tests
if self.failed_tests:
print(f"\033[91mFailed: {failed_count}\033[0m")
for failure in self.failed_tests:
print(f" - {failure}")
# Return False only if there are actual test failures
return False
result = False
else:
# Calculate success rate based on actual executed tests
executed_tests = passed_count + partial_count + failed_count
......@@ -352,10 +341,11 @@ class TestRunner:
print(
f"\n\033[93mTests completed with some implementations missing\033[0m"
)
return True # Skipped/partial tests don't count as failures
else:
print(f"\n\033[92mAll tests passed!\033[0m")
return True
print(f"{'='*60}")
return result
class BaseOperatorTest(ABC):
......@@ -537,6 +527,9 @@ class BaseOperatorTest(ABC):
if torch_result is None:
torch_implemented = False
except NotImplementedError:
if config.verbose:
traceback.print_exc()
return False # Stop test execution immediately
torch_implemented = False
torch_result = None
......@@ -545,6 +538,9 @@ class BaseOperatorTest(ABC):
if infini_result is None:
infini_implemented = False
except NotImplementedError:
if config.verbose:
traceback.print_exc()
return False # Stop test execution immediately
infini_implemented = False
infini_result = None
......
test/infinicore/framework/config.py
View file @ d2afb65b
import argparse
from .devices import InfiniDeviceEnum
# hardware_info.py
"""
Shared hardware platform information for the InfiniCore testing framework
"""
......@@ -61,6 +60,9 @@ Examples:
# Run with debug mode on multiple devices
python test_operator.py --cpu --nvidia --debug
# Run with verbose mode to stop on first error with full traceback
python test_operator.py --cpu --nvidia --verbose
# Run performance profiling with custom iterations
python test_operator.py --nvidia --bench --num_prerun 50 --num_iterations 5000
......@@ -90,6 +92,11 @@ Examples:
action="store_true",
help="Enable debug mode for detailed tensor comparison",
)
parser.add_argument(
"--verbose",
action="store_true",
help="Enable verbose mode to stop on first error with full traceback",
)
# Device options using shared hardware info
hardware_group = get_hardware_args_group(parser)
......
test/infinicore/framework/runner.py
View file @ d2afb65b
......@@ -28,6 +28,7 @@ class GenericTestRunner:
bench=self.args.bench,
num_prerun=self.args.num_prerun,
num_iterations=self.args.num_iterations,
verbose=self.args.verbose, # Pass verbose flag to TestConfig
)
runner = TestRunner(self.operator_test.test_cases, config)
......
test/infinicore/run.py
View file @ d2afb65b
......@@ -122,6 +122,9 @@ def run_all_op_tests(ops_dir=None, specific_ops=None, extra_args=None):
results = {}
# Check if verbose mode is enabled
verbose_mode = extra_args and "--verbose" in extra_args
for test_file in operator_test_files:
test_name = test_file.stem
......@@ -199,14 +202,27 @@ def run_all_op_tests(ops_dir=None, specific_ops=None, extra_args=None):
f"{status_icon} {test_name}: {status_text} (return code: {returncode})"
)
# In verbose mode, stop execution on first failure
if verbose_mode and not success and returncode not in [-2, -3]:
break
except Exception as e:
print(f"💥 {test_name}: ERROR - {str(e)}")
results[test_name] = (False, -1, "", str(e))
# In verbose mode, stop execution on any exception
if verbose_mode:
print(f"\n{'!'*60}")
print(
f"VERBOSE MODE: Stopping execution due to exception in {test_name}"
)
print(f"{'!'*60}")
break
return results
def print_summary(results):
def print_summary(results, verbose_mode=False, total_expected_tests=0):
"""Print a comprehensive summary of test results."""
print(f"\n{'='*80}")
print("CUMULATIVE TEST SUMMARY")
......@@ -242,7 +258,11 @@ def print_summary(results):
total = len(results)
print(f"Total tests: {total}")
print(f"Total tests run: {total}")
if total_expected_tests > 0 and total < total_expected_tests:
print(f"Total tests expected: {total_expected_tests}")
print(f"Tests not executed: {total_expected_tests - total}")
print(f"Passed: {passed}")
print(f"Failed: {failed}")
......@@ -290,6 +310,10 @@ def print_summary(results):
success_rate = passed / executed_tests * 100
print(f"\nSuccess rate: {success_rate:.1f}%")
if verbose_mode and total < total_expected_tests:
print(f"\n💡 Verbose mode: Execution stopped after first failure")
print(f" {total_expected_tests - total} tests were not executed")
if failed == 0:
if skipped > 0 or partial > 0:
print(f"\n⚠️ Tests completed with some operators not implemented")
......@@ -358,6 +382,11 @@ def generate_help_epilog(ops_dir):
epilog_parts.append(" # Run with debug mode on multiple devices")
epilog_parts.append(" python run.py --cpu --nvidia --debug")
epilog_parts.append("")
epilog_parts.append(
" # Run with verbose mode to stop on first error with full traceback"
)
epilog_parts.append(" python run.py --cpu --nvidia --verbose")
epilog_parts.append("")
epilog_parts.append(" # List available tests without running")
epilog_parts.append(" python run.py --list")
epilog_parts.append("")
......@@ -386,6 +415,12 @@ def generate_help_epilog(ops_dir):
epilog_parts.append(
" - --bench option is disabled in batch mode (run individual tests for benchmarking)"
)
epilog_parts.append(
" - --verbose mode stops execution on first error and shows full traceback"
)
epilog_parts.append(
" - In verbose mode, subsequent tests are skipped after first failure"
)
return "\n".join(epilog_parts)
......@@ -413,6 +448,11 @@ def main():
action="store_true",
help="List all available test files without running them",
)
parser.add_argument(
"--verbose",
action="store_true",
help="Enable verbose mode to stop on first error with full traceback (passed to individual tests)",
)
from framework import get_hardware_args_group
......@@ -442,6 +482,10 @@ def main():
print(f"Error: Ops directory '{ops_dir}' does not exist.")
sys.exit(1)
# Add verbose flag to extra arguments if specified
if args.verbose and "--verbose" not in unknown_args:
unknown_args.append("--verbose")
# Show what extra arguments will be passed
if unknown_args:
print(f"Passing extra arguments to test scripts: {unknown_args}")
......@@ -453,6 +497,9 @@ def main():
print(f"Operating directory: {ops_dir}")
print(f"Available operators: {len(available_operators)}")
if args.verbose:
print(f"Verbose mode: ENABLED (will stop on first error with full traceback)")
if args.ops:
# Validate requested operators
valid_ops = []
......@@ -469,10 +516,13 @@ def main():
if valid_ops:
print(f"Testing operators: {', '.join(valid_ops)}")
total_expected_tests = len(valid_ops)
else:
print("No valid operators specified. Running all available tests.")
total_expected_tests = len(available_operators)
else:
print("Testing all available operators")
total_expected_tests = len(available_operators)
print()
......@@ -484,7 +534,7 @@ def main():
)
# Print summary and exit with appropriate code
all_passed = print_summary(results)
all_passed = print_summary(results, args.verbose, total_expected_tests)
# Check if there were any tests with missing implementations
has_missing_implementations = any(
......@@ -495,6 +545,18 @@ def main():
print(f"\n⚠️ Note: Some operators are not fully implemented")
print(f" Run individual tests for details on missing implementations")
if args.verbose and not all_passed:
print(
f"\n💡 Verbose mode tip: Use individual test commands for detailed debugging:"
)
failed_ops = [
name
for name, (success, _, _, _) in results.items()
if not success and name in results
]
for op in failed_ops[:3]: # Show first 3 failed operators
print(f" python {ops_dir / (op + '.py')} --verbose")
sys.exit(0 if all_passed else 1)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment