#!/usr/bin/env python3 """ Performance benchmark runner Comprehensive benchmark runner that executes all performance tests, generates reports, and detects performance regressions. """ import sys import argparse import subprocess import time import json from pathlib import Path from typing import List, Dict, Any, Optional from datetime import datetime # Add project root to path sys.path.insert(0, str(Path(__file__).parent.parent.parent)) from tests.performance.benchmark_config import ( BenchmarkConfig, PerformanceTracker, BenchmarkUtils ) class BenchmarkRunner: """Main benchmark runner class""" def __init__(self, config: Optional[BenchmarkConfig] = None): self.config = config or BenchmarkConfig() self.tracker = PerformanceTracker() self.results: Dict[str, Any] = {} def run_pytest_benchmarks(self, test_pattern: str = "tests/performance/") -> Dict[str, Any]: """Run pytest benchmarks and collect results""" print(f"Running pytest benchmarks: {test_pattern}") # Prepare pytest command cmd = [ sys.executable, "-m", "pytest", test_pattern, "-v", "--benchmark-only", "--benchmark-json=benchmark_results.json", "--benchmark-sort=mean", "--benchmark-columns=min,max,mean,stddev,median", "--benchmark-warmup=on", "--benchmark-disable-gc", "--tb=short" ] # Add markers if specified if hasattr(self, 'markers') and self.markers: cmd.extend(["-m", " or ".join(self.markers)]) try: # Run pytest start_time = time.time() result = subprocess.run(cmd, capture_output=True, text=True, timeout=3600) # 1 hour timeout end_time = time.time() # Parse results benchmark_data = {} if Path("benchmark_results.json").exists(): with open("benchmark_results.json", "r") as f: benchmark_data = json.load(f) return { 'success': result.returncode == 0, 'duration': end_time - start_time, 'stdout': result.stdout, 'stderr': result.stderr, 'benchmark_data': benchmark_data } except subprocess.TimeoutExpired: return { 'success': False, 'duration': 3600, 'error': 'Benchmark execn timed out after 1 hour' } except Exception as e: return { 'success': False, 'duration': 0, 'error': str(e) } def run_file_manager_benchmarks(self) -> Dict[str, Any]: """Run file manager specific benchmarks""" print("Running file manager benchmarks...") return self.run_pytest_benchmarks("tests/performance/test_file_manager_benchmarks.py") def run_ui_benchmarks(self) -> Dict[str, Any]: """Run UI specific benchmarks""" print("Running UI benchmarks...") return self.run_pytest_benchmarks("tests/performance/test_ui_performance.py") def run_ai_benchmarks(self) -> Dict[str, Any]: """Run AI specific benchmarks""" print("Running AI benchmarks...") return self.run_pytest_benchmarks("tests/performance/test_ai_performance.py") def run_stress_tests(self) -> Dict[str, Any]: """Run stress tests""" print("Running stress tests...") self.markers = ["slow"] return self.run_pytest_benchmarks("tests/performance/") def run_all_benchmarks(self) -> Dict[str, Any]: """Run all benchmark suites""" print("=" * 60) print("LAXYFILE PERFORMANCE BENCHMARK SUITE") print("=" * 60) print(f"Started at: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") print(f"System info: {BenchmarkUtils.get_system_info()}") print("=" * 60) all_results = {} total_start_time = time.time() # Run individual benchmark suites benchmark_suites = [ ("file_manager", self.run_file_manager_benchmarks), ("ui", self.run_ui_benchmarks), ("ai", self.run_ai_benchmarks), ] for suite_name, suite_func in benchmark_suites: print(f"\n{'='*20} {suite_name.upper()} BENCHMARKS {'='*20}") suite_start = time.time() try: suite_results = suite_func() suite_duration = time.time() - suite_start all_results[suite_name] = { **suite_results, 'suite_duration': suite_duration } if suite_results['success']: print(f"✅ {suite_name} benchmarks completed in {suite_duration:.2f}s") else: print(f"❌ {suite_name} benchmarks failed") if 'error' in suite_results: print(f"Error: {suite_results['error']}") except Exception as e: print(f"❌ {suite_name} benchmarks crashed: {e}") all_results[suite_name] = { 'success': False, 'error': str(e), 'suite_duration': time.time() - suite_start } total_duration = time.time() - total_start_time # Generate summary successful_suites = sum(1 for r in all_results.values() if r.get('success', False)) total_suites = len(all_results) print(f"\n{'='*60}") print("BENCHMARK SUMMARY") print(f"{'='*60}") print(f"Total duration: {total_duration:.2f}s") print(f"Successful suites: {successful_suites}/{total_suites}") for suite_name, results in all_results.items(): status = "✅ PASSED" if results.get('success', False) else "❌ FAILED" duration = results.get('suite_duration', 0) print(f" {suite_name}: {status} ({duration:.2f}s)") all_results['summary'] = { 'total_duration': total_duration, 'successful_suites': successful_suites, 'total_suites': total_suites, 'timestamp': datetime.now().isoformat(), 'system_info': BenchmarkUtils.get_system_info() } return all_results def generate_html_report(self, results: Dict[str, Any], output_file: Path = Path("benchmark_report.html")) -> None: """Generate HTML benchmark report""" html_content = f"""
Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}
Total Duration: {results.get('summary', {}).get('total_duration', 0):.2f}s
Success Rate: {results.get('summary', {}).get('successful_suites', 0)}/{results.get('summary', {}).get('total_suites', 0)} suites
Duration: {suite_results.get('suite_duration', 0):.2f}s
Status: {'PASSED' if success else 'FAILED'}
""" if not success and 'error' in suite_results: html_content += f'| Test Name | Mean (s) | Min (s) | Max (s) | Std Dev | Iterations |
|---|---|---|---|---|---|
| {benchmark.get('name', 'Unknown')} | {stats.get('mean', 0):.4f} | {stats.get('min', 0):.4f} | {stats.get('max', 0):.4f} | {stats.get('stddev', 0):.4f} | {stats.get('rounds', 0)} |