run_generated_tests.py

#!/usr/bin/env python3
"""
Test Case Runner for Generated Game Tests

This script loads test cases generated by generate_test_cases.py and runs them
using pytest or unittest framework.
"""

import json
import os
import sys
import tempfile
import subprocess
import argparse
import shutil
import uuid
from pathlib import Path
from typing import Dict, List, Any, Tuple
import importlib.util
from pprint import pprint

import conda_env_utils as _conda
from game_config_utils import load_game_configs_from_path


class TestCaseRunner:
    """Runner for AI-generated test cases"""
    
    def __init__(self, test_cases_file: str = "game_test_cases.json", config_file: str = "game_config.json"):
        """Initialize the test runner"""
        self.test_cases_file = test_cases_file
        self.config_file = config_file
        self.test_suites = []
        self.active_conda_envs = set()  # Track active conda environments for cleanup
        self.game_configs = {}
        self.load_test_cases()
        self.load_game_configs()
    
    def load_test_cases(self):
        """Load test cases from JSON file"""
        try:
            with open(self.test_cases_file, 'r', encoding='utf-8') as f:
                data = json.load(f)
                self.test_suites = data.get('test_suites', [])
            print(f"Loaded {len(self.test_suites)} test suites from {self.test_cases_file}")
        except FileNotFoundError:
            print(f"Test cases file {self.test_cases_file} not found!")
            sys.exit(1)
        except Exception as e:
            print(f"Error loading test cases: {e}")
            sys.exit(1)

    def load_game_configs(self):
        """Load game-specific configurations from JSON file"""
        self.game_configs = load_game_configs_from_path(
            self.config_file, style="minimal"
        )

    def find_repo_root(self, file_path: Path) -> Path:
        """
        Find the repository root directory by looking for common indicators

        Args:
            file_path: Path to a file within the repository

        Returns:
            Path to the repository root directory
        """
        current_path = file_path.parent

        # Look for common repository indicators
        repo_indicators = ['.git', 'README.md', 'requirements.txt', 'setup.py', 'pyproject.toml']

        # Walk up the directory tree
        while current_path != current_path.parent:  # Stop at filesystem root
            # Check if this directory contains repository indicators
            for indicator in repo_indicators:
                if (current_path / indicator).exists():
                    return current_path

            # Check if we've reached a common base directory
            if str(current_path).endswith('repos_GAME_python_demo'):
                # If we're at the base directory, look for the immediate subdirectory
                # that contains the file (this is likely the repository root)
                try:
                    # Extract repository name from the path
                    parts = file_path.parts
                    for i, part in enumerate(parts):
                        if part == 'repos_GAME_python_demo':
                            if i + 1 < len(parts):
                                return Path(*parts[:i+2])  # Return base_dir/repo_name
                except ValueError:
                    break

            current_path = current_path.parent

        # If no repository root found, return the immediate parent directory
        return file_path.parent

    def create_conda_environment(self, env_name: str, requirements: List[str], python_version: str = "3.9") -> Tuple[bool, str]:
        """
        Create a conda environment with specified requirements and Python version

        Args:
            env_name: Name of the conda environment
            requirements: List of pip packages to install
            python_version: Python version to use (e.g., "3.8", "3.9", "3.10")

        Returns:
            Tuple of (success, error_message)
        """
        try:
            # Create conda environment with specified Python version
            create_cmd = ["conda", "create", "-n", env_name, f"python={python_version}", "-y"]
            result = subprocess.run(create_cmd, capture_output=True, text=True, timeout=60)
            if result.returncode != 0:
                return False, f"Failed to create conda environment: {result.stderr}"

            # Always install pytest first for test execution
            pytest_cmd = ["conda", "run", "-n", env_name, "pip", "install", "pytest"]
            pytest_result = subprocess.run(pytest_cmd, capture_output=True, text=True, timeout=120)
            if pytest_result.returncode != 0:
                return False, f"Failed to install pytest: {pytest_result.stderr}"

            # Install pip requirements if any
            if requirements:
                pip_cmd = ["conda", "run", "-n", env_name, "pip", "install"] + requirements
                pip_result = subprocess.run(pip_cmd, capture_output=True, text=True, timeout=240)
                if pip_result.returncode != 0:
                    return False, f"Failed to install requirements: {pip_result.stderr}"

            self.active_conda_envs.add(env_name)
            return True, ""

        except subprocess.TimeoutExpired:
            return False, "Conda environment creation timeout"
        except Exception as e:
            return False, str(e)

    def cleanup_conda_environment(self, env_name: str) -> bool:
        """
        Remove a conda environment

        Args:
            env_name: Name of the environment to remove

        Returns:
            Success status
        """
        try:
            remove_cmd = ["conda", "env", "remove", "-n", env_name, "-y"]
            result = subprocess.run(remove_cmd, capture_output=True, text=True, timeout=60)

            if env_name in self.active_conda_envs:
                self.active_conda_envs.remove(env_name)

            return result.returncode == 0
        except Exception:
            return False

    def cleanup_all_conda_environments(self):
        """Clean up all conda environments created during test execution"""
        for env_name in list(self.active_conda_envs):
            self.cleanup_conda_environment(env_name)

    def create_test_file(self, test_suite: Dict[str, Any], output_dir: Path) -> Path:
        """Create a pytest test file from test suite"""
        game_name = test_suite['game_name']
        test_filename = f"test_{game_name.replace(' ', '_').lower()}.py"
        test_file_path = output_dir / test_filename
        
        # Generate test file content
        test_content = f'''"""
Generated test cases for {game_name}
Source: {test_suite['game_path']}
Generated on: {test_suite['generation_timestamp']}
"""

import pytest
import sys
import os
import unittest.mock as mock
from pathlib import Path

# Add the game directory to path for imports
game_dir = Path(__file__).parent.parent / "{test_suite['game_path']}"
if game_dir.parent.exists():
    sys.path.insert(0, str(game_dir.parent))

# Mock common game and GUI dependencies to prevent import errors
{self._setup_mocks()}

'''
        
        # Add each test case
        for i, test_case in enumerate(test_suite['test_cases']):
            # Get the actual test code from the test case
            test_code = test_case['test_code']

            # Clean and fix the test code
            test_code = self._clean_test_code(test_code, test_suite['game_path'])

            test_content += f'''
# Test Case {i+1}: {test_case['test_name']}
# Description: {test_case['test_description']}
# Type: {test_case['test_type']}
# Difficulty: {test_case['difficulty_level']}
# Prerequisites: {', '.join(test_case['prerequisites'])}

{test_code}

'''
        
        # Write test file
        with open(test_file_path, 'w', encoding='utf-8') as f:
            f.write(test_content)
        
        return test_file_path

    def _clean_test_code(self, test_code: str, game_path: str) -> str:
        """Clean and fix test code for better execution"""
        lines = test_code.split('\n')
        cleaned_lines = []
        inside_test_function = False
        current_indent = ""

        for i, line in enumerate(lines):
            original_line = line
            line = line.rstrip()

            # Track if we're inside a test function
            if line.strip().startswith('def test_'):
                inside_test_function = True
                current_indent = ""  # Reset indent tracking
            elif line.strip().startswith('def ') and inside_test_function:
                # New function, might be the end of test function
                inside_test_function = False
                current_indent = ""

            # Get the indentation of the current line
            if line.strip():  # Non-empty line
                line_indent = line[:len(line) - len(line.lstrip())]
                if inside_test_function and not line.strip().startswith('def '):
                    current_indent = line_indent

            # Fix common import issues
            if 'from main import' in line:
                # Comment out direct imports from main to avoid import errors
                cleaned_lines.append(f'{current_indent}# {line}  # Commented out to avoid import issues')
                continue
            elif 'import main' in line:
                # Comment out direct imports from main
                cleaned_lines.append(f'{current_indent}# {line}  # Commented out to avoid import issues')
                continue
            elif 'Ui_MainWindow' in line and '()' in line and inside_test_function:
                # Handle Ui_MainWindow class instantiation - create a mock
                indent = current_indent if current_indent else "    "
                cleaned_lines.append(f'{indent}Ui_MainWindow = mock.MagicMock()')
                cleaned_lines.append(f'{indent}ui = Ui_MainWindow()')
                continue
            elif 'Ui_MainWindow' in line and '=' in line and not '()' in line:
                # Handle Ui_MainWindow class definition
                cleaned_lines.append(line)
                continue
            elif 'MagicMock(spec=' in line and any(cls in line for cls in ['QtWidgets.QMainWindow', 'QtWidgets.QWidget']):
                # Replace problematic spec usage with plain MagicMock
                indent = current_indent if current_indent else ""
                cleaned_lines.append(f'{indent}{line.split("=")[0]}= mock.MagicMock()')
                continue
            elif line.strip().startswith('from ') and any(module in line for module in ['PySide2', 'PyQt5', 'pygame', 'tkinter']):
                # Keep GUI framework imports but they should work now with proper mocks
                cleaned_lines.append(line)
                continue

            cleaned_lines.append(original_line)

        return '\n'.join(cleaned_lines)

    def _setup_mocks(self):
        """Setup proper mocks for common dependencies"""
        mock_code = '''
# Mock pygame and other common game dependencies
try:
    import pygame
except ImportError:
    import unittest.mock as mock
    # Create a mock pygame module with proper submodules
    pygame = mock.MagicMock()
    pygame.init = mock.MagicMock()
    pygame.quit = mock.MagicMock()
    pygame.display = mock.MagicMock()
    pygame.event = mock.MagicMock()
    pygame.font = mock.MagicMock()
    pygame.draw = mock.MagicMock()
    pygame.color = mock.MagicMock()
    pygame.QUIT = 12  # Common pygame constant
    sys.modules['pygame'] = pygame
    sys.modules['pygame.color'] = pygame.color
    sys.modules['pygame.display'] = pygame.display
    sys.modules['pygame.event'] = pygame.event
    sys.modules['pygame.font'] = pygame.font
    sys.modules['pygame.draw'] = pygame.draw

# Mock PySide2/Qt framework
try:
    from PySide2 import QtWidgets, QtCore, QtGui
except ImportError:
    import unittest.mock as mock
    # Create mock PySide2 modules with proper classes
    PySide2 = mock.MagicMock()

    # Create actual classes that can be used as specs
    class MockQMainWindow:
        pass

    class MockQWidget:
        pass

    class MockQApplication:
        pass

    class MockQPushButton:
        pass

    class MockQLabel:
        pass

    class MockQVBoxLayout:
        pass

    class MockQHBoxLayout:
        pass

    class MockQt:
        AlignCenter = mock.MagicMock()
        AlignLeft = mock.MagicMock()

    # Create module mocks
    QtWidgets = mock.MagicMock()
    QtCore = mock.MagicMock()
    QtGui = mock.MagicMock()

    # Assign classes to modules
    QtWidgets.QMainWindow = MockQMainWindow
    QtWidgets.QWidget = MockQWidget
    QtWidgets.QApplication = MockQApplication
    QtWidgets.QPushButton = MockQPushButton
    QtWidgets.QLabel = MockQLabel
    QtWidgets.QVBoxLayout = MockQVBoxLayout
    QtWidgets.QHBoxLayout = MockQHBoxLayout

    QtCore.Qt = MockQt

    sys.modules['PySide2'] = PySide2
    sys.modules['PySide2.QtWidgets'] = QtWidgets
    sys.modules['PySide2.QtCore'] = QtCore
    sys.modules['PySide2.QtGui'] = QtGui

# Mock PyQt5 framework
try:
    from PyQt5 import QtWidgets, QtCore, QtGui
except ImportError:
    import unittest.mock as mock
    # Create mock PyQt5 modules with proper classes
    PyQt5 = mock.MagicMock()

    # Create actual classes that can be used as specs
    class MockQMainWindow:
        pass

    class MockQWidget:
        pass

    class MockQApplication:
        pass

    class MockQPushButton:
        pass

    class MockQLabel:
        pass

    class MockQVBoxLayout:
        pass

    class MockQHBoxLayout:
        pass

    class MockQt:
        AlignCenter = mock.MagicMock()
        AlignLeft = mock.MagicMock()

    # Create module mocks
    QtWidgets = mock.MagicMock()
    QtCore = mock.MagicMock()
    QtGui = mock.MagicMock()

    # Assign classes to modules
    QtWidgets.QMainWindow = MockQMainWindow
    QtWidgets.QWidget = MockQWidget
    QtWidgets.QApplication = MockQApplication
    QtWidgets.QPushButton = MockQPushButton
    QtWidgets.QLabel = MockQLabel
    QtWidgets.QVBoxLayout = MockQVBoxLayout
    QtWidgets.QHBoxLayout = MockQHBoxLayout

    QtCore.Qt = MockQt

    sys.modules['PyQt5'] = PyQt5
    sys.modules['PyQt5.QtWidgets'] = QtWidgets
    sys.modules['PyQt5.QtCore'] = QtCore
    sys.modules['PyQt5.QtGui'] = QtGui

# Mock tkinter
try:
    import tkinter as tk
    from tkinter import ttk
except ImportError:
    import unittest.mock as mock
    tk = mock.MagicMock()
    ttk = mock.MagicMock()
    sys.modules['tkinter'] = tk
    sys.modules['tkinter.ttk'] = ttk
'''
        return mock_code

    def run_tests_for_game(self, test_suite: Dict[str, Any], output_dir: Path) -> Dict[str, Any]:
        """Run tests for a specific game using conda environment and repo root"""
        game_name = test_suite['game_name']
        game_path = test_suite['game_path']
        print(f"\n🎮 Running tests for {game_name}...")

        # Find repository root
        original_game_file = Path(game_path)
        if original_game_file.exists():
            repo_root = self.find_repo_root(original_game_file)
        else:
            # Fallback: try to construct path from repos_GAME_python_demo
            base_dir = Path("repos_GAME_python_demo")
            if base_dir.exists():
                # Extract relative path parts to reconstruct the repo root
                parts = game_path.split('/')
                if len(parts) >= 2:
                    repo_root = base_dir / parts[0] / parts[1]
                else:
                    repo_root = base_dir / game_name
            else:
                repo_root = Path.cwd()

        print(f"📁 Using repository root: {repo_root}")

        # Check if this game has custom configuration
        config = self.game_configs.get(game_path, {})
        pip_requirements = config.get("pip_requirements", [])
        running_arguments = config.get("running_arguments", [])
        timeout = config.get("timeout", 60)
        python_version = config.get("python_version", "3.9")

        with tempfile.TemporaryDirectory() as temp_dir:
            try:
                # Copy entire repository to temp directory
                repo_temp_dir = Path(temp_dir) / "repo"
                try:
                    shutil.copytree(repo_root, repo_temp_dir, ignore=shutil.ignore_patterns(
                        '.git', '__pycache__', '*.pyc', '.DS_Store', '.pytest_cache',
                        '*.egg-info', 'build', 'dist', '.tox', '.coverage'
                    ))
                    print(f"Copied repository from {repo_root} to temporary directory")
                except Exception as e:
                    print(f"Failed to copy repository: {e}, using temp directory as fallback")
                    repo_temp_dir = Path(temp_dir)

                # Create test file in the temp directory
                test_file = self.create_test_file(test_suite, repo_temp_dir)

                # Decide whether to use conda environment
                used_conda_env = False
                env_name = ""
                python_cmd = [sys.executable]

                if pip_requirements or config:
                    # Create conda environment for this execution
                    env_name = f"test_run_{uuid.uuid4().hex[:8]}"
                    used_conda_env = True

                    env_success, env_error = self.create_conda_environment(env_name, pip_requirements, python_version)
                    if not env_success:
                        print(f"❌ Failed to create conda environment: {env_error}")
                        return {
                            'game_name': game_name,
                            'game_path': game_path,
                            'total_tests': len(test_suite['test_cases']),
                            'passed': 0,
                            'failed': 0,
                            'errors': 1,
                            'return_code': -1,
                            'stdout': '',
                            'stderr': f'Conda environment creation failed: {env_error}',
                            'success': False
                        }

                    python_cmd = ["conda", "run", "-n", env_name, "python"]
                    print(f"🐍 Created conda environment: {env_name}")

                # Build pytest command
                pytest_cmd = python_cmd + ["-m", "pytest", str(test_file), "-v"]

                # Add any custom running arguments
                if running_arguments:
                    pytest_cmd.extend(running_arguments)

                print(f"🚀 Executing: {' '.join(pytest_cmd)}")

                # Run pytest in the repository directory
                result = subprocess.run(
                    pytest_cmd,
                    capture_output=True,
                    text=True,
                    timeout=timeout,
                    cwd=repo_temp_dir
                )

                # Clean up conda environment
                if used_conda_env:
                    self.cleanup_conda_environment(env_name)

                # Parse results
                stdout = result.stdout
                stderr = result.stderr
                return_code = result.returncode

                # Count passed/failed tests
                passed_count = stdout.count('PASSED')
                failed_count = stdout.count('FAILED')
                error_count = stdout.count('ERROR')

                test_result = {
                    'game_name': game_name,
                    'game_path': game_path,
                    'total_tests': len(test_suite['test_cases']),
                    'passed': passed_count,
                    'failed': failed_count,
                    'errors': error_count,
                    'return_code': return_code,
                    'stdout': stdout,
                    'stderr': stderr,
                    'success': return_code == 0
                }

                if test_result['success']:
                    print(f"✅ {game_name}: {passed_count}/{len(test_suite['test_cases'])} tests passed")
                else:
                    print(f"❌ {game_name}: {failed_count} failed, {error_count} errors")
                    if stderr:
                        print(f"Error details: {stderr[:300]}...")

                return test_result

            except subprocess.TimeoutExpired:
                if used_conda_env:
                    self.cleanup_conda_environment(env_name)
                print(f"⏰ {game_name}: Tests timed out")
                return {
                    'game_name': game_name,
                    'game_path': game_path,
                    'total_tests': len(test_suite['test_cases']),
                    'passed': 0,
                    'failed': 0,
                    'errors': 1,
                    'return_code': -1,
                    'stdout': '',
                    'stderr': 'Test execution timed out',
                    'success': False
                }
            except Exception as e:
                if used_conda_env:
                    self.cleanup_conda_environment(env_name)
                print(f"💥 {game_name}: Error running tests - {e}")
                return {
                    'game_name': game_name,
                    'game_path': game_path,
                    'total_tests': len(test_suite['test_cases']),
                    'passed': 0,
                    'failed': 0,
                    'errors': 1,
                    'return_code': -1,
                    'stdout': '',
                    'stderr': str(e),
                    'success': False
                }
    
    def run_all_tests(self, max_games: int = None, output_dir: str = "generated_tests") -> List[Dict[str, Any]]:
        """Run tests for all games"""
        output_path = Path(output_dir)
        output_path.mkdir(exist_ok=True)
        
        test_suites = self.test_suites[:max_games] if max_games else self.test_suites
        
        print(f"🚀 Running tests for {len(test_suites)} games...")
        print(f"📁 Test files will be created in: {output_path}")
        
        results = []
        
        for i, test_suite in enumerate(test_suites, 1):
            print(f"\n[{i}/{len(test_suites)}]", end=" ")
            result = self.run_tests_for_game(test_suite, output_path)
            results.append(result)
        
        return results
    
    def print_summary(self, results: List[Dict[str, Any]]):
        """Print test execution summary"""
        if not results:
            print("No test results to display.")
            return
        
        total_games = len(results)
        successful_games = sum(1 for r in results if r['success'])
        total_tests = sum(r['total_tests'] for r in results)
        total_passed = sum(r['passed'] for r in results)
        total_failed = sum(r['failed'] for r in results)
        total_errors = sum(r['errors'] for r in results)
        
        print("\n" + "="*60)
        print("🎯 TEST EXECUTION SUMMARY")
        print("="*60)
        print(f"Games tested: {total_games}")
        print(f"Games with all tests passing: {successful_games} ({successful_games/total_games*100:.1f}%)")
        print(f"Total test cases: {total_tests}")
        print(f"Passed: {total_passed} ({total_passed/total_tests*100:.1f}%)")
        print(f"Failed: {total_failed} ({total_failed/total_tests*100:.1f}%)")
        print(f"Errors: {total_errors} ({total_errors/total_tests*100:.1f}%)")
        
        print(f"\n📊 DETAILED RESULTS:")
        for result in results:
            status = "✅" if result['success'] else "❌"
            print(f"  {status} {result['game_name']}: {result['passed']}/{result['total_tests']} passed")
        
        # Show games with failures
        failed_games = [r for r in results if not r['success']]
        if failed_games:
            print(f"\n❗ GAMES WITH FAILURES:")
            for result in failed_games:
                print(f"  🔍 {result['game_name']} ({result['game_path']}):")
                if result['stderr']:
                    print(f"    Error: {result['stderr'][:100]}...")
        
        print("="*60)
    
    def save_results(self, results: List[Dict[str, Any]], filename: str = "test_execution_results.json"):
        """Save test execution results to JSON file"""
        try:
            with open(filename, 'w', encoding='utf-8') as f:
                json.dump({
                    'execution_timestamp': str(Path().absolute()),
                    'total_games': len(results),
                    'results': results
                }, f, indent=2, ensure_ascii=False)
            print(f"💾 Test results saved to {filename}")
        except Exception as e:
            print(f"Error saving results: {e}")
    
    def generate_html_report(self, results: List[Dict[str, Any]], filename: str = "test_report.html"):
        """Generate an HTML test report"""
        total_tests = sum(r['total_tests'] for r in results)
        total_passed = sum(r['passed'] for r in results)
        
        html_content = f"""
<!DOCTYPE html>
<html>
<head>
    <title>Game Test Report</title>
    <style>
        body {{ font-family: Arial, sans-serif; margin: 40px; }}
        .header {{ background-color: #f0f0f0; padding: 20px; border-radius: 5px; }}
        .summary {{ margin: 20px 0; }}
        .game-result {{ margin: 10px 0; padding: 10px; border-left: 4px solid #ccc; }}
        .success {{ border-left-color: #4CAF50; }}
        .failure {{ border-left-color: #f44336; }}
        .stats {{ display: inline-block; margin-right: 20px; }}
    </style>
</head>
<body>
    <div class="header">
        <h1>🎮 Game Test Execution Report</h1>
        <div class="stats">
            <strong>Total Games:</strong> {len(results)}
        </div>
        <div class="stats">
            <strong>Total Tests:</strong> {total_tests}
        </div>
        <div class="stats">
            <strong>Success Rate:</strong> {total_passed/total_tests*100:.1f}%
        </div>
    </div>
    
    <div class="summary">
        <h2>📊 Results by Game</h2>
"""
        
        for result in results:
            status_class = "success" if result['success'] else "failure"
            status_icon = "✅" if result['success'] else "❌"
            
            html_content += f"""
        <div class="game-result {status_class}">
            <h3>{status_icon} {result['game_name']}</h3>
            <p><strong>Path:</strong> {result['game_path']}</p>
            <p><strong>Tests:</strong> {result['passed']}/{result['total_tests']} passed</p>
            {f'<p><strong>Error:</strong> {result["stderr"][:200]}...</p>' if result['stderr'] else ''}
        </div>
"""
        
        html_content += """
    </div>
</body>
</html>
"""
        
        try:
            with open(filename, 'w', encoding='utf-8') as f:
                f.write(html_content)
            print(f"📄 HTML report generated: {filename}")
        except Exception as e:
            print(f"Error generating HTML report: {e}")


def main():
    """Main function"""
    parser = argparse.ArgumentParser(description="Run generated test cases for games")
    parser.add_argument("--test-cases", default="game_test_cases.json",
                       help="JSON file containing generated test cases")
    parser.add_argument("--config-file", default="game_config.json",
                       help="JSON configuration file for game-specific requirements")
    parser.add_argument("--max-games", type=int,
                       help="Maximum number of games to test")
    parser.add_argument("--output-dir", default="generated_tests",
                       help="Directory to store generated test files")
    parser.add_argument("--save-results", action="store_true",
                       help="Save results to JSON file")
    parser.add_argument("--html-report", action="store_true",
                       help="Generate HTML report")

    args = parser.parse_args()

    # Create test runner
    runner = TestCaseRunner(args.test_cases, args.config_file)

    try:
        # Run tests
        results = runner.run_all_tests(
            max_games=args.max_games,
            output_dir=args.output_dir
        )

        # Print summary
        runner.print_summary(results)

        # Save results if requested
        if args.save_results:
            runner.save_results(results)

        # Generate HTML report if requested
        if args.html_report:
            runner.generate_html_report(results)

        print(f"\n✅ Test execution complete!")
    finally:
        # Ensure all conda environments are cleaned up
        print("Cleaning up conda environments...")
        runner.cleanup_all_conda_environments()
        print("Cleanup completed.")


if __name__ == "__main__":
    main()