diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ebc56cc..8f76aa2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,10 +20,12 @@ jobs: working-directory: cli steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: ${{ matrix.python-version }} @@ -31,4 +33,42 @@ jobs: run: pip install -e ".[dev]" - name: Run tests - run: pytest --cov=localci --cov-report=term-missing + run: pytest --cov=localci --cov-report=term-missing -m "not integration" + + integration: + name: Integration (act + Docker) + runs-on: ubuntu-latest + needs: [test] + timeout-minutes: 30 + + defaults: + run: + working-directory: cli + + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false + + - name: Set up Python 3.12 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.12" + + - name: Install dependencies + run: pip install -e ".[dev]" + + - name: Install act + env: + ACT_VERSION: v0.2.76 + # From https://github.com/nektos/act/releases/download/v0.2.76/checksums.txt + ACT_LINUX_X86_64_SHA256: d4720e05e73ce634239fc0e7fa6b552f9ac79b4e6c878a8756bbfacce3c56720 + run: | + curl -sSfL -o /tmp/act.tar.gz \ + "https://github.com/nektos/act/releases/download/${ACT_VERSION}/act_Linux_x86_64.tar.gz" + echo "${ACT_LINUX_X86_64_SHA256} /tmp/act.tar.gz" | sha256sum --check + sudo tar xzf /tmp/act.tar.gz -C /usr/local/bin act + act --version + + - name: Run integration tests + run: pytest tests/integration -m integration -v --tb=short diff --git a/cli/README.md b/cli/README.md index 70469b1..57318b6 100644 --- a/cli/README.md +++ b/cli/README.md @@ -105,11 +105,16 @@ cli/ # Editable install with dev dependencies pip install -e ".[dev]" -# Run tests +# Run unit tests (default). Integration tests are excluded via norecursedirs +# (pytest does not enter tests/integration/). Use -m "not integration" for +# coverage or if integration tests move outside that directory name. pytest -# Run tests with coverage -pytest --cov=localci +# Run unit tests with coverage (CI unit job uses this marker) +pytest --cov=localci -m "not integration" + +# Integration tests (requires act + Docker; path + marker required) +pytest tests/integration -m integration ``` ## License diff --git a/cli/pyproject.toml b/cli/pyproject.toml index f2cab24..d55ed8f 100644 --- a/cli/pyproject.toml +++ b/cli/pyproject.toml @@ -31,3 +31,7 @@ packages = ["localci"] [tool.pytest.ini_options] testpaths = ["tests"] addopts = "-v --tb=short" +markers = [ + "integration: requires act and Docker; runs real subprocesses", +] +norecursedirs = ["integration"] diff --git a/cli/tests/fixtures/integration/project/.github/workflows/invalid.yml b/cli/tests/fixtures/integration/project/.github/workflows/invalid.yml new file mode 100644 index 0000000..d186642 --- /dev/null +++ b/cli/tests/fixtures/integration/project/.github/workflows/invalid.yml @@ -0,0 +1,5 @@ +name: Invalid +on: [push +jobs: + test: + runs-on: ubuntu-latest diff --git a/cli/tests/fixtures/integration/project/.github/workflows/test-fail.yml b/cli/tests/fixtures/integration/project/.github/workflows/test-fail.yml new file mode 100644 index 0000000..e183b17 --- /dev/null +++ b/cli/tests/fixtures/integration/project/.github/workflows/test-fail.yml @@ -0,0 +1,15 @@ +name: Integration Failure +on: [push] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + include: + - name: "Hello" + runs-on: ubuntu-latest + compiler: gcc + version: "15" + steps: + - run: exit 1 diff --git a/cli/tests/fixtures/integration/project/.github/workflows/test.yml b/cli/tests/fixtures/integration/project/.github/workflows/test.yml new file mode 100644 index 0000000..e4ea8b1 --- /dev/null +++ b/cli/tests/fixtures/integration/project/.github/workflows/test.yml @@ -0,0 +1,15 @@ +name: Integration Success +on: [push] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + include: + - name: "Hello" + runs-on: ubuntu-latest + compiler: gcc + version: "15" + steps: + - run: echo "Hello" diff --git a/cli/tests/fixtures/integration/project/.localci.yml b/cli/tests/fixtures/integration/project/.localci.yml new file mode 100644 index 0000000..15868c9 --- /dev/null +++ b/cli/tests/fixtures/integration/project/.localci.yml @@ -0,0 +1,26 @@ +version: 1 +workflow: .github/workflows/test.yml +event: push + +parallel: + max_jobs: 1 + +platforms: + linux: true + windows: false + macos: false + +images: + auto_build: false + +cache: + enabled: false + +logging: + level: warning + directory: logs + +execution: + timeout: 120 + keep_containers: false + stop_on_first_failure: false diff --git a/cli/tests/integration/__init__.py b/cli/tests/integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cli/tests/integration/conftest.py b/cli/tests/integration/conftest.py new file mode 100644 index 0000000..3917523 --- /dev/null +++ b/cli/tests/integration/conftest.py @@ -0,0 +1,116 @@ +"""Shared fixtures for act/Docker integration tests.""" + +from __future__ import annotations + +import shutil +import subprocess +from pathlib import Path + +import pytest +import yaml + +from localci.core.executor import JobExecutor +from localci.core.image_tag import derive_image_tag +from localci.errors import DockerNotAvailableError + +FIXTURE_PROJECT = ( + Path(__file__).resolve().parent.parent / "fixtures" / "integration" / "project" +) +ACT_RUNNER_IMAGE = "catthehacker/ubuntu:act-24.04" +INTEGRATION_JOB_ID = "test" +INTEGRATION_TIMEOUT = 180 +DOCKER_PULL_TIMEOUT = 600 +DOCKER_TAG_TIMEOUT = 60 + + +def _act_available() -> bool: + return JobExecutor().has_act + + +def _docker_available() -> bool: + try: + JobExecutor().check_docker() + return True + except DockerNotAvailableError: + return False + + +@pytest.fixture(scope="session") +def require_act_and_docker() -> None: + """Skip the entire integration session when act or Docker is unavailable.""" + if not _act_available(): + pytest.skip("act is not installed") + if not _docker_available(): + pytest.skip("Docker is not available") + + +@pytest.fixture(scope="session") +def act_runner_image(require_act_and_docker: None) -> str: + """Pull the act runner image used for ubuntu-latest jobs.""" + try: + pull = subprocess.run( + ["docker", "pull", ACT_RUNNER_IMAGE], + capture_output=True, + text=True, + timeout=DOCKER_PULL_TIMEOUT, + ) + except subprocess.TimeoutExpired: + pytest.skip( + f"timed out pulling {ACT_RUNNER_IMAGE} after {DOCKER_PULL_TIMEOUT}s" + ) + if pull.returncode != 0: + pytest.skip( + f"could not pull {ACT_RUNNER_IMAGE}: " + f"{pull.stderr.strip() or pull.stdout.strip()}" + ) + return ACT_RUNNER_IMAGE + + +@pytest.fixture(scope="session") +def capy_image_tag(act_runner_image: str, require_act_and_docker: None) -> str: + """Tag the act runner image as the derived capy name for localci run.""" + from localci.core.workflow import WorkflowAnalyzer + + # Derive tag from test.yml; test-fail.yml uses the same matrix.include shape today. + # If failure fixture matrix diverges, derive from that workflow (or both) instead. + workflow_path = FIXTURE_PROJECT / ".github/workflows/test.yml" + entry = WorkflowAnalyzer().analyze(workflow_path).jobs[INTEGRATION_JOB_ID].matrix[0] + tag = derive_image_tag(entry) + assert tag is not None + + try: + tag_result = subprocess.run( + ["docker", "tag", act_runner_image, tag], + capture_output=True, + text=True, + timeout=DOCKER_TAG_TIMEOUT, + ) + except subprocess.TimeoutExpired: + pytest.skip( + f"timed out tagging {act_runner_image} as {tag} after {DOCKER_TAG_TIMEOUT}s" + ) + if tag_result.returncode != 0: + pytest.skip( + f"could not tag {act_runner_image} as {tag}: " + f"{tag_result.stderr.strip() or tag_result.stdout.strip()}" + ) + return tag + + +@pytest.fixture +def integration_project(tmp_path: Path) -> tuple[Path, Path]: + """Copy fixture project and return (project_root, logs_dir).""" + dest = tmp_path / "project" + shutil.copytree(FIXTURE_PROJECT, dest) + + logs_dir = tmp_path / "logs" + logs_dir.mkdir(parents=True, exist_ok=True) + + config_path = dest / ".localci.yml" + config = yaml.safe_load(config_path.read_text()) + config["logging"]["directory"] = str(logs_dir) + config_path.write_text( + yaml.dump(config, default_flow_style=False, sort_keys=False) + ) + + return dest, logs_dir diff --git a/cli/tests/integration/test_act_subprocess.py b/cli/tests/integration/test_act_subprocess.py new file mode 100644 index 0000000..d382480 --- /dev/null +++ b/cli/tests/integration/test_act_subprocess.py @@ -0,0 +1,85 @@ +"""Integration tests for JobExecutor + act subprocess boundary.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from localci.core.command_builder import ActCommandBuilder +from localci.core.executor import JobExecutor, JobResult, JobStatus +from localci.core.workflow import MatrixEntry, WorkflowAnalyzer + +from .conftest import INTEGRATION_JOB_ID, INTEGRATION_TIMEOUT + +pytestmark = pytest.mark.integration + + +def _workflow_path(name: str, project_dir: Path) -> Path: + return project_dir / ".github/workflows" / name + + +def _build_and_run( + workflow_name: str, + logs_dir: Path, + act_runner_image: str, + project_dir: Path, +) -> tuple[JobResult, MatrixEntry]: + workflow_path = _workflow_path(workflow_name, project_dir) + analyzer = WorkflowAnalyzer() + workflow = analyzer.analyze(workflow_path) + entry = workflow.jobs[INTEGRATION_JOB_ID].matrix[0] + + builder = ActCommandBuilder( + workflow_file=workflow_path, + project_dir=project_dir, + job_id=INTEGRATION_JOB_ID, + ) + cmd = builder.build(entry, image_tag=act_runner_image) + executor = JobExecutor(logs_dir=logs_dir) + result = executor.run( + cmd, + matrix_index=entry.index, + matrix_name=entry.name, + timeout=INTEGRATION_TIMEOUT, + stream_output=False, + ) + return result, entry + + +def test_successful_job_execution( + integration_project: tuple[Path, Path], + act_runner_image: str, +) -> None: + project, logs_dir = integration_project + result, _ = _build_and_run("test.yml", logs_dir, act_runner_image, project) + + assert result.status == JobStatus.PASSED + assert result.exit_code == 0 + assert result.stdout or result.stderr + assert result.log_file is not None + assert result.log_file.exists() + + +def test_failing_job_extract_error( + integration_project: tuple[Path, Path], + act_runner_image: str, +) -> None: + project, logs_dir = integration_project + result, _ = _build_and_run( + "test-fail.yml", logs_dir, act_runner_image, project + ) + + assert result.status == JobStatus.FAILED + assert result.exit_code is not None + assert result.exit_code != 0 + + captured = result.stderr or result.stdout + assert captured.strip() + + extracted = JobExecutor._extract_error(captured) + assert result.error_message == extracted + assert result.error_message is not None + + lower = result.error_message.lower() + assert any(kw in lower for kw in ("failed", "error", "exit")) diff --git a/cli/tests/integration/test_localci_run.py b/cli/tests/integration/test_localci_run.py new file mode 100644 index 0000000..52f8dbc --- /dev/null +++ b/cli/tests/integration/test_localci_run.py @@ -0,0 +1,128 @@ +"""Integration tests for ``localci run`` against real act/Docker.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +from click.testing import CliRunner + +from localci.cli.main import cli +from localci.core.executor import JobExecutor, JobStatus +from localci.core.results import ExecutionSummary + +from .conftest import INTEGRATION_TIMEOUT + +pytestmark = pytest.mark.integration + +runner = CliRunner() + + +def _executor_capture_from_log_text(log_text: str) -> str: + """Approximate the stderr-or-stdout string passed to ``_extract_error``. + + Job logs interleave streams and include act summary lines that are not in + the stderr-only capture the executor uses when stderr is non-empty. + """ + body_lines = [ + line for line in log_text.splitlines() if not line.startswith("#") + ] + stderr_like = [ + line + for line in body_lines + if "error:" in line.lower() or "fatal:" in line.lower() + ] + if stderr_like: + return "\n".join(stderr_like) + return "\n".join(body_lines) + + +def _run_localci( + project: Path, + workflow: str, + extra_args: list[str] | None = None, +) -> object: + args = [ + "-c", + str(project / ".localci.yml"), + "run", + "--workflow", + workflow, + "--no-cache", + "--parallel", + "1", + "--timeout", + str(INTEGRATION_TIMEOUT), + ] + if extra_args: + args.extend(extra_args) + return runner.invoke(cli, args) + + +@pytest.mark.usefixtures("capy_image_tag") +def test_run_success( + integration_project: tuple[Path, Path], + monkeypatch: pytest.MonkeyPatch, +) -> None: + project, logs_dir = integration_project + monkeypatch.chdir(project) + result = _run_localci(project, ".github/workflows/test.yml") + + assert result.exit_code == 0, result.output + + last_run = logs_dir / "last-run.json" + assert last_run.exists(), "expected last-run.json after successful run" + + summary = ExecutionSummary.load(last_run) + assert summary.all_passed + assert summary.total == 1 + assert summary.results[0].status == JobStatus.PASSED + + +@pytest.mark.usefixtures("capy_image_tag") +def test_run_failure( + integration_project: tuple[Path, Path], + monkeypatch: pytest.MonkeyPatch, +) -> None: + project, logs_dir = integration_project + monkeypatch.chdir(project) + result = _run_localci(project, ".github/workflows/test-fail.yml") + + assert result.exit_code == 1, result.output + assert "Traceback" not in result.output + + last_run = logs_dir / "last-run.json" + assert last_run.exists(), "expected last-run.json after failed run" + + summary = ExecutionSummary.load(last_run) + assert summary.total == 1 + job = summary.results[0] + assert job.status == JobStatus.FAILED + assert job.error_message + + if job.log_file and job.log_file.exists(): + log_text = job.log_file.read_text(encoding="utf-8", errors="replace") + captured = _executor_capture_from_log_text(log_text) + assert job.error_message in JobExecutor._extract_error(captured) + assert job.error_message.splitlines()[0] in log_text + else: + assert job.error_message in JobExecutor._extract_error(result.output) + + +def test_run_invalid_workflow( + integration_project: tuple[Path, Path], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Workflow parse fails before act/Docker; intentionally omits require_act_and_docker. + + Kept in this package (not unit tests) to assert the full CLI path exits cleanly + without a traceback when given a malformed workflow file. + """ + project, _logs_dir = integration_project + monkeypatch.chdir(project) + result = _run_localci(project, ".github/workflows/invalid.yml") + + assert result.exit_code == 1, result.output + assert "Traceback" not in result.output + lower = result.output.lower() + assert "parse" in lower or "workflow" in lower or "invalid" in lower