From 8763ff1715821c06fc2c6900bee4a04d0c77f942 Mon Sep 17 00:00:00 2001 From: mac Date: Fri, 29 May 2026 04:39:01 +0800 Subject: [PATCH 1/6] Add act/Docker integration tests for subprocess boundary and _extract_error --- .github/workflows/ci.yml | 32 +++++- cli/README.md | 9 +- cli/pyproject.toml | 4 + .../project/.github/workflows/invalid.yml | 5 + .../project/.github/workflows/test-fail.yml | 15 +++ .../project/.github/workflows/test.yml | 15 +++ .../fixtures/integration/project/.localci.yml | 26 +++++ cli/tests/integration/__init__.py | 0 cli/tests/integration/conftest.py | 107 ++++++++++++++++++ cli/tests/integration/test_act_subprocess.py | 84 ++++++++++++++ cli/tests/integration/test_localci_run.py | 98 ++++++++++++++++ 11 files changed, 391 insertions(+), 4 deletions(-) create mode 100644 cli/tests/fixtures/integration/project/.github/workflows/invalid.yml create mode 100644 cli/tests/fixtures/integration/project/.github/workflows/test-fail.yml create mode 100644 cli/tests/fixtures/integration/project/.github/workflows/test.yml create mode 100644 cli/tests/fixtures/integration/project/.localci.yml create mode 100644 cli/tests/integration/__init__.py create mode 100644 cli/tests/integration/conftest.py create mode 100644 cli/tests/integration/test_act_subprocess.py create mode 100644 cli/tests/integration/test_localci_run.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ebc56cc..17ed8d8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,4 +31,34 @@ jobs: run: pip install -e ".[dev]" - name: Run tests - run: pytest --cov=localci --cov-report=term-missing + run: pytest --cov=localci --cov-report=term-missing -m "not integration" + + integration: + name: Integration (act + Docker) + runs-on: ubuntu-latest + + defaults: + run: + working-directory: cli + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install dependencies + run: pip install -e ".[dev]" + + - name: Install act + run: | + curl -sSfL https://raw.githubusercontent.com/nektos/act/master/install.sh | bash -s -- -b /usr/local/bin v0.2.76 + act --version + + - name: Pull act runner image + run: docker pull catthehacker/ubuntu:act-24.04 + + - name: Run integration tests + run: pytest tests/integration -m integration -v --tb=short diff --git a/cli/README.md b/cli/README.md index 70469b1..56743f1 100644 --- a/cli/README.md +++ b/cli/README.md @@ -105,11 +105,14 @@ cli/ # Editable install with dev dependencies pip install -e ".[dev]" -# Run tests +# Run unit tests (default; integration tests are excluded) pytest -# Run tests with coverage -pytest --cov=localci +# Run unit tests with coverage +pytest --cov=localci -m "not integration" + +# Integration tests (requires act + Docker) +pytest tests/integration -m integration ``` ## License diff --git a/cli/pyproject.toml b/cli/pyproject.toml index f2cab24..d55ed8f 100644 --- a/cli/pyproject.toml +++ b/cli/pyproject.toml @@ -31,3 +31,7 @@ packages = ["localci"] [tool.pytest.ini_options] testpaths = ["tests"] addopts = "-v --tb=short" +markers = [ + "integration: requires act and Docker; runs real subprocesses", +] +norecursedirs = ["integration"] diff --git a/cli/tests/fixtures/integration/project/.github/workflows/invalid.yml b/cli/tests/fixtures/integration/project/.github/workflows/invalid.yml new file mode 100644 index 0000000..d186642 --- /dev/null +++ b/cli/tests/fixtures/integration/project/.github/workflows/invalid.yml @@ -0,0 +1,5 @@ +name: Invalid +on: [push +jobs: + test: + runs-on: ubuntu-latest diff --git a/cli/tests/fixtures/integration/project/.github/workflows/test-fail.yml b/cli/tests/fixtures/integration/project/.github/workflows/test-fail.yml new file mode 100644 index 0000000..e183b17 --- /dev/null +++ b/cli/tests/fixtures/integration/project/.github/workflows/test-fail.yml @@ -0,0 +1,15 @@ +name: Integration Failure +on: [push] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + include: + - name: "Hello" + runs-on: ubuntu-latest + compiler: gcc + version: "15" + steps: + - run: exit 1 diff --git a/cli/tests/fixtures/integration/project/.github/workflows/test.yml b/cli/tests/fixtures/integration/project/.github/workflows/test.yml new file mode 100644 index 0000000..e4ea8b1 --- /dev/null +++ b/cli/tests/fixtures/integration/project/.github/workflows/test.yml @@ -0,0 +1,15 @@ +name: Integration Success +on: [push] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + include: + - name: "Hello" + runs-on: ubuntu-latest + compiler: gcc + version: "15" + steps: + - run: echo "Hello" diff --git a/cli/tests/fixtures/integration/project/.localci.yml b/cli/tests/fixtures/integration/project/.localci.yml new file mode 100644 index 0000000..15868c9 --- /dev/null +++ b/cli/tests/fixtures/integration/project/.localci.yml @@ -0,0 +1,26 @@ +version: 1 +workflow: .github/workflows/test.yml +event: push + +parallel: + max_jobs: 1 + +platforms: + linux: true + windows: false + macos: false + +images: + auto_build: false + +cache: + enabled: false + +logging: + level: warning + directory: logs + +execution: + timeout: 120 + keep_containers: false + stop_on_first_failure: false diff --git a/cli/tests/integration/__init__.py b/cli/tests/integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cli/tests/integration/conftest.py b/cli/tests/integration/conftest.py new file mode 100644 index 0000000..7c8d316 --- /dev/null +++ b/cli/tests/integration/conftest.py @@ -0,0 +1,107 @@ +"""Shared fixtures for act/Docker integration tests.""" + +from __future__ import annotations + +import shutil +import subprocess +from pathlib import Path + +import pytest +import yaml + +from localci.core.executor import JobExecutor +from localci.core.image_tag import derive_image_tag +from localci.errors import DockerNotAvailableError + +FIXTURE_PROJECT = ( + Path(__file__).resolve().parent.parent / "fixtures" / "integration" / "project" +) +ACT_RUNNER_IMAGE = "catthehacker/ubuntu:act-24.04" +INTEGRATION_JOB_ID = "test" +INTEGRATION_TIMEOUT = 180 + + +def _act_available() -> bool: + return JobExecutor().has_act + + +def _docker_available() -> bool: + try: + JobExecutor().check_docker() + return True + except DockerNotAvailableError: + return False + + +@pytest.fixture(scope="session") +def require_act_and_docker() -> None: + """Skip the entire integration session when act or Docker is unavailable.""" + if not _act_available(): + pytest.skip("act is not installed") + if not _docker_available(): + pytest.skip("Docker is not available") + + +@pytest.fixture(scope="session") +def act_runner_image(require_act_and_docker: None) -> str: + """Pull the act runner image used for ubuntu-latest jobs.""" + pull = subprocess.run( + ["docker", "pull", ACT_RUNNER_IMAGE], + capture_output=True, + text=True, + timeout=600, + ) + if pull.returncode != 0: + pytest.skip( + f"could not pull {ACT_RUNNER_IMAGE}: " + f"{pull.stderr.strip() or pull.stdout.strip()}" + ) + return ACT_RUNNER_IMAGE + + +@pytest.fixture(scope="session") +def capy_image_tag(act_runner_image: str, require_act_and_docker: None) -> str: + """Tag the act runner image as the derived capy name for localci run.""" + from localci.core.workflow import WorkflowAnalyzer + + workflow_path = FIXTURE_PROJECT / ".github/workflows/test.yml" + entry = WorkflowAnalyzer().analyze(workflow_path).jobs[INTEGRATION_JOB_ID].matrix[0] + tag = derive_image_tag(entry) + assert tag is not None + + tag_result = subprocess.run( + ["docker", "tag", act_runner_image, tag], + capture_output=True, + text=True, + timeout=60, + ) + if tag_result.returncode != 0: + pytest.skip( + f"could not tag {act_runner_image} as {tag}: " + f"{tag_result.stderr.strip() or tag_result.stdout.strip()}" + ) + return tag + + +@pytest.fixture +def integration_project(tmp_path: Path) -> Path: + """Copy the integration fixture project into an isolated directory.""" + dest = tmp_path / "project" + shutil.copytree(FIXTURE_PROJECT, dest) + + logs_dir = tmp_path / "logs" + logs_dir.mkdir(parents=True, exist_ok=True) + + config_path = dest / ".localci.yml" + config = yaml.safe_load(config_path.read_text()) + config["logging"]["directory"] = str(logs_dir) + config_path.write_text(yaml.dump(config, default_flow_style=False)) + + return dest + + +@pytest.fixture +def integration_logs_dir(tmp_path: Path, integration_project: Path) -> Path: + """Logs directory configured for the copied integration project.""" + _ = integration_project + return tmp_path / "logs" diff --git a/cli/tests/integration/test_act_subprocess.py b/cli/tests/integration/test_act_subprocess.py new file mode 100644 index 0000000..4ca379d --- /dev/null +++ b/cli/tests/integration/test_act_subprocess.py @@ -0,0 +1,84 @@ +"""Integration tests for JobExecutor + act subprocess boundary.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from localci.core.command_builder import ActCommandBuilder +from localci.core.executor import JobExecutor, JobStatus +from localci.core.workflow import WorkflowAnalyzer + +from .conftest import FIXTURE_PROJECT, INTEGRATION_JOB_ID, INTEGRATION_TIMEOUT + +pytestmark = pytest.mark.integration + + +def _workflow_path(name: str) -> Path: + return FIXTURE_PROJECT / ".github/workflows" / name + + +def _build_and_run( + workflow_name: str, + logs_dir: Path, + act_runner_image: str, +) -> tuple: + workflow_path = _workflow_path(workflow_name) + project_dir = FIXTURE_PROJECT + analyzer = WorkflowAnalyzer() + workflow = analyzer.analyze(workflow_path) + entry = workflow.jobs[INTEGRATION_JOB_ID].matrix[0] + + builder = ActCommandBuilder( + workflow_file=workflow_path, + project_dir=project_dir, + job_id=INTEGRATION_JOB_ID, + ) + cmd = builder.build(entry, image_tag=act_runner_image) + executor = JobExecutor(logs_dir=logs_dir) + result = executor.run( + cmd, + matrix_index=entry.index, + matrix_name=entry.name, + timeout=INTEGRATION_TIMEOUT, + stream_output=False, + ) + return result, entry + + +def test_successful_job_execution( + act_runner_image: str, + tmp_path: Path, +) -> None: + logs_dir = tmp_path / "logs" + result, _ = _build_and_run("test.yml", logs_dir, act_runner_image) + + assert result.status == JobStatus.PASSED + assert result.exit_code == 0 + assert result.stdout or result.stderr + assert result.log_file is not None + assert result.log_file.exists() + + +def test_failing_job_extract_error( + act_runner_image: str, + tmp_path: Path, +) -> None: + logs_dir = tmp_path / "logs" + result, _ = _build_and_run("test-fail.yml", logs_dir, act_runner_image) + + assert result.status == JobStatus.FAILED + assert result.exit_code is not None + assert result.exit_code != 0 + + captured = result.stderr or result.stdout + assert captured.strip() + + extracted = JobExecutor._extract_error(captured) + assert result.error_message == extracted + assert result.error_message is not None + + lower = result.error_message.lower() + assert any(kw in lower for kw in ("failed", "error", "exit")) + diff --git a/cli/tests/integration/test_localci_run.py b/cli/tests/integration/test_localci_run.py new file mode 100644 index 0000000..7640cbd --- /dev/null +++ b/cli/tests/integration/test_localci_run.py @@ -0,0 +1,98 @@ +"""Integration tests for ``localci run`` against real act/Docker.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +from click.testing import CliRunner + +from localci.cli.main import cli +from localci.core.executor import JobStatus +from localci.core.results import ExecutionSummary + +from .conftest import INTEGRATION_TIMEOUT + +pytestmark = pytest.mark.integration + +runner = CliRunner() + + +def _run_localci( + project: Path, + workflow: str, + extra_args: list[str] | None = None, +) -> object: + args = [ + "-c", + str(project / ".localci.yml"), + "run", + "--workflow", + workflow, + "--no-cache", + "--parallel", + "1", + "--timeout", + str(INTEGRATION_TIMEOUT), + ] + if extra_args: + args.extend(extra_args) + return runner.invoke(cli, args) + + +@pytest.mark.usefixtures("capy_image_tag") +def test_run_success( + integration_project: Path, + integration_logs_dir: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.chdir(integration_project) + result = _run_localci( + integration_project, + ".github/workflows/test.yml", + ) + + assert result.exit_code == 0, result.output + + last_run = integration_logs_dir / "last-run.json" + assert last_run.exists(), "expected last-run.json after successful run" + + summary = ExecutionSummary.load(last_run) + assert summary.all_passed + assert summary.total == 1 + assert summary.results[0].status == JobStatus.PASSED + + +@pytest.mark.usefixtures("capy_image_tag") +def test_run_failure( + integration_project: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.chdir(integration_project) + result = _run_localci( + integration_project, + ".github/workflows/test-fail.yml", + ) + + assert result.exit_code == 1, result.output + assert "Traceback" not in result.output + + lower = result.output.lower() + assert any(kw in lower for kw in ("failed", "error", "exit")) + + +def test_run_invalid_workflow( + integration_project: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Parse errors occur before act/Docker; no capy_image_tag fixture required.""" + monkeypatch.chdir(integration_project) + result = _run_localci( + integration_project, + ".github/workflows/invalid.yml", + ) + + assert result.exit_code == 1, result.output + assert "Traceback" not in result.output + lower = result.output.lower() + assert "parse" in lower or "workflow" in lower or "invalid" in lower From fb6a2649e8715797a81e4cd0ec2920c39291c033 Mon Sep 17 00:00:00 2001 From: mac Date: Fri, 29 May 2026 05:21:52 +0800 Subject: [PATCH 2/6] addressed ai reviews --- .github/workflows/ci.yml | 18 ++++++++++++----- cli/tests/integration/test_act_subprocess.py | 21 ++++++++++++-------- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 17ed8d8..f0edc7a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,10 +20,12 @@ jobs: working-directory: cli steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: ${{ matrix.python-version }} @@ -42,10 +44,12 @@ jobs: working-directory: cli steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false - name: Set up Python 3.12 - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.12" @@ -53,8 +57,12 @@ jobs: run: pip install -e ".[dev]" - name: Install act + env: + ACT_VERSION: v0.2.76 run: | - curl -sSfL https://raw.githubusercontent.com/nektos/act/master/install.sh | bash -s -- -b /usr/local/bin v0.2.76 + curl -sSfL -o /tmp/act-install.sh \ + "https://raw.githubusercontent.com/nektos/act/${ACT_VERSION}/install.sh" + bash /tmp/act-install.sh -b /usr/local/bin "${ACT_VERSION}" act --version - name: Pull act runner image diff --git a/cli/tests/integration/test_act_subprocess.py b/cli/tests/integration/test_act_subprocess.py index 4ca379d..29a0368 100644 --- a/cli/tests/integration/test_act_subprocess.py +++ b/cli/tests/integration/test_act_subprocess.py @@ -10,22 +10,22 @@ from localci.core.executor import JobExecutor, JobStatus from localci.core.workflow import WorkflowAnalyzer -from .conftest import FIXTURE_PROJECT, INTEGRATION_JOB_ID, INTEGRATION_TIMEOUT +from .conftest import INTEGRATION_JOB_ID, INTEGRATION_TIMEOUT pytestmark = pytest.mark.integration -def _workflow_path(name: str) -> Path: - return FIXTURE_PROJECT / ".github/workflows" / name +def _workflow_path(name: str, project_dir: Path) -> Path: + return project_dir / ".github/workflows" / name def _build_and_run( workflow_name: str, logs_dir: Path, act_runner_image: str, + project_dir: Path, ) -> tuple: - workflow_path = _workflow_path(workflow_name) - project_dir = FIXTURE_PROJECT + workflow_path = _workflow_path(workflow_name, project_dir) analyzer = WorkflowAnalyzer() workflow = analyzer.analyze(workflow_path) entry = workflow.jobs[INTEGRATION_JOB_ID].matrix[0] @@ -48,11 +48,14 @@ def _build_and_run( def test_successful_job_execution( + integration_project: Path, act_runner_image: str, tmp_path: Path, ) -> None: logs_dir = tmp_path / "logs" - result, _ = _build_and_run("test.yml", logs_dir, act_runner_image) + result, _ = _build_and_run( + "test.yml", logs_dir, act_runner_image, integration_project + ) assert result.status == JobStatus.PASSED assert result.exit_code == 0 @@ -62,11 +65,14 @@ def test_successful_job_execution( def test_failing_job_extract_error( + integration_project: Path, act_runner_image: str, tmp_path: Path, ) -> None: logs_dir = tmp_path / "logs" - result, _ = _build_and_run("test-fail.yml", logs_dir, act_runner_image) + result, _ = _build_and_run( + "test-fail.yml", logs_dir, act_runner_image, integration_project + ) assert result.status == JobStatus.FAILED assert result.exit_code is not None @@ -81,4 +87,3 @@ def test_failing_job_extract_error( lower = result.error_message.lower() assert any(kw in lower for kw in ("failed", "error", "exit")) - From cdbab8e25bef55f959f7a7a0198c74c6f9fc95ff Mon Sep 17 00:00:00 2001 From: mac Date: Fri, 29 May 2026 05:33:40 +0800 Subject: [PATCH 3/6] addressed AI reviews --- .github/workflows/ci.yml | 11 ++++-- cli/tests/integration/conftest.py | 20 ++++------ cli/tests/integration/test_act_subprocess.py | 18 ++++----- cli/tests/integration/test_localci_run.py | 39 +++++++++----------- 4 files changed, 43 insertions(+), 45 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f0edc7a..1ce4fb7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,6 +38,8 @@ jobs: integration: name: Integration (act + Docker) runs-on: ubuntu-latest + needs: [test] + timeout-minutes: 30 defaults: run: @@ -59,10 +61,13 @@ jobs: - name: Install act env: ACT_VERSION: v0.2.76 + # From https://github.com/nektos/act/releases/download/v0.2.76/checksums.txt + ACT_LINUX_X86_64_SHA256: d4720e05e73ce634239fc0e7fa6b552f9ac79b4e6c878a8756bbfacce3c56720 run: | - curl -sSfL -o /tmp/act-install.sh \ - "https://raw.githubusercontent.com/nektos/act/${ACT_VERSION}/install.sh" - bash /tmp/act-install.sh -b /usr/local/bin "${ACT_VERSION}" + curl -sSfL -o /tmp/act.tar.gz \ + "https://github.com/nektos/act/releases/download/${ACT_VERSION}/act_Linux_x86_64.tar.gz" + echo "${ACT_LINUX_X86_64_SHA256} /tmp/act.tar.gz" | sha256sum --check + sudo tar xzf /tmp/act.tar.gz -C /usr/local/bin act act --version - name: Pull act runner image diff --git a/cli/tests/integration/conftest.py b/cli/tests/integration/conftest.py index 7c8d316..7c3b6b7 100644 --- a/cli/tests/integration/conftest.py +++ b/cli/tests/integration/conftest.py @@ -19,6 +19,7 @@ ACT_RUNNER_IMAGE = "catthehacker/ubuntu:act-24.04" INTEGRATION_JOB_ID = "test" INTEGRATION_TIMEOUT = 180 +DOCKER_PULL_TIMEOUT = 600 def _act_available() -> bool: @@ -49,7 +50,7 @@ def act_runner_image(require_act_and_docker: None) -> str: ["docker", "pull", ACT_RUNNER_IMAGE], capture_output=True, text=True, - timeout=600, + timeout=DOCKER_PULL_TIMEOUT, ) if pull.returncode != 0: pytest.skip( @@ -84,8 +85,8 @@ def capy_image_tag(act_runner_image: str, require_act_and_docker: None) -> str: @pytest.fixture -def integration_project(tmp_path: Path) -> Path: - """Copy the integration fixture project into an isolated directory.""" +def integration_project(tmp_path: Path) -> tuple[Path, Path]: + """Copy fixture project and return (project_root, logs_dir).""" dest = tmp_path / "project" shutil.copytree(FIXTURE_PROJECT, dest) @@ -95,13 +96,8 @@ def integration_project(tmp_path: Path) -> Path: config_path = dest / ".localci.yml" config = yaml.safe_load(config_path.read_text()) config["logging"]["directory"] = str(logs_dir) - config_path.write_text(yaml.dump(config, default_flow_style=False)) - - return dest - + config_path.write_text( + yaml.dump(config, default_flow_style=False, sort_keys=False) + ) -@pytest.fixture -def integration_logs_dir(tmp_path: Path, integration_project: Path) -> Path: - """Logs directory configured for the copied integration project.""" - _ = integration_project - return tmp_path / "logs" + return dest, logs_dir diff --git a/cli/tests/integration/test_act_subprocess.py b/cli/tests/integration/test_act_subprocess.py index 29a0368..33df8fc 100644 --- a/cli/tests/integration/test_act_subprocess.py +++ b/cli/tests/integration/test_act_subprocess.py @@ -7,8 +7,8 @@ import pytest from localci.core.command_builder import ActCommandBuilder -from localci.core.executor import JobExecutor, JobStatus -from localci.core.workflow import WorkflowAnalyzer +from localci.core.executor import JobExecutor, JobResult, JobStatus +from localci.core.workflow import MatrixEntry, WorkflowAnalyzer from .conftest import INTEGRATION_JOB_ID, INTEGRATION_TIMEOUT @@ -24,7 +24,7 @@ def _build_and_run( logs_dir: Path, act_runner_image: str, project_dir: Path, -) -> tuple: +) -> tuple[JobResult, MatrixEntry]: workflow_path = _workflow_path(workflow_name, project_dir) analyzer = WorkflowAnalyzer() workflow = analyzer.analyze(workflow_path) @@ -48,14 +48,13 @@ def _build_and_run( def test_successful_job_execution( - integration_project: Path, + integration_project: tuple[Path, Path], act_runner_image: str, tmp_path: Path, ) -> None: + project, _ = integration_project logs_dir = tmp_path / "logs" - result, _ = _build_and_run( - "test.yml", logs_dir, act_runner_image, integration_project - ) + result, _ = _build_and_run("test.yml", logs_dir, act_runner_image, project) assert result.status == JobStatus.PASSED assert result.exit_code == 0 @@ -65,13 +64,14 @@ def test_successful_job_execution( def test_failing_job_extract_error( - integration_project: Path, + integration_project: tuple[Path, Path], act_runner_image: str, tmp_path: Path, ) -> None: + project, _ = integration_project logs_dir = tmp_path / "logs" result, _ = _build_and_run( - "test-fail.yml", logs_dir, act_runner_image, integration_project + "test-fail.yml", logs_dir, act_runner_image, project ) assert result.status == JobStatus.FAILED diff --git a/cli/tests/integration/test_localci_run.py b/cli/tests/integration/test_localci_run.py index 7640cbd..2112623 100644 --- a/cli/tests/integration/test_localci_run.py +++ b/cli/tests/integration/test_localci_run.py @@ -42,19 +42,16 @@ def _run_localci( @pytest.mark.usefixtures("capy_image_tag") def test_run_success( - integration_project: Path, - integration_logs_dir: Path, + integration_project: tuple[Path, Path], monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.chdir(integration_project) - result = _run_localci( - integration_project, - ".github/workflows/test.yml", - ) + project, logs_dir = integration_project + monkeypatch.chdir(project) + result = _run_localci(project, ".github/workflows/test.yml") assert result.exit_code == 0, result.output - last_run = integration_logs_dir / "last-run.json" + last_run = logs_dir / "last-run.json" assert last_run.exists(), "expected last-run.json after successful run" summary = ExecutionSummary.load(last_run) @@ -65,14 +62,12 @@ def test_run_success( @pytest.mark.usefixtures("capy_image_tag") def test_run_failure( - integration_project: Path, + integration_project: tuple[Path, Path], monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.chdir(integration_project) - result = _run_localci( - integration_project, - ".github/workflows/test-fail.yml", - ) + project, _logs_dir = integration_project + monkeypatch.chdir(project) + result = _run_localci(project, ".github/workflows/test-fail.yml") assert result.exit_code == 1, result.output assert "Traceback" not in result.output @@ -82,15 +77,17 @@ def test_run_failure( def test_run_invalid_workflow( - integration_project: Path, + integration_project: tuple[Path, Path], monkeypatch: pytest.MonkeyPatch, ) -> None: - """Parse errors occur before act/Docker; no capy_image_tag fixture required.""" - monkeypatch.chdir(integration_project) - result = _run_localci( - integration_project, - ".github/workflows/invalid.yml", - ) + """Workflow parse fails before act/Docker; intentionally omits require_act_and_docker. + + Kept in this package (not unit tests) to assert the full CLI path exits cleanly + without a traceback when given a malformed workflow file. + """ + project, _logs_dir = integration_project + monkeypatch.chdir(project) + result = _run_localci(project, ".github/workflows/invalid.yml") assert result.exit_code == 1, result.output assert "Traceback" not in result.output From 2e71982f930d762f77a432c2ebb77ae68c887836 Mon Sep 17 00:00:00 2001 From: mac Date: Fri, 29 May 2026 23:44:30 +0800 Subject: [PATCH 4/6] Addressed all of Brad's reviews --- .github/workflows/ci.yml | 3 -- cli/README.md | 8 +++-- cli/tests/integration/conftest.py | 37 +++++++++++++------- cli/tests/integration/test_act_subprocess.py | 8 ++--- cli/tests/integration/test_localci_run.py | 21 ++++++++--- 5 files changed, 49 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1ce4fb7..8f76aa2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,8 +70,5 @@ jobs: sudo tar xzf /tmp/act.tar.gz -C /usr/local/bin act act --version - - name: Pull act runner image - run: docker pull catthehacker/ubuntu:act-24.04 - - name: Run integration tests run: pytest tests/integration -m integration -v --tb=short diff --git a/cli/README.md b/cli/README.md index 56743f1..57318b6 100644 --- a/cli/README.md +++ b/cli/README.md @@ -105,13 +105,15 @@ cli/ # Editable install with dev dependencies pip install -e ".[dev]" -# Run unit tests (default; integration tests are excluded) +# Run unit tests (default). Integration tests are excluded via norecursedirs +# (pytest does not enter tests/integration/). Use -m "not integration" for +# coverage or if integration tests move outside that directory name. pytest -# Run unit tests with coverage +# Run unit tests with coverage (CI unit job uses this marker) pytest --cov=localci -m "not integration" -# Integration tests (requires act + Docker) +# Integration tests (requires act + Docker; path + marker required) pytest tests/integration -m integration ``` diff --git a/cli/tests/integration/conftest.py b/cli/tests/integration/conftest.py index 7c3b6b7..3917523 100644 --- a/cli/tests/integration/conftest.py +++ b/cli/tests/integration/conftest.py @@ -20,6 +20,7 @@ INTEGRATION_JOB_ID = "test" INTEGRATION_TIMEOUT = 180 DOCKER_PULL_TIMEOUT = 600 +DOCKER_TAG_TIMEOUT = 60 def _act_available() -> bool: @@ -46,12 +47,17 @@ def require_act_and_docker() -> None: @pytest.fixture(scope="session") def act_runner_image(require_act_and_docker: None) -> str: """Pull the act runner image used for ubuntu-latest jobs.""" - pull = subprocess.run( - ["docker", "pull", ACT_RUNNER_IMAGE], - capture_output=True, - text=True, - timeout=DOCKER_PULL_TIMEOUT, - ) + try: + pull = subprocess.run( + ["docker", "pull", ACT_RUNNER_IMAGE], + capture_output=True, + text=True, + timeout=DOCKER_PULL_TIMEOUT, + ) + except subprocess.TimeoutExpired: + pytest.skip( + f"timed out pulling {ACT_RUNNER_IMAGE} after {DOCKER_PULL_TIMEOUT}s" + ) if pull.returncode != 0: pytest.skip( f"could not pull {ACT_RUNNER_IMAGE}: " @@ -65,17 +71,24 @@ def capy_image_tag(act_runner_image: str, require_act_and_docker: None) -> str: """Tag the act runner image as the derived capy name for localci run.""" from localci.core.workflow import WorkflowAnalyzer + # Derive tag from test.yml; test-fail.yml uses the same matrix.include shape today. + # If failure fixture matrix diverges, derive from that workflow (or both) instead. workflow_path = FIXTURE_PROJECT / ".github/workflows/test.yml" entry = WorkflowAnalyzer().analyze(workflow_path).jobs[INTEGRATION_JOB_ID].matrix[0] tag = derive_image_tag(entry) assert tag is not None - tag_result = subprocess.run( - ["docker", "tag", act_runner_image, tag], - capture_output=True, - text=True, - timeout=60, - ) + try: + tag_result = subprocess.run( + ["docker", "tag", act_runner_image, tag], + capture_output=True, + text=True, + timeout=DOCKER_TAG_TIMEOUT, + ) + except subprocess.TimeoutExpired: + pytest.skip( + f"timed out tagging {act_runner_image} as {tag} after {DOCKER_TAG_TIMEOUT}s" + ) if tag_result.returncode != 0: pytest.skip( f"could not tag {act_runner_image} as {tag}: " diff --git a/cli/tests/integration/test_act_subprocess.py b/cli/tests/integration/test_act_subprocess.py index 33df8fc..d382480 100644 --- a/cli/tests/integration/test_act_subprocess.py +++ b/cli/tests/integration/test_act_subprocess.py @@ -50,10 +50,8 @@ def _build_and_run( def test_successful_job_execution( integration_project: tuple[Path, Path], act_runner_image: str, - tmp_path: Path, ) -> None: - project, _ = integration_project - logs_dir = tmp_path / "logs" + project, logs_dir = integration_project result, _ = _build_and_run("test.yml", logs_dir, act_runner_image, project) assert result.status == JobStatus.PASSED @@ -66,10 +64,8 @@ def test_successful_job_execution( def test_failing_job_extract_error( integration_project: tuple[Path, Path], act_runner_image: str, - tmp_path: Path, ) -> None: - project, _ = integration_project - logs_dir = tmp_path / "logs" + project, logs_dir = integration_project result, _ = _build_and_run( "test-fail.yml", logs_dir, act_runner_image, project ) diff --git a/cli/tests/integration/test_localci_run.py b/cli/tests/integration/test_localci_run.py index 2112623..7b2ca97 100644 --- a/cli/tests/integration/test_localci_run.py +++ b/cli/tests/integration/test_localci_run.py @@ -8,7 +8,7 @@ from click.testing import CliRunner from localci.cli.main import cli -from localci.core.executor import JobStatus +from localci.core.executor import JobExecutor, JobStatus from localci.core.results import ExecutionSummary from .conftest import INTEGRATION_TIMEOUT @@ -65,15 +65,28 @@ def test_run_failure( integration_project: tuple[Path, Path], monkeypatch: pytest.MonkeyPatch, ) -> None: - project, _logs_dir = integration_project + project, logs_dir = integration_project monkeypatch.chdir(project) result = _run_localci(project, ".github/workflows/test-fail.yml") assert result.exit_code == 1, result.output assert "Traceback" not in result.output - lower = result.output.lower() - assert any(kw in lower for kw in ("failed", "error", "exit")) + last_run = logs_dir / "last-run.json" + assert last_run.exists(), "expected last-run.json after failed run" + + summary = ExecutionSummary.load(last_run) + assert summary.total == 1 + job = summary.results[0] + assert job.status == JobStatus.FAILED + assert job.error_message + + if job.log_file and job.log_file.exists(): + captured = job.log_file.read_text(encoding="utf-8", errors="replace") + else: + captured = result.output + + assert job.error_message == JobExecutor._extract_error(captured) def test_run_invalid_workflow( From d8e4b2a1889d0cf5c580750a5345aff00fce2fe6 Mon Sep 17 00:00:00 2001 From: mac Date: Fri, 29 May 2026 23:51:08 +0800 Subject: [PATCH 5/6] added _executor_capture_from_log_text to fix integration error --- cli/tests/integration/test_localci_run.py | 29 +++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/cli/tests/integration/test_localci_run.py b/cli/tests/integration/test_localci_run.py index 7b2ca97..68eff5d 100644 --- a/cli/tests/integration/test_localci_run.py +++ b/cli/tests/integration/test_localci_run.py @@ -18,6 +18,25 @@ runner = CliRunner() +def _executor_capture_from_log_text(log_text: str) -> str: + """Approximate the stderr-or-stdout string passed to ``_extract_error``. + + Job logs interleave streams and include act summary lines that are not in + the stderr-only capture the executor uses when stderr is non-empty. + """ + body_lines = [ + line for line in log_text.splitlines() if not line.startswith("#") + ] + stderr_like = [ + line + for line in body_lines + if "error:" in line.lower() or "fatal:" in line.lower() + ] + if stderr_like: + return "\n".join(stderr_like) + return "\n".join(body_lines) + + def _run_localci( project: Path, workflow: str, @@ -81,12 +100,14 @@ def test_run_failure( assert job.status == JobStatus.FAILED assert job.error_message + assert job.error_message if job.log_file and job.log_file.exists(): - captured = job.log_file.read_text(encoding="utf-8", errors="replace") + log_text = job.log_file.read_text(encoding="utf-8", errors="replace") + captured = _executor_capture_from_log_text(log_text) + assert job.error_message == JobExecutor._extract_error(captured) + assert job.error_message.splitlines()[0] in log_text else: - captured = result.output - - assert job.error_message == JobExecutor._extract_error(captured) + assert job.error_message in JobExecutor._extract_error(result.output) def test_run_invalid_workflow( From e43d1e2ec4b9c35caa9e1b6c9fd1ea794d396cba Mon Sep 17 00:00:00 2001 From: mac Date: Sat, 30 May 2026 00:02:33 +0800 Subject: [PATCH 6/6] addressed ai review --- cli/tests/integration/test_localci_run.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cli/tests/integration/test_localci_run.py b/cli/tests/integration/test_localci_run.py index 68eff5d..52f8dbc 100644 --- a/cli/tests/integration/test_localci_run.py +++ b/cli/tests/integration/test_localci_run.py @@ -100,11 +100,10 @@ def test_run_failure( assert job.status == JobStatus.FAILED assert job.error_message - assert job.error_message if job.log_file and job.log_file.exists(): log_text = job.log_file.read_text(encoding="utf-8", errors="replace") captured = _executor_capture_from_log_text(log_text) - assert job.error_message == JobExecutor._extract_error(captured) + assert job.error_message in JobExecutor._extract_error(captured) assert job.error_message.splitlines()[0] in log_text else: assert job.error_message in JobExecutor._extract_error(result.output)