Add act/Docker integration tests for subprocess boundary and _extract…

.github/workflows/ci.yml

@@ -20,15 +20,55 @@ jobs:
         working-directory: cli
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        with:
+          persist-credentials: false
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
           python-version: ${{ matrix.python-version }}
 
       - name: Install dependencies
         run: pip install -e ".[dev]"
 
       - name: Run tests
-        run: pytest --cov=localci --cov-report=term-missing
+        run: pytest --cov=localci --cov-report=term-missing -m "not integration"
+
+  integration:
+    name: Integration (act + Docker)
+    runs-on: ubuntu-latest
+    needs: [test]
+    timeout-minutes: 30
+
+    defaults:
+      run:
+        working-directory: cli
+
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        with:
+          persist-credentials: false
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: pip install -e ".[dev]"
+
+      - name: Install act
+        env:
+          ACT_VERSION: v0.2.76
+          # From https://github.com/nektos/act/releases/download/v0.2.76/checksums.txt
+          ACT_LINUX_X86_64_SHA256: d4720e05e73ce634239fc0e7fa6b552f9ac79b4e6c878a8756bbfacce3c56720
+        run: |
+          curl -sSfL -o /tmp/act.tar.gz \
+            "https://github.com/nektos/act/releases/download/${ACT_VERSION}/act_Linux_x86_64.tar.gz"
+          echo "${ACT_LINUX_X86_64_SHA256}  /tmp/act.tar.gz" | sha256sum --check
+          sudo tar xzf /tmp/act.tar.gz -C /usr/local/bin act
+          act --version
+
+      - name: Run integration tests
+        run: pytest tests/integration -m integration -v --tb=short

cli/README.md

@@ -105,11 +105,16 @@ cli/
 # Editable install with dev dependencies
 pip install -e ".[dev]"
 
-# Run tests
+# Run unit tests (default). Integration tests are excluded via norecursedirs
+# (pytest does not enter tests/integration/). Use -m "not integration" for
+# coverage or if integration tests move outside that directory name.
 pytest
 
-# Run tests with coverage
-pytest --cov=localci
+# Run unit tests with coverage (CI unit job uses this marker)
+pytest --cov=localci -m "not integration"
+
+# Integration tests (requires act + Docker; path + marker required)
+pytest tests/integration -m integration
 ```
 
 ## License

cli/pyproject.toml

@@ -31,3 +31,7 @@ packages = ["localci"]
 [tool.pytest.ini_options]
 testpaths = ["tests"]
 addopts = "-v --tb=short"
+markers = [
+    "integration: requires act and Docker; runs real subprocesses",
+]
+norecursedirs = ["integration"]

cli/tests/fixtures/integration/project/.github/workflows/invalid.yml

@@ -0,0 +1,5 @@
+name: Invalid
+on: [push
+jobs:
+  test:
+    runs-on: ubuntu-latest

cli/tests/fixtures/integration/project/.github/workflows/test-fail.yml

@@ -0,0 +1,15 @@
+name: Integration Failure
+on: [push]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - name: "Hello"
+            runs-on: ubuntu-latest
+            compiler: gcc
+            version: "15"
+    steps:
+      - run: exit 1

cli/tests/fixtures/integration/project/.github/workflows/test.yml

@@ -0,0 +1,15 @@
+name: Integration Success
+on: [push]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - name: "Hello"
+            runs-on: ubuntu-latest
+            compiler: gcc
+            version: "15"
+    steps:
+      - run: echo "Hello"

cli/tests/fixtures/integration/project/.localci.yml

@@ -0,0 +1,26 @@
+version: 1
+workflow: .github/workflows/test.yml
+event: push
+
+parallel:
+  max_jobs: 1
+
+platforms:
+  linux: true
+  windows: false
+  macos: false
+
+images:
+  auto_build: false
+
+cache:
+  enabled: false
+
+logging:
+  level: warning
+  directory: logs
+
+execution:
+  timeout: 120
+  keep_containers: false
+  stop_on_first_failure: false

cli/tests/integration/conftest.py

@@ -0,0 +1,116 @@
+"""Shared fixtures for act/Docker integration tests."""
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+from pathlib import Path
+
+import pytest
+import yaml
+
+from localci.core.executor import JobExecutor
+from localci.core.image_tag import derive_image_tag
+from localci.errors import DockerNotAvailableError
+
+FIXTURE_PROJECT = (
+    Path(__file__).resolve().parent.parent / "fixtures" / "integration" / "project"
+)
+ACT_RUNNER_IMAGE = "catthehacker/ubuntu:act-24.04"
+INTEGRATION_JOB_ID = "test"
+INTEGRATION_TIMEOUT = 180
+DOCKER_PULL_TIMEOUT = 600
+DOCKER_TAG_TIMEOUT = 60
+
+
+def _act_available() -> bool:
+    return JobExecutor().has_act
+
+
+def _docker_available() -> bool:
+    try:
+        JobExecutor().check_docker()
+        return True
+    except DockerNotAvailableError:
+        return False
+
+
+@pytest.fixture(scope="session")
+def require_act_and_docker() -> None:
+    """Skip the entire integration session when act or Docker is unavailable."""
+    if not _act_available():
+        pytest.skip("act is not installed")
+    if not _docker_available():
+        pytest.skip("Docker is not available")
+
+
+@pytest.fixture(scope="session")
+def act_runner_image(require_act_and_docker: None) -> str:
+    """Pull the act runner image used for ubuntu-latest jobs."""
+    try:
+        pull = subprocess.run(
+            ["docker", "pull", ACT_RUNNER_IMAGE],
+            capture_output=True,
+            text=True,
+            timeout=DOCKER_PULL_TIMEOUT,
+        )
+    except subprocess.TimeoutExpired:
+        pytest.skip(
+            f"timed out pulling {ACT_RUNNER_IMAGE} after {DOCKER_PULL_TIMEOUT}s"
+        )
+    if pull.returncode != 0:
+        pytest.skip(
+            f"could not pull {ACT_RUNNER_IMAGE}: "
+            f"{pull.stderr.strip() or pull.stdout.strip()}"
+        )
+    return ACT_RUNNER_IMAGE
+
+
+@pytest.fixture(scope="session")
+def capy_image_tag(act_runner_image: str, require_act_and_docker: None) -> str:
+    """Tag the act runner image as the derived capy name for localci run."""
+    from localci.core.workflow import WorkflowAnalyzer
+
+    # Derive tag from test.yml; test-fail.yml uses the same matrix.include shape today.
+    # If failure fixture matrix diverges, derive from that workflow (or both) instead.
+    workflow_path = FIXTURE_PROJECT / ".github/workflows/test.yml"
+    entry = WorkflowAnalyzer().analyze(workflow_path).jobs[INTEGRATION_JOB_ID].matrix[0]
+    tag = derive_image_tag(entry)
+    assert tag is not None
+
+    try:
+        tag_result = subprocess.run(
+            ["docker", "tag", act_runner_image, tag],
+            capture_output=True,
+            text=True,
+            timeout=DOCKER_TAG_TIMEOUT,
+        )
+    except subprocess.TimeoutExpired:
+        pytest.skip(
+            f"timed out tagging {act_runner_image} as {tag} after {DOCKER_TAG_TIMEOUT}s"
+        )
+    if tag_result.returncode != 0:
+        pytest.skip(
+            f"could not tag {act_runner_image} as {tag}: "
+            f"{tag_result.stderr.strip() or tag_result.stdout.strip()}"
+        )
+    return tag
+
+
+@pytest.fixture
+def integration_project(tmp_path: Path) -> tuple[Path, Path]:
+    """Copy fixture project and return (project_root, logs_dir)."""
+    dest = tmp_path / "project"
+    shutil.copytree(FIXTURE_PROJECT, dest)
+
+    logs_dir = tmp_path / "logs"
+    logs_dir.mkdir(parents=True, exist_ok=True)
+
+    config_path = dest / ".localci.yml"
+    config = yaml.safe_load(config_path.read_text())
+    config["logging"]["directory"] = str(logs_dir)
+    config_path.write_text(
+        yaml.dump(config, default_flow_style=False, sort_keys=False)
+    )
+
+    return dest, logs_dir

cli/tests/integration/test_act_subprocess.py

@@ -0,0 +1,85 @@
+"""Integration tests for JobExecutor + act subprocess boundary."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from localci.core.command_builder import ActCommandBuilder
+from localci.core.executor import JobExecutor, JobResult, JobStatus
+from localci.core.workflow import MatrixEntry, WorkflowAnalyzer
+
+from .conftest import INTEGRATION_JOB_ID, INTEGRATION_TIMEOUT
+
+pytestmark = pytest.mark.integration
+
+
+def _workflow_path(name: str, project_dir: Path) -> Path:
+    return project_dir / ".github/workflows" / name
+
+
+def _build_and_run(
+    workflow_name: str,
+    logs_dir: Path,
+    act_runner_image: str,
+    project_dir: Path,
+) -> tuple[JobResult, MatrixEntry]:
+    workflow_path = _workflow_path(workflow_name, project_dir)
+    analyzer = WorkflowAnalyzer()
+    workflow = analyzer.analyze(workflow_path)
+    entry = workflow.jobs[INTEGRATION_JOB_ID].matrix[0]
+
+    builder = ActCommandBuilder(
+        workflow_file=workflow_path,
+        project_dir=project_dir,
+        job_id=INTEGRATION_JOB_ID,
+    )
+    cmd = builder.build(entry, image_tag=act_runner_image)
+    executor = JobExecutor(logs_dir=logs_dir)
+    result = executor.run(
+        cmd,
+        matrix_index=entry.index,
+        matrix_name=entry.name,
+        timeout=INTEGRATION_TIMEOUT,
+        stream_output=False,
+    )
+    return result, entry
+
+
+def test_successful_job_execution(
+    integration_project: tuple[Path, Path],
+    act_runner_image: str,
+) -> None:
+    project, logs_dir = integration_project
+    result, _ = _build_and_run("test.yml", logs_dir, act_runner_image, project)
+
+    assert result.status == JobStatus.PASSED
+    assert result.exit_code == 0
+    assert result.stdout or result.stderr
+    assert result.log_file is not None
+    assert result.log_file.exists()
+
+
+def test_failing_job_extract_error(
+    integration_project: tuple[Path, Path],
+    act_runner_image: str,
+) -> None:
+    project, logs_dir = integration_project
+    result, _ = _build_and_run(
+        "test-fail.yml", logs_dir, act_runner_image, project
+    )
+
+    assert result.status == JobStatus.FAILED
+    assert result.exit_code is not None
+    assert result.exit_code != 0
+
+    captured = result.stderr or result.stdout
+    assert captured.strip()
+
+    extracted = JobExecutor._extract_error(captured)
+    assert result.error_message == extracted
+    assert result.error_message is not None
+
+    lower = result.error_message.lower()
+    assert any(kw in lower for kw in ("failed", "error", "exit"))